From a5c164a6ae6fda750f8052923f7174f7100a52c5 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 9 Mar 2021 11:16:22 +0100 Subject: [PATCH 001/107] Add configurable offset --- slo_generator/cli.py | 2 +- slo_generator/report.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/slo_generator/cli.py b/slo_generator/cli.py index 5fd47033..d4dc1512 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -121,7 +121,7 @@ def parse_args(args): '-t', type=int, default=None, - help="End timestamp for query.") + help="Start timestamp for query.") return parser.parse_args(args) diff --git a/slo_generator/report.py b/slo_generator/report.py index d36cfeb8..11e054f6 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -66,6 +66,7 @@ class SLOReport: # Error Budget step config timestamp: int timestamp_human: str + offset: int = 0 window: int alert: bool alerting_burn_rate_threshold: float @@ -190,6 +191,14 @@ def run_backend(self, config, client=None, delete=False): method = instance.delete LOGGER.warning(f'{info} | Delete mode enabled.') + # Set offset from class attribute if it exists in the class, otherwise + # keep the value defined in config. + self.offset = max(cls.getattr('DEFAULT_OFFSET', 0), self.offset) + LOGGER.debug(f'{info} | Running with offset {self.offset}s') + + # Substract offset from start timestamp + self.timestamp = self.timestamp - self.offset + # Run backend method and return results. data = method(self.timestamp, self.window, config) LOGGER.debug(f'{info} | Backend response: {data}') From c0fc041dabf50d5cd35b980827b31df2e21966bb Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Mon, 31 May 2021 09:37:48 +0200 Subject: [PATCH 002/107] chore: release 1.5.1 (#118) --- CHANGELOG.md | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaaae893..ade52e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +### [1.5.1](https://www.github.com/google/slo-generator/compare/v1.5.0...v1.5.1) (2021-02-12) + + +### Bug Fixes + +* broken setuptools ([#117](https://www.github.com/google/slo-generator/issues/117)) ([f1fa346](https://www.github.com/google/slo-generator/commit/f1fa346d2b8ae618b85a44d3683aa04377bba85f)) + ## [1.5.0](https://www.github.com/google/slo-generator/compare/v1.4.1...v1.5.0) (2021-01-12) diff --git a/setup.py b/setup.py index c96b7bb7..92b30cd9 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ # Package metadata. name = "slo-generator" description = "SLO Generator" -version = "1.5.0" +version = "1.5.1" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' From ac3ee2fed2dacc942200171335e8340ffc8607ba Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 12:53:17 +0200 Subject: [PATCH 003/107] ci: Update CI steps (#136) * Update CI builds * Update Dockerfile, improve CloudBuild deployment * Add CI test configs --- .github/workflows/build.yml | 12 +++-- .github/workflows/release.yml | 2 +- Dockerfile | 16 +++--- cloudbuild.yaml | 9 ++-- tests/unit/fixtures/ci_config.yaml | 56 +++++++++++++++++++++ tests/unit/fixtures/ci_knative_service.yaml | 35 +++++++++++++ tests/unit/fixtures/ci_slo.json | 26 ++++++++++ 7 files changed, 139 insertions(+), 17 deletions(-) create mode 100644 tests/unit/fixtures/ci_config.yaml create mode 100644 tests/unit/fixtures/ci_knative_service.yaml create mode 100644 tests/unit/fixtures/ci_slo.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08b141b1..feef9d8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,10 +15,10 @@ jobs: architecture: 'x64' - name: Install dependencies - run: make install_test install + run: make install - name: Run lint test - run: make flake8 pylint + run: make lint unit: runs-on: ubuntu-latest @@ -30,21 +30,23 @@ jobs: architecture: 'x64' - name: Install dependencies - run: make install_test install + run: make install - name: Run unittests - run: make unittest + run: make unit env: MIN_VALID_EVENTS: "10" GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json - name: Run coverage report - run: make coverage_report + run: make coverage docker: runs-on: ubuntu-latest steps: - uses: actions/checkout@master - uses: docker-practice/actions-setup-docker@master + - name: Build Docker image + run: make docker_build - name: Run Docker tests run: make docker_test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0bbc9bf8..38b7d168 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: architecture: 'x64' - name: Run all tests - run: make all + run: make env: MIN_VALID_EVENTS: "10" GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json diff --git a/Dockerfile b/Dockerfile index e6253f1c..c7550d8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,15 +15,15 @@ FROM python:3.7-slim RUN apt-get update && \ apt-get install -y \ - build-essential \ - make \ - gcc \ - locales \ - libgdal20 \ - libgdal-dev + build-essential \ + make \ + gcc \ + locales \ + libgdal20 \ + libgdal-dev ADD . /app WORKDIR /app RUN pip install -U setuptools -RUN pip install . -CMD ["make"] +RUN pip install ."[api, datadog, dynatrace, prometheus, elasticsearch, pubsub, cloud_monitoring, cloud_service_monitoring, cloud_storage, bigquery, dev]" ENTRYPOINT [ "slo-generator" ] +CMD ["-v"] diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 7d43ab6c..9d70a771 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -16,14 +16,17 @@ steps: - name: gcr.io/cloud-builders/docker id: Build SLO generator - args: ['build', '-t', 'gcr.io/$PROJECT_ID/slo-generator', '.'] + args: ['build', '-t', 'gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION}', '.'] -- name: gcr.io/$PROJECT_ID/slo-generator +- name: gcr.io/$_GCR_PROJECT_ID/slo-generator id: Run all tests entrypoint: make env: - 'MIN_VALID_EVENTS=10' args: [] +substitutions: + _VERSION: latest + images: -- 'gcr.io/$PROJECT_ID/slo-generator' +- 'gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION}' diff --git a/tests/unit/fixtures/ci_config.yaml b/tests/unit/fixtures/ci_config.yaml new file mode 100644 index 00000000..dc4c6198 --- /dev/null +++ b/tests/unit/fixtures/ci_config.yaml @@ -0,0 +1,56 @@ +--- +backends: + cloud_monitoring: + project_id: ${CLOUDRUN_PROJECT_ID} + +exporters: + cloud_monitoring: + project_id: ${CLOUDRUN_PROJECT_ID} + +error_budget_policies: + default: + steps: + - name: 1 hour + burn_rate_threshold: 9 + alert: true + message_alert: Page to defend the SLO + message_ok: Last hour on track + window: 3600 + - name: 12 hours + burn_rate_threshold: 3 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 12 hours on track + window: 43200 + - name: 7 days + burn_rate_threshold: 1.5 + alert: false + message_alert: Dev team dedicates 25% of engineers to the reliability backlog + message_ok: Last week on track + window: 604800 + - name: 28 days + burn_rate_threshold: 1 + alert: false + message_alert: Freeze release, unless related to reliability or security + message_ok: Unfreeze release, per the agreed roll-out policy + window: 2419200 + cloud_service_monitoring: + steps: + - name: 24 hours + burn_rate_threshold: 3 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 24 hours on track + window: 86400 + - name: 7 days + burn_rate_threshold: 1.5 + alert: false + message_alert: Dev team dedicates 25% of engineers to the reliability backlog + message_ok: Last week on track + window: 604800 + - name: 28 days + burn_rate_threshold: 1 + alert: false + message_alert: Freeze release, unless related to reliability or security + message_ok: Unfreeze release, per the agreed roll-out policy + window: 2419200 diff --git a/tests/unit/fixtures/ci_knative_service.yaml b/tests/unit/fixtures/ci_knative_service.yaml new file mode 100644 index 00000000..8ded03e5 --- /dev/null +++ b/tests/unit/fixtures/ci_knative_service.yaml @@ -0,0 +1,35 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: slo-generator +spec: + template: + metadata: + name: slo-generator + annotations: + client.knative.dev/user-image: gcr.io/${PROJECT_ID}/slo-generator:${VERSION} + autoscaling.knative.dev/minScale: '1' + autoscaling.knative.dev/maxScale: '100' + spec: + containerConcurrency: 80 + timeoutSeconds: 300 + containers: + - image: gcr.io/${PROJECT_ID}/slo-generator:${VERSION} + command: + - slo-generator + args: + - api + - --signature-type=http + ports: + - name: http1 + containerPort: 8080 + env: + - name: CONFIG_PATH + value: gs://${PROJECT_ID}/config.yaml + resources: + limits: + cpu: 1000m + memory: 512Mi + traffic: + - percent: 100 + latestRevision: true diff --git a/tests/unit/fixtures/ci_slo.json b/tests/unit/fixtures/ci_slo.json new file mode 100644 index 00000000..f5063adc --- /dev/null +++ b/tests/unit/fixtures/ci_slo.json @@ -0,0 +1,26 @@ +{ + "apiVersion": "sre.google.com/v2", + "kind": "ServiceLevelObjective", + "metadata": { + "name": "gae-app-latency724ms", + "labels": { + "service_name": "gae", + "feature_name": "app", + "slo_name": "latency724ms" + } + }, + "spec": { + "description": "Latency of App Engine app requests < 724ms", + "backend": "cloud_monitoring", + "method": "distribution_cut", + "exporters": [ + "cloud_monitoring" + ], + "service_level_indicator": { + "filter_valid": "project=\"${CLOUDRUN_PROJECT_ID}\" metric.type=\"appengine.googleapis.com/http/server/response_latencies\" resource.type=\"gae_app\" metric.labels.response_code >= 200 metric.labels.response_code < 500", + "good_below_threshold": true, + "threshold_bucket": 19 + }, + "goal": 0.999 + } +} From 8703c1b898ee05be3c67a08a597fa1df55c65df0 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 12:53:50 +0200 Subject: [PATCH 004/107] refactor: Update .gitignore and Makefile (#134) * Update .gitignore * Update Makefile * Remove buildpack builds and replace with Dockerfile-based builds --- .gitignore | 5 +++ Makefile | 94 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 75 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 509b5bb2..05b77b9d 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,8 @@ htmlcov/ *.tfvars *.tfstate.backup *.tfstate.*.backup +.vscode +.env +venv/ +.venv/ +reports/ diff --git a/Makefile b/Makefile index 8ae6dd04..fceab9b6 100644 --- a/Makefile +++ b/Makefile @@ -5,22 +5,21 @@ # # useful targets: # make clean -- clean distutils -# make coverage_report -- code coverage report -# make flake8 -- flake8 checks -# make pylint -- source code checks -# make tests -- run all of the tests -# make unittest -- runs the unit tests +# make coverage -- code coverage report +# make test -- run lint + unit tests +# make lint -- run lint tests separately +# make unit -- runs unit tests separately +# make integration -- runs integration tests ######################################################## # variable section NAME = "slo_generator" PIP=pip3 - PYTHON=python3 TWINE=twine COVERAGE=coverage -NOSE_OPTS = --with-coverage --cover-package=$(NAME) --cover-erase +NOSE_OPTS = --with-coverage --cover-package=$(NAME) --cover-erase --nologcapture --logging-level=ERROR SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") VERSION := $(shell grep "version = " setup.py | cut -d\ -f3) @@ -29,18 +28,11 @@ FLAKE8_IGNORE = E302,E203,E261 ######################################################## -all: clean install install_test test +all: clean install test info: @echo "slo-generator version: ${VERSION}" -flake8: - flake8 --ignore=$(FLAKE8_IGNORE) $(NAME)/ --max-line-length=80 - flake8 --ignore=$(FLAKE8_IGNORE),E402 tests/ --max-line-length=80 - -pylint: - find ./$(NAME) ./tests -name \*.py | xargs pylint --rcfile .pylintrc --ignore-patterns=test_.*?py - clean: @echo "Cleaning up distutils stuff" rm -rf build @@ -69,26 +61,80 @@ develop: $(PIP) install -e . install: clean - $(PIP) install . - -install_test: - $(PIP) install wheel flake8 mock coverage nose pylint + $(PIP) install -e ."[api, datadog, prometheus, elasticsearch, pubsub, cloud_monitoring, bigquery, dev]" -test: install_test flake8 pylint unittest +test: install unit lint -unittest: clean +unit: clean nosetests $(NOSE_OPTS) tests/unit/* -v -coverage_report: +coverage: $(COVERAGE) report --rcfile=".coveragerc" -# Docker +lint: flake8 pylint + +flake8: + flake8 --ignore=$(FLAKE8_IGNORE) $(NAME)/ --max-line-length=80 + flake8 --ignore=$(FLAKE8_IGNORE),E402 tests/ --max-line-length=80 + +pylint: + find ./$(NAME) ./tests -name \*.py | xargs pylint --rcfile .pylintrc --ignore-patterns=test_.*?py + +integration: int_cm int_csm int_custom int_dd int_dt int_es int_prom + +int_cm: + slo-generator compute -f samples/cloud_monitoring -c samples/config.yaml + +int_csm: + slo-generator compute -f samples/cloud_service_monitoring -c samples/config.yaml + +int_custom: + slo-generator compute -f samples/custom -c samples/config.yaml + +int_dd: + slo-generator compute -f samples/datadog -c samples/config.yaml + +int_dt: + slo-generator compute -f samples/dynatrace -c samples/config.yaml + +int_es: + slo-generator compute -f samples/elasticsearch -c samples/config.yaml + +int_prom: + slo-generator compute -f samples/prometheus -c samples/config.yaml + +# Run API locally +run_api: + slo-generator api --target=run_compute --signature-type=http + +# Local Docker build / push docker_build: DOCKER_BUILDKIT=1 docker build -t slo-generator:latest . docker_test: docker_build docker run --entrypoint "make" \ - -e MIN_VALID_EVENTS=10 \ -e GOOGLE_APPLICATION_CREDENTIALS=tests/unit/fixtures/fake_credentials.json \ slo-generator test + +# Cloudbuild +cloudbuild: + gcloud alpha builds submit \ + --config=cloudbuild.yaml \ + --project=${CLOUDBUILD_PROJECT_ID} \ + --substitutions=_GCR_PROJECT_ID=${GCR_PROJECT_ID},_VERSION=${VERSION} + +# Cloudrun +cloudrun: + gcloud run deploy slo-generator \ + --image gcr.io/${GCR_PROJECT_ID}/slo-generator:${VERSION} \ + --region=${REGION} \ + --platform managed \ + --set-env-vars CONFIG_PATH=${CONFIG_URL} \ + --service-account=${SERVICE_ACCOUNT} \ + --project=${CLOUDRUN_PROJECT_ID} \ + --command="slo-generator" \ + --args=api \ + --args=--signature-type="${SIGNATURE_TYPE}" \ + --min-instances 1 \ + --allow-unauthenticated From 8a2525b3311c92414169f83d7eec9df730f13c3a Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 13:26:44 +0200 Subject: [PATCH 005/107] feat!: Split dependencies by backend (#129) --- setup.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 92b30cd9..af7b009c 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ https://packaging.python.org/en/latest/distributing.html https://github.com/pypa/sampleproject """ +# pylint: disable=invalid-name from io import open from os import path @@ -35,14 +36,30 @@ # 'Development Status :: 4 - Beta' # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 3 - Alpha" -dependencies = [ - 'google-api-python-client < 2.0.0', 'oauth2client', - 'google-cloud-monitoring < 2.0.0', 'google-cloud-pubsub==1.7.0', - 'google-cloud-bigquery < 3.0.0', 'prometheus-http-client', - 'prometheus-client', 'pyyaml', 'opencensus', 'elasticsearch', - 'python-dateutil', 'datadog', 'retrying==1.3.3' -] -extras = {} +dependencies = ['pyyaml', 'ruamel.yaml', 'python-dateutil', 'click < 8.0'] +extras = { + 'api': ['Flask', 'gunicorn', 'cloudevents', 'functions-framework'], + 'prometheus': ['prometheus-client', 'prometheus-http-client'], + 'datadog': ['datadog', 'retrying==1.3.3'], + 'dynatrace': ['requests'], + 'bigquery': [ + 'google-api-python-client < 2.0.0', 'google-cloud-bigquery < 3.0.0' + ], + 'cloud_monitoring': [ + 'google-api-python-client < 2.0.0', 'google-cloud-monitoring < 2.0.0' + ], + 'cloud_service_monitoring': [ + 'google-api-python-client < 2.0.0', 'google-cloud-monitoring < 2.0.0' + ], + 'cloud_storage': [ + 'google-api-python-client < 2.0.0', 'google-cloud-storage' + ], + 'pubsub': [ + 'google-api-python-client < 2.0.0', 'google-cloud-pubsub==1.7.0' + ], + 'elasticsearch': ['elasticsearch'], + 'dev': ['wheel', 'flake8', 'mock', 'coverage', 'nose', 'pylint'] +} # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: @@ -70,6 +87,7 @@ ], keywords='slo sli generator gcp', install_requires=dependencies, + extras_require=extras, entry_points={ 'console_scripts': ['slo-generator=slo_generator.cli:main'], }, From ecdf9bde525ab29f33d8b34cb8a78316914335ea Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 13:28:04 +0200 Subject: [PATCH 006/107] feat!: Support slo-generator config v2 format (core changes) (#126) * feat!: Support slo-generator config v2 format (core changes) * Add step property * Fix kind filtering * Fix Istio cluster naming for Service Monitoring API + modularize logging format based on debug flag * Fix linting --- .../{stackdriver.py => cloud_monitoring.py} | 37 ++- ...itoring.py => cloud_service_monitoring.py} | 174 +++++----- slo_generator/backends/datadog.py | 14 +- slo_generator/backends/dynatrace.py | 18 +- slo_generator/backends/elasticsearch.py | 3 +- slo_generator/backends/prometheus.py | 26 +- slo_generator/compute.py | 35 +- slo_generator/constants.py | 69 ++++ slo_generator/exporters/base.py | 44 +-- slo_generator/exporters/bigquery.py | 49 ++- .../{stackdriver.py => cloud_monitoring.py} | 28 +- slo_generator/exporters/pubsub.py | 14 +- slo_generator/report.py | 178 ++++++----- slo_generator/utils.py | 302 ++++++++++++++---- 14 files changed, 653 insertions(+), 338 deletions(-) rename slo_generator/backends/{stackdriver.py => cloud_monitoring.py} (90%) rename slo_generator/backends/{stackdriver_service_monitoring.py => cloud_service_monitoring.py} (79%) rename slo_generator/exporters/{stackdriver.py => cloud_monitoring.py} (82%) diff --git a/slo_generator/backends/stackdriver.py b/slo_generator/backends/cloud_monitoring.py similarity index 90% rename from slo_generator/backends/stackdriver.py rename to slo_generator/backends/cloud_monitoring.py index 6607ae7c..14213c3a 100644 --- a/slo_generator/backends/stackdriver.py +++ b/slo_generator/backends/cloud_monitoring.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -`stackdriver.py` -Stackdriver Monitoring backend implementation. +`cloud_monitoring.py` +Cloud Monitoring backend implementation. """ import logging import pprint @@ -27,15 +27,16 @@ LOGGER = logging.getLogger(__name__) -class StackdriverBackend: - """Backend for querying metrics from Stackdriver Monitoring. +class CloudMonitoringBackend: + """Backend for querying metrics from Cloud Monitoring. Args: - project_id (str): Stackdriver host project id. + project_id (str): Cloud Monitoring host project id. client (google.cloud.monitoring_v3.MetricServiceClient, optional): - Existing Stackdriver Service Monitoring client. Initialize a new - client if omitted. + Existing Cloud Monitoring Metrics client. Initialize a new client + if omitted. """ + def __init__(self, project_id, client=None): self.client = client if client is None: @@ -54,8 +55,7 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: A tuple (good_event_count, bad_event_count) """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] filter_good = measurement['filter_good'] filter_bad = measurement.get('filter_bad') filter_valid = measurement.get('filter_valid') @@ -65,7 +65,7 @@ def good_bad_ratio(self, timestamp, window, slo_config): window=window, filter=filter_good) good_ts = list(good_ts) - good_event_count = SD.count(good_ts) + good_event_count = CM.count(good_ts) # Query 'bad events' timeseries if filter_bad: @@ -73,16 +73,16 @@ def good_bad_ratio(self, timestamp, window, slo_config): window=window, filter=filter_bad) bad_ts = list(bad_ts) - bad_event_count = SD.count(bad_ts) + bad_event_count = CM.count(bad_ts) elif filter_valid: valid_ts = self.query(timestamp=timestamp, window=window, filter=filter_valid) valid_ts = list(valid_ts) - bad_event_count = SD.count(valid_ts) - good_event_count + bad_event_count = CM.count(valid_ts) - good_event_count else: raise Exception( - "Oneof `filter_bad` or `filter_valid` is required.") + "One of `filter_bad` or `filter_valid` is required.") LOGGER.debug(f'Good events: {good_event_count} | ' f'Bad events: {bad_event_count}') @@ -100,8 +100,7 @@ def distribution_cut(self, timestamp, window, slo_config): Returns: tuple: A tuple (good_event_count, bad_event_count). """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] filter_valid = measurement['filter_valid'] threshold_bucket = int(measurement['threshold_bucket']) good_below_threshold = measurement.get('good_below_threshold', True) @@ -168,7 +167,7 @@ def query(self, aligner='ALIGN_SUM', reducer='REDUCE_SUM', group_by=[]): - """Query timeseries from Stackdriver Monitoring. + """Query timeseries from Cloud Monitoring. Args: timestamp (int): Current timestamp. @@ -181,8 +180,8 @@ def query(self, Returns: list: List of timeseries objects. """ - measurement_window = SD.get_window(timestamp, window) - aggregation = SD.get_aggregation(window, + measurement_window = CM.get_window(timestamp, window) + aggregation = CM.get_aggregation(window, aligner=aligner, reducer=reducer, group_by=group_by) @@ -261,4 +260,4 @@ def get_aggregation(window, return aggregation -SD = StackdriverBackend +CM = CloudMonitoringBackend diff --git a/slo_generator/backends/stackdriver_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py similarity index 79% rename from slo_generator/backends/stackdriver_service_monitoring.py rename to slo_generator/backends/cloud_service_monitoring.py index 7e49ef19..6e855241 100644 --- a/slo_generator/backends/stackdriver_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -`stackdriver_service_monitoring.py` -Stackdriver Service Monitoring exporter class. +`cloud_service_monitoring.py` +Cloud Service Monitoring exporter class. """ import difflib import json @@ -25,7 +25,7 @@ from google.cloud.monitoring_v3 import ServiceMonitoringServiceClient from google.protobuf.json_format import MessageToJson -from slo_generator.backends.stackdriver import StackdriverBackend +from slo_generator.backends.cloud_monitoring import CloudMonitoringBackend from slo_generator.constants import NO_DATA from slo_generator.utils import dict_snake_to_caml @@ -34,20 +34,21 @@ SID_GAE = 'gae:{project_id}_{module_id}' SID_CLOUD_ENDPOINT = 'ist:{project_id}-{service}' SID_CLUSTER_ISTIO = ( - 'ist:{project_id}-location-{location}-{cluster_name}-{service_namespace}-' + 'ist:{project_id}-{suffix}-{location}-{cluster_name}-{service_namespace}-' '{service_name}') SID_MESH_ISTIO = ('ist:{mesh_uid}-{service_namespace}-{service_name}') -class StackdriverServiceMonitoringBackend: - """Stackdriver Service Monitoring backend class. +class CloudServiceMonitoringBackend: + """Cloud Service Monitoring backend class. Args: - project_id (str): Stackdriver host project id. + project_id (str): Cloud Monitoring host project id. client (google.cloud.monitoring_v3.ServiceMonitoringServiceClient): Existing Service Monitoring API client. Initialize a new client if omitted. """ + def __init__(self, project_id, client=None): self.project_id = project_id self.client = client @@ -125,7 +126,7 @@ def delete(self, timestamp, window, slo_config): return self.delete_slo(window, slo_config) def retrieve_slo(self, timestamp, window, slo_config): - """Get SLI value from Stackdriver Monitoring API. + """Get SLI value from Cloud Monitoring API. Args: timestamp (int): UNIX timestamp. @@ -147,26 +148,27 @@ def retrieve_slo(self, timestamp, window, slo_config): slo = self.create_slo(window, slo_config) LOGGER.debug(service) - # Now that we have our SLO, retrieve the TimeSeries from Stackdriver + # Now that we have our SLO, retrieve the TimeSeries from Cloud # Monitoring API for that particular SLO id. - metric_filter = SSM.build_slo_id(window, slo_config, full=True) + metric_filter = self.build_slo_id(window, slo_config, full=True) filter = f"select_slo_counts(\"{metric_filter}\")" # Query SLO timeseries - stackdriver = StackdriverBackend(self.project_id) - timeseries = stackdriver.query(timestamp, - window, - filter, - aligner='ALIGN_SUM', - reducer='REDUCE_SUM', - group_by=['metric.labels.event_type']) + cloud_monitoring = CloudMonitoringBackend(self.project_id) + timeseries = cloud_monitoring.query( + timestamp, + window, + filter, + aligner='ALIGN_SUM', + reducer='REDUCE_SUM', + group_by=['metric.labels.event_type']) timeseries = list(timeseries) good_event_count, bad_event_count = SSM.count(timeseries) return (good_event_count, bad_event_count) @staticmethod def count(timeseries): - """Extract good_count, bad_count tuple from Stackdriver Monitoring API + """Extract good_count, bad_count tuple from Cloud Monitoring API response. Args: @@ -186,27 +188,26 @@ def count(timeseries): return good_event_count, bad_event_count def create_service(self, slo_config): - """Create Service object in Stackdriver Service Monitoring API. + """Create Service object in Cloud Service Monitoring API. Args: slo_config (dict): SLO configuration. Returns: - dict: Stackdriver Service Monitoring API response. + dict: Cloud Service Monitoring API response. """ LOGGER.debug("Creating service ...") - service_json = SSM.build_service(slo_config) - service_id = SSM.build_service_id(slo_config) + service_json = self.build_service(slo_config) + service_id = self.build_service_id(slo_config) service = self.client.create_service(self.project_path, service_json, service_id=service_id) - LOGGER.info( - f'Service "{service_id}" created successfully in Stackdriver ' - f'Service Monitoring API.') + LOGGER.info(f'Service "{service_id}" created successfully in Cloud ' + f'Service Monitoring API.') return SSM.to_json(service) def get_service(self, slo_config): - """Get Service object from Stackdriver Service Monitoring API. + """Get Service object from Cloud Service Monitoring API. Args: slo_config (dict): SLO configuration. @@ -216,7 +217,7 @@ def get_service(self, slo_config): """ # Look for API services in workspace matching our config. - service_id = SSM.build_service_id(slo_config) + service_id = self.build_service_id(slo_config) services = list(self.client.list_services(self.workspace_path)) matches = [ service for service in services @@ -229,7 +230,7 @@ def get_service(self, slo_config): if not matches: msg = (f'Service "{service_id}" does not exist in ' f'workspace "{self.project_id}"') - method = slo_config['backend']['method'] + method = slo_config['spec']['method'] if method == 'basic': sids = [service.name.split("/")[-1] for service in services] LOGGER.debug( @@ -244,24 +245,22 @@ def get_service(self, slo_config): LOGGER.debug(f'Found matching service "{service.name}"') return SSM.to_json(service) - @staticmethod - def build_service(slo_config): - """Build service JSON in Stackdriver Monitoring API from SLO + def build_service(self, slo_config): + """Build service JSON in Cloud Monitoring API from SLO configuration. Args: slo_config (dict): SLO configuration. Returns: - dict: Service JSON in Stackdriver Monitoring API. + dict: Service JSON in Cloud Monitoring API. """ - service_id = SSM.build_service_id(slo_config) + service_id = self.build_service_id(slo_config) display_name = slo_config.get('service_display_name', service_id) service = {'display_name': display_name, 'custom': {}} return service - @staticmethod - def build_service_id(slo_config, dest_project_id=None, full=False): + def build_service_id(self, slo_config, dest_project_id=None, full=False): """Build service id from SLO configuration. Args: @@ -274,11 +273,8 @@ def build_service_id(slo_config, dest_project_id=None, full=False): Returns: str: Service id. """ - service_name = slo_config['service_name'] - feature_name = slo_config['feature_name'] - backend = slo_config['backend'] - project_id = backend['project_id'] - measurement = backend['measurement'] + project_id = self.project_id + measurement = slo_config['spec']['service_level_indicator'] app_engine = measurement.get('app_engine') cluster_istio = measurement.get('cluster_istio') mesh_istio = measurement.get('mesh_istio') @@ -292,8 +288,13 @@ def build_service_id(slo_config, dest_project_id=None, full=False): elif cluster_istio: warnings.warn( 'ClusterIstio is deprecated in the Service Monitoring API.' - 'It will be removed in version 2.0, please use MeshIstio ' + 'It will be removed in version 3.0, please use MeshIstio ' 'instead', FutureWarning) + if 'location' in cluster_istio: + cluster_istio['suffix'] = 'location' + elif 'zone' in cluster_istio: + cluster_istio['suffix'] = 'zone' + cluster_istio['location'] = cluster_istio['zone'] service_id = SID_CLUSTER_ISTIO.format_map(cluster_istio) dest_project_id = cluster_istio['project_id'] elif mesh_istio: @@ -301,8 +302,22 @@ def build_service_id(slo_config, dest_project_id=None, full=False): elif cloud_endpoints: service_id = SID_CLOUD_ENDPOINT.format_map(cloud_endpoints) dest_project_id = cluster_istio['project_id'] - else: - service_id = f'{service_name}-{feature_name}' + else: # user-defined service id + service_name = slo_config['metadata']['labels'].get( + 'service_name', '') + feature_name = slo_config['metadata']['labels'].get( + 'feature_name', '') + service_id = slo_config['spec']['service_level_indicator'].get( + 'service_id') + if not service_id: + if not service_name or not feature_name: + raise Exception( + 'Service id not set in SLO configuration. Please set ' + 'either `spec.service_level_indicator.service_id` or ' + 'both `metadata.labels.service_name` and ' + '`metadata.labels.feature_name` in your SLO ' + 'configuration.') + service_id = f'{service_name}-{feature_name}' if full: if dest_project_id: @@ -312,7 +327,7 @@ def build_service_id(slo_config, dest_project_id=None, full=False): return service_id def create_slo(self, window, slo_config): - """Create SLO object in Stackdriver Service Monitoring API. + """Create SLO object in Cloud Service Monitoring API. Args: window (int): Window (in seconds). @@ -322,15 +337,15 @@ def create_slo(self, window, slo_config): dict: Service Management API response. """ slo_json = SSM.build_slo(window, slo_config) - slo_id = SSM.build_slo_id(window, slo_config) - parent = SSM.build_service_id(slo_config, full=True) + slo_id = self.build_slo_id(window, slo_config) + parent = self.build_service_id(slo_config, full=True) slo = self.client.create_service_level_objective( parent, slo_json, service_level_objective_id=slo_id) return SSM.to_json(slo) @staticmethod def build_slo(window, slo_config): # pylint: disable=R0912,R0915 - """Get SLO JSON representation in Service Monitoring API from SLO + """Get SLO JSON representation in Cloud Service Monitoring API from SLO configuration. Args: @@ -340,16 +355,16 @@ def build_slo(window, slo_config): # pylint: disable=R0912,R0915 Returns: dict: SLO JSON configuration. """ - measurement = slo_config['backend'].get('measurement', {}) - method = slo_config['backend']['method'] - description = slo_config['slo_description'] - target = slo_config['slo_target'] + measurement = slo_config['spec'].get('service_level_indicator', {}) + method = slo_config['spec']['method'] + description = slo_config['spec']['description'] + goal = slo_config['spec']['goal'] minutes, _ = divmod(window, 60) hours, _ = divmod(minutes, 60) display_name = f'{description} ({hours}h)' slo = { 'display_name': display_name, - 'goal': target, + 'goal': goal, 'rolling_period': { 'seconds': window } @@ -437,7 +452,7 @@ def build_slo(window, slo_config): # pylint: disable=R0912,R0915 return slo def get_slo(self, window, slo_config): - """Get SLO object from Stackriver Service Monitoring API. + """Get SLO object from Cloud Service Monitoring API. Args: service_id (str): Service identifier. @@ -447,10 +462,10 @@ def get_slo(self, window, slo_config): Returns: dict: API response. """ - service_path = SSM.build_service_id(slo_config, full=True) + service_path = self.build_service_id(slo_config, full=True) LOGGER.debug(f'Getting SLO for for "{service_path}" ...') slos = self.list_slos(service_path) - slo_local_id = SSM.build_slo_id(window, slo_config) + slo_local_id = self.build_slo_id(window, slo_config) slo_json = SSM.build_slo(window, slo_config) slo_json = SSM.convert_slo_to_ssm_format(slo_json) @@ -467,7 +482,7 @@ def get_slo(self, window, slo_config): return slo return self.update_slo(window, slo_config) LOGGER.warning('No SLO found matching configuration.') - LOGGER.debug(f'SLOs from Stackdriver Monitoring API: {slos}') + LOGGER.debug(f'SLOs from Cloud Service Monitoring API: {slos}') LOGGER.debug(f'SLO config converted: {slo_json}') return None @@ -482,14 +497,13 @@ def update_slo(self, window, slo_config): dict: API response. """ slo_json = SSM.build_slo(window, slo_config) - slo_id = SSM.build_slo_id(window, slo_config, full=True) + slo_id = self.build_slo_id(window, slo_config, full=True) LOGGER.warning(f"Updating SLO {slo_id} ...") slo_json['name'] = slo_id - return SSM.to_json( - self.client.update_service_level_objective(slo_json)) + return SSM.to_json(self.client.update_service_level_objective(slo_json)) def list_slos(self, service_path): - """List all SLOs from Stackdriver Service Monitoring API. + """List all SLOs from Cloud Service Monitoring API. Args: service_path (str): Service path in the form @@ -501,12 +515,12 @@ def list_slos(self, service_path): """ slos = self.client.list_service_level_objectives(service_path) slos = list(slos) - LOGGER.debug(f"{len(slos)} SLOs found in Service Monitoring API.") + LOGGER.debug(f"{len(slos)} SLOs found in Cloud Service Monitoring API.") # LOGGER.debug(slos) return [SSM.to_json(slo) for slo in slos] def delete_slo(self, window, slo_config): - """Delete SLO from Stackdriver Monitoring API. + """Delete SLO from Cloud Service Monitoring API. Args: window (int): Window (in seconds). @@ -515,7 +529,7 @@ def delete_slo(self, window, slo_config): Returns: dict: API response. """ - slo_path = SSM.build_slo_id(window, slo_config, full=True) + slo_path = self.build_slo_id(window, slo_config, full=True) LOGGER.info(f'Deleting SLO "{slo_path}"') try: return self.client.delete_service_level_objective(slo_path) @@ -525,8 +539,7 @@ def delete_slo(self, window, slo_config): f'Skipping.') return None - @staticmethod - def build_slo_id(window, slo_config, full=False): + def build_slo_id(self, window, slo_config, full=False): """Build SLO id from SLO configuration. Args: @@ -536,16 +549,19 @@ def build_slo_id(window, slo_config, full=False): Returns: str: SLO id. """ - if 'slo_id' in slo_config: - slo_id_part = slo_config['slo_id'] - slo_id = f'{slo_id_part}-{window}' - else: - slo_name = slo_config['slo_name'] - slo_id = f'{slo_name}-{window}' + sli = slo_config['spec']['service_level_indicator'] + slo_name = slo_config['metadata']['labels'].get('slo_name') + slo_id = sli.get('slo_id', slo_name) + if not slo_id: + raise Exception( + 'SLO id not set in SLO configuration. Please set either ' + '`spec.service_level_indicator.slo_id` or ' + '`metadata.labels.slo_name` in your SLO configuration.') + full_slo_id = f'{slo_id}-{window}' if full: - service_path = SSM.build_service_id(slo_config, full=True) - return f'{service_path}/serviceLevelObjectives/{slo_id}' - return slo_id + service_path = self.build_service_id(slo_config, full=True) + return f'{service_path}/serviceLevelObjectives/{full_slo_id}' + return full_slo_id @staticmethod def compare_slo(slo1, slo2): @@ -601,15 +617,15 @@ def string_diff(string1, string2): @staticmethod def convert_slo_to_ssm_format(slo): - """Convert SLO JSON to Service Monitoring API format. + """Convert SLO JSON to Cloud Service Monitoring API format. Address edge cases, like `duration` object computation. Args: - slo (dict): SLO JSON object to be converted to Stackdriver Service + slo (dict): SLO JSON object to be converted to Cloud Service Monitoring API format. Returns: - dict: SLO configuration in Service Monitoring API format. + dict: SLO configuration in Cloud Service Monitoring API format. """ # Our local JSON is in snake case, convert it to Caml case. data = dict_snake_to_caml(slo) @@ -654,7 +670,7 @@ def convert_duration_to_string(duration): @staticmethod def to_json(response): - """Convert a Stackdriver Service Monitoring API response to JSON + """Convert a Cloud Service Monitoring API response to JSON format. Args: @@ -666,4 +682,4 @@ def to_json(response): return json.loads(MessageToJson(response)) -SSM = StackdriverServiceMonitoringBackend +SSM = CloudServiceMonitoringBackend diff --git a/slo_generator/backends/datadog.py b/slo_generator/backends/datadog.py index 12299527..e9f69bd3 100644 --- a/slo_generator/backends/datadog.py +++ b/slo_generator/backends/datadog.py @@ -33,6 +33,7 @@ class DatadogBackend: app_key (str): Datadog APP key. kwargs (dict): Extra arguments to pass to initialize function. """ + def __init__(self, client=None, api_key=None, app_key=None, **kwargs): self.client = client if not self.client: @@ -52,8 +53,7 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: Good event count, Bad event count. """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] operator = measurement.get('operator', 'sum') operator_suffix = measurement.get('operator_suffix', 'as_count()') start = timestamp - window @@ -88,8 +88,7 @@ def query_sli(self, timestamp, window, slo_config): Returns: float: SLI value. """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] start = timestamp - window end = timestamp query = measurement['query'] @@ -110,11 +109,10 @@ def query_slo(self, timestamp, window, slo_config): Returns: tuple: Good event count, bad event count. """ - slo_id = slo_config['backend']['measurement']['slo_id'] + slo_id = slo_config['spec']['service_level_indicator']['slo_id'] from_ts = timestamp - window slo_data = self.client.ServiceLevelObjective.get(id=slo_id) - LOGGER.debug( - f"SLO data: {slo_id} | Result: {pprint.pformat(slo_data)}") + LOGGER.debug(f"SLO data: {slo_id} | Result: {pprint.pformat(slo_data)}") try: data = self.client.ServiceLevelObjective.history(id=slo_id, from_ts=from_ts, @@ -125,7 +123,7 @@ def query_slo(self, timestamp, window, slo_config): valid_event_count = data['data']['series']['denominator']['sum'] bad_event_count = valid_event_count - good_event_count return (good_event_count, bad_event_count) - except (KeyError) as exception:# monitor-based SLI + except (KeyError) as exception: # monitor-based SLI sli_value = data['data']['overall']['sli_value'] / 100 LOGGER.debug(exception) return sli_value diff --git a/slo_generator/backends/dynatrace.py b/slo_generator/backends/dynatrace.py index f685c92c..59090a62 100644 --- a/slo_generator/backends/dynatrace.py +++ b/slo_generator/backends/dynatrace.py @@ -34,6 +34,7 @@ class DynatraceBackend: api_url (str): Dynatrace API URL. api_token (str): Dynatrace token. """ + def __init__(self, client=None, api_url=None, api_token=None): self.client = client if client is None: @@ -50,8 +51,7 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: Good event count, Bad event count. """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] start = (timestamp - window) * 1000 end = timestamp * 1000 query_good = measurement['query_good'] @@ -83,8 +83,7 @@ def threshold(self, timestamp, window, slo_config): Returns: tuple: Good event count, Bad event count. """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] start = (timestamp - window) * 1000 end = timestamp * 1000 query_valid = measurement['query_valid'] @@ -92,8 +91,7 @@ def threshold(self, timestamp, window, slo_config): good_below_threshold = measurement.get('good_below_threshold', True) response = self.query(start=start, end=end, **query_valid) LOGGER.debug(f"Result valid: {pprint.pformat(response)}") - return DynatraceBackend.count_threshold(response, - threshold, + return DynatraceBackend.count_threshold(response, threshold, good_below_threshold) def query(self, @@ -222,7 +220,8 @@ def __init__(self, api_url, api_key): @retry(retry_on_result=retry_http, wait_exponential_multiplier=1000, - wait_exponential_max=10000) + wait_exponential_max=10000, + stop_max_delay=10000) def request(self, method, endpoint, @@ -255,9 +254,10 @@ def request(self, } if name: url += f'/{name}' - params_str = "&".join("%s=%s" % (k, v) for k, v in params.items() - if v is not None) + params_str = "&".join( + "%s=%s" % (k, v) for k, v in params.items() if v is not None) url += f'?{params_str}' + LOGGER.debug(f'Running "{method}" request to {url} ...') if method in ['put', 'post']: response = req(url, headers=headers, json=post_data) else: diff --git a/slo_generator/backends/elasticsearch.py b/slo_generator/backends/elasticsearch.py index 464b6f34..2b5d4b0e 100644 --- a/slo_generator/backends/elasticsearch.py +++ b/slo_generator/backends/elasticsearch.py @@ -34,6 +34,7 @@ class ElasticsearchBackend: client (elasticsearch.ElasticSearch): Existing ES client. es_config (dict): ES client configuration. """ + def __init__(self, client=None, **es_config): self.client = client if self.client is None: @@ -52,7 +53,7 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: A tuple (good_event_count, bad_event_count) """ - measurement = slo_config['backend']['measurement'] + measurement = slo_config['spec']['service_level_indicator'] index = measurement['index'] query_good = measurement['query_good'] query_bad = measurement.get('query_bad') diff --git a/slo_generator/backends/prometheus.py b/slo_generator/backends/prometheus.py index 4b5074c7..f69a755f 100644 --- a/slo_generator/backends/prometheus.py +++ b/slo_generator/backends/prometheus.py @@ -30,6 +30,7 @@ class PrometheusBackend: """Backend for querying metrics from Prometheus.""" + def __init__(self, client=None, url=None, headers=None): self.client = client if not self.client: @@ -50,8 +51,7 @@ def query_sli(self, timestamp, window, slo_config): Returns: float: SLI value. """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] expr = measurement['expression'] response = self.query(expr, window, timestamp, operators=[]) sli_value = PrometheusBackend.count(response) @@ -72,11 +72,11 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: A tuple of (good_count, bad_count). """ - conf = slo_config['backend'] - good = conf['measurement']['filter_good'] - bad = conf['measurement'].get('filter_bad') - valid = conf['measurement'].get('filter_valid') - operators = conf['measurement'].get('operators', ['increase', 'sum']) + measurement = slo_config['spec']['service_level_indicator'] + good = measurement['filter_good'] + bad = measurement.get('filter_bad') + valid = measurement.get('filter_valid') + operators = measurement.get('operators', ['increase', 'sum']) # Replace window by its value in the error budget policy step res = self.query(good, window, timestamp, operators) @@ -92,8 +92,7 @@ def good_bad_ratio(self, timestamp, window, slo_config): else: raise Exception("`filter_bad` or `filter_valid` is required.") - LOGGER.debug(f'Good events: {good_count} | ' - f'Bad events: {bad_count}') + LOGGER.debug(f'Good events: {good_count} | ' f'Bad events: {bad_count}') return (good_count, bad_count) @@ -109,8 +108,7 @@ def distribution_cut(self, timestamp, window, slo_config): Returns: float: SLI value. """ - conf = slo_config['backend'] - measurement = conf['measurement'] + measurement = slo_config['spec']['service_level_indicator'] expr = measurement['expression'] threshold_bucket = measurement['threshold_bucket'] labels = {"le": threshold_bucket} @@ -130,8 +128,7 @@ def distribution_cut(self, timestamp, window, slo_config): operators=['increase', 'sum']) valid_count = PrometheusBackend.count(res_valid) bad_count = valid_count - good_count - LOGGER.debug(f'Good events: {good_count} | ' - f'Bad events: {bad_count}') + LOGGER.debug(f'Good events: {good_count} | ' f'Bad events: {bad_count}') return (good_count, bad_count) # pylint: disable=unused-argument @@ -148,8 +145,7 @@ def query(self, filter, window, timestamp=None, operators=[], labels={}): Returns: dict: Response. """ - filter = PrometheusBackend._fmt_query(filter, window, operators, - labels) + filter = PrometheusBackend._fmt_query(filter, window, operators, labels) LOGGER.debug(f'Query: {filter}') response = self.client.query(metric=filter) response = json.loads(response) diff --git a/slo_generator/compute.py b/slo_generator/compute.py index a2cd984e..3fd9a8a0 100644 --- a/slo_generator/compute.py +++ b/slo_generator/compute.py @@ -22,12 +22,13 @@ from slo_generator import utils from slo_generator.report import SLOReport +from slo_generator.migrations.migrator import report_v2tov1 LOGGER = logging.getLogger(__name__) def compute(slo_config, - error_budget_policy, + config, timestamp=None, client=None, do_export=False, @@ -37,8 +38,8 @@ def compute(slo_config, Args: slo_config (dict): SLO configuration. - error_budget_policy (dict): Error Budget policy configuration. - timestamp (int, optional): UNIX timestamp. Defaults to now. + config (dict): SLO Generator configuration. + timestamp (float, optional): UNIX timestamp. Defaults to now. client (obj, optional): Existing metrics backend client. do_export (bool, optional): Enable / Disable export. Default: False. delete (bool, optional): Enable / Disable delete mode. Default: False. @@ -47,11 +48,19 @@ def compute(slo_config, if timestamp is None: timestamp = time.time() - # Compute SLO, Error Budget, Burn rates and make report - exporters = slo_config.get('exporters') + if slo_config is None: + LOGGER.error('SLO configuration is empty') + return [] + + # Get exporters, backend and error budget policy + spec = slo_config['spec'] + exporters = utils.get_exporters(config, spec) + error_budget_policy = utils.get_error_budget_policy(config, spec) + backend = utils.get_backend(config, spec) reports = [] - for step in error_budget_policy: + for step in error_budget_policy['steps']: report = SLOReport(config=slo_config, + backend=backend, step=step, timestamp=timestamp, client=client, @@ -73,7 +82,7 @@ def compute(slo_config, end = time.time() run_duration = round(end - start, 1) LOGGER.debug(pprint.pformat(reports)) - LOGGER.debug(f'Run finished successfully in {run_duration}s.') + LOGGER.info(f'Run finished successfully in {run_duration}s.') return reports @@ -91,6 +100,9 @@ def export(data, exporters, raise_on_error=False): LOGGER.debug(f'Data: {pprint.pformat(data)}') responses = [] + # Convert data to export from v1 to v2 for backwards-compatible exports + data = report_v2tov1(data) + # Passing one exporter as a dict will work for convenience if isinstance(exporters, dict): exporters = [exporters] @@ -98,10 +110,13 @@ def export(data, exporters, raise_on_error=False): for config in exporters: try: exporter_class = config.get('class') - LOGGER.info(f'Exporting results to {exporter_class}') + instance = utils.get_exporter_cls(exporter_class) + if not instance: + continue + LOGGER.info( + f'Exporting SLO report using {exporter_class}Exporter ...') LOGGER.debug(f'Exporter config: {pprint.pformat(config)}') - exporter = utils.get_exporter_cls(exporter_class)() - response = exporter.export(data, **config) + response = instance().export(data, **config) if isinstance(response, list): for elem in response: elem['exporter'] = exporter_class diff --git a/slo_generator/constants.py b/slo_generator/constants.py index 60a1f995..58653c54 100644 --- a/slo_generator/constants.py +++ b/slo_generator/constants.py @@ -17,7 +17,76 @@ """ import os +# Compute NO_DATA = -1 MIN_VALID_EVENTS = int(os.environ.get("MIN_VALID_EVENTS", "1")) + +# Global +LATEST_MAJOR_VERSION = 'v2' COLORED_OUTPUT = int(os.environ.get("COLORED_OUTPUT", "0")) DRY_RUN = bool(int(os.environ.get("DRY_RUN", "0"))) +DEBUG = int(os.environ.get("DEBUG", "0")) + +# Config skeletons +CONFIG_SCHEMA = { + 'backends': {}, + 'exporters': {}, + 'error_budget_policies': {}, +} +SLO_CONFIG_SCHEMA = { + 'apiVersion': '', + 'kind': '', + 'metadata': {}, + 'spec': { + 'description': '', + 'backend': '', + 'method': '', + 'exporters': [], + 'service_level_indicator': {} + } +} + +# Providers that have changed with v2 YAML config format. This mapping helps +# migrate them to their updated names. +PROVIDERS_COMPAT = { + 'Stackdriver': 'CloudMonitoring', + 'StackdriverServiceMonitoring': 'CloudServiceMonitoring' +} + +# Fields that have changed name with v2 YAML config format. This mapping helps +# migrate them back to their former name, so that exporters are backward- +# compatible with v1. +METRIC_LABELS_COMPAT = { + 'goal': 'slo_target', + 'description': 'slo_description', + 'burn_rate_threshold': 'alerting_burn_rate_threshold' +} + +# Fields that used to be specified in top-level of YAML config are now specified +# in metadata fields. This mapping helps migrate them back to the top level when +# exporting reports, so that so that exporters are backward-compatible with v1. +METRIC_METADATA_LABELS_TOP_COMPAT = ['service_name', 'feature_name', 'slo_name'] + + +# Colors / Status +# pylint: disable=too-few-public-methods +class Colors: + """Colors for console output.""" + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +GREEN = Colors.OKGREEN +RED = Colors.FAIL +ENDC = Colors.ENDC +BOLD = Colors.BOLD +WARNING = Colors.WARNING +FAIL = '❌' +SUCCESS = '✅' +RIGHT_ARROW = '➞' diff --git a/slo_generator/exporters/base.py b/slo_generator/exporters/base.py index d40b074c..2846fd66 100644 --- a/slo_generator/exporters/base.py +++ b/slo_generator/exporters/base.py @@ -19,13 +19,17 @@ import warnings from abc import ABCMeta, abstractmethod +from slo_generator import constants + LOGGER = logging.getLogger(__name__) +# Default metric labels exported by all metrics exporters DEFAULT_METRIC_LABELS = [ 'error_budget_policy_step_name', 'service_name', 'feature_name', 'slo_name', 'metadata' ] +# Default metrics that are exported by metrics exporters. DEFAULT_METRICS = [ { 'name': 'error_budget_burn_rate', @@ -33,15 +37,17 @@ 'labels': DEFAULT_METRIC_LABELS }, { - 'name': 'alerting_burn_rate_threshold', + 'name': 'error_budget_burn_rate_threshold', 'description': 'Error Budget burn rate threshold.', 'labels': DEFAULT_METRIC_LABELS }, { - 'name': 'events_count', - 'description': 'Number of events', - 'labels': DEFAULT_METRIC_LABELS + [ - 'good_events_count', 'bad_events_count'] + 'name': + 'events_count', + 'description': + 'Number of events', + 'labels': + DEFAULT_METRIC_LABELS + ['good_events_count', 'bad_events_count'] }, { 'name': 'sli_measurement', @@ -55,6 +61,7 @@ }, ] + class MetricsExporter: """Abstract class to export metrics to different backends. Common format for YAML configuration to configure which metrics should be exported.""" @@ -78,7 +85,7 @@ class `export_metric` method. LOGGER.debug( f'Exporting {len(metrics)} metrics with {self.__class__.__name__}') for metric_cfg in metrics: - if isinstance(metric_cfg, str): # short form + if isinstance(metric_cfg, str): # short form metric_cfg = { 'name': metric_cfg, 'alias': metric_cfg, @@ -87,11 +94,11 @@ class `export_metric` method. } if metric_cfg['name'] == 'error_budget_burn_rate': metric_cfg = MetricsExporter.use_deprecated_fields( - config=config, - metric=metric_cfg) + config=config, metric=metric_cfg) metric = metric_cfg.copy() fields = { - key: value for key, value in config.items() + key: value + for key, value in config.items() if key in required_fields or key in optional_fields } metric.update(fields) @@ -99,20 +106,21 @@ class `export_metric` method. name = metric['name'] labels = metric['labels'] labels_str = ', '.join([f'{k}={v}' for k, v in labels.items()]) - LOGGER.info( - f'Exporting "{name}" with {len(labels.keys())} labels: ' - f'{labels_str}...') ret = self.export_metric(metric) metric_info = { - k: v for k, v in metric.items() + k: v + for k, v in metric.items() if k in ['name', 'alias', 'description', 'labels'] } - response = { - 'response': ret, - 'metric': metric_info - } + response = {'response': ret, 'metric': metric_info} + status = f' Export {name} {{{labels_str}}}' if ret and 'error' in ret: + status = constants.FAIL + status + LOGGER.error(status) LOGGER.error(response) + else: + status = constants.SUCCESS + status + LOGGER.info(status) all_data.append(response) return all_data @@ -181,7 +189,7 @@ def build_data_labels(data, labels): @staticmethod def use_deprecated_fields(config, metric): - """Old format to new format with DeprecationWarning for 2.0.0. + """Old format to new format with FutureWarning for 2.0.0. Update error_budget_burn_rate metric with `metric_type`, `metric_labels`, and `metric_description`. diff --git a/slo_generator/exporters/bigquery.py b/slo_generator/exporters/bigquery.py index 726d4f59..dc22868a 100644 --- a/slo_generator/exporters/bigquery.py +++ b/slo_generator/exporters/bigquery.py @@ -23,12 +23,14 @@ import google.api_core from google.cloud import bigquery -from slo_generator.constants import DRY_RUN +from slo_generator import constants LOGGER = logging.getLogger(__name__) + class BigqueryExporter: """BigQuery exporter class.""" + def __init__(self): self.client = bigquery.Client(project='unset') @@ -74,14 +76,14 @@ def export(self, data, **config): json_data = {k: v for k, v in data.items() if k in schema_fields} metadata = json_data.get('metadata', {}) if isinstance(metadata, dict): - metadata_fields = [ - {'key': key, 'value': value} - for key, value in metadata.items() - ] + metadata_fields = [{ + 'key': key, + 'value': value + } for key, value in metadata.items()] json_data['metadata'] = metadata_fields # Write results to BQ table - if DRY_RUN: + if constants.DRY_RUN: LOGGER.info(f'[DRY RUN] Writing data to BigQuery: \n{json_data}') return [] LOGGER.debug(f'Writing data to BigQuery:\n{json_data}') @@ -90,9 +92,12 @@ def export(self, data, **config): json_rows=[json_data], row_ids=[row_ids], retry=google.api_core.retry.Retry(deadline=30)) + status = f' Export data to {str(table_ref)}' if results != []: + status = constants.FAIL + status raise BigQueryError(results) - + status = constants.SUCCESS + status + LOGGER.info(status) return results @staticmethod @@ -111,12 +116,15 @@ def build_schema(schema): subschema = [] if 'fields' in row: subschema = [ - bigquery.SchemaField(subrow['name'], subrow['type'], + bigquery.SchemaField(subrow['name'], + subrow['type'], mode=subrow['mode']) for subrow in row['fields'] ] - field = bigquery.SchemaField(row['name'], row['type'], - mode=row['mode'], fields=subschema) + field = bigquery.SchemaField(row['name'], + row['type'], + mode=row['mode'], + fields=subschema) final_schema.append(field) return final_schema @@ -140,7 +148,7 @@ def create_table(self, project_id, dataset_id, table_id, schema=None): LOGGER.debug(f'Table schema: {pyschema}') table = bigquery.Table(table_name, schema=pyschema) table.time_partitioning = bigquery.TimePartitioning( - type_=bigquery.TimePartitioningType.DAY, ) + type_=bigquery.TimePartitioningType.DAY,) return self.client.create_table(table) def update_schema(self, table_ref, keep=[]): @@ -161,7 +169,8 @@ def update_schema(self, table_ref, keep=[]): # Fields in TABLE_SCHEMA to add / remove updated_fields = [ - field['name'] for field in TABLE_SCHEMA + field['name'] + for field in TABLE_SCHEMA if field not in existing_schema ] extra_remote_fields = [ @@ -179,7 +188,7 @@ def update_schema(self, table_ref, keep=[]): if updated_fields: LOGGER.info(f'Updated BigQuery fields: {updated_fields}') table.schema = BigqueryExporter.build_schema(TABLE_SCHEMA) - if DRY_RUN: + if constants.DRY_RUN: LOGGER.info('[DRY RUN] Updating BigQuery schema.') else: LOGGER.info('Updating BigQuery schema.') @@ -187,12 +196,14 @@ def update_schema(self, table_ref, keep=[]): self.client.update_table(table, ['schema']) return table + class BigQueryError(Exception): """Exception raised whenever a BigQuery error happened. Args: errors (list): List of errors. """ + def __init__(self, errors): super().__init__(BigQueryError._format(errors)) self.errors = errors @@ -321,10 +332,14 @@ def _format(errors): 'type': 'BOOLEAN', 'mode': 'NULLABLE' }, { - 'name': 'metadata', - 'description': None, - 'type': 'RECORD', - 'mode': 'REPEATED', + 'name': + 'metadata', + 'description': + None, + 'type': + 'RECORD', + 'mode': + 'REPEATED', 'fields': [{ 'description': None, 'name': 'key', diff --git a/slo_generator/exporters/stackdriver.py b/slo_generator/exporters/cloud_monitoring.py similarity index 82% rename from slo_generator/exporters/stackdriver.py rename to slo_generator/exporters/cloud_monitoring.py index e72f5369..de4285ad 100644 --- a/slo_generator/exporters/stackdriver.py +++ b/slo_generator/exporters/cloud_monitoring.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -`stackdriver.py` -Stackdriver Monitoring exporter class. +`cloud_monitoring.py` +Cloud Monitoring exporter class. """ import logging @@ -24,8 +24,9 @@ LOGGER = logging.getLogger(__name__) -class StackdriverExporter(MetricsExporter): - """Stackdriver Monitoring exporter class.""" + +class CloudMonitoringExporter(MetricsExporter): + """Cloud Monitoring exporter class.""" METRIC_PREFIX = "custom.googleapis.com/" REQUIRED_FIELDS = ['project_id'] @@ -33,22 +34,22 @@ def __init__(self): self.client = monitoring_v3.MetricServiceClient() def export_metric(self, data): - """Export metric to Stackdriver Monitoring. Create metric descriptor if + """Export metric to Cloud Monitoring. Create metric descriptor if it doesn't exist. Args: - data (dict): Data to send to Stackdriver Monitoring. - project_id (str): Stackdriver Monitoring project id. + data (dict): Data to send to Cloud Monitoring. + project_id (str): Cloud Monitoring project id. Returns: - object: Stackdriver Monitoring API result. + object: Cloud Monitoring API result. """ if not self.get_metric_descriptor(data): self.create_metric_descriptor(data) self.create_timeseries(data) def create_timeseries(self, data): - """Create Stackdriver Monitoring timeseries. + """Create Cloud Monitoring timeseries. Args: data (dict): Metric data. @@ -75,18 +76,17 @@ def create_timeseries(self, data): # Set the metric value. point.value.double_value = data['value'] - # Record the timeseries to Stackdriver Monitoring. + # Record the timeseries to Cloud Monitoring. project = self.client.project_path(data['project_id']) - result = self.client.create_time_series(project, [series]) + self.client.create_time_series(project, [series]) labels = series.metric.labels LOGGER.debug( f"timestamp: {timestamp} value: {point.value.double_value}" f"{labels['service_name']}-{labels['feature_name']}-" f"{labels['slo_name']}-{labels['error_budget_policy_step_name']}") - return result def get_metric_descriptor(self, data): - """Get Stackdriver Monitoring metric descriptor. + """Get Cloud Monitoring metric descriptor. Args: data (dict): Metric data. @@ -102,7 +102,7 @@ def get_metric_descriptor(self, data): return None def create_metric_descriptor(self, data): - """Create Stackdriver Monitoring metric descriptor. + """Create Cloud Monitoring metric descriptor. Args: data (dict): Metric data. diff --git a/slo_generator/exporters/pubsub.py b/slo_generator/exporters/pubsub.py index 138a160a..f353298b 100644 --- a/slo_generator/exporters/pubsub.py +++ b/slo_generator/exporters/pubsub.py @@ -16,9 +16,14 @@ Pubsub exporter class. """ import json +import logging from google.cloud import pubsub_v1 +from slo_generator import constants + +LOGGER = logging.getLogger(__name__) + class PubsubExporter: # pylint: disable=too-few-public-methods """Pubsub exporter class.""" @@ -43,4 +48,11 @@ def export(self, data, **config): topic_path = self.publisher.topic_path(project_id, topic_name) data = json.dumps(data, indent=4) data = data.encode('utf-8') - return self.publisher.publish(topic_path, data=data).result() + res = self.publisher.publish(topic_path, data=data).result() + status = f' Export data to {topic_path}' + if not isinstance(res, str): + status = constants.FAIL + status + else: + status = constants.SUCCESS + status + LOGGER.info(status) + return res diff --git a/slo_generator/report.py b/slo_generator/report.py index 11e054f6..620ea4a4 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -20,7 +20,8 @@ from dataclasses import asdict, dataclass, fields, field from slo_generator import utils -from slo_generator.constants import COLORED_OUTPUT, MIN_VALID_EVENTS, NO_DATA +from slo_generator.constants import (COLORED_OUTPUT, MIN_VALID_EVENTS, NO_DATA, + Colors) LOGGER = logging.getLogger(__name__) @@ -32,6 +33,7 @@ class SLOReport: Args: config (dict): SLO configuration. + backend (dict): Backend configuration. step (dict): Error budget policy step configuration. timestamp (int): Timestamp. client (obj): Existing backend client. @@ -43,11 +45,14 @@ class SLOReport: metadata: dict = field(default_factory=dict) # SLO - service_name: str - feature_name: str - slo_name: str - slo_target: float - slo_description: str + name: str + description: str + goal: str + backend: str + exporters: list + error_budget_policy: str = 'default' + + # SLI sli_measurement: float = 0 events_count: int = 0 bad_events_count: int = 0 @@ -59,40 +64,52 @@ class SLOReport: error_budget_target: float error_budget_measurement: float error_budget_burn_rate: float + error_budget_burn_rate_threshold: float error_budget_minutes: float error_budget_remaining_minutes: float error_minutes: float - # Error Budget step config + # Global (from error budget policy) timestamp: int timestamp_human: str offset: int = 0 window: int alert: bool - alerting_burn_rate_threshold: float + consequence_message: str # Data validation valid: bool - def __init__(self, config, step, timestamp, client=None, delete=False): + def __init__(self, + config, + backend, + step, + timestamp, + client=None, + delete=False): # Init dataclass fields from SLO config and Error Budget Policy - self.__set_fields(**config, + spec = config['spec'] + self.__set_fields(**spec, **step, lambdas={ - 'slo_target': float, - 'alerting_burn_rate_threshold': float + 'goal': float, + 'step': int, + 'error_budget_burn_rate_threshold': float }) # Set other fields - self.window = int(step['measurement_window_seconds']) + self.metadata = config['metadata'] self.timestamp = int(timestamp) + self.name = self.metadata['name'] + self.error_budget_policy_step_name = step['name'] + self.error_budget_burn_rate_threshold = float( + step['burn_rate_threshold']) self.timestamp_human = utils.get_human_time(timestamp) self.valid = True - self.metadata = config.get('metadata', {}) # Get backend results - data = self.run_backend(config, client=client, delete=delete) + data = self.run_backend(config, backend, client=client, delete=delete) if not self._validate(data): self.valid = False return @@ -105,7 +122,7 @@ def __init__(self, config, step, timestamp, client=None, delete=False): self.valid = False def build(self, step, data): - """Compute all data necessary for the SLO report. + """Compute all data necessary to build the SLO report. Args: step (dict): Error Budget Policy step configuration. @@ -114,15 +131,14 @@ def build(self, step, data): See https://landing.google.com/sre/workbook/chapters/implementing-slos/ for details on the calculations. """ - info = self.__get_info() - LOGGER.debug(f"{info} | SLO report starting ...") + LOGGER.debug(f"{self.info} | SLO report starting ...") # SLI, Good count, Bad count, Gap from backend results sli, good_count, bad_count = self.get_sli(data) - gap = sli - self.slo_target + gap = sli - self.goal # Error Budget calculations - eb_target = 1 - self.slo_target + eb_target = 1 - self.goal eb_value = 1 - sli eb_remaining_minutes = self.window * gap / 60 eb_target_minutes = self.window * eb_target / 60 @@ -133,13 +149,13 @@ def build(self, step, data): eb_burn_rate = round(eb_value / eb_target, 1) # Alert boolean on burn rate excessive speed. - alert = eb_burn_rate > self.alerting_burn_rate_threshold + alert = eb_burn_rate > self.error_budget_burn_rate_threshold # Manage alerting message. if alert: - consequence_message = step['overburned_consequence_message'] + consequence_message = step['message_alert'] elif eb_burn_rate <= 1: - consequence_message = step['achieved_consequence_message'] + consequence_message = step['message_ok'] else: consequence_message = \ 'Missed for this measurement window, but not enough to alert' @@ -159,12 +175,13 @@ def build(self, step, data): alert=alert, consequence_message=consequence_message) - def run_backend(self, config, client=None, delete=False): + def run_backend(self, config, backend, client=None, delete=False): """Get appropriate backend method from SLO configuration and run it on current SLO config and Error Budget Policy step. Args: config (dict): SLO configuration. + backend (dict): Backend configuration. client (obj, optional): Backend client initiated beforehand. delete (bool, optional): Set to True if we're running a delete action. @@ -172,24 +189,28 @@ def run_backend(self, config, client=None, delete=False): Returns: obj: Backend data. """ - info = self.__get_info() - # Grab backend class and method dynamically. - cfg = config.get('backend', {}) - cls = cfg.get('class') - method = cfg.get('method') - excluded_keys = ['class', 'method', 'measurement'] - backend_cfg = {k: v for k, v in cfg.items() if k not in excluded_keys} - instance = utils.get_backend_cls(cls)(client=client, **backend_cfg) + cls_name = backend.get('class') + method = config['spec']['method'] + excluded_keys = ['class', 'service_level_indicator', 'name'] + backend_cfg = { + k: v for k, v in backend.items() if k not in excluded_keys + } + cls = utils.get_backend_cls(cls_name) + if not cls: + LOGGER.warning(f'{self.info} | Backend {cls_name} not loaded.') + self.valid = False + return None + instance = cls(client=client, **backend_cfg) method = getattr(instance, method) - LOGGER.debug(f'{info} | ' - f'Using backend {cls}.{method.__name__} (from ' + LOGGER.debug(f'{self.info} | ' + f'Using backend {cls_name}.{method.__name__} (from ' f'SLO config file).') # Delete mode activation. if delete and hasattr(instance, 'delete'): method = instance.delete - LOGGER.warning(f'{info} | Delete mode enabled.') + LOGGER.info(f'{self.info} | Delete mode enabled.') # Set offset from class attribute if it exists in the class, otherwise # keep the value defined in config. @@ -200,8 +221,14 @@ def run_backend(self, config, client=None, delete=False): self.timestamp = self.timestamp - self.offset # Run backend method and return results. - data = method(self.timestamp, self.window, config) - LOGGER.debug(f'{info} | Backend response: {data}') + try: + data = method(self.timestamp, self.window, config) + LOGGER.debug(f'{self.info} | Backend response: {data}') + except Exception as exc: # pylint:disable=broad-except + LOGGER.exception(exc) + LOGGER.error( + f'{self.info} | Backend error occured while fetching data.') + return None return data def get_sli(self, data): @@ -223,14 +250,13 @@ def get_sli(self, data): Raises: Exception: When the backend does not return a proper result. """ - info = self.__get_info() if isinstance(data, tuple): # good, bad count good_count, bad_count = data if good_count == NO_DATA: good_count = 0 if bad_count == NO_DATA: bad_count = 0 - LOGGER.debug(f"{info} | Good: {good_count} | Bad: {bad_count}") + LOGGER.debug(f'{self.info} | Good: {good_count} | Bad: {bad_count}') sli_measurement = round(good_count / (good_count + bad_count), 6) else: # sli value sli_measurement = round(data, 6) @@ -252,15 +278,16 @@ def _validate(self, data): Returns: bool: True if data is valid, False """ - info = self.__get_info() + # Backend not found + if data is None: + return False - # Backend data type should be one of tuple, float, or int + # Backend result is the wrong type if not isinstance(data, (tuple, float, int)): LOGGER.error( - f'{info} | Backend method returned an object of type ' + f'{self.info} | Backend method returned an object of type ' f'{type(data).__name__}. It should instead return a tuple ' - '(good_count, bad_count) or a numeric SLI value (float / int).' - ) + '(good_count, bad_count) or a numeric SLI value (float / int).') return False # Good / Bad tuple @@ -269,27 +296,28 @@ def _validate(self, data): # Tuple length should be 2 if len(data) != 2: LOGGER.error( - f'{info} | Backend method returned a tuple with {len(data)}' - ' elements. Expected 2 elements.') + f'{self.info} | Backend method returned a tuple with ' + f'{len(data)} elements. Expected 2 elements.') return False good, bad = data # Tuple should contain only elements of type int or float if not all(isinstance(n, (float, int)) for n in data): - LOGGER.error('f{info} | Backend method returned' + LOGGER.error(f'{self.info} | Backend method returned' 'a tuple with some elements having ' 'a type different than float / int') return False # Tuple should not contain any element with value None. if good is None or bad is None: - LOGGER.error(f'{info} | Backend method returned a valid tuple ' - '{data} but one of the values is None.') + LOGGER.error( + f'{self.info} | Backend method returned a valid tuple ' + f'{data} but one of the values is None.') return False # Tuple should not have NO_DATA everywhere if (good + bad) == (NO_DATA, NO_DATA): - LOGGER.error(f'{info} | Backend method returned a valid ' + LOGGER.error(f'{self.info} | Backend method returned a valid ' f'tuple {data} but the good and bad count ' 'is NO_DATA (-1).') return False @@ -297,18 +325,18 @@ def _validate(self, data): # Tuple should not have elements where the sum is inferior to our # minimum valid events threshold if (good + bad) < MIN_VALID_EVENTS: - LOGGER.error(f"{info} | Not enough valid events found | " - f"Minimum valid events: {MIN_VALID_EVENTS}") + LOGGER.error(f'{self.info} | Not enough valid events found | ' + f'Minimum valid events: {MIN_VALID_EVENTS}') return False # Check backend float / int value if isinstance(data, (float, int)) and data == NO_DATA: - LOGGER.error(f'{info} | Backend returned NO_DATA (-1).') + LOGGER.error(f'{self.info} | Backend returned NO_DATA (-1).') return False # Check backend None if data is None: - LOGGER.error(f'{info} | Backend returned None.') + LOGGER.error(f'{self.info} | Backend returned None.') return False return True @@ -318,8 +346,8 @@ def _post_validate(self): # SLI measurement should be 0 <= x <= 1 if not 0 <= self.sli_measurement <= 1: - LOGGER.error( - f"SLI is not between 0 and 1 (value = {self.sli_measurement})") + LOGGER.error(f'{self.info} | SLI is not between 0 and 1 (value = ' + f'{self.sli_measurement})') return False return True @@ -341,40 +369,30 @@ def __set_fields(self, lambdas={}, **kwargs): value = lambdas[name](value) setattr(self, name, value) - def __get_info(self): - """Get info message describing current SLO andcurrent Error Budget Step. - """ - slo_full_name = self.__get_slo_full_name() - step_name = self.error_budget_policy_step_name - return f"{slo_full_name :<32} | {step_name :<8}" - - def __get_slo_full_name(self): - """Compile full SLO name from SLO configuration. - - Returns: - str: Full SLO name. - """ - return f'{self.service_name}/{self.feature_name}/{self.slo_name}' + @property + def info(self): + """Step information.""" + return f"{self.name :<32} | {self.error_budget_policy_step_name :<8}" def __str__(self): report = self.to_json() - info = self.__get_info() - slo_target_per = self.slo_target * 100 + goal_per = self.goal * 100 sli_per = round(self.sli_measurement * 100, 6) gap = round(self.gap * 100, 2) gap_str = str(gap) if gap >= 0: gap_str = f'+{gap}' - sli_str = (f'SLI: {sli_per:<7} % | SLO: {slo_target_per} % | ' + + sli_str = (f'SLI: {sli_per:<7} % | SLO: {goal_per} % | ' f'Gap: {gap_str:<6}%') - result_str = ("BR: {error_budget_burn_rate:<2} / " - "{alerting_burn_rate_threshold} | " - "Alert: {alert:<1} | Good: {good_events_count:<8} | " - "Bad: {bad_events_count:<8}").format_map(report) - full_str = f'{info} | {sli_str} | {result_str}' + result_str = ('BR: {error_budget_burn_rate:<2} / ' + '{error_budget_burn_rate_threshold} | ' + 'Alert: {alert:<1} | Good: {good_events_count:<8} | ' + 'Bad: {bad_events_count:<8}').format_map(report) + full_str = f'{self.info} | {sli_str} | {result_str}' if COLORED_OUTPUT == 1: if self.alert: - full_str = utils.Colors.FAIL + full_str + utils.Colors.ENDC + full_str = Colors.FAIL + full_str + Colors.ENDC else: - full_str = utils.Colors.OKGREEN + full_str + utils.Colors.ENDC + full_str = Colors.OKGREEN + full_str + Colors.ENDC return full_str diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 255b334c..bbb2e7d0 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -18,7 +18,7 @@ from datetime import datetime import argparse import collections -import glob +import errno import importlib import logging import os @@ -26,39 +26,86 @@ import re import sys import warnings -import yaml +from pathlib import Path from dateutil import tz +import yaml -from google.auth._default import _CLOUD_SDK_CREDENTIALS_WARNING +from slo_generator.constants import DEBUG + +try: + from google.cloud import storage + GCS_ENABLED = True +except ImportError: + GCS_ENABLED = False LOGGER = logging.getLogger(__name__) -def list_slo_configs(path): - """List all SLO configs from path. +def load_configs(path, ctx=os.environ, kind=None): + """Load multiple slo-generator configs from a folder path. - If path is a file, normalize the path and return it as a list with one - element. + Args: + path (str): Folder path. + ctx (dict): Context for variable replacement. + kind (str): Config kind filter. - If path is a folder, get all SLO configs from folder (files starting with - slo_*), normalize their paths and return them as a list. + Returns: + list: List of configs downloaded and parsed. """ - path = normalize(path) - if os.path.isfile(path): - paths = [path] - elif os.path.isdir(path): - paths = sorted(glob.glob(f'{path}/slo_*.yaml')) - else: - raise Exception(f'SLO Config path "{path}" is not a file or folder.') - return paths + configs = [ + load_config(str(p), ctx=ctx, kind=kind) + for p in sorted(Path(path).glob('*.yaml')) + ] + return [cfg for cfg in configs if cfg] -def parse_config(path, ctx=os.environ): +def load_config(path, ctx=os.environ, kind=None): + """Load any slo-generator config, from a local path, a GCS URL, or directly + from a string content. + + Args: + path (str): GCS URL, file path, or data as string. + ctx (dict): Context for variable replacement. + kind (str): Config kind filter. + + Returns: + dict: Config downloaded and parsed. + """ + abspath = Path(path) + try: + if path.startswith('gs://'): + if not GCS_ENABLED: + warnings.warn( + 'To load a file from GCS, you need `google-cloud-storage` ' + 'installed. Please install it using pip by running ' + '`pip install google-cloud-storage`', ImportWarning) + sys.exit(1) + config = parse_config(content=download_gcs_file(str(path)), ctx=ctx) + elif abspath.is_file(): + config = parse_config(path=str(abspath.resolve()), ctx=ctx) + else: + LOGGER.warning(f'Path {path} not found. Trying to load from string') + config = parse_config(content=str(path), ctx=ctx) + + # Filter on 'kind' + if kind: + if not isinstance(config, dict) or kind != config.get('kind', ''): + config = None + return config + + except OSError as exc: + if exc.errno == errno.ENAMETOOLONG: # filename too long, string content + return parse_config(content=str(path), ctx=ctx) + raise + + +def parse_config(path=None, content=None, ctx=os.environ): """Load a yaml configuration file and resolve environment variables in it. Args: path (str): the path to the yaml file. + content (str): the config content as a dict string. ctx (dict): Context to replace env variables from (defaults to `os.environ`). @@ -84,17 +131,18 @@ def replace_env_vars(content, ctx): try: full_value = full_value.replace(f'${{{var}}}', ctx[var]) except KeyError as exception: - LOGGER.error( - f'Environment variable "{var}" should be set.', - exc_info=True) + LOGGER.error(f'Environment variable "{var}" should be set.', + exc_info=True) raise exception content = full_value return content - with open(path) as config: - content = config.read() + if path: + with Path(path).open() as config: + content = config.read() + if ctx: content = replace_env_vars(content, ctx) - data = yaml.safe_load(content) + data = yaml.safe_load(content) LOGGER.debug(pprint.pformat(data)) return data @@ -102,20 +150,27 @@ def replace_env_vars(content, ctx): def setup_logging(): """Setup logging for the CLI.""" - debug = os.environ.get("DEBUG", "0") - print("DEBUG: %s" % debug) - if debug == "1": + if DEBUG == 1: + print(f'DEBUG mode is enabled. DEBUG={DEBUG}') level = logging.DEBUG + format_str = '%(name)s - %(levelname)s - %(message)s' else: level = logging.INFO + format_str = '%(levelname)s - %(message)s' logging.basicConfig(stream=sys.stdout, level=level, - format='%(name)s - %(levelname)s - %(message)s', + format=format_str, datefmt='%m/%d/%Y %I:%M:%S') logging.getLogger('googleapiclient').setLevel(logging.ERROR) # Ignore Cloud SDK warning when using a user instead of service account - warnings.filterwarnings("ignore", message=_CLOUD_SDK_CREDENTIALS_WARNING) + try: + # pylint: disable=import-outside-toplevel + from google.auth._default import _CLOUD_SDK_CREDENTIALS_WARNING + warnings.filterwarnings("ignore", + message=_CLOUD_SDK_CREDENTIALS_WARNING) + except ImportError: + pass def get_human_time(timestamp, timezone=None): @@ -143,21 +198,76 @@ def get_human_time(timestamp, timezone=None): dt_tz = dt_utc.replace(tzinfo=to_zone) timeformat = '%Y-%m-%dT%H:%M:%S.%f%z' date_str = datetime.strftime(dt_tz, timeformat) - date_str = "{0}:{1}".format( - date_str[:-2], date_str[-2:] - ) + date_str = "{0}:{1}".format(date_str[:-2], date_str[-2:]) return date_str -def normalize(path): - """Converts a path to an absolute path. + +def get_exporters(config, spec): + """Get SLO exporters configs from spec and global config. Args: - path (str): Input path. + config (dict): Global config. + spec (dict): SLO config. Returns: - str: Absolute path. + list: List of dict containing exporters configurations. """ - return os.path.abspath(path) + all_exporters = config.get('exporters', {}) + spec_exporters = spec.get('exporters', []) + exporters = [] + for exporter in spec_exporters: + if exporter not in all_exporters.keys(): + LOGGER.warning( + f'Exporter "{exporter}" not found in config. Skipping.') + continue + exporter_data = all_exporters[exporter] + exporter_data['name'] = exporter + exporter_data['class'] = capitalize( + snake_to_caml(exporter.split('/')[0])) + exporters.append(exporter_data) + return exporters + + +def get_backend(config, spec): + """Get SLO backend config from spec and global config. + + Args: + config (dict): Global config. + spec (dict): SLO config. + + Returns: + list: List of dict containing exporters configurations. + """ + all_backends = config.get('backends', {}) + spec_backend = spec['backend'] + backend_data = {} + if spec_backend not in all_backends.keys(): + LOGGER.error(f'Backend "{spec_backend}" not found in config. Exiting.') + sys.exit(0) + backend_data = all_backends[spec_backend] + backend_data['name'] = spec_backend + backend_data['class'] = capitalize(snake_to_caml( + spec_backend.split('/')[0])) + return backend_data + + +def get_error_budget_policy(config, spec): + """Get error budget policy from spec and global config. + + Args: + config (dict): Global config. + spec (dict): SLO config. + + Returns: + list: List of dict containing exporters configurations. + """ + all_ebp = config.get('error_budget_policies', {}) + spec_ebp = spec.get('error_budget_policy', 'default') + if spec_ebp not in all_ebp.keys(): + LOGGER.error( + f'Error budget policy "{spec_ebp}" not found in config. Exiting.') + sys.exit(0) + return all_ebp[spec_ebp] def get_backend_cls(backend): @@ -186,27 +296,28 @@ def get_exporter_cls(exporter): return import_cls(exporter, expected_type) -def import_cls(klass_name, expected_type): +def import_cls(cls_name, expected_type): """Import class or method dynamically from full name. - If the classname is not fully qualified, import in current sub modules. + If `cls_name` is not part of the core, try import from local path (plugins). Args: - klass_name: the class name to import - expected_type: the type of class expected + cls_name: Class name to import. + expected_type: Type of class expected. Returns: obj: Imported class or method object. """ - if "." in klass_name: - package, name = klass_name.rsplit(".", maxsplit=1) + # plugin class + if "." in cls_name: + package, name = cls_name.rsplit(".", maxsplit=1) return import_dynamic(package, name, prefix=expected_type) - # else + # slo-generator core class modules_name = f"{expected_type.lower()}s" - full_klass_name = f'{klass_name}{expected_type}' - filename = re.sub(r'(? Date: Mon, 31 May 2021 13:29:07 +0200 Subject: [PATCH 007/107] feat!: Upgrade slo-generator CLI to Click library (#131) * feat!: Upgrade slo-geneartor CLI to Click library * Change --version implementation to use pkg_resources instead of importlib which is different in Python 3.6 and above --- slo_generator/cli.py | 224 +++++++++++++++++++++++++++---------------- 1 file changed, 139 insertions(+), 85 deletions(-) diff --git a/slo_generator/cli.py b/slo_generator/cli.py index d4dc1512..ab012e23 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -16,114 +16,168 @@ Command-Line interface of `slo-generator`. """ -import argparse import logging import os import sys import time +from pathlib import Path +from pkg_resources import get_distribution -import slo_generator.utils as utils -from slo_generator.compute import compute +import click +from slo_generator import utils +from slo_generator.compute import compute as _compute +from slo_generator.migrations import migrator +from slo_generator.constants import LATEST_MAJOR_VERSION sys.path.append(os.getcwd()) # dynamic backend loading LOGGER = logging.getLogger(__name__) -def main(): - """slo-generator CLI entrypoint.""" - args = parse_args(sys.argv[1:]) - cli(args) - - -def cli(args): - """Main CLI function. - - Args: - args (Namespace): Argparsed CLI parameters. - - Returns: - dict: Dict of all reports indexed by config file path. - """ +@click.group(invoke_without_command=True) +@click.option('--version', + '-v', + is_flag=True, + help='Show slo-generator version.') +@click.pass_context +def main(ctx, version): + """CLI entrypoint.""" utils.setup_logging() - export = args.export - delete = args.delete - timestamp = args.timestamp + if ctx.invoked_subcommand is None or version: + ver = get_distribution('slo-generator').version + print(f'slo-generator v{ver}') + sys.exit(0) + + +@main.command() +@click.option('--slo-config', + '-f', + type=click.Path(), + required=True, + help='SLO config path') +@click.option('--config', + '-c', + type=click.Path(exists=True), + default='config.yaml', + show_default=True, + help='slo-generator config path') +@click.option('--export', + '-e', + is_flag=True, + help='Export SLO report to exporters') +@click.option('--delete', + '-d', + is_flag=True, + help='Delete mode (used for backends with SLO APIs)') +@click.option('--timestamp', + '-t', + type=float, + default=time.time(), + help='End timestamp for query.') +def compute(slo_config, config, export, delete, timestamp): + """Compute SLO report.""" start = time.time() - # Load error budget policy - LOGGER.debug(f"Loading Error Budget config from {args.error_budget_policy}") - eb_path = utils.normalize(args.error_budget_policy) - eb_policy = utils.parse_config(eb_path) + # Load slo-generator config + LOGGER.debug(f"Loading slo-generator config from {config}") + config_dict = utils.load_config(config) + + # Load SLO config(s) + if Path(slo_config).is_dir(): + slo_configs = utils.load_configs(slo_config, + kind='ServiceLevelObjective') + else: + slo_configs = [ + utils.load_config(slo_config, kind='ServiceLevelObjective') + ] - # Parse SLO folder for configs - slo_configs = utils.list_slo_configs(args.slo_config) if not slo_configs: - LOGGER.error(f'No SLO configs found in SLO folder {args.slo_config}.') + LOGGER.error(f'No SLO configs found in {slo_config}.') + sys.exit(1) # Load SLO configs and compute SLO reports all_reports = {} - for path in slo_configs: - slo_config_name = path.split("/")[-1] - LOGGER.debug(f'Loading SLO config "{slo_config_name}"') - slo_config = utils.parse_config(path) - reports = compute(slo_config, - eb_policy, - timestamp=timestamp, - do_export=export, - delete=delete) - all_reports[path] = reports + for slo_config_dict in slo_configs: + reports = _compute(slo_config_dict, + config_dict, + timestamp=timestamp, + do_export=export, + delete=delete) + if reports: + name = slo_config_dict['metadata']['name'] + all_reports[name] = reports end = time.time() duration = round(end - start, 1) LOGGER.info(f'Run summary | SLO Configs: {len(slo_configs)} | ' - f'Error Budget Policy Steps: {len(eb_policy)} | ' - f'Total: {len(slo_configs) * len(eb_policy)} | ' f'Duration: {duration}s') + LOGGER.debug(all_reports) return all_reports -def parse_args(args): - """Parse CLI arguments. - - Args: - args (list): List of args passed from CLI. - - Returns: - obj: Args parsed by ArgumentParser. - """ - parser = argparse.ArgumentParser() - parser.add_argument('--slo-config', - '-f', - type=str, - required=False, - help='SLO configuration file (JSON / YAML)') - parser.add_argument('--error-budget-policy', - '-b', - type=str, - required=False, - default='error_budget_policy.yaml', - help='Error budget policy file (JSON / YAML)') - parser.add_argument('--export', - '-e', - type=utils.str2bool, - nargs='?', - const=True, - default=False, - help="Export SLO reports") - parser.add_argument('--delete', - '-d', - type=utils.str2bool, - nargs='?', - const=True, - default=False, - help="Delete SLO (use for backends with APIs).") - parser.add_argument('--timestamp', - '-t', - type=int, - default=None, - help="Start timestamp for query.") - return parser.parse_args(args) - - -if __name__ == '__main__': - main() +# pylint: disable=import-error,import-outside-toplevel +@main.command() +@click.pass_context +@click.option('--config', + envvar='CONFIG_PATH', + required=True, + help='slo-generator configuration file path.') +@click.option('--signature-type', + envvar='GOOGLE_FUNCTION_SIGNATURE_TYPE', + default='http', + type=click.Choice(['http', 'cloudevent']), + help='Signature type') +@click.option('--target', + envvar='GOOGLE_FUNCTION_SIGNATURE_TYPE', + default='run_compute', + help='Target function name') +def api(ctx, config, signature_type, target): + """Run an API that can receive requests (supports both 'http' and + 'cloudevents' signature types).""" + from functions_framework._cli import _cli + os.environ['CONFIG_PATH'] = config + os.environ['GOOGLE_FUNCTION_SIGNATURE_TYPE'] = signature_type + os.environ['GOOGLE_FUNCTION_TARGET'] = target + ctx.invoke(_cli, + target=target, + source=Path(__file__).parent / 'api' / 'main.py', + signature_type=signature_type) + + +@main.command() +@click.option('--source', + '-s', + type=click.Path(exists=True, resolve_path=True, readable=True), + required=True, + default=Path.cwd(), + help='Source SLO configs folder') +@click.option('--target', + '-t', + type=click.Path(resolve_path=True), + default=Path.cwd(), + required=True, + help='Target SLO configs folder') +@click.option('--error-budget-policy-path', + '-b', + type=click.Path(exists=True, resolve_path=True, readable=True), + required=False, + default='error_budget_policy.yaml', + help='Error budget policy path') +@click.option('--glob', + type=str, + required=False, + default='**/slo_*.yaml', + help='Glob expression to seek SLO configs in subpaths') +@click.option('--version', + type=str, + required=False, + default=LATEST_MAJOR_VERSION, + show_default=True, + help='SLO generate major version to migrate towards') +@click.option('--quiet', + '-q', + is_flag=True, + default=False, + help='Do not ask for user input and auto-generate config keys') +def migrate(**kwargs): + """Migrate SLO configs from v1 to v2.""" + migrator.do_migrate(**kwargs) From cd304899025d095d2aa7c5615f7a3fb6bbe57eeb Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 13:33:26 +0200 Subject: [PATCH 008/107] docs: Update documentation for v2 (#133) --- README.md | 250 ++++++++++++------ docs/deploy/cloudfunctions.md | 8 +- .../{stackdriver.md => cloud_monitoring.md} | 119 +++++---- ...itoring.md => cloud_service_monitoring.md} | 188 +++++++------ docs/providers/custom.md | 44 ++- docs/providers/datadog.md | 81 +++--- docs/providers/dynatrace.md | 59 ++--- docs/providers/elasticsearch.md | 13 +- docs/providers/prometheus.md | 90 +++---- docs/providers/pubsub.md | 14 +- docs/shared/metrics.md | 10 +- docs/shared/migration.md | 33 +++ docs/shared/troubleshooting.md | 2 +- samples/README.md | 22 +- 14 files changed, 533 insertions(+), 400 deletions(-) rename docs/providers/{stackdriver.md => cloud_monitoring.md} (52%) rename docs/providers/{stackdriver_service_monitoring.md => cloud_service_monitoring.md} (51%) create mode 100644 docs/shared/migration.md diff --git a/README.md b/README.md index b10e4a22..c0c78bed 100644 --- a/README.md +++ b/README.md @@ -5,111 +5,213 @@ [![PyPI version](https://badge.fury.io/py/slo-generator.svg)](https://badge.fury.io/py/slo-generator) `slo-generator` is a tool to compute and export **Service Level Objectives** ([SLOs](https://landing.google.com/sre/sre-book/chapters/service-level-objectives/)), -**Error Budgets** and **Burn Rates**, using policies written in JSON or YAML format. +**Error Budgets** and **Burn Rates**, using configurations written in YAML (or JSON) format. + +## Table of contents +- [Description](#description) +- [Local usage](#local-usage) + - [Requirements](#requirements) + - [Installation](#installation) + - [CLI usage](#cli-usage) + - [API usage](#api-usage) +- [Configuration](#configuration) + - [SLO configuration](#slo-configuration) + - [Shared configuration](#shared-configuration) +- [More documentation](#more-documentation) + - [Build an SLO achievements report with BigQuery and DataStudio](#build-an-slo-achievements-report-with-bigquery-and-datastudio) + - [Deploy the SLO Generator in Cloud Run](#deploy-the-slo-generator-in-cloud-run) + - [Deploy the SLO Generator in Kubernetes (Alpha)](#deploy-the-slo-generator-in-kubernetes-alpha) + - [Deploy the SLO Generator in a CloudBuild pipeline](#deploy-the-slo-generator-in-a-cloudbuild-pipeline) + - [DEPRECATED: Deploy the SLO Generator on Google Cloud Functions (Terraform)](#deprecated-deploy-the-slo-generator-on-google-cloud-functions-terraform) + - [Contribute to the SLO Generator](#contribute-to-the-slo-generator) ## Description -`slo-generator` will query metrics backend and compute the following metrics: +The `slo-generator` runs backend queries computing **Service Level Indicators**, +compares them with the **Service Level Objectives** defined and generates a report +by computing important metrics: -* **Service Level Objective** defined as `SLO (%) = GOOD_EVENTS / VALID_EVENTS` -* **Error Budget** defined as `ERROR_BUDGET = 100 - SLO (%)` -* **Burn Rate** defined as `BURN_RATE = ERROR_BUDGET / ERROR_BUDGET_TARGET` +* **Service Level Indicator** (SLI) defined as **SLI = Ngood_events / Nvalid_events** +* **Error Budget** (EB) defined as **EB = 1 - SLI** +* **Error Budget Burn Rate** (EBBR) defined as **EBBR = EB / EBtarget** +* **... and more**, see the [example SLO report](./test/unit/../../tests/unit/fixtures/slo_report_v2.json). + +The **Error Budget Burn Rate** is often used for [**alerting on SLOs**](https://sre.google/workbook/alerting-on-slos/), as it demonstrates in practice to be more **reliable** and **stable** than +alerting directly on metrics or on **SLI > SLO** thresholds. ## Local usage -**Requirements** +### Requirements -* Python 3 +* `python3.7` and above +* `pip3` -**Installation** +### Installation -`slo-generator` is published on PyPI. To install it, run: +`slo-generator` is a Python library published on [PyPI](https://pypi.org). To install it, run: ```sh pip3 install slo-generator ``` -**Run the `slo-generator`** - -``` -slo-generator -f -b --export -``` - * `` is the [SLO config](#slo-configuration) file or folder. - If a folder path is passed, the SLO configs filenames should match the pattern `slo_*.yaml` to be loaded. - - * `` is the [Error Budget Policy](#error-budget-policy) file. - - * `--export` enables exporting data using the `exporters` defined in the SLO - configuration file. - -Use `slo-generator --help` to list all available arguments. - ***Notes:*** +* To install **[providers](./docs/providers)**, use `pip3 install slo-generator[, , ... -c --export +``` +where: + * `` is the [SLO configuration](#slo-configuration) file or folder path. -* **SLO metadata**: - * `slo_name`: Name of this SLO. - * `slo_description`: Description of this SLO. - * `slo_target`: SLO target (between 0 and 1). - * `service_name`: Name of the monitored service. - * `feature_name`: Name of the monitored subsystem. - * `metadata`: Dict of user metadata. + * `` is the [Shared configuration](#shared-configuration) file path. + * `--export` | `-e` enables exporting data using the `exporters` specified in the SLO + configuration file. -* **SLI configuration**: - * `backend`: Specific documentation and examples are available for each supported backends: - * [Stackdriver Monitoring](docs/providers/stackdriver.md#backend) - * [Stackdriver Service Monitoring](docs/providers/stackdriver_service_monitoring.md#backend) - * [Prometheus](docs/providers/prometheus.md#backend) - * [ElasticSearch](docs/providers/elasticsearch.md#backend) - * [Datadog](docs/providers/datadog.md#backend) - * [Dynatrace](docs/providers/dynatrace.md#backend) - * [Custom](docs/providers/custom.md#backend) +Use `slo-generator compute --help` to list all available arguments. -- **Exporter configuration**: - * `exporters`: A list of exporters to export results to. Specific documentation is available for each supported exporters: - * [Cloud Pub/Sub](docs/providers/pubsub.md#exporter) to stream SLO reports. - * [BigQuery](docs/providers/bigquery.md#exporter) to export SLO reports to BigQuery for historical analysis and DataStudio reporting. - * [Stackdriver Monitoring](docs/providers/stackdriver.md#exporter) to export metrics to Stackdriver Monitoring. - * [Prometheus](docs/providers/prometheus.md#exporter) to export metrics to Prometheus. - * [Datadog](docs/providers/datadog.md#exporter) to export metrics to Datadog. - * [Dynatrace](docs/providers/dynatrace.md#exporter) to export metrics to Dynatrace. - * [Custom](docs/providers/custom.md#exporter) to export SLO data or metrics to a custom destination. +### API usage -***Note:*** *you can use environment variables in your SLO configs by using `${MY_ENV_VAR}` syntax to avoid having sensitive data in version control. Environment variables will be replaced at run time.* +On top of the CLI, the `slo-generator` can also be run as an API using the Cloud +Functions Framework SDK (Flask): +``` +slo-generator api -c +``` +where: + * `` is the [Shared configuration](#shared-configuration) file path or GCS URL. -==> An example SLO configuration file is available [here](samples/stackdriver/slo_gae_app_availability.yaml). +Once the API is up-and-running, you can `HTTP POST` SLO configurations to it. -#### Error Budget policy +***Notes:*** +* The API responds by default to HTTP requests. An alternative mode is to +respond to [`CloudEvents`](https://cloudevents.io/) instead, by setting +`--signature-type cloudevent`. -The **Error Budget policy** (JSON or YAML) is a list of multiple error budgets, each one composed of the following fields: +* Use `--target export` to run the API in export mode only (former `slo-pipeline`). -* `window`: Rolling time window for this error budget. -* `alerting_burn_rate_threshold`: Target burnrate threshold over which alerting is needed. -* `urgent_notification`: boolean whether violating this error budget should trigger a page. -* `overburned_consequence_message`: message to show when the error budget is above the target. -* `achieved_consequence_message`: message to show when the error budget is within the target. +## Configuration -==> An example Error Budget policy is available [here](samples/error_budget_policy.yaml). +The `slo-generator` requires two configuration files to run, an **SLO configuration** +file, describing your SLO, and the **Shared configuration** file (common +configuration for all SLOs). + +### SLO configuration + +The **SLO configuration** (JSON or YAML) is following the Kubernetes format and +is composed of the following fields: + +* `api`: `sre.google.com/v2` +* `kind`: `ServiceLevelObjective` +* `metadata`: + * `name`: [**required**] *string* - Full SLO name (**MUST** be unique). + * `labels`: [*optional*] *map* - Metadata labels, **for example**: + * `slo_name`: SLO name (e.g `availability`, `latency128ms`, ...). + * `service_name`: Monitored service (to group SLOs by service). + * `feature_name`: Monitored feature (to group SLOs by feature). + +* `spec`: + * `description`: [**required**] *string* - Description of this SLO. + * `goal`: [**required**] *string* - SLO goal (or target) (**MUST** be between 0 and 1). + * `backend`: [**required**] *string* - Backend name (**MUST** exist in SLO Generator Configuration). + * `service_level_indicator`: [**required**] *map* - SLI configuration. The content of this section is + specific to each provider, see [`docs/providers`](./docs/providers). + * `error_budget_policy`: [*optional*] *string* - Error budget policy name + (**MUST** exist in SLO Generator Configuration). If not specified, defaults to `default`. + * `exporters`: [*optional*] *string* - List of exporter names (**MUST** exist in SLO Generator Configuration). + +***Note:*** *you can use environment variables in your SLO configs by using +`${MY_ENV_VAR}` syntax to avoid having sensitive data in version control. +Environment variables will be replaced automatically at run time.* + +**→ See [example SLO configuration](samples/cloud_monitoring/slo_gae_app_availability.yaml).** + +### Shared configuration +The shared configuration (JSON or YAML) configures the `slo-generator` and acts +as a shared config for all SLO configs. It is composed of the following fields: + +* `backends`: [**required**] *map* - Data backends configurations. Each backend + alias is defined as a key `/`, and a configuration map. + ```yaml + backends: + cloud_monitoring/dev: + project_id: proj-cm-dev-a4b7 + datadog/test: + app_key: ${APP_SECRET_KEY} + api_key: ${API_SECRET_KEY} + ``` + See specific providers documentation for detailed configuration: + * [`cloud_monitoring`](docs/providers/cloud_monitoring.md#backend) + * [`cloud_service_monitoring`](docs/providers/cloud_service_monitoring.md#backend) + * [`prometheus`](docs/providers/prometheus.md#backend) + * [`elasticsearch`](docs/providers/elasticsearch.md#backend) + * [`datadog`](docs/providers/datadog.md#backend) + * [`dynatrace`](docs/providers/dynatrace.md#backend) + * [``](docs/providers/custom.md#backend) + +* `exporters`: A map of exporters to export results to. Each exporter is defined + as a key formatted as `/`, and a map value detailing the + exporter configuration. + ```yaml + exporters: + bigquery/dev: + project_id: proj-bq-dev-a4b7 + dataset_id: my-test-dataset + table_id: my-test-table + prometheus/test: + url: ${PROMETHEUS_URL} + ``` + See specific providers documentation for detailed configuration: + * [`pubsub`](docs/providers/pubsub.md#exporter) to stream SLO reports. + * [`bigquery`](docs/providers/bigquery.md#exporter) to export SLO reports to BigQuery for historical analysis and DataStudio reporting. + * [`cloud_monitoring`](docs/providers/cloud_monitoring.md#exporter) to export metrics to Cloud Monitoring. + * [`prometheus`](docs/providers/prometheus.md#exporter) to export metrics to Prometheus. + * [`datadog`](docs/providers/datadog.md#exporter) to export metrics to Datadog. + * [`dynatrace`](docs/providers/dynatrace.md#exporter) to export metrics to Dynatrace. + * [``](docs/providers/custom.md#exporter) to export SLO data or metrics to a custom destination. + +* `error_budget_policies`: [**required**] A map of various error budget policies. + * ``: Name of the error budget policy. + * `steps`: List of error budget policy steps, each containing the following fields: + * `window`: Rolling time window for this error budget. + * `alerting_burn_rate_threshold`: Target burnrate threshold over which alerting is needed. + * `urgent_notification`: boolean whether violating this error budget should trigger a page. + * `overburned_consequence_message`: message to show when the error budget is above the target. + * `achieved_consequence_message`: message to show when the error budget is within the target. + + ```yaml + error_budget_policies: + default: + steps: + - name: 1 hour + burn_rate_threshold: 9 + alert: true + message_alert: Page to defend the SLO + message_ok: Last hour on track + window: 3600 + - name: 12 hours + burn_rate_threshold: 3 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 12 hours on track + window: 43200 + ``` + +**→ See [example Shared configuration](samples/config.yaml).** ## More documentation To go further with the SLO Generator, you can read: -* [Build an SLO achievements report with BigQuery and DataStudio](docs/deploy/datastudio_slo_report.md) - -* [Deploy the SLO Generator on Google Cloud Functions (Terraform)](docs/deploy/cloudfunctions.md) - -* [Deploy the SLO Generator on Kubernetes (Alpha)](docs/deploy/kubernetes.md) - -* [Deploy the SLO Generator in a CloudBuild pipeline](docs/deploy/cloudbuild.md) - -* [Contribute to the SLO Generator](CONTRIBUTING.md) +### [Build an SLO achievements report with BigQuery and DataStudio](docs/deploy/datastudio_slo_report.md) +### [Deploy the SLO Generator in Cloud Run](docs/deploy/cloudrun.md) +### [Deploy the SLO Generator in Kubernetes (Alpha)](docs/deploy/kubernetes.md) +### [Deploy the SLO Generator in a CloudBuild pipeline](docs/deploy/cloudbuild.md) +### [DEPRECATED: Deploy the SLO Generator on Google Cloud Functions (Terraform)](docs/deploy/cloudfunctions.md) +### [Contribute to the SLO Generator](CONTRIBUTING.md) diff --git a/docs/deploy/cloudfunctions.md b/docs/deploy/cloudfunctions.md index 301043d8..72c032c4 100644 --- a/docs/deploy/cloudfunctions.md +++ b/docs/deploy/cloudfunctions.md @@ -9,8 +9,8 @@ Other components can be added to make results available to other destinations: -* A **Cloud Function** to export SLO reports (e.g: to BigQuery and Stackdriver Monitoring), running `slo-generator`. -* A **Stackdriver Monitoring Policy** to alert on high budget Burn Rates. +* A **Cloud Function** to export SLO reports (e.g: to BigQuery and Cloud Monitoring), running `slo-generator`. +* A **Cloud Monitoring Policy** to alert on high budget Burn Rates. Below is a diagram of what this pipeline looks like: @@ -22,9 +22,9 @@ Below is a diagram of what this pipeline looks like: * **Historical analytics** by analyzing SLO data in Bigquery. -* **Real-time alerting** by setting up Stackdriver Monitoring alerts based on +* **Real-time alerting** by setting up Cloud Monitoring alerts based on wanted SLOs. * **Real-time, daily, monthly, yearly dashboards** by streaming BigQuery SLO reports to DataStudio (see [here](datastudio_slo_report.md)) and building dashboards. -An example of pipeline automation with Terraform can be found in the corresponding [Terraform module](https://github.com/terraform-google-modules/terraform-google-slo/tree/master/examples/simple_example). +An example of pipeline automation with Terraform can be found in the corresponding [Terraform module](https://github.com/terraform-google-modules/terraform-google-slo/tree/master/examples/slo-generator/simple_example). diff --git a/docs/providers/stackdriver.md b/docs/providers/cloud_monitoring.md similarity index 52% rename from docs/providers/stackdriver.md rename to docs/providers/cloud_monitoring.md index 2e107d6c..26054ad2 100644 --- a/docs/providers/stackdriver.md +++ b/docs/providers/cloud_monitoring.md @@ -1,16 +1,23 @@ -# Stackdriver Monitoring +# Cloud Monitoring ## Backend -Using the `Stackdriver` backend class, you can query any metrics available in -Stackdriver Monitoring to create an SLO. +Using the `cloud_monitoring` backend class, you can query any metrics available +in `Cloud Monitoring` to create an SLO. -The following methods are available to compute SLOs with the `Stackdriver` +```yaml +backends: + cloud_monitoring: + project_id: "${WORKSPACE_PROJECT_ID}" +``` + +The following methods are available to compute SLOs with the `cloud_monitoring` backend: * `good_bad_ratio` for metrics of type `DELTA`, `GAUGE`, or `CUMULATIVE`. * `distribution_cut` for metrics of type `DELTA` and unit `DISTRIBUTION`. + ### Good / bad ratio The `good_bad_ratio` method is used to compute the ratio between two metrics: @@ -23,84 +30,75 @@ SLO. This method is often used for availability SLOs, but can be used for other purposes as well (see examples). -**Config example:** +**SLO config blob:** ```yaml -backend: - class: Stackdriver - project_id: "${STACKDRIVER_HOST_PROJECT_ID}" - method: good_bad_ratio - measurement: - filter_good: > - project="${GAE_PROJECT_ID}" - metric.type="appengine.googleapis.com/http/server/response_count" - metric.labels.response_code >= 200 - metric.labels.response_code < 500 - filter_valid: > - project="${GAE_PROJECT_ID}" - metric.type="appengine.googleapis.com/http/server/response_count" +backend: cloud_monitoring +method: good_bad_ratio +service_level_indicator: + filter_good: > + project="${GAE_PROJECT_ID}" + metric.type="appengine.googleapis.com/http/server/response_count" + metric.labels.response_code >= 200 + metric.labels.response_code < 500 + filter_valid: > + project="${GAE_PROJECT_ID}" + metric.type="appengine.googleapis.com/http/server/response_count" ``` You can also use the `filter_bad` field which identifies bad events instead of the `filter_valid` field which identifies all valid events. -**→ [Full SLO config](../../samples/stackdriver/slo_gae_app_availability.yaml)** +**→ [Full SLO config](../../samples/cloud_monitoring/slo_gae_app_availability.yaml)** ### Distribution cut -The `distribution_cut` method is used for Stackdriver distribution-type metrics, -which are usually used for latency metrics. +The `distribution_cut` method is used for Cloud Monitoring distribution-type +metrics, which are usually used for latency metrics. A distribution metric records the **statistical distribution of the extracted values** in **histogram buckets**. The extracted values are not recorded individually, but their distribution across the configured buckets are recorded, along with the `count`, `mean`, and `sum` of squared deviation of the values. -In `Stackdriver Monitoring`, there are three different ways to specify bucket +In Cloud Monitoring, there are three different ways to specify bucket boundaries: * **Linear:** Every bucket has the same width. * **Exponential:** Bucket widths increases for higher values, using an exponential growth factor. * **Explicit:** Bucket boundaries are set for each bucket using a bounds array. -**Config example:** +**SLO config blob:** ```yaml -backend: - class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - method: exponential_distribution_cut - measurement: - filter_valid: > - project=${GAE_PROJECT_ID} AND - metric.type=appengine.googleapis.com/http/server/response_latencies AND - metric.labels.response_code >= 200 AND - metric.labels.response_code < 500 - good_below_threshold: true - threshold_bucket: 19 +backend: cloud_monitoring +method: exponential_distribution_cut +service_level_indicator: + filter_valid: > + project=${GAE_PROJECT_ID} AND + metric.type=appengine.googleapis.com/http/server/response_latencies AND + metric.labels.response_code >= 200 AND + metric.labels.response_code < 500 + good_below_threshold: true + threshold_bucket: 19 ``` -**→ [Full SLO config](../../samples/stackdriver/slo_gae_app_latency.yaml)** +**→ [Full SLO config](../../samples/cloud_monitoring/slo_gae_app_latency.yaml)** The `threshold_bucket` number to reach our 724ms target latency will depend on how the buckets boundaries are set. Learn how to [inspect your distribution metrics](https://cloud.google.com/logging/docs/logs-based-metrics/distribution-metrics#inspecting_distribution_metrics) to figure out the bucketization. ## Exporter -The `Stackdriver` exporter allows to export SLO metrics to Cloud Monitoring API. - -**Example config:** - -The following configuration will create the custom metric -`error_budget_burn_rate` in `Stackdriver Monitoring`: +The `cloud_monitoring` exporter allows to export SLO metrics to Cloud Monitoring API. ```yaml -exporters: - - class: Stackdriver - project_id: "${STACKDRIVER_HOST_PROJECT_ID}" +backends: + cloud_monitoring: + project_id: "${WORKSPACE_PROJECT_ID}" ``` Optional fields: - * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`custom:error_budget_burn_rate`, `custom:sli_measurement`]. + * `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). -**→ [Full SLO config](../../samples/stackdriver/slo_lb_request_availability.yaml)** +**→ [Full SLO config](../../samples/cloud_monitoring/slo_lb_request_availability.yaml)** ## Alerting @@ -109,6 +107,7 @@ being able to alert on them is simply useless. **Too many alerts** can be daunting, and page your SRE engineers for no valid reasons. + **Too little alerts** can mean that your applications are not monitored at all (no application have 100% reliability). @@ -117,24 +116,24 @@ reduce the noise and page only when it's needed. **Example:** -We will define a `Stackdriver Monitoring` alert that we will **filter out on the +We will define a `Cloud Monitoring` alert that we will **filter out on the corresponding error budget step**. -Consider the following error budget policy config: +Consider the following error budget policy step config: ```yaml -- error_budget_policy_step_name: 1 hour - measurement_window_seconds: 3600 - alerting_burn_rate_threshold: 9 - urgent_notification: true - overburned_consequence_message: Page the SRE team to defend the SLO - achieved_consequence_message: Last hour on track +- name: 1 hour + window: 3600 + burn_rate_threshold: 9 + alert: true + message_alert: Page the SRE team to defend the SLO + message_ok: Last hour on track ``` -Using Stackdriver UI, let's set up an alert when our error budget burn rate is -burning **9X faster** than it should in the last hour: +Using Cloud Monitoring UI, let's set up an alert when our error budget burn rate +is burning **9X faster** than it should in the last hour: -* Open `Stackdriver Monitoring` and click on `Alerting > Create Policy` +* Open `Cloud Monitoring` and click on `Alerting > Create Policy` * Fill the alert name and click on `Add Condition`. @@ -163,5 +162,5 @@ differentiate the alert messages. ## Examples -Complete SLO samples using `Stackdriver` are available in -[samples/stackdriver](../../samples/stackdriver). Check them out ! +Complete SLO samples using Cloud Monitoring are available in +[samples/cloud_monitoring](../../samples/cloud_monitoring). Check them out ! diff --git a/docs/providers/stackdriver_service_monitoring.md b/docs/providers/cloud_service_monitoring.md similarity index 51% rename from docs/providers/stackdriver_service_monitoring.md rename to docs/providers/cloud_service_monitoring.md index b72d71c6..bea7d835 100644 --- a/docs/providers/stackdriver_service_monitoring.md +++ b/docs/providers/cloud_service_monitoring.md @@ -1,15 +1,21 @@ -# Stackdriver Service Monitoring +# Cloud Service Monitoring ## Backend -Using the `StackdriverServiceMonitoring` backend class, you can use the -`Stackdriver Service Monitoring API` to manage your SLOs. +Using the `cloud_service_monitoring` backend, you can use the +`Cloud Service Monitoring API` to manage your SLOs. -SLOs are created from standard metrics available in Stackdriver Monitoring and -the data is stored in `Stackdriver Service Monitoring API` (see +```yaml +backends: + cloud_service_monitoring: + project_id: "${WORKSPACE_PROJECT_ID}" +``` + +SLOs are created from standard metrics available in Cloud Monitoring and +the data is stored in `Cloud Service Monitoring API` (see [docs](https://cloud.google.com/monitoring/service-monitoring/using-api)). -The following methods are available to compute SLOs with the `Stackdriver` +The following methods are available to compute SLOs with the `cloud_service_monitoring` backend: * `basic` to create standard SLOs for Google App Engine, Google Kubernetes @@ -17,91 +23,84 @@ Engine, and Cloud Endpoints. * `good_bad_ratio` for metrics of type `DELTA` or `CUMULATIVE`. * `distribution_cut` for metrics of type `DELTA` and unit `DISTRIBUTION`. + ### Basic -The `basic` method is used to let the `Stackdriver Service Monitoring API` +The `basic` method is used to let the `Cloud Service Monitoring API` automatically generate standardized SLOs for the following GCP services: * **Google App Engine** * **Google Kubernetes Engine** (with Istio) * **Google Cloud Endpoints** -The SLO configuration uses Stackdriver +The SLO configuration uses Cloud Monitoring [GCP metrics](https://cloud.google.com/monitoring/api/metrics_gcp) and only requires minimal configuration compared to custom SLOs. **Example config (App Engine availability):** ```yaml -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - app_engine: - project_id: ${GAE_PROJECT_ID} - module_id: ${GAE_MODULE_ID} - availability: {} +backend: cloud_service_monitoring +method: basic +service_level_indicator: + app_engine: + project_id: ${GAE_PROJECT_ID} + module_id: ${GAE_MODULE_ID} + availability: {} ``` For details on filling the `app_engine` fields, see [AppEngine](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#appengine) spec. -**→ [Full SLO config](../../samples/stackdriver_service_monitoring/slo_gae_app_availability_basic.yaml)** +**→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml)** **Example config (Cloud Endpoint latency):** ```yaml -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - cloud_endpoints: - service: ${ENDPOINT_URL} - latency: - threshold: 724 # ms +backend: cloud_service_monitoring +method: basic +service_level_indicator: + cloud_endpoints: + service_name: ${ENDPOINT_URL} + latency: + threshold: 724 # ms ``` For details on filling the `cloud_endpoints` fields, see [CloudEndpoint](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#cloudendpoints) spec. -**Example config (Istio service latency) [NOT YET RELEASED]:** +**Example config (Istio service latency):** ```yaml -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - mesh_istio: - mesh_uid: ${GKE_MESH_UID} - service_namespace: ${GKE_SERVICE_NAMESPACE} - service_name: ${GKE_SERVICE_NAME} - latency: - threshold: 500 # ms +backend: cloud_service_monitoring +method: basic +service_level_indicator: + mesh_istio: + mesh_uid: ${GKE_MESH_UID} + service_namespace: ${GKE_SERVICE_NAMESPACE} + service_name: ${GKE_SERVICE_NAME} + latency: + threshold: 500 # ms ``` For details on filling the `mesh_istio` fields, see [MeshIstio](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#meshistio) spec. -**→ [Full SLO config](../../samples/stackdriver_service_monitoring/slo_gke_app_latency_basic.yaml)** +**→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gke_app_latency_basic.yaml)** -**Example config (Istio service latency) [DEPRECATED SOON]:** +**Example config (Istio service latency) [DEPRECATED]:** ```yaml -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - cluster_istio: - project_id: ${GKE_PROJECT_ID} - location: ${GKE_LOCATION} - cluster_name: ${GKE_CLUSTER_NAME} - service_namespace: ${GKE_SERVICE_NAMESPACE} - service_name: ${GKE_SERVICE_NAME} - latency: - threshold: 500 # ms +backend: cloud_service_monitoring +method: basic +service_level_indicator: + cluster_istio: + project_id: ${GKE_PROJECT_ID} + location: ${GKE_LOCATION} + cluster_name: ${GKE_CLUSTER_NAME} + service_namespace: ${GKE_SERVICE_NAMESPACE} + service_name: ${GKE_SERVICE_NAME} + latency: + threshold: 500 # ms ``` For details on filling the `cluster_istio` fields, see [ClusterIstio](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#clusteristio) spec. -**→ [Full SLO config](../../samples/stackdriver_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml)** +**→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml)** ### Good / bad ratio @@ -118,30 +117,28 @@ purposes as well (see examples). **Example config:** ```yaml -backend: - class: StackdriverServiceMonitoring - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - method: good_bad_ratio - measurement: - filter_good: > - project="${GAE_PROJECT_ID}" - metric.type="appengine.googleapis.com/http/server/response_count" - resource.type="gae_app" - metric.labels.response_code >= 200 - metric.labels.response_code < 500 - filter_valid: > - project="${GAE_PROJECT_ID}" - metric.type="appengine.googleapis.com/http/server/response_count" +backend: cloud_service_monitoring +method: good_bad_ratio +service_level_indicator: + filter_good: > + project="${GAE_PROJECT_ID}" + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + metric.labels.response_code >= 200 + metric.labels.response_code < 500 + filter_valid: > + project="${GAE_PROJECT_ID}" + metric.type="appengine.googleapis.com/http/server/response_count" ``` You can also use the `filter_bad` field which identifies bad events instead of the `filter_valid` field which identifies all valid events. -**→ [Full SLO config](../../samples/stackdriver_service_monitoring/slo_gae_app_availability.yaml)** +**→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gae_app_availability.yaml)** ## Distribution cut -The `distribution_cut` method is used for Stackdriver distribution-type metrics, +The `distribution_cut` method is used for Cloud distribution-type metrics, which are usually used for latency metrics. A distribution metric records the **statistical distribution of the extracted @@ -152,30 +149,28 @@ along with the `count`, `mean`, and `sum` of squared deviation of the values. **Example config:** ```yaml -backend: - class: StackdriverServiceMonitoring - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - method: distribution_cut - measurement: - filter_valid: > - project=${GAE_PROJECT_ID} - metric.type=appengine.googleapis.com/http/server/response_latencies - metric.labels.response_code >= 200 - metric.labels.response_code < 500 - range_min: 0 - range_max: 724 # ms +backend: cloud_service_monitoring +method: distribution_cut +service_level_indicator: + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type=appengine.googleapis.com/http/server/response_latencies + metric.labels.response_code >= 200 + metric.labels.response_code < 500 + range_min: 0 + range_max: 724 # ms ``` The `range_min` and `range_max` are used to specify the latency range that we consider 'good'. -**→ [Full SLO config](../../samples/stackdriver_service_monitoring/slo_gae_app_latency.yaml)** +**→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gae_app_latency.yaml)** ## Service Monitoring API considerations ### Tracking objects -Since `Stackdriver Service Monitoring API` persists `Service` and +Since `Cloud Service Monitoring API` persists `Service` and `ServiceLevelObjective` objects, we need ways to keep our local SLO YAML configuration synced with the remote objects. @@ -214,7 +209,7 @@ unique id to an auto-imported `Service`: * **Cluster Istio [DEPRECATED SOON]:** ``` - ist:{project_id}-zone-{location}-{cluster_name}-{service_namespace}-{service_name} + ist:{project_id}-{suffix}-{location}-{cluster_name}-{service_namespace}-{service_name} ``` → *Make sure that the `cluster_istio` block in your config has the correct fields corresponding to your Istio service.* @@ -225,19 +220,19 @@ random id. **Custom** Custom services are the ones you create yourself using the -`Service Monitoring API` and the `slo-generator`. +`Cloud Service Monitoring API` and the `slo-generator`. The following conventions are used by the `slo-generator` to give a unique id to a custom `Service` and `Service Level Objective` objects: -* `service_id = ${service_name}-${feature_name}` +* `service_id = ${metadata.service_name}-${metadata.feature_name}` -* `slo_id = ${service_name}-${feature_name}-${slo_name}-${window}` +* `slo_id = ${metadata.service_name}-${metadata.feature_name}-${metadata.slo_name}-${window}` To keep track of those, **do not update any of the following fields** in your configs: - * `service_name`, `feature_name` and `slo_name` in the SLO config. + * `metadata.service_name`, `metadata.feature_name` and `metadata.slo_name` in the SLO config. * `window` in the Error Budget Policy. @@ -246,15 +241,12 @@ If you need to make updates to any of those fields, first run the [#deleting-objects](#deleting-objects)), then re-run normally. To import an existing custom `Service` objects, find out your service id from -the API and fill the `service_id` in the SLO configuration. - -You cannot import an existing custom `ServiceLevelObjective` unless it complies -to the naming convention. +the API and fill the `service_id` in the `service_level_indicator` configuration. ### Deleting objects -To delete an SLO object in `Stackdriver Monitoring API` using the -`StackdriverServiceMonitoringBackend` class, run the `slo-generator` with the +To delete an SLO object in `Cloud Monitoring API` using the +`cloud_service_monitoring` class, run the `slo-generator` with the `-d` (or `--delete`) flag: ``` @@ -263,10 +255,10 @@ slo-generator -f -b --delete ## Alerting -See the Stackdriver Service Monitoring [docs](https://cloud.google.com/monitoring/service-monitoring/alerting-on-budget-burn-rate) +See the Cloud Service Monitoring [docs](https://cloud.google.com/monitoring/service-monitoring/alerting-on-budget-burn-rate) for instructions on alerting. ### Examples -Complete SLO samples using `Stackdriver Service Monitoring` are available in [ samples/stackdriver_service_monitoring](../../samples/stackdriver_service_monitoring). +Complete SLO samples using `Cloud Service Monitoring` are available in [ samples/cloud_service_monitoring](../../samples/cloud_service_monitoring). Check them out ! diff --git a/docs/providers/custom.md b/docs/providers/custom.md index 8da87640..bb180a58 100644 --- a/docs/providers/custom.md +++ b/docs/providers/custom.md @@ -41,14 +41,20 @@ class CustomBackend: In order to call the `good_bad_ratio` method in the custom backend above, the -`backend` block would look like this: +`backends` block would look like this: ```yaml -backend: - class: custom.custom_backend.CustomBackend # relative Python path to the backend. Make sure __init__.py is created in subdirectories for this to work. - method: good_bad_ratio # name of the method to run - arg_1: test_arg_1 # passed to kwargs in __init__ - arg_2: test_arg_2 # passed to kwargs in __init__ +backends: + custom.custom_backend.CustomBackend: # relative Python path to the backend. Make sure __init__.py is created in subdirectories for this to work. + arg_1: test_arg_1 # passed to kwargs in __init__ + arg_2: test_arg_2 # passed to kwargs in __init__ +``` + +The `spec` section in the SLO config would look like: +```yaml +backend: custom.custom_backend.CustomBackend +method: good_bad_ratio # name of the method to run +service_level_indicator: {} ``` **→ [Full SLO config](../../samples/custom/slo_custom_app_availability_ratio.yaml)** @@ -92,10 +98,17 @@ class CustomExporter: and the corresponding `exporters` section in your SLO config: +The `exporters` block in the shared config would look like this: + ```yaml exporters: -- class: custom.custom_exporter.CustomExporter - arg_1: test + custom.custom_exporter.CustomExporter: # relative Python path to the backend. Make sure __init__.py is created in subdirectories for this to work. + arg_1: test_arg_1 # passed to kwargs in __init__ +``` + +The `spec` section in the SLO config would look like: +```yaml +exporters: [custom.custom_exporter.CustomExporter] ``` ### Metrics @@ -103,7 +116,9 @@ exporters: A metrics exporter: * must inherit from `slo_generator.exporters.base.MetricsExporter`. -* must implement the `export_metric` method which exports **one** metric as a dict like: +* must implement the `export_metric` method which exports **one** metric. +The `export_metric` function takes a metric dict as input, such as: + ```py { "name": , @@ -129,13 +144,13 @@ class CustomExporter(MetricsExporter): # derive from base class """Custom exporter.""" def export_metric(self, data): - """Export data to Stackdriver Monitoring. + """Export data to Custom Monitoring API. Args: data (dict): Metric data. Returns: - object: Stackdriver Monitoring API result. + object: Custom Monitoring API result. """ # implement how to export 1 metric here... return { @@ -144,11 +159,12 @@ class CustomExporter(MetricsExporter): # derive from base class } ``` -and the exporters section in your SLO config: +The `exporters` block in the shared config would look like this: + ```yaml exporters: - - class: custom.custom_exporter.CustomExporter - arg_1: test + custom.custom_exporter.CustomExporter: # relative Python path to the backend. Make sure __init__.py is created in subdirectories for this to work. + arg_1: test_arg_1 # passed to kwargs in __init__ ``` **Note:** diff --git a/docs/providers/datadog.md b/docs/providers/datadog.md index fb6b2128..f6e13f61 100644 --- a/docs/providers/datadog.md +++ b/docs/providers/datadog.md @@ -2,16 +2,28 @@ ## Backend -Using the `Datadog` backend class, you can query any metrics available in +Using the `datadog` backend class, you can query any metrics available in Datadog to create an SLO. -The following methods are available to compute SLOs with the `Datadog` +```yaml +backends: + datadog: + api_key: ${DATADOG_API_KEY} + app_key: ${DATADOG_APP_KEY} +``` + +The following methods are available to compute SLOs with the `datadog` backend: * `good_bad_ratio` for computing good / bad metrics ratios. * `query_sli` for computing SLIs directly with Datadog. * `query_slo` for getting SLO value from Datadog SLO endpoint. +Optional arguments to configure Datadog are documented in the Datadog +`initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). +You can pass them in the `backend` section, such as specifying +`api_host: api.datadoghq.eu` in order to use the EU site. + ### Good / bad ratio The `good_bad_ratio` method is used to compute the ratio between two metrics: @@ -27,45 +39,29 @@ purposes as well (see examples). **Config example:** ```yaml -backend: - class: Datadog - method: good_bad_ratio - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} - measurement: - filter_good: app.requests.count{http.path:/, http.status_code_class:2xx} - filter_valid: app.requests.count{http.path:/} +backend: datadog +method: good_bad_ratio +service_level_indicator: + filter_good: app.requests.count{http.path:/, http.status_code_class:2xx} + filter_valid: app.requests.count{http.path:/} ``` **→ [Full SLO config](../../samples/datadog/slo_dd_app_availability_ratio.yaml)** -Optional arguments to configure Datadog are documented in the Datadog -`initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). -You can pass them in the `backend` section, such as specifying -`api_host: api.datadoghq.eu` in order to use the EU site. - ### Query SLI The `query_sli` method is used to directly query the needed SLI with Datadog: Datadog's query language is powerful enough that it can do ratios natively. -This method makes it more flexible to input any `Datadog` SLI computation and +This method makes it more flexible to input any `datadog` SLI computation and eventually reduces the number of queries made to Datadog. ```yaml -backend: - class: Datadog - method: query_sli - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} - measurement: - expression: sum:app.requests.count{http.path:/, http.status_code_class:2xx} / sum:app.requests.count{http.path:/} +backend: datadog +method: query_sli +service_level_indicator: + expression: sum:app.requests.count{http.path:/, http.status_code_class:2xx} / sum:app.requests.count{http.path:/} ``` -Optional arguments to configure Datadog are documented in the Datadog -`initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). -You can pass them in the `backend` section, such as specifying -`api_host: api.datadoghq.eu` in order to use the EU site. - **→ [Full SLO config](../../samples/datadog/slo_dd_app_availability_query_sli.yaml)** ### Query SLO @@ -73,46 +69,43 @@ You can pass them in the `backend` section, such as specifying The `query_slo` method is used to directly query the needed SLO with Datadog: indeed, Datadog has SLO objects that you can directly refer to in your config by inputing their `slo_id`. -This method makes it more flexible to input any `Datadog` SLI computation and +This method makes it more flexible to input any `datadog` SLI computation and eventually reduces the number of queries made to Datadog. To query the value from Datadog SLO, simply add a `slo_id` field in the `measurement` section: ```yaml -... -backend: - class: Datadog - method: query_slo - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} - measurement: - slo_id: ${DATADOG_SLO_ID} +backend: datadog +method: query_slo +service_level_indicator: + slo_id: ${DATADOG_SLO_ID} ``` **→ [Full SLO config](../../samples/datadog/slo_dd_app_availability_query_slo.yaml)** ### Examples -Complete SLO samples using `Datadog` are available in +Complete SLO samples using `datadog` are available in [samples/datadog](../../samples/datadog). Check them out! ## Exporter -The `Datadog` exporter allows to export SLO metrics to the Datadog API. - -**Example config:** +The `datadog` exporter allows to export SLO metrics to the Datadog API. ```yaml exporters: - - class: Datadog + datadog: api_key: ${DATADOG_API_KEY} app_key: ${DATADOG_APP_KEY} ``` +Optional arguments to configure Datadog are documented in the Datadog +`initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). +You can pass them in the `backend` section, such as specifying +`api_host: api.datadoghq.eu` in order to use the EU site. Optional fields: - * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`custom:error_budget_burn_rate`, `custom:sli_measurement`]. - + * `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). **→ [Full SLO config](../../samples/datadog/slo_dd_app_availability_ratio.yaml)** diff --git a/docs/providers/dynatrace.md b/docs/providers/dynatrace.md index c2fcbbac..90f1a3db 100644 --- a/docs/providers/dynatrace.md +++ b/docs/providers/dynatrace.md @@ -2,10 +2,17 @@ ## Backend -Using the `Dynatrace` backend class, you can query any metrics available in +Using the `dynatrace` backend class, you can query any metrics available in Dynatrace to create an SLO. -The following methods are available to compute SLOs with the `Dynatrace` +```yaml +backends: + dynatrace: + api_token: ${DYNATRACE_API_TOKEN} + api_url: ${DYNATRACE_API_URL} +``` + +The following methods are available to compute SLOs with the `dynatrace` backend: * `good_bad_ratio` for computing good / bad metrics ratios. @@ -25,16 +32,13 @@ purposes as well (see examples). **Config example:** ```yaml -backend: - class: Dynatrace - method: good_bad_ratio - api_token: ${DYNATRACE_API_TOKEN} - api_url: ${DYNATRACE_API_URL} - measurement: - query_good: - metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) - entity_selector: type(HOST) - query_valid: +backend: dynatrace +method: good_bad_ratio +service_level_indicator: + query_good: + metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) + entity_selector: type(HOST) + query_valid: metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod))) entity_selector: type(HOST) ``` @@ -52,16 +56,13 @@ This method can be used for latency SLOs, by defining a latency threshold. **Config example:** ```yaml -backend: - class: Dynatrace - method: threshold - api_token: ${DYNATRACE_API_TOKEN} - api_url: ${DYNATRACE_API_URL} - measurement: - query_valid: - metric_selector: ext:app.request_latency:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) - entity_selector: type(HOST) - threshold: 40000 # us +backend: dynatrace +method: threshold +service_level_indicator: + query_valid: + metric_selector: ext:app.request_latency:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) + entity_selector: type(HOST) + threshold: 40000 # us ``` **→ [Full SLO config](../../samples/dynatrace/slo_dt_app_latency_threshold.yaml)** @@ -71,24 +72,22 @@ Optional fields: ### Examples -Complete SLO samples using `Dynatrace` are available in +Complete SLO samples using `dynatrace` are available in [samples/dynatrace](../../samples/dynatrace). Check them out! ## Exporter -The `Dynatrace` exporter allows to export SLO metrics to Dynatrace API. - -**Example config:** +The `dynatrace` exporter allows to export SLO metrics to Dynatrace API. ```yaml exporters: - - class: Dynatrace - api_token: ${DYNATRACE_API_TOKEN} - api_url: ${DYNATRACE_API_URL} + dynatrace: + api_token: ${DYNATRACE_API_TOKEN} + api_url: ${DYNATRACE_API_URL} ``` Optional fields: - * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`custom:error_budget_burn_rate`, `custom:sli_measurement`]. + * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`custom:error_budget_burn_rate`, `custom:sli_service_level_indicator`]. **→ [Full SLO config](../../samples/dynatrace/slo_dt_app_availability_ratio.yaml)** diff --git a/docs/providers/elasticsearch.md b/docs/providers/elasticsearch.md index 5e160bc3..c30b1dbb 100644 --- a/docs/providers/elasticsearch.md +++ b/docs/providers/elasticsearch.md @@ -2,10 +2,17 @@ ## Backend -Using the `Elasticsearch` backend class, you can query any metrics available in +Using the `elasticsearch` backend class, you can query any metrics available in Elasticsearch to create an SLO. -The following methods are available to compute SLOs with the `Elasticsearch` +```yaml +backends: + elasticsearch: + api_token: ${DYNATRACE_API_TOKEN} + api_url: ${DYNATRACE_API_URL} +``` + +The following methods are available to compute SLOs with the `elasticsearch` backend: * `good_bad_ratio` for computing good / bad metrics ratios. @@ -81,5 +88,5 @@ look like: ### Examples -Complete SLO samples using the `Elasticsearch` backend are available in +Complete SLO samples using the `elasticsearch` backend are available in [samples/elasticsearch](../../samples/elasticsearch). Check them out ! diff --git a/docs/providers/prometheus.md b/docs/providers/prometheus.md index 1ceb4a62..1120b777 100644 --- a/docs/providers/prometheus.md +++ b/docs/providers/prometheus.md @@ -2,10 +2,22 @@ ## Backend -Using the `Prometheus` backend class, you can query any metrics available in +Using the `prometheus` backend class, you can query any metrics available in Prometheus to create an SLO. -The following methods are available to compute SLOs with the `Prometheus` +```yaml +backends: + prometheus: + url: http://localhost:9090 + # headers: + # Content-Type: application/json + # Authorization: Basic b2s6cGFzcW== +``` + +Optional fields: +* `headers` allows to specify Basic Authentication credentials if needed. + +The following methods are available to compute SLOs with the `prometheus` backend: * `good_bad_ratio` for computing good / bad metrics ratios. @@ -26,24 +38,16 @@ purposes as well (see examples). **Config example:** ```yaml -backend: - class: Prometheus - method: good_bad_ratio - url: http://localhost:9090 - # headers: - # Content-Type: application/json - # Authorization: Basic b2s6cGFzcW== - measurement: - filter_good: http_requests_total{handler="/metrics", code=~"2.."}[window] - filter_valid: http_requests_total{handler="/metrics"}[window] - # operators: ['sum', 'rate'] +backend: prometheus +method: good_bad_ratio +service_level_indicator: + filter_good: http_requests_total{handler="/metrics", code=~"2.."}[window] + filter_valid: http_requests_total{handler="/metrics"}[window] + # operators: ['sum', 'rate'] ``` * The `window` placeholder is needed in the query and will be replaced by the corresponding `window` field set in each step of the Error Budget Policy. -* The `headers` section (commented) allows to specify Basic Authentication -credentials if needed. - * The `operators` section defines which PromQL functions to apply on the timeseries. The default is to compute `sum(increase([METRIC_NAME][window]))` to get an accurate count of good and bad events. Be aware that changing will likely @@ -64,26 +68,20 @@ eventually reduces the number of queries made to Prometheus. See Bitnami's [article](https://engineering.bitnami.com/articles/implementing-slos-using-prometheus.html) on engineering SLOs with Prometheus. +**Config example:** + ```yaml -backend: - class: Prometheus - method: query_sli - url: ${PROMETHEUS_URL} - # headers: - # Content-Type: application/json - # Authorization: Basic b2s6cGFzcW== - measurement: - expression: > - sum(rate(http_requests_total{handler="/metrics", code=~"2.."}[window])) - / - sum(rate(http_requests_total{handler="/metrics"}[window])) +backend: prometheus +method: query_sli +service_level_indicator: + expression: > + sum(rate(http_requests_total{handler="/metrics", code=~"2.."}[window])) + / + sum(rate(http_requests_total{handler="/metrics"}[window])) ``` * The `window` placeholder is needed in the query and will be replaced by the corresponding `window` field set in each step of the Error Budget Policy. -* The `headers` section (commented) allows to specify Basic Authentication -credentials if needed. - **→ [Full SLO config (availability)](../../samples/prometheus/slo_prom_metrics_availability_query_sli.yaml)** **→ [Full SLO config (latency)](../../samples/prometheus/slo_prom_metrics_latency_query_sli.yaml)** @@ -121,13 +119,11 @@ expressing it, as shown in the config example below. **Config example:** ```yaml -backend: - class: Prometheus - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - method: distribution_cut - measurement: - expression: http_requests_duration_bucket{path='/', code=~"2.."} - threshold_bucket: 0.25 # corresponds to 'le' attribute in Prometheus histograms +backend: prometheus +method: distribution_cut +service_level_indicator: + expression: http_requests_duration_bucket{path='/', code=~"2.."} + threshold_bucket: 0.25 # corresponds to 'le' attribute in Prometheus histograms ``` **→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_latency_distribution_cut.yaml)** @@ -137,31 +133,29 @@ set for your metric. Learn more in the [Prometheus docs](https://prometheus.io/d ## Exporter -The `Prometheus` exporter allows to export SLO metrics to the +The `prometheus` exporter allows to export SLO metrics to the [Prometheus Pushgateway](https://prometheus.io/docs/practices/pushing/) which needs to be running. -`Prometheus` needs to be setup to **scrape metrics from `Pushgateway`** (see - [documentation](https://github.com/prometheus/pushgateway) for more details). - -**Example config:** - ```yaml exporters: - - class: Prometheus - url: ${PUSHGATEWAY_URL} + prometheus: + url: ${PUSHGATEWAY_URL} ``` Optional fields: - * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`error_budget_burn_rate`, `sli_measurement`]. + * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`error_budget_burn_rate`, `sli_service_level_indicator`]. * `username`: Username for Basic Auth. * `password`: Password for Basic Auth. * `job`: Name of `Pushgateway` job. Defaults to `slo-generator`. +***Note:*** `prometheus` needs to be setup to **scrape metrics from `Pushgateway`** +(see [documentation](https://github.com/prometheus/pushgateway) for more details). + **→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_availability_query_sli.yaml)** ### Examples -Complete SLO samples using `Prometheus` are available in +Complete SLO samples using `prometheus` are available in [samples/prometheus](../../samples/prometheus). Check them out ! diff --git a/docs/providers/pubsub.md b/docs/providers/pubsub.md index 197274f4..0f9f2437 100644 --- a/docs/providers/pubsub.md +++ b/docs/providers/pubsub.md @@ -2,18 +2,16 @@ ## Exporter -The `Pubsub` exporter will export SLO reports to a Pub/Sub topic, in JSON format. - -This allows teams to consume SLO reports in real-time, and take appropriate -actions when they see a need. - -**Example config:** +The `pubsub` exporter will export SLO reports to a Pub/Sub topic, in JSON format. ```yaml exporters: - - class: Pubsub + pubsub: project_id: "${PUBSUB_PROJECT_ID}" topic_name: "${PUBSUB_TOPIC_NAME}" ``` -**→ [Full SLO config](../../samples/stackdriver/slo_pubsub_subscription_throughput.yaml)** +This allows teams to consume SLO reports in real-time, and take appropriate +actions when they see a need. + +**→ [Full SLO config](../../samples/cloud_monitoring/slo_pubsub_subscription_throughput.yaml)** diff --git a/docs/shared/metrics.md b/docs/shared/metrics.md index a6cecf4d..b50c95ed 100644 --- a/docs/shared/metrics.md +++ b/docs/shared/metrics.md @@ -63,23 +63,23 @@ metrics: ``` where: -* `name`: name of the [SLO Report](../../tests/unit/fixtures/slo_report.json) +* `name`: name of the [SLO Report](../../tests/unit/fixtures/slo_report_v2.json) field to export as a metric. The field MUST exist in the SLO report. * `description`: description of the metric (if the metrics exporter supports it) * `alias` (optional): rename the metric before writing to the monitoring backend. * `additional_labels` (optional) allow you to specify other labels to the timeseries written. Each label name must correspond to a field of the -[SLO Report](../../tests/unit/fixtures/slo_report.json). +[SLO Report](../../tests/unit/fixtures/slo_report_v2.json). ## Metric exporters Some metrics exporters have a specific `prefix` that is pre-prepended to the metric name: -* `StackdriverExporter` prefix: `custom.googleapis.com/` -* `DatadogExporter` prefix: `custom:` +* `cloud_monitoring` exporter prefix: `custom.googleapis.com/` +* `datadog` prefix: `custom:` Some metrics exporters have a limit of `labels` that can be written to their metrics timeseries: -* `StackdriverExporter` labels limit: `10`. +* `cloud_monitoring` labels limit: `10`. Those are standards and cannot be modified. diff --git a/docs/shared/migration.md b/docs/shared/migration.md new file mode 100644 index 00000000..65f9ef72 --- /dev/null +++ b/docs/shared/migration.md @@ -0,0 +1,33 @@ +# Migrating `slo-generator` to the next major version + +## v1 to v2 + +Version `v2` of the slo-generator introduces some changes to the structure of +the SLO configurations. + +To migrate your SLO configurations from v1 to v3, please execute the following +instructions: + +**Upgrade `slo-generator`:** +``` +pip3 install slo-generator -U # upgrades slo-generator version to the latest version +``` + +**Run the `slo-generator-migrate` command:** +``` +slo-generator-migrate -s -t -b +``` +where: +* is the source folder containg SLO configurations in v1 format. +This folder can have nested subfolders containing SLOs. The subfolder structure +will be reproduced on the target folder. + +* is the target folder to drop the SLO configurations in v2 +format. If the target folder is identical to the source folder, the existing SLO +configurations will be updated in-place. + +* is the path to your error budget policy configuration. + +**Follow the instructions printed to finish the migration:** +This includes committing the resulting files to git and updating your Terraform +modules to the version that supports the v2 configuration format. diff --git a/docs/shared/troubleshooting.md b/docs/shared/troubleshooting.md index 45fa562b..29d6eef5 100644 --- a/docs/shared/troubleshooting.md +++ b/docs/shared/troubleshooting.md @@ -2,7 +2,7 @@ ## Problem -**`StackdriverExporter`: Labels limit (10) reached.** +**`cloud_monitoring` exporter: Labels limit (10) reached.** ``` The new labels would cause the metric custom.googleapis.com/slo_target to have over 10 labels.: timeSeries[0]" diff --git a/samples/README.md b/samples/README.md index e7198ba8..201181fe 100644 --- a/samples/README.md +++ b/samples/README.md @@ -14,17 +14,17 @@ running it. The following is listing all environmental variables found in the SLO configs, per backend: -`stackdriver/`: - - `STACKDRIVER_HOST_PROJECT_ID`: Stackdriver host project id. - - `STACKDRIVER_LOG_METRIC_NAME`: Stackdriver log-based metric name. +`cloud_monitoring/`: + - `WORKSPACE_PROJECT_ID`: Cloud Monitoring host project id. + - `LOG_METRIC_NAME`: Cloud Logging log-based metric name. - `GAE_PROJECT_ID`: Google App Engine application project id. - `GAE_MODULE_ID`: Google App Engine application module id. - `PUBSUB_PROJECT_ID`: Pub/Sub project id. - `PUBSUB_TOPIC_NAME`: Pub/Sub topic name. -`stackdriver_service_monitoring/`: - - `STACKDRIVER_HOST_PROJECT_ID`: Stackdriver host project id. - - `STACKDRIVER_LOG_METRIC_NAME`: Stackdriver log-based metric name. +`cloud_service_monitoring/`: + - `WORKSPACE_PROJECT_ID`: Cloud Monitoring host project id. + - `LOG_METRIC_NAME`: Cloud Logging log-based metric name. - `GAE_PROJECT_ID`: Google App Engine application project id. - `GAE_MODULE_ID`: Google App Engine application module id. - `PUBSUB_PROJECT_ID`: Pub/Sub project id. @@ -50,7 +50,7 @@ you're pointing to need to exist. To run one sample: ``` -slo-generator -f samples/stackdriver/.yaml +slo-generator -f samples/cloud_monitoring/.yaml ``` To run all the samples for a backend: @@ -68,14 +68,14 @@ slo-generator -f samples/ -b samples/ ### Examples -##### Stackdriver +##### Cloud Monitoring ``` -slo-generator -f samples/stackdriver -b error_budget_policy.yaml +slo-generator -f samples/cloud_monitoring -b error_budget_policy.yaml ``` -##### Stackdriver Service Monitoring +##### Cloud Service Monitoring ``` -slo-generator -f samples/stackdriver_service_monitoring -b error_budget_policy_ssm.yaml +slo-generator -f samples/cloud_service_monitoring -b error_budget_policy_ssm.yaml ``` ***Note:*** *the Error Budget Policy is different for this backend, because it only From be27c4a0f319f592aa8838f351749afaaaa081f3 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 13:35:43 +0200 Subject: [PATCH 009/107] refactor: Upgrade unit tests to v2 (#132) --- tests/unit/__init__.py | 6 + tests/unit/fixtures/config.yaml | 34 +++++ tests/unit/fixtures/dummy_backend.py | 19 +++ tests/unit/fixtures/dummy_config.json | 15 ++ tests/unit/fixtures/dummy_slo_config.json | 19 +++ tests/unit/fixtures/dummy_tests.json | 66 ++++++++ tests/unit/fixtures/exporters.yaml | 10 -- tests/unit/fixtures/fail_exporter.py | 13 ++ tests/unit/fixtures/slo_config_v1.yaml | 35 +++++ tests/unit/fixtures/slo_config_v2.yaml | 38 +++++ tests/unit/fixtures/slo_report.json | 30 ---- tests/unit/fixtures/slo_report_v1.json | 30 ++++ tests/unit/fixtures/slo_report_v2.json | 37 +++++ tests/unit/test_cli.py | 73 ++++----- tests/unit/test_compute.py | 93 ++++-------- tests/unit/test_migrate.py | 36 +++++ tests/unit/test_report.py | 4 +- tests/unit/test_stubs.py | 177 ++++++---------------- tests/unit/test_utils.py | 39 ++--- 19 files changed, 483 insertions(+), 291 deletions(-) create mode 100644 tests/unit/fixtures/config.yaml create mode 100644 tests/unit/fixtures/dummy_backend.py create mode 100644 tests/unit/fixtures/dummy_config.json create mode 100644 tests/unit/fixtures/dummy_slo_config.json create mode 100644 tests/unit/fixtures/dummy_tests.json create mode 100644 tests/unit/fixtures/fail_exporter.py create mode 100644 tests/unit/fixtures/slo_config_v1.yaml create mode 100644 tests/unit/fixtures/slo_config_v2.yaml delete mode 100644 tests/unit/fixtures/slo_report.json create mode 100644 tests/unit/fixtures/slo_report_v1.json create mode 100644 tests/unit/fixtures/slo_report_v2.json create mode 100644 tests/unit/test_migrate.py diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index fab6aabf..bb72f5be 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -11,3 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""__init__.py + +Init test environment variables. +""" +import os +os.environ['MIN_VALID_EVENTS'] = '10' diff --git a/tests/unit/fixtures/config.yaml b/tests/unit/fixtures/config.yaml new file mode 100644 index 00000000..057c8cf4 --- /dev/null +++ b/tests/unit/fixtures/config.yaml @@ -0,0 +1,34 @@ +--- +backends: + cloud_monitoring: + project_id: ${PROJECT_ID} +exporters: + cloud_monitoring: + project_id: ${PROJECT_ID} +error_budget_policies: + standard: + steps: + - name: 1 hour + window: 3600 + burn_rate_threshold: 9 + alert: true + message_alert: Page to defend the SLO + message_ok: Last hour on track + - name: 12 hours + window: 43200 + burn_rate_threshold: 3 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 12 hours on track + - name: 7 days + window: 604800 + burn_rate_threshold: 1.5 + alert: false + message_alert: Dev team dedicates 25% of engineers to the reliability backlog + message_ok: Last week on track + - name: 28 days + window: 2419200 + burn_rate_threshold: 1 + alert: false + message_alert: Freeze release, unless related to reliability or security + message_ok: Unfreeze release, per the agreed roll-out policy diff --git a/tests/unit/fixtures/dummy_backend.py b/tests/unit/fixtures/dummy_backend.py new file mode 100644 index 00000000..0022e541 --- /dev/null +++ b/tests/unit/fixtures/dummy_backend.py @@ -0,0 +1,19 @@ +"""dummy_backend.py + +Dummy backend implementation for testing. +""" +# pylint:disable=missing-class-docstring,missing-function-docstring,unused-argument + + +class DummyBackend: + + def __init__(self, client=None, **config): + self.good_events = config.get('good_events', None) + self.bad_events = config.get('bad_events', None) + self.sli_value = config.get('sli', None) + + def good_bad_ratio(self, timestamp, window, slo_config): + return (self.good_events, self.bad_events) + + def sli(self, timestamp, window, slo_config): + return self.sli_value diff --git a/tests/unit/fixtures/dummy_config.json b/tests/unit/fixtures/dummy_config.json new file mode 100644 index 00000000..862f266a --- /dev/null +++ b/tests/unit/fixtures/dummy_config.json @@ -0,0 +1,15 @@ +{ + "backends": { + "dummy": {} + }, + "error_budget_policies": { + "default": [{ + "name": "1 hour", + "window": 3600, + "burn_rate_threshold": 1, + "alert": true, + "message_alert": "Page to defend the SLO", + "message_ok": "Last hour on track" + }] + } +} diff --git a/tests/unit/fixtures/dummy_slo_config.json b/tests/unit/fixtures/dummy_slo_config.json new file mode 100644 index 00000000..b5015816 --- /dev/null +++ b/tests/unit/fixtures/dummy_slo_config.json @@ -0,0 +1,19 @@ +{ + "kind": "ServiceLevelObjective", + "version": "sre.google.com/v2", + "metadata": { + "name": "test-test-test", + "labels": { + "service_name": "test", + "feature_name": "test", + "slo_name": "test" + } + }, + "spec": { + "description": "Test dummy backend", + "goal": 0.99, + "backend": "dummy", + "method": "good_bad_ratio", + "service_level_indicator": {} + } +} diff --git a/tests/unit/fixtures/dummy_tests.json b/tests/unit/fixtures/dummy_tests.json new file mode 100644 index 00000000..a5c7537a --- /dev/null +++ b/tests/unit/fixtures/dummy_tests.json @@ -0,0 +1,66 @@ +{ + "enough_events": { + "method": "good_bad_ratio", + "good_events": 5, + "bad_events": 5, + }, + "no_good_events": { + "method": "good_bad_ratio", + "good_events": -1, + "bad_events": 15, + }, + "no_bad_events": { + "method": "good_bad_ratio", + "good_events": 15, + "bad_events": -1, + }, + "valid_sli_value": { + "method": "sli", + "good_events": -1, + "bad_events": -1, + "sli": 0.991 + }, + "no_events": { + "method": "good_bad_ratio", + "good_events": 0, + "bad_events": 0, + }, + "no_good_bad_events": { + "method": "good_bad_ratio", + "good_events": -1, + "bad_events": -1, + }, + "not_enough_events": { + "method": "good_bad_ratio", + "good_events": 5, + "bad_events": 4, + }, + "no_sli_value": { + "method": "sli", + "good_events": -1, + "bad_events": -1, + }, + "no_backend_response_sli": { + "method": "sli", + "sli": null + }, + "no_backend_response_ratio": { + "method": "good_bad_ratio", + "good_events": null, + "bad_events": null, + }, + "invalid_backend_response_type": { + "method": "good_bad_ratio", + "good_events": { + "data": { + "value": 30 + } + }, + "bad_events": { + "data": { + "value": 400 + } + }, + "sli": null + } +} diff --git a/tests/unit/fixtures/exporters.yaml b/tests/unit/fixtures/exporters.yaml index f4a281b5..d26adf51 100644 --- a/tests/unit/fixtures/exporters.yaml +++ b/tests/unit/fixtures/exporters.yaml @@ -35,16 +35,6 @@ api_url: ${DYNATRACE_API_URL} api_token: ${DYNATRACE_API_TOKEN} - # Old format that will be deprecated in 2.0.0 in favor of the `metrics` block - - class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - metric_type: custom.googleapis.com/ebp - metric_description: Test old format - metric_labels: [good_events_count, bad_events_count] - metrics: - - error_budget_burn_rate - - # New format ('metrics' block) - class: Stackdriver project_id: ${STACKDRIVER_HOST_PROJECT_ID} metrics: diff --git a/tests/unit/fixtures/fail_exporter.py b/tests/unit/fixtures/fail_exporter.py new file mode 100644 index 00000000..430c3da9 --- /dev/null +++ b/tests/unit/fixtures/fail_exporter.py @@ -0,0 +1,13 @@ +"""dummy_exporter.py + +Dummy exporter implementation for testing. +""" +# pylint: disable=missing-class-docstring + +from slo_generator.exporters.base import MetricsExporter + + +class FailExporter(MetricsExporter): + + def export_metric(self, data): + raise ValueError("Oops !") diff --git a/tests/unit/fixtures/slo_config_v1.yaml b/tests/unit/fixtures/slo_config_v1.yaml new file mode 100644 index 00000000..5b1baf95 --- /dev/null +++ b/tests/unit/fixtures/slo_config_v1.yaml @@ -0,0 +1,35 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +service_name: gae +feature_name: app +slo_description: Availability of App Engine app +slo_name: availability +slo_target: 0.95 +backend: + class: Stackdriver + method: good_bad_ratio + project_id: ${STACKDRIVER_HOST_PROJECT_ID} + measurement: + filter_good: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + metric.labels.response_code = 200 + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" +exporters: +- class: Stackdriver + project_id: ${STACKDRIVER_HOST_PROJECT_ID} diff --git a/tests/unit/fixtures/slo_config_v2.yaml b/tests/unit/fixtures/slo_config_v2.yaml new file mode 100644 index 00000000..0dfa7cde --- /dev/null +++ b/tests/unit/fixtures/slo_config_v2.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-availability + labels: + service_name: gae + feature_name: app + slo_name: availability +spec: + description: Availability of App Engine app + backend: cloud_monitoring + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + metric.labels.response_code = 200 + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + goal: 0.95 diff --git a/tests/unit/fixtures/slo_report.json b/tests/unit/fixtures/slo_report.json deleted file mode 100644 index 5c9626e2..00000000 --- a/tests/unit/fixtures/slo_report.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "service_name": "test", - "feature_name": "test", - "slo_name": "slo-pubsub-acked-msg", - "slo_target": 0.9, - "slo_description": "Acked Pub/Sub messages over total number of Pub/Sub messages", - "error_budget_policy_step_name": "b.Last 12 hours", - "error_budget_remaining_minutes": -288.0, - "consequence_message": "Page the SRE team to defend the SLO", - "error_budget_minutes": 71.99999999999999, - "error_minutes": "360.0", - "error_budget_target": 0.09999999999999998, - "timestamp_human": "2019-09-05 11:55:01.004603 UTC", - "timestamp": 1567762279.287761, - "cadence": null, - "window": 43200, - "events_count": 7112, - "bad_events_count": 3556, - "good_events_count": 3556, - "sli_measurement": 0.5, - "gap": -0.4, - "error_budget_measurement": 0.5, - "error_budget_burn_rate": 5.000000000000001, - "alerting_burn_rate_threshold": 3.0, - "alert": "true", - "metadata": { - "env": "test", - "team": "test" - } -} diff --git a/tests/unit/fixtures/slo_report_v1.json b/tests/unit/fixtures/slo_report_v1.json new file mode 100644 index 00000000..3c8fdbf6 --- /dev/null +++ b/tests/unit/fixtures/slo_report_v1.json @@ -0,0 +1,30 @@ +{ + "service_name": "test", + "feature_name": "test", + "slo_name": "slo-pubsub-acked-msg", + "slo_target": 0.9, + "slo_description": "Acked Pub/Sub messages over total number of Pub/Sub messages", + "error_budget_policy_step_name": "b.Last 12 hours", + "error_budget_remaining_minutes": -288.0, + "consequence_message": "Page the SRE team to defend the SLO", + "error_budget_minutes": 71.99999999999999, + "error_minutes": "360.0", + "error_budget_target": 0.09999999999999998, + "timestamp_human": "2019-09-05 11:55:01.004603 UTC", + "timestamp": 1567762279.287761, + "cadence": null, + "window": 43200, + "events_count": 7112, + "bad_events_count": 3556, + "good_events_count": 3556, + "sli_measurement": 0.5, + "gap": -0.4, + "error_budget_measurement": 0.5, + "error_budget_burn_rate": 5.000000000000001, + "alerting_burn_rate_threshold": 3.0, + "alert": "true", + "metadata": { + "env": "test", + "team": "test" + } +} diff --git a/tests/unit/fixtures/slo_report_v2.json b/tests/unit/fixtures/slo_report_v2.json new file mode 100644 index 00000000..c6db547a --- /dev/null +++ b/tests/unit/fixtures/slo_report_v2.json @@ -0,0 +1,37 @@ +{ + "alert": true, + "backend": "cloud_monitoring", + "bad_events_count": 3556, + "description": "Acked Pub/Sub messages over total number of Pub/Sub messages", + "exporters": [ + "cloud_monitoring" + ], + "consequence_message": "Page the SRE team to defend the SLO", + "error_budget_burn_rate": 5.000000000000001, + "error_budget_burn_rate_threshold": 3, + "error_budget_measurement": 0.5, + "error_budget_policy": "default", + "error_budget_policy_step_name": "1h", + "error_budget_minutes": 71.99999999999999, + "error_budget_remaining_minutes": -288, + "error_budget_target": 0.09999999999999998, + "error_minutes": "360.0", + "events_count": 7112, + "gap": -0.4, + "goal": 0.9, + "good_events_count": 3556, + "metadata": { + "name": "test-slo", + "labels": { + "service_name": "test", + "feature_name": "test", + "slo_name": "test", + "env": "test", + "team": "test" + } + }, + "sli_measurement": 0.5, + "timestamp": 1567762279.287761, + "timestamp_human": "2019-09-05 11:55:01.004603 UTC", + "window": 43200 +} diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 0424ee20..036731f0 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -17,7 +17,9 @@ from mock import patch -from slo_generator.cli import cli, parse_args +from click.testing import CliRunner +from slo_generator.cli import main +from slo_generator.utils import load_config from .test_stubs import CTX, mock_sd @@ -26,49 +28,50 @@ class TestCLI(unittest.TestCase): + def setUp(self): - for k, v in CTX.items(): - os.environ[k] = v - slo_config = f'{root}/samples/stackdriver/slo_gae_app_availability.yaml' - eb_policy = f'{root}/samples/error_budget_policy.yaml' + for key, value in CTX.items(): + os.environ[key] = value + slo_config = f'{root}/samples/cloud_monitoring/slo_gae_app_availability.yaml' # noqa: E501 + config = f'{root}/samples/config.yaml' self.slo_config = slo_config - self.eb_policy = eb_policy - - def test_parse_args(self): - args = parse_args([ - '--slo-config', self.slo_config, '--error-budget-policy', - self.eb_policy, '--export' - ]) - self.assertEqual(args.slo_config, self.slo_config) - self.assertEqual(args.error_budget_policy, self.eb_policy) - self.assertEqual(args.export, True) + self.slo_metadata_name = load_config(slo_config, + ctx=CTX)['metadata']['name'] + self.config = config + self.cli = CliRunner() @patch('google.api_core.grpc_helpers.create_channel', return_value=mock_sd(8)) - def test_cli(self, mock): - args = parse_args(['-f', self.slo_config, '-b', self.eb_policy]) - all_reports = cli(args) - len_first_report = len(all_reports[self.slo_config]) - self.assertIn(self.slo_config, all_reports.keys()) - self.assertEqual(len_first_report, 4) + def test_cli_compute(self, mock): + args = ['compute', '-f', self.slo_config, '-c', self.config] + result = self.cli.invoke(main, args) + self.assertEqual(result.exit_code, 0) @patch('google.api_core.grpc_helpers.create_channel', return_value=mock_sd(40)) - def test_cli_folder(self, mock): - args = parse_args( - ['-f', f'{root}/samples/stackdriver', '-b', self.eb_policy]) - all_reports = cli(args) - len_first_report = len(all_reports[self.slo_config]) - self.assertIn(self.slo_config, all_reports.keys()) - self.assertEqual(len_first_report, 4) + def test_cli_compute_folder(self, mock): + args = [ + 'compute', '-f', f'{root}/samples/cloud_monitoring', '-c', + self.config + ] + result = self.cli.invoke(main, args) + self.assertEqual(result.exit_code, 0) + + def test_cli_compute_no_config(self): + args = [ + 'compute', '-f', f'{root}/samples', '-c', + f'{root}/samples/config.yaml' + ] + result = self.cli.invoke(main, args) + self.assertEqual(result.exit_code, 1) + + def test_cli_api(self): + # TODO: Write test + pass - def test_cli_no_config(self): - args = parse_args([ - '-f', f'{root}/samples', '-b', - f'{root}/samples/error_budget_policy.yaml' - ]) - all_reports = cli(args) - self.assertEqual(all_reports, {}) + def test_cli_migrate(self): + # TODO: Write test + pass if __name__ == '__main__': diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index 461fe705..5aa31257 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -23,25 +23,26 @@ from slo_generator.backends.dynatrace import DynatraceClient from slo_generator.compute import compute, export from slo_generator.exporters.bigquery import BigQueryError -from slo_generator.exporters.base import MetricsExporter, DEFAULT_METRIC_LABELS +from slo_generator.exporters.base import MetricsExporter from .test_stubs import (CTX, load_fixture, load_sample, load_slo_samples, mock_dd_metric_query, mock_dd_metric_send, - mock_dd_slo_get, mock_dd_slo_history, - mock_dt, mock_dt_errors, mock_es, mock_prom, mock_sd, + mock_dd_slo_get, mock_dd_slo_history, mock_dt, + mock_dt_errors, mock_es, mock_prom, mock_sd, mock_ssm_client) warnings.filterwarnings("ignore", message=_CLOUD_SDK_CREDENTIALS_WARNING) -ERROR_BUDGET_POLICY = load_sample('error_budget_policy.yaml', **CTX) -STEPS = len(ERROR_BUDGET_POLICY) -SLO_CONFIGS_SD = load_slo_samples('stackdriver', **CTX) -SLO_CONFIGS_SDSM = load_slo_samples('stackdriver_service_monitoring', **CTX) -SLO_CONFIGS_PROM = load_slo_samples('prometheus', **CTX) -SLO_CONFIGS_ES = load_slo_samples('elasticsearch', **CTX) -SLO_CONFIGS_DD = load_slo_samples('datadog', **CTX) -SLO_CONFIGS_DT = load_slo_samples('dynatrace', **CTX) -SLO_REPORT = load_fixture('slo_report.json') -EXPORTERS = load_fixture('exporters.yaml', **CTX) +CONFIG = load_sample('config.yaml', CTX) +STEPS = len(CONFIG['error_budget_policies']['default']['steps']) +SLO_CONFIGS_SD = load_slo_samples('cloud_monitoring', CTX) +SLO_CONFIGS_SDSM = load_slo_samples('cloud_service_monitoring', CTX) +SLO_CONFIGS_PROM = load_slo_samples('prometheus', CTX) +SLO_CONFIGS_ES = load_slo_samples('elasticsearch', CTX) +SLO_CONFIGS_DD = load_slo_samples('datadog', CTX) +SLO_CONFIGS_DT = load_slo_samples('dynatrace', CTX) +SLO_REPORT = load_fixture('slo_report_v2.json') +SLO_REPORT_V1 = load_fixture('slo_report_v1.json') +EXPORTERS = load_fixture('exporters.yaml', CTX) BQ_ERROR = load_fixture('bq_error.json') # Pub/Sub methods to patch @@ -53,8 +54,8 @@ # Service Monitoring method to patch # pylint: ignore=E501 SSM_MOCKS = [ - "slo_generator.backends.stackdriver_service_monitoring.ServiceMonitoringServiceClient", # noqa: E501 - "slo_generator.backends.stackdriver_service_monitoring.SSM.to_json" + "slo_generator.backends.cloud_service_monitoring.ServiceMonitoringServiceClient", # noqa: E501 + "slo_generator.backends.cloud_service_monitoring.SSM.to_json" ] @@ -66,7 +67,7 @@ class TestCompute(unittest.TestCase): def test_compute_stackdriver(self, mock): for config in SLO_CONFIGS_SD: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch(SSM_MOCKS[0], return_value=mock_ssm_client()) @patch(SSM_MOCKS[1], @@ -76,7 +77,7 @@ def test_compute_stackdriver(self, mock): def test_compute_ssm(self, *mocks): for config in SLO_CONFIGS_SDSM: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch(SSM_MOCKS[0], return_value=mock_ssm_client()) @patch(SSM_MOCKS[1], @@ -88,22 +89,19 @@ def test_compute_ssm(self, *mocks): def test_compute_ssm_delete_export(self, *mocks): for config in SLO_CONFIGS_SDSM: with self.subTest(config=config): - compute(config, - ERROR_BUDGET_POLICY, - delete=True, - do_export=True) + compute(config, CONFIG, delete=True, do_export=True) @patch.object(Prometheus, 'query', mock_prom) def test_compute_prometheus(self): for config in SLO_CONFIGS_PROM: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch.object(Elasticsearch, 'search', mock_es) def test_compute_elasticsearch(self): for config in SLO_CONFIGS_ES: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch.object(Metric, 'query', mock_dd_metric_query) @patch.object(ServiceLevelObjective, 'history', mock_dd_slo_history) @@ -111,13 +109,13 @@ def test_compute_elasticsearch(self): def test_compute_datadog(self): for config in SLO_CONFIGS_DD: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch.object(DynatraceClient, 'request', side_effect=mock_dt) def test_compute_dynatrace(self, mock): for config in SLO_CONFIGS_DT: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch(PUBSUB_MOCKS[0]) @patch(PUBSUB_MOCKS[1]) @@ -163,51 +161,16 @@ def test_export_dynatrace_error(self, mock): codes = [r[0]['response']['error']['code'] for r in responses] self.assertTrue(all(code == 429 for code in codes)) - @patch("google.api_core.grpc_helpers.create_channel", - return_value=mock_sd(STEPS)) - def test_export_deprecated(self, mock): - with self.assertWarns(FutureWarning): - export(SLO_REPORT, EXPORTERS[6]) - - def test_metrics_exporter_build_metrics(self): - exporter = MetricsExporter() - metric = EXPORTERS[7]['metrics'][0] - labels = {} - metric_labels = { - label: str(SLO_REPORT[label]) - for label in DEFAULT_METRIC_LABELS - if label != 'metadata' - } - metadata_labels = SLO_REPORT['metadata'].items() - additional_labels = { - 'good_events_count': str(SLO_REPORT['good_events_count']), - 'bad_events_count': str(SLO_REPORT['bad_events_count']), - } - labels.update(metric_labels) - labels.update(additional_labels) - labels.update(metadata_labels) - metric_expected = { - 'name': 'error_budget_burn_rate', - 'description': "", - 'value': SLO_REPORT['error_budget_burn_rate'], - 'timestamp': SLO_REPORT['timestamp'], - 'labels': labels, - 'additional_labels': metric['additional_labels'] - } - metric = exporter.build_metric(data=SLO_REPORT, metric=metric) - self.assertEqual(labels, metric['labels']) - self.assertEqual(metric, metric_expected) - def test_metrics_exporter_build_data_labels(self): exporter = MetricsExporter() - data = SLO_REPORT + data = SLO_REPORT_V1 labels = ['service_name', 'slo_name', 'metadata'] result = exporter.build_data_labels(data, labels) expected = { - 'service_name': SLO_REPORT['service_name'], - 'slo_name': SLO_REPORT['slo_name'], - 'env': SLO_REPORT['metadata']['env'], - 'team': SLO_REPORT['metadata']['team'] + 'service_name': SLO_REPORT_V1['service_name'], + 'slo_name': SLO_REPORT_V1['slo_name'], + 'env': SLO_REPORT_V1['metadata']['env'], + 'team': SLO_REPORT_V1['metadata']['team'] } self.assertEqual(result, expected) diff --git a/tests/unit/test_migrate.py b/tests/unit/test_migrate.py new file mode 100644 index 00000000..c3ac5498 --- /dev/null +++ b/tests/unit/test_migrate.py @@ -0,0 +1,36 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from slo_generator.migrations.migrator import slo_config_v1tov2 +from .test_stubs import load_fixture + + +class TestMigrator(unittest.TestCase): + + def setUp(self): + self.slo_config_v1 = load_fixture('slo_config_v1.yaml') + self.slo_config_v2 = load_fixture('slo_config_v2.yaml') + self.shared_config = { + 'backends': {}, + 'exporters': {}, + 'error_budget_policies': {} + } + + def test_migrate_v1_to_v2(self): + slo_config_migrated = slo_config_v1tov2(self.slo_config_v1, + self.shared_config, + quiet=True) + self.assertDictEqual(slo_config_migrated, self.slo_config_v2) diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index 22f302d2..347ae8b9 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -20,6 +20,7 @@ class TestReport(unittest.TestCase): + def test_report_enough_events(self): report_cfg = mock_slo_report("enough_events") report = SLOReport(**report_cfg) @@ -43,8 +44,7 @@ def test_report_valid_sli_value(self): report_cfg = mock_slo_report("valid_sli_value") report = SLOReport(**report_cfg) self.assertTrue(report.valid) - self.assertEqual(report.sli_measurement, - report_cfg['config']['backend']['sli']) + self.assertEqual(report.sli_measurement, report_cfg['backend']['sli']) self.assertEqual(report.alert, False) def test_report_no_events(self): diff --git a/tests/unit/test_stubs.py b/tests/unit/test_stubs.py index 23bcd28d..5b43d72b 100644 --- a/tests/unit/test_stubs.py +++ b/tests/unit/test_stubs.py @@ -15,7 +15,6 @@ Stubs for mocking backends and exporters. """ -import copy import json import os import sys @@ -23,13 +22,14 @@ from types import ModuleType from google.cloud.monitoring_v3.proto import metric_service_pb2 -from slo_generator.utils import list_slo_configs, parse_config +from slo_generator.utils import load_configs, load_config TEST_DIR = os.path.dirname(os.path.abspath(__file__)) SAMPLE_DIR = os.path.join(os.path.dirname(os.path.dirname(TEST_DIR)), "samples/") CTX = { + 'PROJECT_ID': 'fake', 'PUBSUB_PROJECT_ID': 'fake', 'PUBSUB_TOPIC_NAME': 'fake', 'GAE_PROJECT_ID': 'fake', @@ -57,26 +57,6 @@ 'DYNATRACE_API_TOKEN': 'fake' } -CUSTOM_BACKEND_CODE = """ -class DummyBackend: - def __init__(self, client=None, **config): - self.good_events = config.get('good_events', None) - self.bad_events = config.get('bad_events', None) - self.sli_value = config.get('sli', None) - - def good_bad_ratio(self, timestamp, window, slo_config): - return (self.good_events, self.bad_events) - - def sli(self, timestamp, window, slo_config): - return self.sli_value -""" - -FAIL_EXPORTER_CODE = """ -from slo_generator.exporters.base import MetricsExporter -class FailExporter(MetricsExporter): - def export_metric(self, data): - raise ValueError("Oops !") -""" def add_dynamic(name, code, type): """Dynamically add a backend or exporter to slo-generator. @@ -92,114 +72,28 @@ def add_dynamic(name, code, type): exec(code, mod.__dict__) -# Add backends / exporters for testing purposes -add_dynamic('dummy', CUSTOM_BACKEND_CODE, 'backends') -add_dynamic('fail', FAIL_EXPORTER_CODE, 'exporters') - - -CUSTOM_BASE_CONFIG = { - "service_name": "test", - "feature_name": "test", - "slo_name": "test", - "slo_description": "Test dummy backend", - "slo_target": 0.99, - "backend": { - "class": "Dummy", - } -} - -CUSTOM_STEP = { - "error_budget_policy_step_name": "1 hour", - "measurement_window_seconds": 3600, - "alerting_burn_rate_threshold": 1, - "urgent_notification": True, - "overburned_consequence_message": "Page to defend the SLO", - "achieved_consequence_message": "Last hour on track" -} - -CUSTOM_TESTS = { - "enough_events": { - 'method': 'good_bad_ratio', - 'good_events': 5, - 'bad_events': 5, - }, - "no_good_events": { - 'method': 'good_bad_ratio', - 'good_events': -1, - 'bad_events': 15, - }, - "no_bad_events": { - 'method': 'good_bad_ratio', - 'good_events': 15, - 'bad_events': -1, - }, - "valid_sli_value": { - 'method': 'sli', - 'good_events': -1, - 'bad_events': -1, - 'sli': 0.991 - }, - "no_events": { - 'method': 'good_bad_ratio', - 'good_events': 0, - 'bad_events': 0, - }, - "no_good_bad_events": { - 'method': 'good_bad_ratio', - 'good_events': -1, - 'bad_events': -1, - }, - "not_enough_events": { - 'method': 'good_bad_ratio', - 'good_events': 5, - 'bad_events': 4, - }, - "no_sli_value": { - 'method': 'sli', - 'good_events': -1, - 'bad_events': -1, - }, - "no_backend_response_sli": { - 'method': 'sli', - 'sli': None - }, - "no_backend_response_ratio": { - 'method': 'good_bad_ratio', - 'good_events': None, - 'bad_events': None, - }, - 'invalid_backend_response_type': { - 'method': 'good_bad_ratio', - 'good_events': { - 'data': { - 'value': 30 - } - }, - 'bad_events': { - 'data': { - 'value': 400 - } - }, - 'sli': None - } -} - - def mock_slo_report(key): - """Mock SLO report config with edge cases contained in CUSTOM_TESTS. + """Mock SLO report config with edge cases contained in DUMMY_TESTS. Args: - key (str): Key identifying which config to pick from CUSTOM_TESTS. + key (str): Key identifying which config to pick from DUMMY_TESTS. Returns: dict: Dict configuration for SLOReport class. """ - config = copy.deepcopy(CUSTOM_BASE_CONFIG) - config["backend"].update(CUSTOM_TESTS[key]) + slo_config = load_fixture('dummy_slo_config.json') + ebp_step = load_fixture( + 'dummy_config.json')['error_budget_policies']['default'][0] + dummy_tests = load_fixture('dummy_tests.json') + backend = dummy_tests[key] + slo_config['spec']['method'] = backend['method'] + backend['name'] = 'dummy' + backend['class'] = 'Dummy' timestamp = time.time() return { - "config": config, - "step": CUSTOM_STEP, + "config": slo_config, + "backend": backend, + "step": ebp_step, "timestamp": timestamp, "client": None, "delete": False @@ -209,6 +103,7 @@ def mock_slo_report(key): # pylint: disable=too-few-public-methods class MultiCallableStub: """Stub for the grpc.UnaryUnaryMultiCallable interface.""" + def __init__(self, method, channel_stub): self.method = method self.channel_stub = channel_stub @@ -231,6 +126,7 @@ def __call__(self, request, timeout=None, metadata=None, credentials=None): # pylint: disable=R0903 class ChannelStub: """Stub for the grpc.Channel interface.""" + def __init__(self, responses=[]): self.responses = responses self.requests = [] @@ -344,6 +240,7 @@ def mock_dt(*args, **kwargs): elif args[0] == 'put' and args[1] == 'timeseries': return {} + def mock_dt_errors(*args, **kwargs): """Mock Dynatrace response with errors.""" if args[0] == 'get' and args[1] == 'timeseries': @@ -358,6 +255,7 @@ def mock_dt_errors(*args, **kwargs): elif args[0] == 'put' and args[1] == 'timeseries': return load_fixture('dt_error_rate.json') + class dotdict(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get @@ -383,6 +281,7 @@ def dotize(data): class mock_ssm_client: """Fake Service Monitoring API client.""" + def __init__(self): self.services = [dotize(s) for s in load_fixture('ssm_services.json')] self.service_level_objectives = [ @@ -425,7 +324,19 @@ def to_json(data): return data -def load_fixture(filename, **ctx): +def get_fixture_path(filename): + """Get path for a fixture file. + + Args: + filename (str): Filename of file in fixtures/. + + Returns: + str: Full path of file in fixtures/. + """ + return os.path.join(TEST_DIR, "fixtures/", filename) + + +def load_fixture(filename, ctx=os.environ): """Load a fixture from the test/fixtures/ directory and replace context environmental variables in it. @@ -436,11 +347,11 @@ def load_fixture(filename, **ctx): Returns: dict: Loaded fixture. """ - filename = os.path.join(TEST_DIR, "fixtures/", filename) - return parse_config(filename, ctx) + path = get_fixture_path(filename) + return load_config(path, ctx=ctx) -def load_sample(filename, **ctx): +def load_sample(filename, ctx=os.environ): """Load a sample from the samples/ directory and replace context environmental variables in it. @@ -452,10 +363,10 @@ def load_sample(filename, **ctx): dict: Loaded sample. """ filename = os.path.join(SAMPLE_DIR, filename) - return parse_config(filename, ctx) + return load_config(filename, ctx=ctx) -def load_slo_samples(folder_path, **ctx): +def load_slo_samples(folder_path, ctx=os.environ): """List and load all SLO samples from folder path. Args: @@ -465,7 +376,11 @@ def load_slo_samples(folder_path, **ctx): Returns: list: List of loaded SLO configs. """ - return [ - load_sample(filename, **ctx) - for filename in list_slo_configs(f'{SAMPLE_DIR}/{folder_path}') - ] + return load_configs(f'{SAMPLE_DIR}/{folder_path}', ctx) + + +# Add custom backends / exporters for testing purposes +DUMMY_BACKEND_CODE = open(get_fixture_path('dummy_backend.py')).read() +FAIL_EXPORTER_CODE = open(get_fixture_path('fail_exporter.py')).read() +add_dynamic('dummy', DUMMY_BACKEND_CODE, 'backends') +add_dynamic('fail', FAIL_EXPORTER_CODE, 'exporters') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7acb9346..012a4ab6 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -19,6 +19,7 @@ class TestUtils(unittest.TestCase): + def test_get_human_time(self): # Timezones tz_1 = 'Europe/Paris' @@ -42,54 +43,56 @@ def test_get_human_time(self): self.assertEqual(human_chicago_2, utc_time_2 + "-05:00") def test_get_backend_cls(self): - res1 = get_backend_cls("Stackdriver") + res1 = get_backend_cls("CloudMonitoring") res2 = get_backend_cls("Prometheus") - self.assertEqual(res1.__name__, "StackdriverBackend") - self.assertEqual(res1.__module__, "slo_generator.backends.stackdriver") + self.assertEqual(res1.__name__, "CloudMonitoringBackend") + self.assertEqual(res1.__module__, + "slo_generator.backends.cloud_monitoring") self.assertEqual(res2.__name__, "PrometheusBackend") self.assertEqual(res2.__module__, "slo_generator.backends.prometheus") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_backend_cls("UndefinedBackend") def test_get_backend_dynamic_cls(self): res1 = get_backend_cls("pathlib.Path") self.assertEqual(res1.__name__, "Path") self.assertEqual(res1.__module__, "pathlib") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_exporter_cls("foo.bar.DoesNotExist") def test_get_exporter_cls(self): - res1 = get_exporter_cls("Stackdriver") + res1 = get_exporter_cls("CloudMonitoring") res2 = get_exporter_cls("Pubsub") res3 = get_exporter_cls("Bigquery") - self.assertEqual(res1.__name__, "StackdriverExporter") - self.assertEqual(res1.__module__, "slo_generator.exporters.stackdriver") + self.assertEqual(res1.__name__, "CloudMonitoringExporter") + self.assertEqual(res1.__module__, + "slo_generator.exporters.cloud_monitoring") self.assertEqual(res2.__name__, "PubsubExporter") self.assertEqual(res2.__module__, "slo_generator.exporters.pubsub") self.assertEqual(res3.__name__, "BigqueryExporter") self.assertEqual(res3.__module__, "slo_generator.exporters.bigquery") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_exporter_cls("UndefinedExporter") def test_get_exporter_dynamic_cls(self): res1 = get_exporter_cls("pathlib.Path") self.assertEqual(res1.__name__, "Path") self.assertEqual(res1.__module__, "pathlib") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_exporter_cls("foo.bar.DoesNotExist") def test_import_dynamic(self): - res1 = import_dynamic("slo_generator.backends.stackdriver", - "StackdriverBackend", + res1 = import_dynamic("slo_generator.backends.cloud_monitoring", + "CloudMonitoringBackend", prefix="backend") - res2 = import_dynamic("slo_generator.exporters.stackdriver", - "StackdriverExporter", + res2 = import_dynamic("slo_generator.exporters.cloud_monitoring", + "CloudMonitoringExporter", prefix="exporter") - self.assertEqual(res1.__name__, "StackdriverBackend") - self.assertEqual(res2.__name__, "StackdriverExporter") - with self.assertRaises(ModuleNotFoundError): + self.assertEqual(res1.__name__, "CloudMonitoringBackend") + self.assertEqual(res2.__name__, "CloudMonitoringExporter") + with self.assertWarns(ImportWarning): import_dynamic("slo_generator.backends.unknown", - "StackdriverUnknown", + "CloudMonitoringUnknown", prefix="unknown") From 191f36c2c09be41c2e7f3e82481974155d03f6dc Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 13:36:57 +0200 Subject: [PATCH 010/107] feat: Add migrator for v1 to v2 migration (#127) --- slo_generator/migrations/__init__.py | 13 + slo_generator/migrations/migrator.py | 442 +++++++++++++++++++++++++++ 2 files changed, 455 insertions(+) create mode 100644 slo_generator/migrations/__init__.py create mode 100644 slo_generator/migrations/migrator.py diff --git a/slo_generator/migrations/__init__.py b/slo_generator/migrations/__init__.py new file mode 100644 index 00000000..31812446 --- /dev/null +++ b/slo_generator/migrations/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py new file mode 100644 index 00000000..d68da1eb --- /dev/null +++ b/slo_generator/migrations/migrator.py @@ -0,0 +1,442 @@ +# Copyright 2021 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +`v1tov2.py` +Migrate utilities for migrating slo-generator configs from v1 to v2. +""" +# pylint: disable=line-too-long, too-many-statements, too-many-ancestors, too-many-locals, too-many-nested-blocks, unused-argument +# flake8: noqa +import copy +import itertools +import pprint +import random +import string +import sys +from collections import OrderedDict +from pathlib import Path + +import click +import ruamel.yaml as yaml + +from slo_generator import utils +from slo_generator.constants import (METRIC_LABELS_COMPAT, + METRIC_METADATA_LABELS_TOP_COMPAT, + PROVIDERS_COMPAT, CONFIG_SCHEMA, + SLO_CONFIG_SCHEMA, GREEN, RED, BOLD, + WARNING, ENDC, SUCCESS, FAIL, RIGHT_ARROW) + +yaml.explicit_start = True +yaml.default_flow_style = None +yaml.preserve_quotes = True + + +def do_migrate(source, + target, + error_budget_policy_path, + glob, + version, + quiet=False, + verbose=0): + """Process all SLO configs in folder and generate new SLO configurations. + + Args: + source (str): Source SLO configs folder. + target (str): Target SLO configs folder. + error_budget_policy_path (str): Error budget policy path. + glob (str): Glob expression to add to source path. + version (str): slo-generator major version string (e.g: v1, v2, ...) + quiet (bool, optional): If true, do not prompt for user input. + verbose (int, optional): Verbose level. + """ + shared_config = CONFIG_SCHEMA + cwd = Path.cwd() + source = Path(source).resolve() + target = Path(target).resolve() + source_str = source.relative_to(cwd) # human-readable path + target_str = target.relative_to(cwd) # human-readable path + error_budget_policy_path = Path(error_budget_policy_path) + + # Create target folder if it doesn't exist + target.mkdir(parents=True, exist_ok=True) + + # Process SLO configs + click.secho('=' * 50) + click.secho(f"Migrating slo-generator configs to {version} ...", + fg='cyan', + bold=True) + + paths = Path(source).glob(glob) + + if not peek(paths): + click.secho(f"{FAIL} No SLO configs found in {source}", + fg='red', + bold=True) + sys.exit(1) + + for source_path in paths: + source_path_str = source_path.relative_to(cwd) + if source == target == cwd: + target_path = target.joinpath(*source_path.relative_to(cwd).parts) + else: + target_path = target.joinpath( + *source_path.relative_to(cwd).parts[1:]) + target_path_str = target_path.relative_to(cwd) + slo_config_str = source_path.open().read() + slo_config, ind, blc = yaml.util.load_yaml_guess_indent(slo_config_str) + curver = get_config_version(slo_config) + + # Source path info + click.secho("-" * 50) + click.secho(f"{WARNING}{source_path_str}{ENDC} [{curver}] ") + + # If config version is same as target version, continue + if curver == version: + click.secho( + f'{FAIL} {source_path_str} is already in {version} format', + fg='red', + bold=True) + continue + + # Create target dirs if needed + target_path.parent.mkdir(parents=True, exist_ok=True) + + # Run vx to vy migrator method + func = getattr(sys.modules[__name__], f"slo_config_{curver}to{version}") + slo_config_v2 = func(slo_config, shared_config, quiet=quiet) + + # Write resulting config to target path + extra = '(replaced)' if target_path_str == source_path_str else '' + click.secho( + f"{RIGHT_ARROW} {GREEN}{target_path_str}{ENDC} [{version}] {extra}") + with target_path.open('w') as conf: + yaml.round_trip_dump( + slo_config_v2, + conf, + indent=ind, + block_seq_indent=blc, + default_flow_style=None, + ) + click.secho(f'{SUCCESS} Success !', fg='green', bold=True) + + # Translate error budget policy to v2 and put into shared config + error_budget_policy = yaml.load(open(error_budget_policy_path), + Loader=yaml.Loader) + for step in error_budget_policy: + step['name'] = step.pop('error_budget_policy_step_name') + step['burn_rate_threshold'] = step.pop('alerting_burn_rate_threshold') + step['alert'] = step.pop('urgent_notification') + step['message_alert'] = step.pop('overburned_consequence_message') + step['message_ok'] = step.pop('achieved_consequence_message') + step['window'] = step.pop('measurement_window_seconds') + + ebp = {'steps': error_budget_policy} + if error_budget_policy_path.name == 'error_budget_policy.yaml': + ebp_key = 'default' + else: + ebp_key = error_budget_policy_path.name + shared_config['error_budget_policies'][ebp_key] = ebp + shared_config_path = target / 'config.yaml' + shared_config_path_str = shared_config_path.relative_to(cwd) + + # Write shared config to file + click.secho('=' * 50) + with shared_config_path.open('w') as conf: + click.secho( + f'Writing slo-generator config to {shared_config_path_str} ...', + fg='cyan', + bold=True) + yaml.round_trip_dump( + shared_config, + conf, + Dumper=CustomDumper, + indent=2, + block_seq_indent=0, + explicit_start=True, + ) + click.secho(f'{SUCCESS} Success !', fg='green', bold=True) + + # Remove error budget policy file + click.secho('=' * 50) + click.secho(f'Removing {error_budget_policy_path} ...', + fg='cyan', + bold=True) + error_budget_policy_path.unlink() + click.secho(f'{SUCCESS} Success !', fg='green', bold=True) + + # Print next steps + click.secho('=' * 50) + click.secho( + f'\n{SUCCESS} Migration of `slo-generator` configs to v2 completed successfully ! Configs path: {target_str}/.\n', + fg='green', + bold=True) + click.secho('=' * 50) + click.secho( + f'{BOLD}PLEASE FOLLOW THE MANUAL STEPS BELOW TO FINISH YOUR MIGRATION:', + fg='red', + bold=True) + click.secho(f""" + 1 - Commit the updated SLO configs and your shared SLO config to version control. + 2 - [local/k8s/cloudbuild] Update your slo-generator command: + {RED} [-] slo-generator -f {source_str} -b {error_budget_policy_path}{ENDC} + {GREEN} [+] slo-generator -f {target_str} -c {target_str}/config.yaml{ENDC} + 3 - [terraform] Upgrade your `terraform-google-slo` modules: + 3.1 - Upgrade the module `version` to 2.0.0. + 3.2 - Replace `error_budget_policy` field in your `slo` and `slo-pipeline` modules by `shared_config` + 3.3 - Replace `error_budget_policy.yaml` local variable to `config.yaml` + """) + + +def slo_config_v1tov2(slo_config, shared_config={}, quiet=False, verbose=0): + """Process old SLO config v1 and generate SLO config v2. + + Args: + slo_config (dict): SLO Config v1. + shared_config (dict): SLO Generator config. + quiet (bool): If true, do not ask for user input. + verbose (int): Verbose level. + + Returns: + dict: SLO Config v2. + """ + # SLO config v2 skeleton + slo_config_v2 = OrderedDict(copy.deepcopy(SLO_CONFIG_SCHEMA)) + slo_config_v2['apiVersion'] = 'sre.google.com/v2' + slo_config_v2['kind'] = 'ServiceLevelObjective' + + # Get fields from old config + slo_metadata_name = '{service_name}-{feature_name}-{slo_name}'.format( + **slo_config) + slo_description = slo_config.pop('slo_description') + slo_target = slo_config.pop('slo_target') + service_level_indicator = slo_config['backend'].pop('measurement', {}) + backend = slo_config['backend'] + method = backend.pop('method') + exporters = slo_config.get('exporters', []) + if isinstance(exporters, dict): # single exporter, deprecated + exporters = [exporters] + + # Fill spec + slo_config_v2['metadata']['name'] = slo_metadata_name + slo_config_v2['metadata']['labels'] = { + 'service_name': slo_config['service_name'], + 'feature_name': slo_config['feature_name'], + 'slo_name': slo_config['slo_name'], + } + other_labels = { + k: v for k, v in slo_config.items() if k not in + ['service_name', 'feature_name', 'slo_name', 'backend', 'exporters'] + } + slo_config_v2['metadata']['labels'].update(other_labels) + slo_config_v2['spec']['description'] = slo_description + slo_config_v2['spec']['goal'] = slo_target + + # Process backend + backend = OrderedDict(backend) + backend_key = add_to_shared_config(backend, + shared_config, + 'backends', + quiet=quiet) + slo_config_v2['spec']['backend'] = backend_key + slo_config_v2['spec']['method'] = method + + # If exporter not in general config, add it and add an alias for the + # exporter. Refer to the alias in the SLO config file. + for exporter in exporters: + exporter = OrderedDict(exporter) + exp_key = add_to_shared_config(exporter, + shared_config, + 'exporters', + quiet=quiet) + slo_config_v2['spec']['exporters'].append(exp_key) + + # Fill spec + slo_config_v2['spec']['service_level_indicator'] = service_level_indicator + + if verbose > 0: + pprint.pprint(dict(slo_config_v2)) + return dict(slo_config_v2) + + +def report_v2tov1(report): + """Convert SLO report from v2 to v1 format, for exporters to be + backward-compatible with v1 data format. + + Args: + report (dict): SLO report. + + Returns: + dict: Converted SLO report. + """ + mapped_report = {} + for key, value in report.items(): + + # If a metadata label is passed, use the metadata label mapping + if key == 'metadata': + mapped_report['metadata'] = {} + for subkey, subvalue in value.items(): + + # v2 `metadata.labels` attributes map to `metadata` attributes + # in v1 + if subkey == 'labels': + labels = subvalue + for labelkey, labelval in labels.items(): + + # Top-level labels like 'service_name', 'feature_name', + # and 'slo_name'. + if labelkey in METRIC_METADATA_LABELS_TOP_COMPAT: + mapped_report[labelkey] = labelval + + # Other labels that are mapped to 'metadata' in the v1 + # report + else: + mapped_report['metadata'][labelkey] = labelval + + # ignore the name attribute which is just a concatenation of + # service_name, feature_name and slo_name + elif subkey == 'name': + continue + + # other metadata labels are still mapped to the v1 `metadata` + # attributes + else: + mapped_report['metadata'][subkey] = subvalue + + # If a key in the default label mapping is passed, use the default + # label mapping + elif key in METRIC_LABELS_COMPAT.keys(): + mapped_report.update({METRIC_LABELS_COMPAT[key]: value}) + + # Otherwise, write the label as is + else: + mapped_report.update({key: value}) + return mapped_report + + +def get_random_suffix(): + """Get random suffix for our backends / exporters when configs clash.""" + return ''.join(random.choices(string.digits, k=4)) + + +def add_to_shared_config(new_obj, shared_config, section, quiet=False): + """Add an object to the shared_config. + + If the object with the same config already exists in the shared config, + simply return its key. + + If the object does not exist in the shared config: + * If the default key is already taken, add a random suffix to it. + * If the default key is not taken, add the new object to the config. + + Args: + new_obj (OrderedDict): Object to add to shared_config. + shared_config (dict): Shared config to add object to. + section (str): Section name in shared config to add the object under. + quiet (bool): If True, do not ask for user input. + + Returns: + str: Object key in the shared config. + """ + shared_obj = shared_config[section] + key = new_obj.pop('class') + if '.' not in key: + key = utils.caml_to_snake(PROVIDERS_COMPAT.get(key, key)) + + existing_obj = { + k: v + for k, v in shared_obj.items() + if k.startswith(key.split('/')[0]) and str(v) == str(dict(new_obj)) + } + if existing_obj: + key = next(iter(existing_obj)) + # click.secho(f'Found existing {section} {key}') + else: + if key in shared_obj.keys(): # key conflicts + if quiet: + key += '/' + get_random_suffix() + else: + name = section.rstrip('s') + cfg = pprint.pformat({key: dict(new_obj)}) + valid = False + while not valid: + click.secho( + f'\nNew {name} found with the following config:\n{cfg}', + fg='cyan', + blink=True) + user_input = click.prompt( + f'\n{RED}{BOLD}Please give this {name} a name:{ENDC}', + type=str) + key += '/' + user_input.lower() + if key in shared_obj.keys(): + click.secho( + f'{name.capitalize()} "{key}" already exists in shared config', + fg='red', + bold=True) + else: + valid = True + click.secho(f'Backend {key} was added to shared config.', + fg='green', + bold=True) + + # click.secho(f"Adding new {section} {key}") + shared_obj[key] = dict(new_obj) + shared_config[section] = dict(sorted(shared_obj.items())) + return key + + +def get_config_version(config): + """Return version of an slo-generator config based on the format. + + Args: + config (dict): slo-generator configuration. + + Returns: + str: SLO config version. + """ + api_version = config.get('apiVersion', '') + kind = config.get('kind', '') + if not kind: # old v1 format + return 'v1' + return api_version.split('/')[-1] + + +def peek(iterable): + """Check if iterable is empty. + + Args: + iterable (collections.Iterable): an iterable + + Returns: + iterable (collections.Iterable): the same iterable, or None if empty. + """ + try: + first = next(iterable) + except StopIteration: + return None + return first, itertools.chain([first], iterable) + + +class CustomDumper(yaml.RoundTripDumper): + """Dedicated YAML dumper to insert lines between top-level objects. + + Args: + data (str): Line data. + """ + + # HACK: insert blank lines between top-level objects + # inspired by https://stackoverflow.com/a/44284819/3786245 + def write_line_break(self, data=None): + super().write_line_break(data) + + if len(self.indents) == 1: + super().write_line_break() From 73a3ee030556505adfec9959cb622669799ade36 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 13:41:45 +0200 Subject: [PATCH 011/107] feat: Add slo-generator Functions Framework API (#130) --- slo_generator/api/__init__.py | 0 slo_generator/api/main.py | 112 ++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 slo_generator/api/__init__.py create mode 100644 slo_generator/api/main.py diff --git a/slo_generator/api/__init__.py b/slo_generator/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py new file mode 100644 index 00000000..29d0ede7 --- /dev/null +++ b/slo_generator/api/main.py @@ -0,0 +1,112 @@ +# Copyright 2021 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +`main.py` +Functions Framework API (Flask). +See https://github.com/GoogleCloudPlatform/functions-framework-python for +details on the Functions Framework. +""" +import base64 +import os +import logging +import pprint + +from datetime import datetime +from flask import jsonify + +import yaml + +from slo_generator.compute import compute, export +from slo_generator.utils import setup_logging, load_config, get_exporters + +CONFIG_PATH = os.environ['CONFIG_PATH'] +EXPORTERS_PATH = os.environ.get('EXPORTERS_PATH', None) +LOGGER = logging.getLogger(__name__) +TIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' +API_SIGNATURE_TYPE = os.environ['GOOGLE_FUNCTION_SIGNATURE_TYPE'] +setup_logging() + + +def run_compute(request): + """Run slo-generator compute function. Can be configured to export data as + well, using the `exporters` key of the SLO config. + + Args: + request (cloudevent.CloudEvent, flask.Request): Request object. + + Returns: + list: List of SLO reports. + """ + # Get SLO config + if API_SIGNATURE_TYPE == 'http': + timestamp = None + data = str(request.get_json()) + LOGGER.info('Loading SLO config from Flask request') + elif API_SIGNATURE_TYPE == 'cloudevent': + timestamp = int( + datetime.strptime(request["time"], TIME_FORMAT).timestamp()) + data = base64.b64decode(request.data).decode('utf-8') + LOGGER.info(f'Loading SLO config from Cloud Event "{request["id"]}"') + + slo_config = load_config(data) + + # Get slo-generator config + LOGGER.info(f'Loading slo-generator config from {CONFIG_PATH}') + config = load_config(CONFIG_PATH) + + # Compute SLO report + LOGGER.debug(f'Config: {pprint.pformat(config)}') + LOGGER.debug(f'SLO Config: {pprint.pformat(slo_config)}') + reports = compute(slo_config, + config, + timestamp=timestamp, + client=None, + do_export=True) + if API_SIGNATURE_TYPE == 'http': + reports = jsonify(reports) + return reports + + +def run_export(request): + """Run slo-generator export function. Get the SLO report data from a request + object. + + Args: + request (cloudevent.CloudEvent, flask.Request): Request object. + + Returns: + list: List of SLO reports. + """ + # Get export data + if API_SIGNATURE_TYPE == 'http': + slo_report = request.get_json() + elif API_SIGNATURE_TYPE == 'cloudevent': + slo_report = yaml.safe_load(base64.b64decode(request.data)) + + # Get SLO config + LOGGER.info(f'Downloading SLO config from {CONFIG_PATH}') + config = load_config(CONFIG_PATH) + + # Build exporters list + if EXPORTERS_PATH: + LOGGER.info(f'Loading exporters from {EXPORTERS_PATH}') + exporters = load_config(EXPORTERS_PATH) + else: + LOGGER.info(f'Loading exporters from SLO report data {EXPORTERS_PATH}') + exporters = slo_report['exporters'] + spec = {"exporters": exporters} + exporters = get_exporters(config, spec) + + # Export data + export(slo_report, exporters) From 1705d21dc7bcdf8bb56b5049e04965e4ff7a86b8 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 18:47:15 +0200 Subject: [PATCH 012/107] fix: Migrate sample configurations to v2 (#128) * fix: Migrate sample configurations to v2 --- samples/.env.sample | 21 +++++ .../slo_gae_app_availability.yaml | 35 ++++++++ .../slo_gae_app_latency.yaml | 30 ++++--- .../slo_lb_request_availability.yaml | 27 ++++++ .../slo_lb_request_latency.yaml | 25 ++++++ .../slo_pubsub_subscription_throughput.yaml | 24 ++++++ .../slo_gae_app_availability.yaml | 36 ++++++++ .../slo_gae_app_availability_basic.yaml | 20 +++++ .../slo_gae_app_latency.yaml | 24 ++++++ .../slo_gae_app_latency_basic.yaml | 21 +++++ .../slo_gke_app_availability_basic.yaml | 21 +++++ ...gke_app_availability_basic_deprecated.yaml | 23 +++++ .../slo_gke_app_latency_basic.yaml | 22 +++++ .../slo_gke_app_latency_basic_deprecated.yaml | 24 ++++++ .../slo_lb_request_availability.yaml | 27 ++++++ .../slo_lb_request_latency.yaml | 25 ++++++ samples/config.yaml | 83 +++++++++++++++++++ ...slo_custom_app_availability_query_sli.yaml | 42 ++++------ .../slo_custom_app_availability_ratio.yaml | 42 ++++------ .../slo_dd_app_availability_query_sli.yaml | 49 ++++------- .../slo_dd_app_availability_query_slo.yaml | 43 ++++------ .../slo_dd_app_availability_ratio.yaml | 50 ++++------- .../slo_dt_app_availability_ratio.yaml | 54 +++++------- .../slo_dt_app_latency_threshold.yaml | 51 +++++------- samples/elasticsearch/slo_elk_test_ratio.yaml | 60 +++++--------- samples/error_budget_policy.yaml | 43 ---------- ...o_prom_metrics_availability_query_sli.yaml | 49 ++++------- .../slo_prom_metrics_availability_ratio.yaml | 49 ++++------- ...prom_metrics_latency_distribution_cut.yaml | 45 ++++------ .../slo_prom_metrics_latency_query_sli.yaml | 49 ++++------- .../stackdriver/slo_gae_app_availability.yaml | 46 ---------- samples/stackdriver/slo_gae_app_latency.yaml | 35 -------- .../slo_lb_request_availability.yaml | 38 --------- .../stackdriver/slo_lb_request_latency.yaml | 36 -------- .../slo_pubsub_subscription_throughput.yaml | 39 --------- .../slo_gae_app_availability.yaml | 44 ---------- .../slo_gae_app_availability_basic.yaml | 28 ------- .../slo_gae_app_latency_basic.yaml | 29 ------- .../slo_gke_app_availability_basic.yaml | 29 ------- ...gke_app_availability_basic_deprecated.yaml | 31 ------- .../slo_gke_app_latency_basic.yaml | 30 ------- .../slo_gke_app_latency_basic_deprecated.yaml | 32 ------- .../slo_lb_request_availability.yaml | 35 -------- .../slo_lb_request_latency.yaml | 33 -------- ...zz_slo_pubsub_subscription_throughput.yaml | 35 -------- 45 files changed, 692 insertions(+), 942 deletions(-) create mode 100644 samples/.env.sample create mode 100644 samples/cloud_monitoring/slo_gae_app_availability.yaml rename samples/{stackdriver_service_monitoring => cloud_monitoring}/slo_gae_app_latency.yaml (65%) create mode 100644 samples/cloud_monitoring/slo_lb_request_availability.yaml create mode 100644 samples/cloud_monitoring/slo_lb_request_latency.yaml create mode 100644 samples/cloud_monitoring/slo_pubsub_subscription_throughput.yaml create mode 100644 samples/cloud_service_monitoring/slo_gae_app_availability.yaml create mode 100644 samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml create mode 100644 samples/cloud_service_monitoring/slo_gae_app_latency.yaml create mode 100644 samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml create mode 100644 samples/cloud_service_monitoring/slo_gke_app_availability_basic.yaml create mode 100644 samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml create mode 100644 samples/cloud_service_monitoring/slo_gke_app_latency_basic.yaml create mode 100644 samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml create mode 100644 samples/cloud_service_monitoring/slo_lb_request_availability.yaml create mode 100644 samples/cloud_service_monitoring/slo_lb_request_latency.yaml create mode 100644 samples/config.yaml delete mode 100644 samples/error_budget_policy.yaml delete mode 100644 samples/stackdriver/slo_gae_app_availability.yaml delete mode 100644 samples/stackdriver/slo_gae_app_latency.yaml delete mode 100644 samples/stackdriver/slo_lb_request_availability.yaml delete mode 100644 samples/stackdriver/slo_lb_request_latency.yaml delete mode 100644 samples/stackdriver/slo_pubsub_subscription_throughput.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_gae_app_availability.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_gae_app_availability_basic.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_gae_app_latency_basic.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_gke_app_availability_basic.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_gke_app_latency_basic.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_lb_request_availability.yaml delete mode 100644 samples/stackdriver_service_monitoring/slo_lb_request_latency.yaml delete mode 100644 samples/stackdriver_service_monitoring/zzz_slo_pubsub_subscription_throughput.yaml diff --git a/samples/.env.sample b/samples/.env.sample new file mode 100644 index 00000000..bb90163d --- /dev/null +++ b/samples/.env.sample @@ -0,0 +1,21 @@ +export STACKDRIVER_HOST_PROJECT_ID= +export GAE_PROJECT_ID= +export GAE_MODULE_ID= +export LB_PROJECT_ID= +export PUBSUB_PROJECT_ID= +export PUBSUB_TOPIC_NAME= +export GKE_PROJECT_ID= +export GKE_LOCATION= +export GKE_CLUSTER_NAME= +export GKE_SERVICE_NAMESPACE= +export GKE_SERVICE_NAME= +export GKE_MESH_UID= +export ELASTICSEARCH_URL= +export PROMETHEUS_URL= +export PROMETHEUS_PUSHGATEWAY_URL= +export DATADOG_SLO_ID= +export DATADOG_API_KEY= +export DATADOG_APP_KEY= +export DYNATRACE_API_URL= +export DYNATRACE_API_TOKEN= +export BIGQUERY_PROJECT_ID= diff --git a/samples/cloud_monitoring/slo_gae_app_availability.yaml b/samples/cloud_monitoring/slo_gae_app_availability.yaml new file mode 100644 index 00000000..675f290f --- /dev/null +++ b/samples/cloud_monitoring/slo_gae_app_availability.yaml @@ -0,0 +1,35 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-availability + labels: + service_name: gae + feature_name: app + slo_name: availability +spec: + description: Availability of App Engine app + backend: cloud_monitoring + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + ( metric.labels.response_code = 429 OR + metric.labels.response_code = 200 OR + metric.labels.response_code = 201 OR + metric.labels.response_code = 202 OR + metric.labels.response_code = 203 OR + metric.labels.response_code = 204 OR + metric.labels.response_code = 205 OR + metric.labels.response_code = 206 OR + metric.labels.response_code = 207 OR + metric.labels.response_code = 208 OR + metric.labels.response_code = 226 OR + metric.labels.response_code = 304 ) + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + goal: 0.95 diff --git a/samples/stackdriver_service_monitoring/slo_gae_app_latency.yaml b/samples/cloud_monitoring/slo_gae_app_latency.yaml similarity index 65% rename from samples/stackdriver_service_monitoring/slo_gae_app_latency.yaml rename to samples/cloud_monitoring/slo_gae_app_latency.yaml index 777e386a..13d1022f 100644 --- a/samples/stackdriver_service_monitoring/slo_gae_app_latency.yaml +++ b/samples/cloud_monitoring/slo_gae_app_latency.yaml @@ -12,21 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -service_name: gae -feature_name: app -slo_description: Latency of App Engine app requests < 724ms -slo_name: latency724ms -slo_target: 0.999 -backend: - class: StackdriverServiceMonitoring - method: distribution_cut - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-latency724ms + labels: + service_name: gae + feature_name: app + slo_name: latency724ms +spec: + description: Latency of App Engine app requests < 724ms + backend: cloud_monitoring + method: distribution_cut + exporters: + - cloud_monitoring + service_level_indicator: filter_valid: > project=${GAE_PROJECT_ID} metric.type="appengine.googleapis.com/http/server/response_latencies" resource.type="gae_app" metric.labels.response_code >= 200 metric.labels.response_code < 500 - range_min: 0 - range_max: 724 + good_below_threshold: true + threshold_bucket: 19 + goal: 0.999 diff --git a/samples/cloud_monitoring/slo_lb_request_availability.yaml b/samples/cloud_monitoring/slo_lb_request_availability.yaml new file mode 100644 index 00000000..0a9b6f57 --- /dev/null +++ b/samples/cloud_monitoring/slo_lb_request_availability.yaml @@ -0,0 +1,27 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: lb-request-availability + labels: + service_name: lb + feature_name: request + slo_name: availability +spec: + description: Availability of HTTP Load Balancer + backend: cloud_monitoring + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + project=${LB_PROJECT_ID} + metric.type="loadbalancing.googleapis.com/https/request_count" + resource.type="https_lb_rule" + ( metric.label.response_code_class="200" OR + metric.label.response_code_class="300" OR + metric.label.response_code_class="400" ) + filter_valid: > + project=${LB_PROJECT_ID} + metric.type="loadbalancing.googleapis.com/https/request_count" + resource.type="https_lb_rule" + goal: 0.98 diff --git a/samples/cloud_monitoring/slo_lb_request_latency.yaml b/samples/cloud_monitoring/slo_lb_request_latency.yaml new file mode 100644 index 00000000..56df0de2 --- /dev/null +++ b/samples/cloud_monitoring/slo_lb_request_latency.yaml @@ -0,0 +1,25 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: lb-request-latency724ms + labels: + service_name: lb + feature_name: request + slo_name: latency724ms +spec: + description: Latency of HTTP Load Balancer < 724ms + backend: cloud_monitoring + method: distribution_cut + exporters: + - cloud_monitoring + service_level_indicator: + filter_valid: > + project=${LB_PROJECT_ID} + metric.type="loadbalancing.googleapis.com/https/total_latencies" + resource.type="https_lb_rule" + ( metric.label.response_code_class="200" OR + metric.label.response_code_class="300" OR + metric.label.response_code_class="400" ) + good_below_threshold: true + threshold_bucket: 19 + goal: 0.98 diff --git a/samples/cloud_monitoring/slo_pubsub_subscription_throughput.yaml b/samples/cloud_monitoring/slo_pubsub_subscription_throughput.yaml new file mode 100644 index 00000000..917e6fed --- /dev/null +++ b/samples/cloud_monitoring/slo_pubsub_subscription_throughput.yaml @@ -0,0 +1,24 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: pubsub-subscription-throughput + labels: + service_name: pubsub + feature_name: subscription + slo_name: throughput +spec: + description: Throughput of Pub/Sub subscription + backend: cloud_monitoring + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + project="${PUBSUB_PROJECT_ID}" + metric.type="pubsub.googleapis.com/subscription/ack_message_count" + resource.type="pubsub_subscription" + filter_bad: > + project="${PUBSUB_PROJECT_ID}" + metric.type="pubsub.googleapis.com/subscription/num_outstanding_messages" + resource.type="pubsub_subscription" + goal: 0.95 diff --git a/samples/cloud_service_monitoring/slo_gae_app_availability.yaml b/samples/cloud_service_monitoring/slo_gae_app_availability.yaml new file mode 100644 index 00000000..a2e97de3 --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gae_app_availability.yaml @@ -0,0 +1,36 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-availability + labels: + service_name: gae + feature_name: app + slo_name: availability +spec: + description: Availability of App Engine app + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: good_bad_ratio + exporters: [] + service_level_indicator: + filter_good: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + ( metric.labels.response_code = 429 OR + metric.labels.response_code = 200 OR + metric.labels.response_code = 201 OR + metric.labels.response_code = 202 OR + metric.labels.response_code = 203 OR + metric.labels.response_code = 204 OR + metric.labels.response_code = 205 OR + metric.labels.response_code = 206 OR + metric.labels.response_code = 207 OR + metric.labels.response_code = 208 OR + metric.labels.response_code = 226 OR + metric.labels.response_code = 304 ) + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + goal: 0.95 diff --git a/samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml b/samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml new file mode 100644 index 00000000..dd6507ff --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml @@ -0,0 +1,20 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-availability + labels: + service_name: gae + feature_name: app + slo_name: availability +spec: + description: Availability of App Engine app + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: basic + exporters: [] + service_level_indicator: + app_engine: + project_id: ${GAE_PROJECT_ID} + module_id: ${GAE_MODULE_ID} + availability: {} + goal: 0.98 diff --git a/samples/cloud_service_monitoring/slo_gae_app_latency.yaml b/samples/cloud_service_monitoring/slo_gae_app_latency.yaml new file mode 100644 index 00000000..c949fb88 --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gae_app_latency.yaml @@ -0,0 +1,24 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-latency724ms + labels: + service_name: gae + feature_name: app + slo_name: latency724ms +spec: + description: Latency of App Engine app requests < 724ms + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: distribution_cut + exporters: [] + service_level_indicator: + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_latencies" + resource.type="gae_app" + metric.labels.response_code >= 200 + metric.labels.response_code < 500 + range_min: 0 + range_max: 724 + goal: 0.999 diff --git a/samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml b/samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml new file mode 100644 index 00000000..e648e99d --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml @@ -0,0 +1,21 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-latency724ms + labels: + service_name: gae + feature_name: app + slo_name: latency724ms +spec: + description: Latency of App Engine app requests < 724ms + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: basic + exporters: [] + service_level_indicator: + app_engine: + project_id: ${GAE_PROJECT_ID} + module_id: ${GAE_MODULE_ID} + latency: + threshold: 724 # ms + goal: 0.999 diff --git a/samples/cloud_service_monitoring/slo_gke_app_availability_basic.yaml b/samples/cloud_service_monitoring/slo_gke_app_availability_basic.yaml new file mode 100644 index 00000000..ab4b675c --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gke_app_availability_basic.yaml @@ -0,0 +1,21 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gke-service-availability + labels: + service_name: gke + feature_name: service + slo_name: availability +spec: + description: Availability of GKE service + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: basic + exporters: [] + service_level_indicator: + mesh_istio: + mesh_uid: ${GKE_MESH_UID} + service_namespace: ${GKE_SERVICE_NAMESPACE} + service_name: ${GKE_SERVICE_NAME} + availability: {} + goal: 0.98 diff --git a/samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml b/samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml new file mode 100644 index 00000000..16fc4ec0 --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml @@ -0,0 +1,23 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gke-service-availability + labels: + service_name: gke + feature_name: service + slo_name: availability +spec: + description: Availability of GKE service + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: basic + exporters: [] + service_level_indicator: + cluster_istio: + project_id: ${GKE_PROJECT_ID} + zone: ${GKE_LOCATION} + cluster_name: ${GKE_CLUSTER_NAME} + service_namespace: ${GKE_SERVICE_NAMESPACE} + service_name: ${GKE_SERVICE_NAME} + availability: {} + goal: 0.98 diff --git a/samples/cloud_service_monitoring/slo_gke_app_latency_basic.yaml b/samples/cloud_service_monitoring/slo_gke_app_latency_basic.yaml new file mode 100644 index 00000000..b3acf134 --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gke_app_latency_basic.yaml @@ -0,0 +1,22 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gke-service-latency724ms + labels: + service_name: gke + feature_name: service + slo_name: latency724ms +spec: + description: Latency of GKE service requests < 724ms + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: basic + exporters: [] + service_level_indicator: + mesh_istio: + mesh_uid: ${GKE_MESH_UID} + service_namespace: ${GKE_SERVICE_NAMESPACE} + service_name: ${GKE_SERVICE_NAME} + latency: + threshold: 724 # ms + goal: 0.999 diff --git a/samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml b/samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml new file mode 100644 index 00000000..ced3999d --- /dev/null +++ b/samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml @@ -0,0 +1,24 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gke-service-latency724ms + labels: + service_name: gke + feature_name: service + slo_name: latency724ms +spec: + description: Latency of GKE service requests < 724ms + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: basic + exporters: [] + service_level_indicator: + cluster_istio: + project_id: ${GKE_PROJECT_ID} + zone: ${GKE_LOCATION} + cluster_name: ${GKE_CLUSTER_NAME} + service_namespace: ${GKE_SERVICE_NAMESPACE} + service_name: ${GKE_SERVICE_NAME} + latency: + threshold: 724 # ms + goal: 0.999 diff --git a/samples/cloud_service_monitoring/slo_lb_request_availability.yaml b/samples/cloud_service_monitoring/slo_lb_request_availability.yaml new file mode 100644 index 00000000..6bc1df9c --- /dev/null +++ b/samples/cloud_service_monitoring/slo_lb_request_availability.yaml @@ -0,0 +1,27 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: lb-request-availability + labels: + service_name: lb + feature_name: request + slo_name: availability +spec: + description: Availability of HTTP Load Balancer + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: good_bad_ratio + exporters: [] + service_level_indicator: + filter_good: > + project=${LB_PROJECT_ID} + metric.type="loadbalancing.googleapis.com/https/request_count" + resource.type="https_lb_rule" + ( metric.label.response_code_class="200" OR + metric.label.response_code_class="300" OR + metric.label.response_code_class="400" ) + filter_valid: > + project=${LB_PROJECT_ID} + metric.type="loadbalancing.googleapis.com/https/request_count" + resource.type="https_lb_rule" + goal: 0.98 diff --git a/samples/cloud_service_monitoring/slo_lb_request_latency.yaml b/samples/cloud_service_monitoring/slo_lb_request_latency.yaml new file mode 100644 index 00000000..2b326176 --- /dev/null +++ b/samples/cloud_service_monitoring/slo_lb_request_latency.yaml @@ -0,0 +1,25 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: lb-request-latency724ms + labels: + service_name: lb + feature_name: request + slo_name: latency724ms +spec: + description: Latency of HTTP Load Balancer < 724ms + error_budget_policy: cloud_service_monitoring + backend: cloud_service_monitoring + method: distribution_cut + exporters: [] + service_level_indicator: + filter_valid: > + project=${LB_PROJECT_ID} + metric.type="loadbalancing.googleapis.com/https/total_latencies" + resource.type="https_lb_rule" + ( metric.label.response_code_class="200" OR + metric.label.response_code_class="300" OR + metric.label.response_code_class="400" ) + range_min: 0 + range_max: 724 # ms + goal: 0.98 diff --git a/samples/config.yaml b/samples/config.yaml new file mode 100644 index 00000000..ca8e7fef --- /dev/null +++ b/samples/config.yaml @@ -0,0 +1,83 @@ +--- + +backends: + cloud_monitoring: + project_id: ${STACKDRIVER_HOST_PROJECT_ID} + cloud_service_monitoring: + project_id: ${STACKDRIVER_HOST_PROJECT_ID} + custom.custom_backend.CustomBackend: {} + datadog: + api_key: ${DATADOG_API_KEY} + app_key: ${DATADOG_APP_KEY} + dynatrace: + api_url: ${DYNATRACE_API_URL} + api_token: ${DYNATRACE_API_TOKEN} + elasticsearch: + url: ${ELASTICSEARCH_URL} + prometheus: + url: ${PROMETHEUS_URL} + +exporters: + cloud_monitoring: + project_id: ${STACKDRIVER_HOST_PROJECT_ID} + cloud_monitoring/test: + project_id: ${PUBSUB_PROJECT_ID} + custom.custom_exporter.CustomMetricExporter: {} + custom.custom_exporter.CustomSLOExporter: {} + datadog: + api_key: ${DATADOG_API_KEY} + app_key: ${DATADOG_APP_KEY} + dynatrace: + api_url: ${DYNATRACE_API_URL} + api_token: ${DYNATRACE_API_TOKEN} + metric_timeseries_id: custom:slo.error_budget_burn_rate + dynatrace/test: + api_url: ${DYNATRACE_API_URL} + api_token: ${DYNATRACE_API_TOKEN} + prometheus: + url: ${PROMETHEUS_PUSHGATEWAY_URL} + pubsub: + project_id: ${PUBSUB_PROJECT_ID} + topic_name: ${PUBSUB_TOPIC_NAME} + +error_budget_policies: + default: + steps: + - name: 1 hour + burn_rate_threshold: 9 + alert: true + message_alert: Page to defend the SLO + message_ok: Last hour on track + window: 3600 + - name: 12 hours + burn_rate_threshold: 3 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 12 hours on track + window: 43200 + - name: 7 days + burn_rate_threshold: 1.5 + alert: false + message_alert: Dev team dedicates 25% of engineers to the reliability backlog + message_ok: Last week on track + window: 604800 + - name: 28 days + burn_rate_threshold: 1 + alert: false + message_alert: Freeze release, unless related to reliability or security + message_ok: Unfreeze release, per the agreed roll-out policy + window: 2419200 + cloud_service_monitoring: + steps: + - name: 24 hours + burn_rate_threshold: 4 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 24 hours on track + window: 86400 + - name: 48 hours + burn_rate_threshold: 2 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 48 hours on track + window: 172800 diff --git a/samples/custom/slo_custom_app_availability_query_sli.yaml b/samples/custom/slo_custom_app_availability_query_sli.yaml index acfb64dd..4869144d 100644 --- a/samples/custom/slo_custom_app_availability_query_sli.yaml +++ b/samples/custom/slo_custom_app_availability_query_sli.yaml @@ -1,25 +1,17 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: custom -feature_name: test -slo_description: 99.99% of fake requests to custom backends are valid -slo_name: availability-sli -slo_target: 0.999 -backend: - class: custom.custom_backend.CustomBackend - method: query_sli -exporters: -- class: custom.custom_exporter.CustomMetricExporter - class: custom.custom_exporter.CustomSLOExporter +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: custom-test-availability-sli + labels: + service_name: custom + feature_name: test + slo_name: availability-sli +spec: + description: 99.99% of fake requests to custom backends are valid + backend: custom.custom_backend.CustomBackend + method: query_sli + exporters: + - custom.custom_exporter.CustomMetricExporter + - custom.custom_exporter.CustomSLOExporter + service_level_indicator: {} + goal: 0.999 diff --git a/samples/custom/slo_custom_app_availability_ratio.yaml b/samples/custom/slo_custom_app_availability_ratio.yaml index 6ba841f5..06a9c957 100644 --- a/samples/custom/slo_custom_app_availability_ratio.yaml +++ b/samples/custom/slo_custom_app_availability_ratio.yaml @@ -1,25 +1,17 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: custom -feature_name: test -slo_description: 99.99% of fake requests to custom backends are valid -slo_name: availability-ratio -slo_target: 0.999 -backend: - class: custom.custom_backend.CustomBackend - method: good_bad_ratio -exporters: -- class: custom.custom_exporter.CustomMetricExporter - class: custom.custom_exporter.CustomSLOExporter +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: custom-test-availability-ratio + labels: + service_name: custom + feature_name: test + slo_name: availability-ratio +spec: + description: 99.99% of fake requests to custom backends are valid + backend: custom.custom_backend.CustomBackend + method: good_bad_ratio + exporters: + - custom.custom_exporter.CustomMetricExporter + - custom.custom_exporter.CustomSLOExporter + service_level_indicator: {} + goal: 0.999 diff --git a/samples/datadog/slo_dd_app_availability_query_sli.yaml b/samples/datadog/slo_dd_app_availability_query_sli.yaml index 47ad78b5..2fe9ed36 100644 --- a/samples/datadog/slo_dd_app_availability_query_sli.yaml +++ b/samples/datadog/slo_dd_app_availability_query_sli.yaml @@ -1,31 +1,18 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: dd -feature_name: app -slo_name: availability -slo_description: 99% of app requests return a valid HTTP code -slo_target: 0.99 -backend: - class: Datadog - method: query_sli - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} - # api_host: api.datadoghq.eu # uncomment to use EU site - measurement: - query: sum:app.requests.count{http.path:/, http.status_code_class:2xx}.as_count() / sum:app.requests.count{http.path:/}.as_count() -exporters: - - class: Datadog - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: dd-app-availability + labels: + service_name: dd + feature_name: app + slo_name: availability +spec: + description: 99% of app requests return a valid HTTP code + backend: datadog + method: query_sli + exporters: + - datadog + service_level_indicator: + query: sum:app.requests.count{http.path:/, http.status_code_class:2xx}.as_count() + / sum:app.requests.count{http.path:/}.as_count() + goal: 0.99 diff --git a/samples/datadog/slo_dd_app_availability_query_slo.yaml b/samples/datadog/slo_dd_app_availability_query_slo.yaml index bc1c5b19..3465a64d 100644 --- a/samples/datadog/slo_dd_app_availability_query_slo.yaml +++ b/samples/datadog/slo_dd_app_availability_query_slo.yaml @@ -1,27 +1,16 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: dd -feature_name: app -slo_name: availability -slo_description: 99% of app requests return a valid HTTP code -slo_target: 0.99 -backend: - class: Datadog - method: query_slo - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} - # api_host: api.datadoghq.eu # uncomment to use EU site - measurement: - slo_id: ${DATADOG_SLO_ID} +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: dd-app-availability + labels: + service_name: dd + feature_name: app + slo_name: availability +spec: + description: 99% of app requests return a valid HTTP code + backend: datadog + method: query_slo + exporters: [] + service_level_indicator: + slo_id: ${DATADOG_SLO_ID} + goal: 0.99 diff --git a/samples/datadog/slo_dd_app_availability_ratio.yaml b/samples/datadog/slo_dd_app_availability_ratio.yaml index e524b23d..aa8be867 100644 --- a/samples/datadog/slo_dd_app_availability_ratio.yaml +++ b/samples/datadog/slo_dd_app_availability_ratio.yaml @@ -1,32 +1,18 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: dd -feature_name: app -slo_name: availability -slo_description: 99% of app requests return a valid HTTP code -slo_target: 0.99 -backend: - class: Datadog - method: good_bad_ratio - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} - # api_host: api.datadoghq.eu # uncomment to use EU site - measurement: - query_good: app.requests.count{http.path:/, http.status_code_class:2xx} - query_valid: app.requests.count{http.path:/} -exporters: - - class: Datadog - api_key: ${DATADOG_API_KEY} - app_key: ${DATADOG_APP_KEY} +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: dd-app-availability + labels: + service_name: dd + feature_name: app + slo_name: availability +spec: + description: 99% of app requests return a valid HTTP code + backend: datadog + method: good_bad_ratio + exporters: + - datadog + service_level_indicator: + query_good: app.requests.count{http.path:/, http.status_code_class:2xx} + query_valid: app.requests.count{http.path:/} + goal: 0.99 diff --git a/samples/dynatrace/slo_dt_app_availability_ratio.yaml b/samples/dynatrace/slo_dt_app_availability_ratio.yaml index ac705550..6a678abc 100644 --- a/samples/dynatrace/slo_dt_app_availability_ratio.yaml +++ b/samples/dynatrace/slo_dt_app_availability_ratio.yaml @@ -1,36 +1,22 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: dt -feature_name: app -slo_name: availability -slo_description: 99.9% of app requests return a good HTTP code -slo_target: 0.999 -backend: - class: Dynatrace - method: good_bad_ratio - api_url: ${DYNATRACE_API_URL} - api_token: ${DYNATRACE_API_TOKEN} - measurement: +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: dt-app-availability + labels: + service_name: dt + feature_name: app + slo_name: availability +spec: + description: 99.9% of app requests return a good HTTP code + backend: dynatrace + method: good_bad_ratio + exporters: + - dynatrace + service_level_indicator: query_good: - metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) - entity_selector: type(HOST) + metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) + entity_selector: type(HOST) query_valid: - metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod))) - entity_selector: type(HOST) -exporters: -- class: Dynatrace - api_url: ${DYNATRACE_API_URL} - api_token: ${DYNATRACE_API_TOKEN} - metric_timeseries_id: custom:slo.error_budget_burn_rate + metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod))) + entity_selector: type(HOST) + goal: 0.999 diff --git a/samples/dynatrace/slo_dt_app_latency_threshold.yaml b/samples/dynatrace/slo_dt_app_latency_threshold.yaml index bd97bb3d..0eb77aec 100644 --- a/samples/dynatrace/slo_dt_app_latency_threshold.yaml +++ b/samples/dynatrace/slo_dt_app_latency_threshold.yaml @@ -1,33 +1,20 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: dt -feature_name: app -slo_name: latency -slo_description: 99.9% of app 2xx requests return within 50ms -slo_target: 0.999 -backend: - class: Dynatrace - method: threshold - api_url: ${DYNATRACE_API_URL} - api_token: ${DYNATRACE_API_TOKEN} - measurement: +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: dt-app-latency + labels: + service_name: dt + feature_name: app + slo_name: latency +spec: + description: 99.9% of app 2xx requests return within 50ms + backend: dynatrace + method: threshold + exporters: + - dynatrace/test + service_level_indicator: query_valid: - metric_selector: ext:app.request_latency:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) - entity_selector: type(HOST) - threshold: 50000 # us -exporters: -- class: Dynatrace - api_url: ${DYNATRACE_API_URL} - api_token: ${DYNATRACE_API_TOKEN} + metric_selector: ext:app.request_latency:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) + entity_selector: type(HOST) + threshold: 50000 # us + goal: 0.999 diff --git a/samples/elasticsearch/slo_elk_test_ratio.yaml b/samples/elasticsearch/slo_elk_test_ratio.yaml index 2114f902..adb834c0 100644 --- a/samples/elasticsearch/slo_elk_test_ratio.yaml +++ b/samples/elasticsearch/slo_elk_test_ratio.yaml @@ -1,41 +1,27 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: elk -feature_name: test -slo_description: > - SLO for random test data generated with the - https://github.com/oliver006/elasticsearch-test-data -slo_name: errors -slo_target: 1 -backend: - class: Elasticsearch - url: ${ELASTICSEARCH_URL} - method: good_bad_ratio - measurement: - index: test_data - date_field: last_updated - query_good: {} +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: elk-test-errors + labels: + service_name: elk + feature_name: test + slo_name: errors +spec: + description: > + SLO for random test data generated with the + https://github.com/oliver006/elasticsearch-test-data + backend: elasticsearch + method: good_bad_ratio + exporters: + - pubsub + - cloud_monitoring/test + service_level_indicator: + index: test_data + date_field: last_updated + query_good: {} query_bad: must: term: - name: JAgOZE8 + name: JAgOZE8 -exporters: -- class: Pubsub - project_id: ${PUBSUB_PROJECT_ID} - topic_name: ${PUBSUB_TOPIC_NAME} - -- class: Stackdriver - project_id: ${PUBSUB_PROJECT_ID} + goal: 1 diff --git a/samples/error_budget_policy.yaml b/samples/error_budget_policy.yaml deleted file mode 100644 index d491fc97..00000000 --- a/samples/error_budget_policy.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- error_budget_policy_step_name: 1 hour - measurement_window_seconds: 3600 - alerting_burn_rate_threshold: 9 - urgent_notification: true - overburned_consequence_message: Page to defend the SLO - achieved_consequence_message: Last hour on track - -- error_budget_policy_step_name: 12 hours - measurement_window_seconds: 43200 - alerting_burn_rate_threshold: 3 - urgent_notification: true - overburned_consequence_message: Page to defend the SLO - achieved_consequence_message: Last 12 hours on track - -- error_budget_policy_step_name: 7 days - measurement_window_seconds: 604800 - alerting_burn_rate_threshold: 1.5 - urgent_notification: false - overburned_consequence_message: Dev team dedicates 25% of engineers to the - reliability backlog - achieved_consequence_message: Last week on track - -- error_budget_policy_step_name: 28 days - measurement_window_seconds: 2419200 - alerting_burn_rate_threshold: 1 - urgent_notification: false - overburned_consequence_message: Freeze release, unless related to reliability - or security - achieved_consequence_message: Unfreeze release, per the agreed roll-out policy diff --git a/samples/prometheus/slo_prom_metrics_availability_query_sli.yaml b/samples/prometheus/slo_prom_metrics_availability_query_sli.yaml index 05076a04..7f4f6718 100644 --- a/samples/prometheus/slo_prom_metrics_availability_query_sli.yaml +++ b/samples/prometheus/slo_prom_metrics_availability_query_sli.yaml @@ -1,35 +1,20 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: prom -feature_name: metrics -slo_name: availability -slo_description: 99.9% of Prometheus requests return a good HTTP code -slo_target: 0.999 -backend: - class: Prometheus - method: query_sli - url: ${PROMETHEUS_URL} - # Basic auth example: - # headers: - # Content-Type: application/json - # Authorization: Basic b2s6cGFzcW== # username:password base64-encoded - measurement: - expression: > +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: prom-metrics-availability + labels: + service_name: prom + feature_name: metrics + slo_name: availability +spec: + description: 99.9% of Prometheus requests return a good HTTP code + backend: prometheus + method: query_sli + exporters: + - prometheus + service_level_indicator: + expression: > sum(rate(prometheus_http_requests_total{handler="/metrics", code=~"2.."}[window])) / sum(rate(prometheus_http_requests_total{handler="/metrics"}[window])) -exporters: -- class: Prometheus - url: ${PROMETHEUS_PUSHGATEWAY_URL} + goal: 0.999 diff --git a/samples/prometheus/slo_prom_metrics_availability_ratio.yaml b/samples/prometheus/slo_prom_metrics_availability_ratio.yaml index 23b5171e..7e4db0cf 100644 --- a/samples/prometheus/slo_prom_metrics_availability_ratio.yaml +++ b/samples/prometheus/slo_prom_metrics_availability_ratio.yaml @@ -1,34 +1,19 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: prom -feature_name: metrics -slo_name: availability -slo_description: 99.9% of Prometheus requests return a good HTTP code -slo_target: 0.999 -backend: - class: Prometheus - method: good_bad_ratio - url: ${PROMETHEUS_URL} - # Basic auth example: - # headers: - # Content-Type: application/json - # Authorization: Basic b2s6cGFzcW== # username:password base64-encoded - measurement: - filter_good: prometheus_http_requests_total{handler="/metrics", code=~"2.."} +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: prom-metrics-availability + labels: + service_name: prom + feature_name: metrics + slo_name: availability +spec: + description: 99.9% of Prometheus requests return a good HTTP code + backend: prometheus + method: good_bad_ratio + exporters: + - prometheus + service_level_indicator: + filter_good: prometheus_http_requests_total{handler="/metrics", code=~"2.."} filter_valid: prometheus_http_requests_total{handler="/metrics"} # filter_bad: prometheus_http_requests_total{code=~"5..", handler="/metrics"} # alternative to filter_valid field -exporters: -- class: Prometheus - url: ${PROMETHEUS_PUSHGATEWAY_URL} + goal: 0.999 diff --git a/samples/prometheus/slo_prom_metrics_latency_distribution_cut.yaml b/samples/prometheus/slo_prom_metrics_latency_distribution_cut.yaml index b2b8da92..95ea93d2 100644 --- a/samples/prometheus/slo_prom_metrics_latency_distribution_cut.yaml +++ b/samples/prometheus/slo_prom_metrics_latency_distribution_cut.yaml @@ -1,29 +1,18 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: prom -feature_name: metrics -slo_description: 99.99% of Prometheus requests return in less than 250ms -slo_name: latency -slo_target: 0.9999 -backend: - class: Prometheus - url: ${PROMETHEUS_URL} - method: distribution_cut - measurement: - expression: http_request_duration_seconds_bucket{handler="/metrics", code=~"2.."} +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: prom-metrics-latency + labels: + service_name: prom + feature_name: metrics + slo_name: latency +spec: + description: 99.99% of Prometheus requests return in less than 250ms + backend: prometheus + method: distribution_cut + exporters: + - prometheus + service_level_indicator: + expression: http_request_duration_seconds_bucket{handler="/metrics", code=~"2.."} threshold_bucket: 0.25 # in seconds, corresponds to the `le` (less than) PromQL label -exporters: -- class: Prometheus - url: ${PROMETHEUS_PUSHGATEWAY_URL} + goal: 0.9999 diff --git a/samples/prometheus/slo_prom_metrics_latency_query_sli.yaml b/samples/prometheus/slo_prom_metrics_latency_query_sli.yaml index 5b384404..35a2bd92 100644 --- a/samples/prometheus/slo_prom_metrics_latency_query_sli.yaml +++ b/samples/prometheus/slo_prom_metrics_latency_query_sli.yaml @@ -1,28 +1,19 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: prom -feature_name: metrics -slo_description: 99.99% of Prometheus requests return in less than 250ms -slo_name: latency -slo_target: 0.9999 -backend: - class: Prometheus - url: ${PROMETHEUS_URL} - method: query_sli - measurement: - expression: > +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: prom-metrics-latency + labels: + service_name: prom + feature_name: metrics + slo_name: latency +spec: + description: 99.99% of Prometheus requests return in less than 250ms + backend: prometheus + method: query_sli + exporters: + - prometheus + service_level_indicator: + expression: > increase( http_request_duration_seconds_bucket{handler="/metrics", code=~"2..",le="0.25"}[window] ) @@ -30,10 +21,4 @@ backend: increase( http_request_duration_seconds_count{handler="/metrics", code=~"2.."}[window] ) -exporters: - - class: Bigquery - project_id: rnm-shared-monitoring - dataset_id: slos - table_id: reports - - class: Stackdriver - project_id: rnm-shared-monitoring + goal: 0.9999 diff --git a/samples/stackdriver/slo_gae_app_availability.yaml b/samples/stackdriver/slo_gae_app_availability.yaml deleted file mode 100644 index 9da3ab32..00000000 --- a/samples/stackdriver/slo_gae_app_availability.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gae -feature_name: app -slo_description: Availability of App Engine app -slo_name: availability -slo_target: 0.95 -backend: - class: Stackdriver - method: good_bad_ratio - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - filter_good: > - project=${GAE_PROJECT_ID} - metric.type="appengine.googleapis.com/http/server/response_count" - resource.type="gae_app" - ( metric.labels.response_code = 429 OR - metric.labels.response_code = 200 OR - metric.labels.response_code = 201 OR - metric.labels.response_code = 202 OR - metric.labels.response_code = 203 OR - metric.labels.response_code = 204 OR - metric.labels.response_code = 205 OR - metric.labels.response_code = 206 OR - metric.labels.response_code = 207 OR - metric.labels.response_code = 208 OR - metric.labels.response_code = 226 OR - metric.labels.response_code = 304 ) - filter_valid: > - project=${GAE_PROJECT_ID} - metric.type="appengine.googleapis.com/http/server/response_count" -exporters: -- class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} diff --git a/samples/stackdriver/slo_gae_app_latency.yaml b/samples/stackdriver/slo_gae_app_latency.yaml deleted file mode 100644 index fdbcfbed..00000000 --- a/samples/stackdriver/slo_gae_app_latency.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gae -feature_name: app -slo_description: Latency of App Engine app requests < 724ms -slo_name: latency724ms -slo_target: 0.999 -backend: - class: Stackdriver - method: distribution_cut - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - filter_valid: > - project=${GAE_PROJECT_ID} - metric.type="appengine.googleapis.com/http/server/response_latencies" - resource.type="gae_app" - metric.labels.response_code >= 200 - metric.labels.response_code < 500 - good_below_threshold: true - threshold_bucket: 19 -exporters: -- class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} diff --git a/samples/stackdriver/slo_lb_request_availability.yaml b/samples/stackdriver/slo_lb_request_availability.yaml deleted file mode 100644 index 000d48c2..00000000 --- a/samples/stackdriver/slo_lb_request_availability.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: lb -feature_name: request -slo_description: Availability of HTTP Load Balancer -slo_name: availability -slo_target: 0.98 -backend: - class: Stackdriver - method: good_bad_ratio - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - filter_good: > - project=${LB_PROJECT_ID} - metric.type="loadbalancing.googleapis.com/https/request_count" - resource.type="https_lb_rule" - ( metric.label.response_code_class="200" OR - metric.label.response_code_class="300" OR - metric.label.response_code_class="400" ) - filter_valid: > - project=${LB_PROJECT_ID} - metric.type="loadbalancing.googleapis.com/https/request_count" - resource.type="https_lb_rule" -exporters: -- class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} diff --git a/samples/stackdriver/slo_lb_request_latency.yaml b/samples/stackdriver/slo_lb_request_latency.yaml deleted file mode 100644 index 1216825b..00000000 --- a/samples/stackdriver/slo_lb_request_latency.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: lb -feature_name: request -slo_description: Latency of HTTP Load Balancer < 724ms -slo_name: latency724ms -slo_target: 0.98 -backend: - class: Stackdriver - method: distribution_cut - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - filter_valid: > - project=${LB_PROJECT_ID} - metric.type="loadbalancing.googleapis.com/https/total_latencies" - resource.type="https_lb_rule" - ( metric.label.response_code_class="200" OR - metric.label.response_code_class="300" OR - metric.label.response_code_class="400" ) - good_below_threshold: true - threshold_bucket: 19 -exporters: -- class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} diff --git a/samples/stackdriver/slo_pubsub_subscription_throughput.yaml b/samples/stackdriver/slo_pubsub_subscription_throughput.yaml deleted file mode 100644 index 58ab1943..00000000 --- a/samples/stackdriver/slo_pubsub_subscription_throughput.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: pubsub -feature_name: subscription -slo_description: Throughput of Pub/Sub subscription -slo_name: throughput -slo_target: 0.95 -backend: - class: Stackdriver - project_id: "${STACKDRIVER_HOST_PROJECT_ID}" - method: good_bad_ratio - measurement: - filter_good: > - project="${PUBSUB_PROJECT_ID}" - metric.type="pubsub.googleapis.com/subscription/ack_message_count" - resource.type="pubsub_subscription" - filter_bad: > - project="${PUBSUB_PROJECT_ID}" - metric.type="pubsub.googleapis.com/subscription/num_outstanding_messages" - resource.type="pubsub_subscription" -exporters: -- class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - -- class: Pubsub - project_id: ${PUBSUB_PROJECT_ID} - topic_name: ${PUBSUB_TOPIC_NAME} diff --git a/samples/stackdriver_service_monitoring/slo_gae_app_availability.yaml b/samples/stackdriver_service_monitoring/slo_gae_app_availability.yaml deleted file mode 100644 index 8ee66be7..00000000 --- a/samples/stackdriver_service_monitoring/slo_gae_app_availability.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gae -feature_name: app -slo_description: Availability of App Engine app -slo_name: availability -slo_target: 0.95 -backend: - class: StackdriverServiceMonitoring - method: good_bad_ratio - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - filter_good: > - project=${GAE_PROJECT_ID} - metric.type="appengine.googleapis.com/http/server/response_count" - resource.type="gae_app" - ( metric.labels.response_code = 429 OR - metric.labels.response_code = 200 OR - metric.labels.response_code = 201 OR - metric.labels.response_code = 202 OR - metric.labels.response_code = 203 OR - metric.labels.response_code = 204 OR - metric.labels.response_code = 205 OR - metric.labels.response_code = 206 OR - metric.labels.response_code = 207 OR - metric.labels.response_code = 208 OR - metric.labels.response_code = 226 OR - metric.labels.response_code = 304 ) - filter_valid: > - project=${GAE_PROJECT_ID} - metric.type="appengine.googleapis.com/http/server/response_count" - resource.type="gae_app" diff --git a/samples/stackdriver_service_monitoring/slo_gae_app_availability_basic.yaml b/samples/stackdriver_service_monitoring/slo_gae_app_availability_basic.yaml deleted file mode 100644 index 2a43c4eb..00000000 --- a/samples/stackdriver_service_monitoring/slo_gae_app_availability_basic.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gae -feature_name: app -slo_description: Availability of App Engine app -slo_name: availability -slo_target: 0.98 -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - app_engine: - project_id: ${GAE_PROJECT_ID} - module_id: ${GAE_MODULE_ID} - availability: {} diff --git a/samples/stackdriver_service_monitoring/slo_gae_app_latency_basic.yaml b/samples/stackdriver_service_monitoring/slo_gae_app_latency_basic.yaml deleted file mode 100644 index 3d91dffb..00000000 --- a/samples/stackdriver_service_monitoring/slo_gae_app_latency_basic.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gae -feature_name: app -slo_description: Latency of App Engine app requests < 724ms -slo_name: latency724ms -slo_target: 0.999 -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - app_engine: - project_id: ${GAE_PROJECT_ID} - module_id: ${GAE_MODULE_ID} - latency: - threshold: 724 # ms diff --git a/samples/stackdriver_service_monitoring/slo_gke_app_availability_basic.yaml b/samples/stackdriver_service_monitoring/slo_gke_app_availability_basic.yaml deleted file mode 100644 index ad4dd6ee..00000000 --- a/samples/stackdriver_service_monitoring/slo_gke_app_availability_basic.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gke -feature_name: service -slo_description: Availability of GKE service -slo_name: availability -slo_target: 0.98 -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - mesh_istio: - mesh_uid: ${GKE_MESH_UID} - service_namespace: ${GKE_SERVICE_NAMESPACE} - service_name: ${GKE_SERVICE_NAME} - availability: {} diff --git a/samples/stackdriver_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml b/samples/stackdriver_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml deleted file mode 100644 index b4ab973f..00000000 --- a/samples/stackdriver_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gke -feature_name: service -slo_description: Availability of GKE service -slo_name: availability -slo_target: 0.98 -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - cluster_istio: - project_id: ${GKE_PROJECT_ID} - location: ${GKE_LOCATION} - cluster_name: ${GKE_CLUSTER_NAME} - service_namespace: ${GKE_SERVICE_NAMESPACE} - service_name: ${GKE_SERVICE_NAME} - availability: {} diff --git a/samples/stackdriver_service_monitoring/slo_gke_app_latency_basic.yaml b/samples/stackdriver_service_monitoring/slo_gke_app_latency_basic.yaml deleted file mode 100644 index c2de6af2..00000000 --- a/samples/stackdriver_service_monitoring/slo_gke_app_latency_basic.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gke -feature_name: service -slo_description: Latency of GKE service requests < 724ms -slo_name: latency724ms -slo_target: 0.999 -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - mesh_istio: - mesh_uid: ${GKE_MESH_UID} - service_namespace: ${GKE_SERVICE_NAMESPACE} - service_name: ${GKE_SERVICE_NAME} - latency: - threshold: 724 # ms diff --git a/samples/stackdriver_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml b/samples/stackdriver_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml deleted file mode 100644 index e7643909..00000000 --- a/samples/stackdriver_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: gke -feature_name: service -slo_description: Latency of GKE service requests < 724ms -slo_name: latency724ms -slo_target: 0.999 -backend: - class: StackdriverServiceMonitoring - method: basic - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - cluster_istio: - project_id: ${GKE_PROJECT_ID} - location: ${GKE_LOCATION} - cluster_name: ${GKE_CLUSTER_NAME} - service_namespace: ${GKE_SERVICE_NAMESPACE} - service_name: ${GKE_SERVICE_NAME} - latency: - threshold: 724 # ms diff --git a/samples/stackdriver_service_monitoring/slo_lb_request_availability.yaml b/samples/stackdriver_service_monitoring/slo_lb_request_availability.yaml deleted file mode 100644 index 428d411b..00000000 --- a/samples/stackdriver_service_monitoring/slo_lb_request_availability.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: lb -feature_name: request -slo_description: Availability of HTTP Load Balancer -slo_name: availability -slo_target: 0.98 -backend: - class: StackdriverServiceMonitoring - method: good_bad_ratio - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - filter_good: > - project=${LB_PROJECT_ID} - metric.type="loadbalancing.googleapis.com/https/request_count" - resource.type="https_lb_rule" - ( metric.label.response_code_class="200" OR - metric.label.response_code_class="300" OR - metric.label.response_code_class="400" ) - filter_valid: > - project=${LB_PROJECT_ID} - metric.type="loadbalancing.googleapis.com/https/request_count" - resource.type="https_lb_rule" diff --git a/samples/stackdriver_service_monitoring/slo_lb_request_latency.yaml b/samples/stackdriver_service_monitoring/slo_lb_request_latency.yaml deleted file mode 100644 index 3e19d8e6..00000000 --- a/samples/stackdriver_service_monitoring/slo_lb_request_latency.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -service_name: lb -feature_name: request -slo_description: Latency of HTTP Load Balancer < 724ms -slo_name: latency724ms -slo_target: 0.98 -backend: - class: StackdriverServiceMonitoring - method: distribution_cut - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - measurement: - filter_valid: > - project=${LB_PROJECT_ID} - metric.type="loadbalancing.googleapis.com/https/total_latencies" - resource.type="https_lb_rule" - ( metric.label.response_code_class="200" OR - metric.label.response_code_class="300" OR - metric.label.response_code_class="400" ) - range_min: 0 - range_max: 724 # ms diff --git a/samples/stackdriver_service_monitoring/zzz_slo_pubsub_subscription_throughput.yaml b/samples/stackdriver_service_monitoring/zzz_slo_pubsub_subscription_throughput.yaml deleted file mode 100644 index cb678d3b..00000000 --- a/samples/stackdriver_service_monitoring/zzz_slo_pubsub_subscription_throughput.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# TODO: Doesn't work at the moment because Stackdriver Service Monitoring API -# does not support Gauge-type metrics. - -# --- -# service_name: pubsub -# feature_name: subscription -# slo_description: Throughput of Pub/Sub subscription -# slo_name: throughput -# slo_target: 0.95 -# backend: -# class: StackdriverServiceMonitoring -# project_id: "${STACKDRIVER_HOST_PROJECT_ID}" -# method: good_bad_ratio -# measurement: -# filter_good: > -# project="${PUBSUB_PROJECT_ID}" -# metric.type="pubsub.googleapis.com/subscription/ack_message_count" -# resource.type="pubsub_subscription" -# filter_bad: > -# project="${PUBSUB_PROJECT_ID}" -# metric.type="pubsub.googleapis.com/subscription/num_outstanding_messages" -# resource.type="pubsub_subscription" From a596d579e4c8aecba34f1a24a73a347ca1e1b1ac Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 18:56:40 +0200 Subject: [PATCH 013/107] fix: Minor v2 fixes (#142) --- slo_generator/backends/cloud_service_monitoring.py | 6 +++--- slo_generator/migrations/migrator.py | 4 +++- tests/unit/fixtures/ci_knative_service.yaml | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 6e855241..8032a00b 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -290,11 +290,11 @@ def build_service_id(self, slo_config, dest_project_id=None, full=False): 'ClusterIstio is deprecated in the Service Monitoring API.' 'It will be removed in version 3.0, please use MeshIstio ' 'instead', FutureWarning) - if 'location' in cluster_istio: - cluster_istio['suffix'] = 'location' - elif 'zone' in cluster_istio: + if 'zone' in cluster_istio: cluster_istio['suffix'] = 'zone' cluster_istio['location'] = cluster_istio['zone'] + elif 'location' in cluster_istio: + cluster_istio['suffix'] = 'location' service_id = SID_CLUSTER_ISTIO.format_map(cluster_istio) dest_project_id = cluster_istio['project_id'] elif mesh_istio: diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index d68da1eb..08eb8bc2 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -374,14 +374,16 @@ def add_to_shared_config(new_obj, shared_config, section, quiet=False): fg='cyan', blink=True) user_input = click.prompt( - f'\n{RED}{BOLD}Please give this {name} a name:{ENDC}', + f'\n{RED}{BOLD}Please give this {name} a name{ENDC}', type=str) + former_key = key key += '/' + user_input.lower() if key in shared_obj.keys(): click.secho( f'{name.capitalize()} "{key}" already exists in shared config', fg='red', bold=True) + key = former_key else: valid = True click.secho(f'Backend {key} was added to shared config.', diff --git a/tests/unit/fixtures/ci_knative_service.yaml b/tests/unit/fixtures/ci_knative_service.yaml index 8ded03e5..8323dae8 100644 --- a/tests/unit/fixtures/ci_knative_service.yaml +++ b/tests/unit/fixtures/ci_knative_service.yaml @@ -2,6 +2,10 @@ apiVersion: serving.knative.dev/v1 kind: Service metadata: name: slo-generator + annotations: + client.knative.dev/user-image: gcr.io/${PROJECT_ID}/slo-generator:${VERSION} + serving.knative.dev/creator: tf-root@rnm-shared-devops.iam.gserviceaccount.com + serving.knative.dev/lastModifier: tf-root@rnm-shared-devops.iam.gserviceaccount.com spec: template: metadata: From d58c4bbd337e68904e55d8073d2e97100388cc9d Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 2 Jun 2021 12:00:10 +0200 Subject: [PATCH 014/107] fix: v2 deployment fixes (#143) * Add --project-id to Cloud Run deployment * Add GCR release * Fix API Cloud Scheduler * Update release yaml * Update github workflow --- .github/workflows/cloudbuild.yml | 19 -------------- .github/workflows/rc-release.yml | 45 ++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 21 ++++++++++++--- Makefile | 3 ++- slo_generator/api/main.py | 3 +-- 5 files changed, 66 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/cloudbuild.yml create mode 100644 .github/workflows/rc-release.yml diff --git a/.github/workflows/cloudbuild.yml b/.github/workflows/cloudbuild.yml deleted file mode 100644 index dee4d277..00000000 --- a/.github/workflows/cloudbuild.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: cloudbuild -on: - push: - branches: - - master - - release-* -jobs: - cloudbuild: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - with: - fetch-depth: 1 - - uses: actions-hub/gcloud@master - env: - PROJECT_ID: ${{ secrets.PROJECT_ID }} - APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} - with: - args: builds submit . --timeout="10m" diff --git a/.github/workflows/rc-release.yml b/.github/workflows/rc-release.yml new file mode 100644 index 00000000..c065a70e --- /dev/null +++ b/.github/workflows/rc-release.yml @@ -0,0 +1,45 @@ +name: rc-release-version +on: + push: + branches: + - release-v*.*.*-rc* +jobs: + release-pypi: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: "Verify version is a release candidate" + run: | + cat "setup.py" | grep "version = .*-rc*" + - uses: actions/setup-python@master + with: + python-version: '3.x' + architecture: 'x64' + + - name: Run all tests + run: make + env: + MIN_VALID_EVENTS: "10" + GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json + + - name: Release PyPI package + run: make deploy + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ + release-gcr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.PROJECT_ID }} + service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} + export_default_credentials: true + - name: Build Docker container and publish on GCR + run: make cloudbuild + env: + GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} + CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38b7d168..cfd69274 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,11 +4,11 @@ on: tags: - v*.*.* jobs: - release-version: + release-pypi: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 + - uses: actions/checkout@v2 + - uses: actions/setup-python@master with: python-version: '3.x' architecture: 'x64' @@ -25,3 +25,18 @@ jobs: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ + release-gcr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.PROJECT_ID }} + service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} + export_default_credentials: true + - name: Build Docker container and publish on GCR + run: make cloudbuild + env: + GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} + CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} diff --git a/Makefile b/Makefile index fceab9b6..cb7b0691 100644 --- a/Makefile +++ b/Makefile @@ -137,4 +137,5 @@ cloudrun: --args=api \ --args=--signature-type="${SIGNATURE_TYPE}" \ --min-instances 1 \ - --allow-unauthenticated + --allow-unauthenticated \ + --project=${CLOUDRUN_PROJECT_ID} diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py index 29d0ede7..78963115 100644 --- a/slo_generator/api/main.py +++ b/slo_generator/api/main.py @@ -51,14 +51,13 @@ def run_compute(request): # Get SLO config if API_SIGNATURE_TYPE == 'http': timestamp = None - data = str(request.get_json()) + data = str(request.data.decode('utf-8')) LOGGER.info('Loading SLO config from Flask request') elif API_SIGNATURE_TYPE == 'cloudevent': timestamp = int( datetime.strptime(request["time"], TIME_FORMAT).timestamp()) data = base64.b64decode(request.data).decode('utf-8') LOGGER.info(f'Loading SLO config from Cloud Event "{request["id"]}"') - slo_config = load_config(data) # Get slo-generator config From 686b6b9f2ef266907dc02affca8627a04bc1b777 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 2 Jun 2021 12:25:22 +0200 Subject: [PATCH 015/107] ci: Add quiet flag to alpha builds submit command (#144) --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb7b0691..201d4567 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,8 @@ cloudbuild: gcloud alpha builds submit \ --config=cloudbuild.yaml \ --project=${CLOUDBUILD_PROJECT_ID} \ - --substitutions=_GCR_PROJECT_ID=${GCR_PROJECT_ID},_VERSION=${VERSION} + --substitutions=_GCR_PROJECT_ID=${GCR_PROJECT_ID},_VERSION=${VERSION} \ + --quiet # Cloudrun cloudrun: From 84556997acb2707b0ac99c7d3901e8f0b21835b3 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 2 Jun 2021 12:30:50 +0200 Subject: [PATCH 016/107] ci: Add gcloud components install alpha cmd (#145) --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 201d4567..c7867090 100644 --- a/Makefile +++ b/Makefile @@ -118,12 +118,11 @@ docker_test: docker_build slo-generator test # Cloudbuild -cloudbuild: +cloudbuild: gcloud_alpha gcloud alpha builds submit \ --config=cloudbuild.yaml \ --project=${CLOUDBUILD_PROJECT_ID} \ - --substitutions=_GCR_PROJECT_ID=${GCR_PROJECT_ID},_VERSION=${VERSION} \ - --quiet + --substitutions=_GCR_PROJECT_ID=${GCR_PROJECT_ID},_VERSION=${VERSION} # Cloudrun cloudrun: @@ -140,3 +139,6 @@ cloudrun: --min-instances 1 \ --allow-unauthenticated \ --project=${CLOUDRUN_PROJECT_ID} + +gcloud_alpha: + gcloud components install alpha --quiet From 306a26b87f668482ebc9804a2506db193a932afd Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 2 Jun 2021 15:16:51 +0200 Subject: [PATCH 017/107] ci: Update workflow to return true (#146) --- .github/workflows/rc-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc-release.yml b/.github/workflows/rc-release.yml index c065a70e..f99fd08d 100644 --- a/.github/workflows/rc-release.yml +++ b/.github/workflows/rc-release.yml @@ -39,7 +39,7 @@ jobs: service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} export_default_credentials: true - name: Build Docker container and publish on GCR - run: make cloudbuild + run: make cloudbuild || true env: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} From 392c1ae8ab3afd0305996f5ad06f20882ca92e5f Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 2 Jun 2021 18:39:13 +0200 Subject: [PATCH 018/107] fix: Support JSON or text data in API (#147) --- slo_generator/api/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py index 78963115..61fe2604 100644 --- a/slo_generator/api/main.py +++ b/slo_generator/api/main.py @@ -51,7 +51,7 @@ def run_compute(request): # Get SLO config if API_SIGNATURE_TYPE == 'http': timestamp = None - data = str(request.data.decode('utf-8')) + data = str(request.get_data().decode('utf-8')) LOGGER.info('Loading SLO config from Flask request') elif API_SIGNATURE_TYPE == 'cloudevent': timestamp = int( From d7f9743bde4146dd2f3c699715c8443f74c39ebc Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 21 Jun 2021 17:28:17 +0200 Subject: [PATCH 019/107] fix: update migrator to fail softly when invalid YAMLs are found (#154) --- slo_generator/cli.py | 2 +- slo_generator/migrations/migrator.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/slo_generator/cli.py b/slo_generator/cli.py index ab012e23..2ec06708 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -165,7 +165,7 @@ def api(ctx, config, signature_type, target): @click.option('--glob', type=str, required=False, - default='**/slo_*.yaml', + default='**/*.yaml', help='Glob expression to seek SLO configs in subpaths') @click.option('--version', type=str, diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 08eb8bc2..f65f8b80 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -95,6 +95,8 @@ def do_migrate(source, slo_config_str = source_path.open().read() slo_config, ind, blc = yaml.util.load_yaml_guess_indent(slo_config_str) curver = get_config_version(slo_config) + if not curver: + continue # Source path info click.secho("-" * 50) @@ -114,6 +116,8 @@ def do_migrate(source, # Run vx to vy migrator method func = getattr(sys.modules[__name__], f"slo_config_{curver}to{version}") slo_config_v2 = func(slo_config, shared_config, quiet=quiet) + if not slo_config_v2: + continue # Write resulting config to target path extra = '(replaced)' if target_path_str == source_path_str else '' @@ -213,6 +217,15 @@ def slo_config_v1tov2(slo_config, shared_config={}, quiet=False, verbose=0): slo_config_v2 = OrderedDict(copy.deepcopy(SLO_CONFIG_SCHEMA)) slo_config_v2['apiVersion'] = 'sre.google.com/v2' slo_config_v2['kind'] = 'ServiceLevelObjective' + missing_keys = [ + key for key in ['service_name', 'feature_name', 'slo_name', 'backend'] + if key not in slo_config + ] + if missing_keys: + click.secho( + f'Invalid configuration: missing required key(s) {missing_keys}.', + fg='red') + return None # Get fields from old config slo_metadata_name = '{service_name}-{feature_name}-{slo_name}'.format( @@ -405,6 +418,11 @@ def get_config_version(config): Returns: str: SLO config version. """ + if not isinstance(config, dict): + click.secho( + 'Config does not correspond to any known SLO config versions.', + fg='red') + return None api_version = config.get('apiVersion', '') kind = config.get('kind', '') if not kind: # old v1 format From ae06013df3bee27cf5b13c7577370dcfc06e88f8 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Sep 2021 14:16:43 +0200 Subject: [PATCH 020/107] Trigger new build From 74a25016be92f0be819c246d9413606213689f23 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Sep 2021 14:57:19 +0200 Subject: [PATCH 021/107] fix: Migrator and dependency issues fixes (#160) * Update migrator to fail softly when invalid YAMLs are found * Add support for multiple error budget policies * Fix migrator * Fix lint * Fix dependency issue * Remove libgdal from Dockerfile --- Dockerfile | 4 +- setup.py | 16 +- .../backends/cloud_service_monitoring.py | 5 +- slo_generator/cli.py | 9 +- slo_generator/migrations/migrator.py | 190 ++++++++++++++---- slo_generator/report.py | 2 +- 6 files changed, 164 insertions(+), 62 deletions(-) diff --git a/Dockerfile b/Dockerfile index c7550d8c..43abdbf3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,7 @@ RUN apt-get update && \ build-essential \ make \ gcc \ - locales \ - libgdal20 \ - libgdal-dev + locales ADD . /app WORKDIR /app RUN pip install -U setuptools diff --git a/setup.py b/setup.py index af7b009c..497ca4b7 100644 --- a/setup.py +++ b/setup.py @@ -42,21 +42,15 @@ 'prometheus': ['prometheus-client', 'prometheus-http-client'], 'datadog': ['datadog', 'retrying==1.3.3'], 'dynatrace': ['requests'], - 'bigquery': [ - 'google-api-python-client < 2.0.0', 'google-cloud-bigquery < 3.0.0' - ], + 'bigquery': ['google-api-python-client <2', 'google-cloud-bigquery <3'], 'cloud_monitoring': [ - 'google-api-python-client < 2.0.0', 'google-cloud-monitoring < 2.0.0' + 'google-api-python-client <2', 'google-cloud-monitoring <2' ], 'cloud_service_monitoring': [ - 'google-api-python-client < 2.0.0', 'google-cloud-monitoring < 2.0.0' - ], - 'cloud_storage': [ - 'google-api-python-client < 2.0.0', 'google-cloud-storage' - ], - 'pubsub': [ - 'google-api-python-client < 2.0.0', 'google-cloud-pubsub==1.7.0' + 'google-api-python-client <2', 'google-cloud-monitoring <2' ], + 'cloud_storage': ['google-api-python-client <2', 'google-cloud-storage'], + 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], 'elasticsearch': ['elasticsearch'], 'dev': ['wheel', 'flake8', 'mock', 'coverage', 'nose', 'pylint'] } diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 8032a00b..119a952d 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -607,11 +607,12 @@ def string_diff(string1, string2): for idx, string in enumerate(difflib.ndiff(string1, string2)): if string[0] == ' ': continue + last = string[-1] if string[0] == '-': - info = u'Delete "{}" from position {}'.format(string[-1], idx) + info = f'Delete "{last}" from position {idx}' lines.append(info) elif string[0] == '+': - info = u'Add "{}" to position {}'.format(string[-1], idx) + info = f'Add "{last}" to position {idx}' lines.append(info) return lines diff --git a/slo_generator/cli.py b/slo_generator/cli.py index 2ec06708..fd36832f 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -160,8 +160,15 @@ def api(ctx, config, signature_type, target): '-b', type=click.Path(exists=True, resolve_path=True, readable=True), required=False, - default='error_budget_policy.yaml', + multiple=True, + default=['error_budget_policy.yaml'], help='Error budget policy path') +@click.option('--exporters-path', + '-e', + type=click.Path(exists=True, resolve_path=True, readable=True), + required=False, + multiple=True, + help='Exporters path') @click.option('--glob', type=str, required=False, diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index f65f8b80..848225a0 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -27,7 +27,7 @@ from pathlib import Path import click -import ruamel.yaml as yaml +from ruamel import yaml from slo_generator import utils from slo_generator.constants import (METRIC_LABELS_COMPAT, @@ -44,6 +44,7 @@ def do_migrate(source, target, error_budget_policy_path, + exporters_path, glob, version, quiet=False, @@ -53,37 +54,58 @@ def do_migrate(source, Args: source (str): Source SLO configs folder. target (str): Target SLO configs folder. - error_budget_policy_path (str): Error budget policy path. + error_budget_policy_path (list): Error budget policy paths. + exporters_path (list): Exporters paths. glob (str): Glob expression to add to source path. version (str): slo-generator major version string (e.g: v1, v2, ...) quiet (bool, optional): If true, do not prompt for user input. verbose (int, optional): Verbose level. """ + curver = 'v1' shared_config = CONFIG_SCHEMA cwd = Path.cwd() source = Path(source).resolve() target = Path(target).resolve() source_str = source.relative_to(cwd) # human-readable path target_str = target.relative_to(cwd) # human-readable path - error_budget_policy_path = Path(error_budget_policy_path) + ebp_paths = [Path(ebp) for ebp in error_budget_policy_path] + exporters_paths = [Path(exp) for exp in exporters_path] # Create target folder if it doesn't exist target.mkdir(parents=True, exist_ok=True) + # Translate error budget policy to v2 and put into shared config + if ebp_paths: + ebp_func = getattr(sys.modules[__name__], f"ebp_{curver}to{version}") + ebp_func( + ebp_paths, + shared_config=shared_config, + quiet=quiet, + ) + + # Translate exporters to v2 and put into shared config + if exporters_paths: + exporters_func = getattr(sys.modules[__name__], + f"exporters_{curver}to{version}") + exp_keys = exporters_func( + exporters_paths, + shared_config=shared_config, + quiet=quiet, + ) + # Process SLO configs click.secho('=' * 50) click.secho(f"Migrating slo-generator configs to {version} ...", fg='cyan', bold=True) - - paths = Path(source).glob(glob) - + paths = Path(source).glob(glob) # find all files in source path if not peek(paths): click.secho(f"{FAIL} No SLO configs found in {source}", fg='red', bold=True) sys.exit(1) + curver = '' for source_path in paths: source_path_str = source_path.relative_to(cwd) if source == target == cwd: @@ -94,7 +116,7 @@ def do_migrate(source, target_path_str = target_path.relative_to(cwd) slo_config_str = source_path.open().read() slo_config, ind, blc = yaml.util.load_yaml_guess_indent(slo_config_str) - curver = get_config_version(slo_config) + curver = detect_config_version(slo_config) if not curver: continue @@ -114,8 +136,14 @@ def do_migrate(source, target_path.parent.mkdir(parents=True, exist_ok=True) # Run vx to vy migrator method - func = getattr(sys.modules[__name__], f"slo_config_{curver}to{version}") - slo_config_v2 = func(slo_config, shared_config, quiet=quiet) + slo_func = getattr(sys.modules[__name__], + f"slo_config_{curver}to{version}") + slo_config_v2 = slo_func( + slo_config, + shared_config=shared_config, + shared_exporters=exp_keys, + quiet=quiet, + ) if not slo_config_v2: continue @@ -133,28 +161,10 @@ def do_migrate(source, ) click.secho(f'{SUCCESS} Success !', fg='green', bold=True) - # Translate error budget policy to v2 and put into shared config - error_budget_policy = yaml.load(open(error_budget_policy_path), - Loader=yaml.Loader) - for step in error_budget_policy: - step['name'] = step.pop('error_budget_policy_step_name') - step['burn_rate_threshold'] = step.pop('alerting_burn_rate_threshold') - step['alert'] = step.pop('urgent_notification') - step['message_alert'] = step.pop('overburned_consequence_message') - step['message_ok'] = step.pop('achieved_consequence_message') - step['window'] = step.pop('measurement_window_seconds') - - ebp = {'steps': error_budget_policy} - if error_budget_policy_path.name == 'error_budget_policy.yaml': - ebp_key = 'default' - else: - ebp_key = error_budget_policy_path.name - shared_config['error_budget_policies'][ebp_key] = ebp - shared_config_path = target / 'config.yaml' - shared_config_path_str = shared_config_path.relative_to(cwd) - # Write shared config to file click.secho('=' * 50) + shared_config_path = target / 'config.yaml' + shared_config_path_str = shared_config_path.relative_to(cwd) with shared_config_path.open('w') as conf: click.secho( f'Writing slo-generator config to {shared_config_path_str} ...', @@ -171,14 +181,15 @@ def do_migrate(source, click.secho(f'{SUCCESS} Success !', fg='green', bold=True) # Remove error budget policy file - click.secho('=' * 50) - click.secho(f'Removing {error_budget_policy_path} ...', - fg='cyan', - bold=True) - error_budget_policy_path.unlink() - click.secho(f'{SUCCESS} Success !', fg='green', bold=True) + # click.secho('=' * 50) + # click.secho(f'Removing {error_budget_policy_path} ...', + # fg='cyan', + # bold=True) + # error_budget_policy_path.unlink() + # click.secho(f'{SUCCESS} Success !', fg='green', bold=True) # Print next steps + relative_ebp_path = ebp_paths[0].relative_to(cwd) click.secho('=' * 50) click.secho( f'\n{SUCCESS} Migration of `slo-generator` configs to v2 completed successfully ! Configs path: {target_str}/.\n', @@ -192,21 +203,101 @@ def do_migrate(source, click.secho(f""" 1 - Commit the updated SLO configs and your shared SLO config to version control. 2 - [local/k8s/cloudbuild] Update your slo-generator command: - {RED} [-] slo-generator -f {source_str} -b {error_budget_policy_path}{ENDC} + {RED} [-] slo-generator -f {source_str} -b {relative_ebp_path}{ENDC} {GREEN} [+] slo-generator -f {target_str} -c {target_str}/config.yaml{ENDC} - 3 - [terraform] Upgrade your `terraform-google-slo` modules: - 3.1 - Upgrade the module `version` to 2.0.0. - 3.2 - Replace `error_budget_policy` field in your `slo` and `slo-pipeline` modules by `shared_config` - 3.3 - Replace `error_budget_policy.yaml` local variable to `config.yaml` """) + # 3 - [terraform] Upgrade your `terraform-google-slo` modules: + # 3.1 - Upgrade the module `version` to 2.0.0. + # 3.2 - Replace `error_budget_policy` field in your `slo` and `slo-pipeline` modules by `shared_config` + # 3.3 - Replace `error_budget_policy.yaml` local variable to `config.yaml` -def slo_config_v1tov2(slo_config, shared_config={}, quiet=False, verbose=0): +def exporters_v1tov2(exporters_paths, shared_config={}, quiet=False): + """Translate exporters to v2 and put into shared config. + + Args: + exporters_path (list): List of exporters file paths. + shared_config (dict): Shared config to add exporters to. + quiet (bool): Quiet mode. + + Returns: + list: List of exporters keys added to shared config. + """ + exp_keys = [] + for exp_path in exporters_paths: + with open(exp_path, encoding='utf-8') as conf: + content = yaml.load(conf, Loader=yaml.Loader) + exporters = content + + # If exporters file has sections, concatenate all of them + if isinstance(content, dict): + exporters = [] + for _, value in content.items(): + exporters.extend(value) + print(exporters) + + # If exporter not in general config, add it and add an alias for the + # exporter. Refer to the alias in the SLO config file. + for exporter in exporters: + print(exporter) + exporter = OrderedDict(exporter) + exp_key = add_to_shared_config(exporter, + shared_config, + 'exporters', + quiet=quiet) + exp_keys.append(exp_key) + return exp_keys + + +def ebp_v1tov2(ebp_paths, shared_config={}, quiet=False): + """Translate error budget policies to v2 and put into shared config + + Args: + ebp_paths (list): List of error budget policies file paths. + shared_config (dict): Shared config to add exporters to. + quiet (bool): Quiet mode. + + Returns: + list: List of error budget policies keys added to shared config. + """ + ebp_keys = [] + for ebp_path in ebp_paths: + with open(ebp_path, encoding='utf-8') as conf: + error_budget_policy = yaml.load(conf, Loader=yaml.Loader) + for step in error_budget_policy: + step['name'] = step.pop('error_budget_policy_step_name') + step['burn_rate_threshold'] = step.pop( + 'alerting_burn_rate_threshold') + step['alert'] = step.pop('urgent_notification') + step['message_alert'] = step.pop('overburned_consequence_message') + step['message_ok'] = step.pop('achieved_consequence_message') + step['window'] = step.pop('measurement_window_seconds') + + ebp = {'steps': error_budget_policy} + if ebp_path.name == 'error_budget_policy.yaml': + ebp_key = 'default' + else: + ebp_key = ebp_path.stem.replace('error_budget_policy_', '') + ebp_key = add_to_shared_config(ebp, + shared_config, + 'error_budget_policies', + ebp_key, + quiet=quiet) + ebp_keys.append(ebp_key) + return ebp_keys + + +def slo_config_v1tov2(slo_config, + shared_config={}, + shared_exporters=[], + quiet=False, + verbose=0): """Process old SLO config v1 and generate SLO config v2. Args: slo_config (dict): SLO Config v1. shared_config (dict): SLO Generator config. + shared_exporters (list): Shared exporters keys to add to SLO configs. quiet (bool): If true, do not ask for user input. verbose (int): Verbose level. @@ -273,6 +364,10 @@ def slo_config_v1tov2(slo_config, shared_config={}, quiet=False, verbose=0): quiet=quiet) slo_config_v2['spec']['exporters'].append(exp_key) + # Add shared exporters to slo config + for exp_key in shared_exporters: + slo_config_v2['spec']['exporters'].append(exp_key) + # Fill spec slo_config_v2['spec']['service_level_indicator'] = service_level_indicator @@ -341,7 +436,11 @@ def get_random_suffix(): return ''.join(random.choices(string.digits, k=4)) -def add_to_shared_config(new_obj, shared_config, section, quiet=False): +def add_to_shared_config(new_obj, + shared_config, + section, + key=None, + quiet=False): """Add an object to the shared_config. If the object with the same config already exists in the shared config, @@ -355,13 +454,16 @@ def add_to_shared_config(new_obj, shared_config, section, quiet=False): new_obj (OrderedDict): Object to add to shared_config. shared_config (dict): Shared config to add object to. section (str): Section name in shared config to add the object under. + key (str): Key if cannot be infered. quiet (bool): If True, do not ask for user input. Returns: str: Object key in the shared config. """ shared_obj = shared_config[section] - key = new_obj.pop('class') + key = key or new_obj.pop('class', None) + if not key: + raise ValueError("Object key is undefined.") if '.' not in key: key = utils.caml_to_snake(PROVIDERS_COMPAT.get(key, key)) @@ -409,7 +511,7 @@ def add_to_shared_config(new_obj, shared_config, section, quiet=False): return key -def get_config_version(config): +def detect_config_version(config): """Return version of an slo-generator config based on the format. Args: diff --git a/slo_generator/report.py b/slo_generator/report.py index 620ea4a4..9033b1e2 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -365,7 +365,7 @@ def __set_fields(self, lambdas={}, **kwargs): if name not in kwargs: continue value = kwargs[name] - if name in lambdas.keys(): + if name in lambdas: value = lambdas[name](value) setattr(self, name, value) From 2d3115d9b19b5d5a583c964c0490646d3546edff Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Sep 2021 15:11:45 +0200 Subject: [PATCH 022/107] ci: Remove release workflows (#163) --- .github/workflows/rc-release.yml | 45 -------------------------------- .github/workflows/release.yml | 36 ++++++++++++++++++++++++- cloudbuild.yaml | 2 +- 3 files changed, 36 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/rc-release.yml diff --git a/.github/workflows/rc-release.yml b/.github/workflows/rc-release.yml deleted file mode 100644 index f99fd08d..00000000 --- a/.github/workflows/rc-release.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: rc-release-version -on: - push: - branches: - - release-v*.*.*-rc* -jobs: - release-pypi: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: "Verify version is a release candidate" - run: | - cat "setup.py" | grep "version = .*-rc*" - - uses: actions/setup-python@master - with: - python-version: '3.x' - architecture: 'x64' - - - name: Run all tests - run: make - env: - MIN_VALID_EVENTS: "10" - GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json - - - name: Release PyPI package - run: make deploy - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ - release-gcr: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master - with: - project_id: ${{ secrets.PROJECT_ID }} - service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} - export_default_credentials: true - - name: Build Docker container and publish on GCR - run: make cloudbuild || true - env: - GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} - CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cfd69274..fe99afbf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@master with: python-version: '3.x' @@ -25,18 +26,51 @@ jobs: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ + release-gcr: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@master with: project_id: ${{ secrets.PROJECT_ID }} service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} export_default_credentials: true - - name: Build Docker container and publish on GCR + + - name: Check release version + id: check-tag + run: | + echo ::set-output name=version::$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-) + if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=match::true + fi + + - name: Build Docker container and publish on GCR [v*.*.*] + run: make cloudbuild + env: + GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} + CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} + VERSION: ${{ steps.check-tag.outputs.version }} + + - name: Build Docker container and publish on GCR [latest] run: make cloudbuild + if: ${{ steps.check-tag.outputs.match == 'true' }} + env: + GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} + CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} + VERSION: latest + + - name: Deploy Docker container to Cloud Run + run: make cloudrun + if: ${{ steps.check-tag.outputs.match == 'true' }} env: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} + CLOUDRUN_PROJECT_ID: ${{ secrets.CLOUDRUN_PROJECT_ID }} + VERSION: ${{ steps.check-tag.outputs.version }} + CONFIG_URL: gs://${{ secrets.CLOUDRUN_PROJECT_ID }}/config.yaml + SIGNATURE_TYPE: http + REGION: ${{ secrets.REGION }} + SERVICE_ACCOUNT: ${{ secrets.CLOUDRUN_SERVICE_ACCOUNT }} diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 9d70a771..230d7e7f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -18,7 +18,7 @@ steps: id: Build SLO generator args: ['build', '-t', 'gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION}', '.'] -- name: gcr.io/$_GCR_PROJECT_ID/slo-generator +- name: gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION} id: Run all tests entrypoint: make env: From ee3cac5f4e734a430400e737cdb0702ad299a7e3 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Sep 2021 16:00:30 +0200 Subject: [PATCH 023/107] ci: Add || true to make cloudbuild cmd (#164) --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe99afbf..ccfc6e11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,14 +48,14 @@ jobs: fi - name: Build Docker container and publish on GCR [v*.*.*] - run: make cloudbuild + run: make cloudbuild || true env: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} VERSION: ${{ steps.check-tag.outputs.version }} - name: Build Docker container and publish on GCR [latest] - run: make cloudbuild + run: make cloudbuild || true if: ${{ steps.check-tag.outputs.match == 'true' }} env: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} From d72cfc247024936b4580b3d945658ebf5e6f9d1e Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Sep 2021 17:42:20 +0200 Subject: [PATCH 024/107] docs: Add Cloudrun docs (#165) * Add CloudRun documentation * Update CloudRun doc * Update CloudBuild / CloudFunctions docs * Update docs, fix migrator output --- docs/deploy/cloudbuild.md | 32 +++---- docs/deploy/cloudfunctions.md | 2 +- docs/deploy/cloudrun.md | 61 ++++++++++++++ docs/shared/migration.md | 119 +++++++++++++++++++++++++-- slo_generator/migrations/migrator.py | 2 - 5 files changed, 192 insertions(+), 24 deletions(-) create mode 100644 docs/deploy/cloudrun.md diff --git a/docs/deploy/cloudbuild.md b/docs/deploy/cloudbuild.md index 490c3f92..53cce786 100644 --- a/docs/deploy/cloudbuild.md +++ b/docs/deploy/cloudbuild.md @@ -4,38 +4,38 @@ To do so, you need to build an image for the `slo-generator` and push it to `Google Container Registry` in your project. -To build and push the image, run: +## [Optional] Build and push the image to GCR + +If you are not allowed to use the public container image, you can build and push +the image to your project using CloudBuild: ```sh git clone https://github.com/google/slo-generator cd slo-generator/ -gcloud config set project -gcloud builds submit --tag gcr.io//slo-generator . +export CLOUDBUILD_PROJECT_ID= +export GCR_PROJECT_ID= +make cloudbuild ``` -Once the image is built, you can call the SLO generator using the following snippet in your `cloudbuild.yaml`: +## Run `slo-generator` as a build step + +Once the image is built, you can call the SLO generator using the following +snippet in your `cloudbuild.yaml`: ```yaml --- steps: -- name: gcr.io/${_PROJECT_NAME}/slo-generator +- name: gcr.io/slo-generator-ci-a2b4/slo-generator + command: slo-generator args: - -f - - ${_SLO_CONFIG_FILE} - - -b - - ${_ERROR_BUDGET_POLICY_FILE} + - slo.yaml + - -c + - config.yaml - --export ``` -Then, in another repo containing your SLO definitions, simply run the pipeline, substituting the needed variables: - -```sh -gcloud builds submit . --config=cloudbuild.yaml --substitutions \ - _SLO_CONFIG_FILE= \ - _ERROR_BUDGET_POLICY_FILE=<_ERROR_BUDGET_POLICY_FILE> \ -``` - If your repo is a Cloud Source Repository, you can also configure a trigger for Cloud Build, so that the pipeline is run automatically when a commit is made. diff --git a/docs/deploy/cloudfunctions.md b/docs/deploy/cloudfunctions.md index 72c032c4..8980a2f6 100644 --- a/docs/deploy/cloudfunctions.md +++ b/docs/deploy/cloudfunctions.md @@ -1,4 +1,4 @@ -# Deploy SLO Generator in Cloud Functions +# Deploy SLO Generator in Cloud Functions [DEPRECATED] `slo-generator` is frequently used as part of an SLO Reporting Pipeline made of: diff --git a/docs/deploy/cloudrun.md b/docs/deploy/cloudrun.md new file mode 100644 index 00000000..59a1c887 --- /dev/null +++ b/docs/deploy/cloudrun.md @@ -0,0 +1,61 @@ +# Deploy SLO Generator as a Cloud Run service + +`slo-generator` can also be deployed as a Cloud Run service by following the +instructions below. + +## Setup a Cloud Storage bucket + +Create the GCS bucket that will hold our SLO configurations: + +``` +gsutil mb -p ${PROJECT_ID} gs://${BUCKET_NAME} +``` + +Upload the slo-generator configuration to the GCS bucket: + +``` +gsutil cp config.yaml gs://${BUCKET_NAME}/ +``` + +See sample [config.yaml](../samples/config.yaml) + +## Deploy the CloudRun service + +``` +gcloud run deploy slo-generator \ + --image gcr.io/slo-generator-ci-a2b4/slo-generator:2.0.0-rc3 \ + --region=europe-west1 \ + --project=${PROJECT_ID} \ + --set-env-vars CONFIG_PATH=gs://${BUCKET_NAME}/config.yaml \ + --platform managed \ + --command="slo-generator" \ + --args=api \ + --args=--signature-type=http \ + --min-instances 1 \ + --allow-unauthenticated +``` + +Once the deployment is finished, get the service URL from the log output. + +## [Optional] Test an SLO +``` +curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml ${SERVICE_URL} # yaml +curl -X POST -H "Content-Type: application/json" -d @${SLO_PATH} ${SERVICE_URL} # json +``` + +See sample [slo.yaml](../samples/cloud_monitoring/slo_gae_app_availability.yaml) + +## Schedule SLO reports every minute + +Upload your SLO config to the GCS bucket: +``` +gsutil cp slo.yaml gs://${BUCKET_NAME}/ +``` + +Create a Cloud Scheduler job that will hit the service with the SLO config URL: +``` +gcloud scheduler jobs create http slo --schedule=”* * * * */1” \ + --uri=${SERVICE_URL} \ + --message-body=”gs://${BUCKET_NAME}/slo.yaml” + --project=${PROJECT_ID} +``` diff --git a/docs/shared/migration.md b/docs/shared/migration.md index 65f9ef72..75cd84ec 100644 --- a/docs/shared/migration.md +++ b/docs/shared/migration.md @@ -13,21 +13,130 @@ instructions: pip3 install slo-generator -U # upgrades slo-generator version to the latest version ``` -**Run the `slo-generator-migrate` command:** +**Run the `slo-generator migrate` command:** ``` -slo-generator-migrate -s -t -b +slo-generator migrate -s -t -b -e ``` where: -* is the source folder containg SLO configurations in v1 format. +* `` is the source folder containg SLO configurations in v1 format. This folder can have nested subfolders containing SLOs. The subfolder structure will be reproduced on the target folder. -* is the target folder to drop the SLO configurations in v2 +* `` is the target folder to drop the SLO configurations in v2 format. If the target folder is identical to the source folder, the existing SLO configurations will be updated in-place. -* is the path to your error budget policy configuration. +* `` is the path to your error budget policy configuration. You can add more by specifying another `-b ` + +* `` (OPTIONAL) is the path to your exporters configurations. + **Follow the instructions printed to finish the migration:** + This includes committing the resulting files to git and updating your Terraform modules to the version that supports the v2 configuration format. + +## Example + +Example bulk migration of [slo-repository](https://github.com/ocervell/slo-repository) SLOs: + +``` +$ slo-generator migrate -s slos/ -t slos/ -b slos/error_budget_policy.yaml -b slos/error_budget_policy_ssm.yaml -e slos/exporters.yaml + +Migrating slo-generator configs to v2 ... +Config does not correspond to any known SLO config versions. +-------------------------------------------------- +slos/exporters.yaml [v1] +Invalid configuration: missing required key(s) ['service_name', 'feature_name', 'slo_name', 'backend']. +Config does not correspond to any known SLO config versions. +-------------------------------------------------- +slos/platform-slos/slo_pubsub_coverage.yaml [v1] +➞ slos/platform-slos/slo_pubsub_coverage.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/custom-example/slo_test_custom.yaml [v1] +➞ slos/custom-example/slo_test_custom.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/flask-app-prometheus/slo_flask_latency_query_sli.yaml [v1] +➞ slos/flask-app-prometheus/slo_flask_latency_query_sli.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/flask-app-prometheus/slo_flask_availability_ratio.yaml [v1] +➞ slos/flask-app-prometheus/slo_flask_availability_ratio.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/flask-app-prometheus/slo_flask_availability_query_sli.yaml [v1] +➞ slos/flask-app-prometheus/slo_flask_availability_query_sli.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/flask-app-prometheus/slo_flask_latency_distribution_cut.yaml [v1] +➞ slos/flask-app-prometheus/slo_flask_latency_distribution_cut.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/flask-app-datadog/slo_dd_app_availability_query_sli.yaml [v1] +➞ slos/flask-app-datadog/slo_dd_app_availability_query_sli.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/flask-app-datadog/slo_dd_app_availability_query_slo.yaml [v1] +➞ slos/flask-app-datadog/slo_dd_app_availability_query_slo.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/flask-app-datadog/slo_dd_app_availability_ratio.yaml [v1] +➞ slos/flask-app-datadog/slo_dd_app_availability_ratio.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/slo-generator/slo_bq_latency.yaml [v1] +➞ slos/slo-generator/slo_bq_latency.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/slo-generator/slo_pubsub_coverage.yaml [v1] +➞ slos/slo-generator/slo_pubsub_coverage.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/slo-generator/slo_gcf_throughput.yaml [v1] +➞ slos/slo-generator/slo_gcf_throughput.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/slo-generator/slo_gcf_latency.yaml [v1] +➞ slos/slo-generator/slo_gcf_latency.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/slo-generator/slo_gcf_latency pipeline.yaml [v1] +➞ slos/slo-generator/slo_gcf_latency pipeline.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/slo-generator/slo_gcf_errors.yaml [v1] +➞ slos/slo-generator/slo_gcf_errors.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/online-boutique/slo_ob_adservice_availability.yaml [v1] +➞ slos/online-boutique/slo_ob_adservice_availability.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/online-boutique/slo_ob_adservice_latency.yaml [v1] +➞ slos/online-boutique/slo_ob_adservice_latency.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/online-boutique/slo_ob_all_latency_distribution_cut.yaml [v1] +➞ slos/online-boutique/slo_ob_all_latency_distribution_cut.yaml [v2] (replaced) +✅ Success ! +-------------------------------------------------- +slos/online-boutique/slo_ob_all_availability_basic.yaml [v1] +➞ slos/online-boutique/slo_ob_all_availability_basic.yaml [v2] (replaced) +✅ Success ! +================================================== +Writing slo-generator config to slos/config.yaml ... +✅ Success ! +================================================== + +✅ Migration of `slo-generator` configs to v2 completed successfully ! Configs path: slos/. + +================================================== +PLEASE FOLLOW THE MANUAL STEPS BELOW TO FINISH YOUR MIGRATION: + + 1 - Commit the updated SLO configs and your shared SLO config to version control. + 2 - [local/k8s/cloudbuild] Update your slo-generator command: + [-] slo-generator -f slos -b slos/error_budget_policy.yaml + [+] slo-generator -f slos -c slos/config.yaml +``` diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 848225a0..0de578b8 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -234,12 +234,10 @@ def exporters_v1tov2(exporters_paths, shared_config={}, quiet=False): exporters = [] for _, value in content.items(): exporters.extend(value) - print(exporters) # If exporter not in general config, add it and add an alias for the # exporter. Refer to the alias in the SLO config file. for exporter in exporters: - print(exporter) exporter = OrderedDict(exporter) exp_key = add_to_shared_config(exporter, shared_config, From 2fb3f4882665da490f887a213f6e60e80f0a30a6 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 10 Sep 2021 10:30:58 +0200 Subject: [PATCH 025/107] Update cloudrun.md --- docs/deploy/cloudrun.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy/cloudrun.md b/docs/deploy/cloudrun.md index 59a1c887..0aaffe32 100644 --- a/docs/deploy/cloudrun.md +++ b/docs/deploy/cloudrun.md @@ -43,7 +43,7 @@ curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml ${SERVICE_UR curl -X POST -H "Content-Type: application/json" -d @${SLO_PATH} ${SERVICE_URL} # json ``` -See sample [slo.yaml](../samples/cloud_monitoring/slo_gae_app_availability.yaml) +See sample [slo.yaml](../../samples/cloud_monitoring/slo_gae_app_availability.yaml) ## Schedule SLO reports every minute From 9f0c2bc61e1f92500f4ef69cf32de59c42bee560 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 10 Sep 2021 10:31:16 +0200 Subject: [PATCH 026/107] Update cloudrun.md --- docs/deploy/cloudrun.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy/cloudrun.md b/docs/deploy/cloudrun.md index 0aaffe32..6780a8d7 100644 --- a/docs/deploy/cloudrun.md +++ b/docs/deploy/cloudrun.md @@ -17,7 +17,7 @@ Upload the slo-generator configuration to the GCS bucket: gsutil cp config.yaml gs://${BUCKET_NAME}/ ``` -See sample [config.yaml](../samples/config.yaml) +See sample [config.yaml](../../samples/config.yaml) ## Deploy the CloudRun service From 52ffd7c964c0258fbae89abefda55828702e334f Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 14 Sep 2021 10:57:51 +0200 Subject: [PATCH 027/107] fix: migrator glob catch .yml, .yaml and .json ext --- slo_generator/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo_generator/cli.py b/slo_generator/cli.py index fd36832f..7c7f9d89 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -172,7 +172,7 @@ def api(ctx, config, signature_type, target): @click.option('--glob', type=str, required=False, - default='**/*.yaml', + default='**/*.*(yml|yaml|json)', help='Glob expression to seek SLO configs in subpaths') @click.option('--version', type=str, From d5d89a91f464b4ae950af387c3e8cd7a0c977028 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 21 Sep 2021 13:36:49 +0200 Subject: [PATCH 028/107] Fix typo in converted config --- samples/dynatrace/slo_dt_app_latency_threshold.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/dynatrace/slo_dt_app_latency_threshold.yaml b/samples/dynatrace/slo_dt_app_latency_threshold.yaml index 0eb77aec..d12eda9a 100644 --- a/samples/dynatrace/slo_dt_app_latency_threshold.yaml +++ b/samples/dynatrace/slo_dt_app_latency_threshold.yaml @@ -11,7 +11,7 @@ spec: backend: dynatrace method: threshold exporters: - - dynatrace/test + - dynatrace service_level_indicator: query_valid: metric_selector: ext:app.request_latency:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) From a89498476493aff2e2a44cde95f3a534e5e33e02 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 11:11:55 +0200 Subject: [PATCH 029/107] style: fix lint after pylint upgrade (#170) * Fix linting after upgrading to pylint2.11 * Fix encoding in open call --- slo_generator/backends/cloud_service_monitoring.py | 4 ++-- slo_generator/backends/dynatrace.py | 4 ++-- slo_generator/exporters/bigquery.py | 5 ++--- slo_generator/migrations/migrator.py | 10 +++++----- slo_generator/utils.py | 6 ++++-- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 119a952d..2c431864 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -452,7 +452,7 @@ def build_slo(window, slo_config): # pylint: disable=R0912,R0915 return slo def get_slo(self, window, slo_config): - """Get SLO object from Cloud Service Monitoring API. + """Get SLO object from Cloud Service Monssitoring API. Args: service_id (str): Service identifier. @@ -666,7 +666,7 @@ def convert_duration_to_string(duration): if duration_seconds.is_integer(): duration_str = int(duration_seconds) else: - duration_str = "{:0.3f}".format(duration_seconds) + duration_str = f'{duration_seconds:0.3f}' return str(duration_str) + 's' @staticmethod diff --git a/slo_generator/backends/dynatrace.py b/slo_generator/backends/dynatrace.py index 59090a62..aea9f46d 100644 --- a/slo_generator/backends/dynatrace.py +++ b/slo_generator/backends/dynatrace.py @@ -254,8 +254,8 @@ def request(self, } if name: url += f'/{name}' - params_str = "&".join( - "%s=%s" % (k, v) for k, v in params.items() if v is not None) + params_str = '&'.join( + f'{key}={val}' for key, val in params.items() if val is not None) url += f'?{params_str}' LOGGER.debug(f'Running "{method}" request to {url} ...') if method in ['put', 'post']: diff --git a/slo_generator/exporters/bigquery.py b/slo_generator/exporters/bigquery.py index dc22868a..c532c29c 100644 --- a/slo_generator/exporters/bigquery.py +++ b/slo_generator/exporters/bigquery.py @@ -68,9 +68,8 @@ def export(self, data, **config): dataset_id, table_id, schema=TABLE_SCHEMA) - row_ids = "%s%s%s%s%s" % (data['service_name'], data['feature_name'], - data['slo_name'], data['timestamp_human'], - data['window']) + row_ids_fmt = '{service_name}{feature_name}{slo_name}{timestamp_human}{window}' # pylint: disable=line-too-long # noqa: E501 + row_ids = row_ids_fmt.format(**data) # Format user metadata if needed json_data = {k: v for k, v in data.items() if k in schema_fields} diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 0de578b8..00949a5e 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -151,7 +151,7 @@ def do_migrate(source, extra = '(replaced)' if target_path_str == source_path_str else '' click.secho( f"{RIGHT_ARROW} {GREEN}{target_path_str}{ENDC} [{version}] {extra}") - with target_path.open('w') as conf: + with target_path.open('w', encoding='utf8') as conf: yaml.round_trip_dump( slo_config_v2, conf, @@ -165,7 +165,7 @@ def do_migrate(source, click.secho('=' * 50) shared_config_path = target / 'config.yaml' shared_config_path_str = shared_config_path.relative_to(cwd) - with shared_config_path.open('w') as conf: + with shared_config_path.open('w', encoding='utf8') as conf: click.secho( f'Writing slo-generator config to {shared_config_path_str} ...', fg='cyan', @@ -317,8 +317,8 @@ def slo_config_v1tov2(slo_config, return None # Get fields from old config - slo_metadata_name = '{service_name}-{feature_name}-{slo_name}'.format( - **slo_config) + slo_metadata_name_fmt = '{service_name}-{feature_name}-{slo_name}' + slo_metadata_name = slo_metadata_name_fmt.format(**slo_config) slo_description = slo_config.pop('slo_description') slo_target = slo_config.pop('slo_target') service_level_indicator = slo_config['backend'].pop('measurement', {}) @@ -420,7 +420,7 @@ def report_v2tov1(report): # If a key in the default label mapping is passed, use the default # label mapping - elif key in METRIC_LABELS_COMPAT.keys(): + elif key in METRIC_LABELS_COMPAT: mapped_report.update({METRIC_LABELS_COMPAT[key]: value}) # Otherwise, write the label as is diff --git a/slo_generator/utils.py b/slo_generator/utils.py index bbb2e7d0..9e61e4df 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -138,7 +138,7 @@ def replace_env_vars(content, ctx): return content if path: - with Path(path).open() as config: + with Path(path).open(encoding='utf8') as config: content = config.read() if ctx: content = replace_env_vars(content, ctx) @@ -198,7 +198,9 @@ def get_human_time(timestamp, timezone=None): dt_tz = dt_utc.replace(tzinfo=to_zone) timeformat = '%Y-%m-%dT%H:%M:%S.%f%z' date_str = datetime.strftime(dt_tz, timeformat) - date_str = "{0}:{1}".format(date_str[:-2], date_str[-2:]) + core_str = date_str[:-2] + tz_str = date_str[-2:] + date_str = f'{core_str}:{tz_str}' return date_str From 2d0e03d8b48ade303b766a2a83e6fd648075476b Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 11:19:23 +0200 Subject: [PATCH 030/107] feat: add slo-generator JSON schemas (#169) --- tests/unit/fixtures/schemas/config.json | 440 ++++++++++++++++++++++++ tests/unit/fixtures/schemas/slo.json | 103 ++++++ 2 files changed, 543 insertions(+) create mode 100644 tests/unit/fixtures/schemas/config.json create mode 100644 tests/unit/fixtures/schemas/slo.json diff --git a/tests/unit/fixtures/schemas/config.json b/tests/unit/fixtures/schemas/config.json new file mode 100644 index 00000000..c466955d --- /dev/null +++ b/tests/unit/fixtures/schemas/config.json @@ -0,0 +1,440 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "backends": { + "type": "object", + "properties": { + "cloud_monitoring": { + "type": "object", + "properties": { + "project_id": { + "type": "string" + } + }, + "required": [ + "project_id" + ] + }, + "cloud_service_monitoring": { + "type": "object", + "properties": { + "project_id": { + "type": "string" + } + }, + "required": [ + "project_id" + ] + }, + "custom.custom_backend.CustomBackend": { + "type": "object" + }, + "datadog": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "app_key": { + "type": "string" + } + }, + "required": [ + "api_key", + "app_key" + ] + }, + "dynatrace": { + "type": "object", + "properties": { + "api_url": { + "type": "string" + }, + "api_token": { + "type": "string" + } + }, + "required": [ + "api_url", + "api_token" + ] + }, + "elasticsearch": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ] + }, + "prometheus": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ] + } + }, + "required": [ + "cloud_monitoring", + "cloud_service_monitoring", + "custom.custom_backend.CustomBackend", + "datadog", + "dynatrace", + "elasticsearch", + "prometheus" + ] + }, + "exporters": { + "type": "object", + "properties": { + "cloud_monitoring": { + "type": "object", + "properties": { + "project_id": { + "type": "string" + } + }, + "required": [ + "project_id" + ] + }, + "cloud_monitoring/test": { + "type": "object", + "properties": { + "project_id": { + "type": "string" + } + }, + "required": [ + "project_id" + ] + }, + "custom.custom_exporter.CustomMetricExporter": { + "type": "object" + }, + "custom.custom_exporter.CustomSLOExporter": { + "type": "object" + }, + "datadog": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "app_key": { + "type": "string" + } + }, + "required": [ + "api_key", + "app_key" + ] + }, + "dynatrace": { + "type": "object", + "properties": { + "api_url": { + "type": "string" + }, + "api_token": { + "type": "string" + }, + "metric_timeseries_id": { + "type": "string" + } + }, + "required": [ + "api_url", + "api_token", + "metric_timeseries_id" + ] + }, + "dynatrace/test": { + "type": "object", + "properties": { + "api_url": { + "type": "string" + }, + "api_token": { + "type": "string" + } + }, + "required": [ + "api_url", + "api_token" + ] + }, + "prometheus": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ] + }, + "pubsub": { + "type": "object", + "properties": { + "project_id": { + "type": "string" + }, + "topic_name": { + "type": "string" + } + }, + "required": [ + "project_id", + "topic_name" + ] + } + }, + "required": [ + "cloud_monitoring", + "cloud_monitoring/test", + "custom.custom_exporter.CustomMetricExporter", + "custom.custom_exporter.CustomSLOExporter", + "datadog", + "dynatrace", + "dynatrace/test", + "prometheus", + "pubsub" + ] + }, + "error_budget_policies": { + "type": "object", + "properties": { + "default": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": [{ + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "burn_rate_threshold": { + "type": "integer" + }, + "alert": { + "type": "boolean" + }, + "message_alert": { + "type": "string" + }, + "message_ok": { + "type": "string" + }, + "window": { + "type": "integer" + } + }, + "required": [ + "name", + "burn_rate_threshold", + "alert", + "message_alert", + "message_ok", + "window" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "burn_rate_threshold": { + "type": "integer" + }, + "alert": { + "type": "boolean" + }, + "message_alert": { + "type": "string" + }, + "message_ok": { + "type": "string" + }, + "window": { + "type": "integer" + } + }, + "required": [ + "name", + "burn_rate_threshold", + "alert", + "message_alert", + "message_ok", + "window" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "burn_rate_threshold": { + "type": "number" + }, + "alert": { + "type": "boolean" + }, + "message_alert": { + "type": "string" + }, + "message_ok": { + "type": "string" + }, + "window": { + "type": "integer" + } + }, + "required": [ + "name", + "burn_rate_threshold", + "alert", + "message_alert", + "message_ok", + "window" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "burn_rate_threshold": { + "type": "integer" + }, + "alert": { + "type": "boolean" + }, + "message_alert": { + "type": "string" + }, + "message_ok": { + "type": "string" + }, + "window": { + "type": "integer" + } + }, + "required": [ + "name", + "burn_rate_threshold", + "alert", + "message_alert", + "message_ok", + "window" + ] + } + ] + } + }, + "required": [ + "steps" + ] + }, + "cloud_service_monitoring": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": [{ + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "burn_rate_threshold": { + "type": "integer" + }, + "alert": { + "type": "boolean" + }, + "message_alert": { + "type": "string" + }, + "message_ok": { + "type": "string" + }, + "window": { + "type": "integer" + } + }, + "required": [ + "name", + "burn_rate_threshold", + "alert", + "message_alert", + "message_ok", + "window" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "burn_rate_threshold": { + "type": "integer" + }, + "alert": { + "type": "boolean" + }, + "message_alert": { + "type": "string" + }, + "message_ok": { + "type": "string" + }, + "window": { + "type": "integer" + } + }, + "required": [ + "name", + "burn_rate_threshold", + "alert", + "message_alert", + "message_ok", + "window" + ] + } + ] + } + }, + "required": [ + "steps" + ] + } + }, + "required": [ + "default", + "cloud_service_monitoring" + ] + } + }, + "required": [ + "backends", + "exporters", + "error_budget_policies" + ] +} diff --git a/tests/unit/fixtures/schemas/slo.json b/tests/unit/fixtures/schemas/slo.json new file mode 100644 index 00000000..cab12735 --- /dev/null +++ b/tests/unit/fixtures/schemas/slo.json @@ -0,0 +1,103 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/ServiceLevelObjective", + "definitions": { + "ServiceLevelObjective": { + "type": "object", + "additionalProperties": false, + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "$ref": "#/definitions/Metadata" + }, + "spec": { + "$ref": "#/definitions/Spec" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata", + "spec" + ], + "title": "ServiceLevelObjective" + }, + "Metadata": { + "type": "object", + "additionalProperties": true, + "properties": { + "name": { + "type": "string" + }, + "labels": { + "$ref": "#/definitions/Labels" + } + }, + "required": [ + "name" + ], + "title": "Metadata" + }, + "Labels": { + "type": "object", + "additionalProperties": false, + "properties": {}, + "required": [], + "title": "Labels" + }, + "Spec": { + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "backend": { + "type": "string" + }, + "method": { + "type": "string" + }, + "exporters": { + "type": "array", + "items": { + "type": "string" + } + }, + "service_level_indicator": { + "$ref": "#/definitions/ServiceLevelIndicator" + }, + "goal": { + "type": "number" + } + }, + "required": [ + "backend", + "description", + "goal", + "method", + "service_level_indicator" + ], + "title": "Spec" + }, + "ServiceLevelIndicator": { + "type": "object", + "additionalProperties": false, + "properties": { + "filter_good": { + "type": "string" + }, + "filter_valid": { + "type": "string" + } + }, + "required": [], + "title": "ServiceLevelIndicator" + } + } +} From b61f51e570fd307d8949030630e7aa224d0c8b12 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 11:19:39 +0200 Subject: [PATCH 031/107] fix: migrator not listing all files (#167) --- slo_generator/cli.py | 5 ----- slo_generator/migrations/migrator.py | 9 +++++---- slo_generator/utils.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/slo_generator/cli.py b/slo_generator/cli.py index 7c7f9d89..e65dc474 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -169,11 +169,6 @@ def api(ctx, config, signature_type, target): required=False, multiple=True, help='Exporters path') -@click.option('--glob', - type=str, - required=False, - default='**/*.*(yml|yaml|json)', - help='Glob expression to seek SLO configs in subpaths') @click.option('--version', type=str, required=False, diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 00949a5e..cf11fdf7 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -45,7 +45,6 @@ def do_migrate(source, target, error_budget_policy_path, exporters_path, - glob, version, quiet=False, verbose=0): @@ -98,8 +97,8 @@ def do_migrate(source, click.secho(f"Migrating slo-generator configs to {version} ...", fg='cyan', bold=True) - paths = Path(source).glob(glob) # find all files in source path - if not peek(paths): + paths = utils.get_files(source) + if not paths: click.secho(f"{FAIL} No SLO configs found in {source}", fg='red', bold=True) @@ -107,6 +106,8 @@ def do_migrate(source, curver = '' for source_path in paths: + if source_path in ebp_paths + exporters_paths: + continue source_path_str = source_path.relative_to(cwd) if source == target == cwd: target_path = target.joinpath(*source_path.relative_to(cwd).parts) @@ -312,7 +313,7 @@ def slo_config_v1tov2(slo_config, ] if missing_keys: click.secho( - f'Invalid configuration: missing required key(s) {missing_keys}.', + f'Invalid SLO configuration: missing key(s) {missing_keys}.', fg='red') return None diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 9e61e4df..edaac8a7 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -463,3 +463,18 @@ def decode_gcs_url(url): bucket_name = split_url[2] file_path = '/'.join(split_url[3:]) return (bucket_name, file_path) + + +def get_files(source, extensions=['yaml', 'yml', 'json']): + """Get all files matching extensions. + + Args: + extensions (list): List of extensions to match. + + Returns: + list: List of all files matching extensions relative to source folder. + """ + all_files = [] + for ext in extensions: + all_files.extend(Path(source).rglob(f'*.{ext}')) + return all_files From 4888f84b470be21dc7e0dbda2605819fd6044878 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 13:34:07 +0200 Subject: [PATCH 032/107] fix: migrator target path locations (#171) --- slo_generator/migrations/migrator.py | 13 ++++++------- slo_generator/utils.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index cf11fdf7..7634c04e 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -109,12 +109,11 @@ def do_migrate(source, if source_path in ebp_paths + exporters_paths: continue source_path_str = source_path.relative_to(cwd) - if source == target == cwd: - target_path = target.joinpath(*source_path.relative_to(cwd).parts) - else: - target_path = target.joinpath( - *source_path.relative_to(cwd).parts[1:]) - target_path_str = target_path.relative_to(cwd) + target_path = utils.get_target_path(source, + target, + source_path, + mkdir=True) + target_path_str = target_path.resolve().relative_to(cwd) slo_config_str = source_path.open().read() slo_config, ind, blc = yaml.util.load_yaml_guess_indent(slo_config_str) curver = detect_config_version(slo_config) @@ -142,7 +141,7 @@ def do_migrate(source, slo_config_v2 = slo_func( slo_config, shared_config=shared_config, - shared_exporters=exp_keys, + shared_exporters=exp_keys if exporters_paths else [], quiet=quiet, ) if not slo_config_v2: diff --git a/slo_generator/utils.py b/slo_generator/utils.py index edaac8a7..40263ce9 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -478,3 +478,27 @@ def get_files(source, extensions=['yaml', 'yml', 'json']): for ext in extensions: all_files.extend(Path(source).rglob(f'*.{ext}')) return all_files + + +def get_target_path(source_dir, target_dir, relative_path, mkdir=True): + """Get target file path from a source directory, a target directory and a + path relative to the source directory. + + Args: + source_dir (Path): path to source directory. + target_dir (pathlib.Path): path to target directory. + relative_path (pathlib.Path): path relative to source directory. + mkdir (bool): Create directory structure for target path. + + Returns: + pathlib.Path: path to target file. + """ + source_dir = source_dir.resolve() + target_dir = target_dir.resolve() + relative_path = relative_path.relative_to(source_dir) + common_path = os.path.commonpath([source_dir, target_dir]) + target_path = common_path / target_dir.relative_to( + common_path) / relative_path + if mkdir: + target_path.parent.mkdir(parents=True, exist_ok=True) + return target_path From d5393749e751341a7544f334190df367ad924e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment?= <41067234+clement-pruvot@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:34:30 +0200 Subject: [PATCH 033/107] feat: add Dynatrace method to query SLO (#116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Pruvot Co-authored-by: Olivier Cervello --- .../slo_dt_app_availability_slo.yaml | 17 ++++++ slo_generator/backends/dynatrace.py | 52 ++++++++++++++++++- tests/unit/fixtures/dt_slo_get.json | 21 ++++++++ tests/unit/test_stubs.py | 6 ++- 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 samples/dynatrace/slo_dt_app_availability_slo.yaml create mode 100644 tests/unit/fixtures/dt_slo_get.json diff --git a/samples/dynatrace/slo_dt_app_availability_slo.yaml b/samples/dynatrace/slo_dt_app_availability_slo.yaml new file mode 100644 index 00000000..0a0d6c37 --- /dev/null +++ b/samples/dynatrace/slo_dt_app_availability_slo.yaml @@ -0,0 +1,17 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: dt-app-availability + labels: + service_name: dt + feature_name: app + slo_name: availability +spec: + description: 99.9% of app requests return a good HTTP code + backend: dynatrace + method: query_sli + exporters: + - dynatrace + service_level_indicator: + slo_id: ${DYNATRACE_SLO_ID} + goal: 0.999 diff --git a/slo_generator/backends/dynatrace.py b/slo_generator/backends/dynatrace.py index aea9f46d..3d288aa2 100644 --- a/slo_generator/backends/dynatrace.py +++ b/slo_generator/backends/dynatrace.py @@ -40,6 +40,27 @@ def __init__(self, client=None, api_url=None, api_token=None): if client is None: self.client = DynatraceClient(api_url, api_token) + def query_sli(self, timestamp, window, slo_config): + """Query SLI value from a given Dynatrace SLO. + + Args: + timestamp (int): UNIX timestamp. + window (int): Window (in seconds). + slo_config (dict): SLO configuration. + + Returns: + float: SLI value. + """ + conf = slo_config['backend'] + start = (timestamp - window) * 1000 + end = timestamp * 1000 + measurement = conf['measurement'] + slo_id = measurement['slo_id'] + data = self.retrieve_slo(start, end, slo_id) + LOGGER.debug(f"Result SLO: {pprint.pformat(data)}") + sli_value = round(data['evaluatedPercentage'] / 100, 4) + return sli_value + def good_bad_ratio(self, timestamp, window, slo_config): """Query SLI value from good and valid queries. @@ -125,6 +146,30 @@ def query(self, version='v2', **params) + def retrieve_slo(self, + start, + end, + slo_id): + """Query Dynatrace SLO V2. + + Args: + start (int): Start timestamp (in milliseconds). + end (int): End timestamp (in milliseconds). + slo_id (int): SLO ID. + + Returns: + dict: Dynatrace API response. + """ + params = { + 'from': start, + 'to': end + } + endpoint = 'slo/' + slo_id + return self.client.request('get', + endpoint, + version='v2', + **params) + @staticmethod def count(response): """Count events in time series data. @@ -199,7 +244,11 @@ def retry_http(response): bool: True to retry, False otherwise. """ retry_codes = [429] - code = int(response.get('error', {}).get('code', 200)) + returned_code = response.get('error', {}) + if isinstance(returned_code, str): + code = 200 + else: + code = int(returned_code.get('code', 200)) return code in retry_codes @@ -262,6 +311,7 @@ def request(self, response = req(url, headers=headers, json=post_data) else: response = req(url, headers=headers) + LOGGER.debug(f'Response: {response}') data = DynatraceClient.to_json(response) next_page_key = data.get('nextPageKey') if next_page_key: diff --git a/tests/unit/fixtures/dt_slo_get.json b/tests/unit/fixtures/dt_slo_get.json new file mode 100644 index 00000000..d888c53a --- /dev/null +++ b/tests/unit/fixtures/dt_slo_get.json @@ -0,0 +1,21 @@ +{ + "id": "XXXXXX", + "enabled": true, + "name": "XXXXXX", + "description": "Service Successes Server Rate > 99.9%", + "evaluatedPercentage": 94.57628498405988, + "errorBudget": -5.323715015940124, + "status": "FAILURE", + "error": "NONE", + "useRateMetric": true, + "metricRate": "builtin:service.successes.server.rate", + "metricNumerator": "", + "numeratorValue": 0, + "metricDenominator": "", + "denominatorValue": 0, + "target": 99.9, + "warning": 99.95, + "evaluationType": "AGGREGATE", + "timeframe": "-30m", + "filter": "type(\"sERVICE\"),tag(\"XXXXXX\")" +} \ No newline at end of file diff --git a/tests/unit/test_stubs.py b/tests/unit/test_stubs.py index 5b43d72b..77e74953 100644 --- a/tests/unit/test_stubs.py +++ b/tests/unit/test_stubs.py @@ -54,7 +54,8 @@ 'DATADOG_APP_KEY': 'fake', 'DATADOG_SLO_ID': 'fake', 'DYNATRACE_API_URL': 'fake', - 'DYNATRACE_API_TOKEN': 'fake' + 'DYNATRACE_API_TOKEN': 'fake', + 'DYNATRACE_SLO_ID': 'fake' } @@ -234,6 +235,9 @@ def mock_dt(*args, **kwargs): elif args[0] == 'get' and args[1] == 'metrics/query': return load_fixture('dt_timeseries_get.json') + elif args[0] == 'get' and args[1].startswith('slo/'): + return load_fixture('dt_slo_get.json') + elif args[0] == 'post' and args[1] == 'entity/infrastructure/custom': return load_fixture('dt_metric_send.json') From 3b71339535eedabfc3872f77e0c7b4fa97cda397 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 13:39:37 +0200 Subject: [PATCH 034/107] ci: add cloudbuild, rename build to test and deploy to Cloud Run (#168) --- .github/workflows/build.yml | 68 +++++++++++++--------------- .github/workflows/deploy.yml | 65 ++++++++++++++++++++++++++ .github/workflows/release-please.yml | 2 + .github/workflows/release.yml | 55 ++-------------------- .github/workflows/test.yml | 51 +++++++++++++++++++++ Makefile | 4 +- 6 files changed, 156 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index feef9d8f..eed8d37f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,52 +1,48 @@ name: build + on: push: branches: - master - pull_request: + - deploy-* + tags: + - v*.*.* + pull_request: jobs: - lint: + cloudbuild: runs-on: ubuntu-latest + environment: prod + concurrency: prod steps: - uses: actions/checkout@master - uses: actions/setup-python@v2 - with: - python-version: '3.x' - architecture: 'x64' - - name: Install dependencies - run: make install + - name: Check release version + id: check-tag + run: | + echo ::set-output name=version::$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-) + if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=match::true + fi - - name: Run lint test - run: make lint - - unit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master with: - python-version: '3.x' - architecture: 'x64' + project_id: ${{ secrets.PROJECT_ID }} + service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} + export_default_credentials: true - - name: Install dependencies - run: make install - - - name: Run unittests - run: make unit + - name: Build Docker container and publish on GCR + run: make cloudbuild || true env: - MIN_VALID_EVENTS: "10" - GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json - - - name: Run coverage report - run: make coverage + GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} + CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} + VERSION: ${{ steps.check-tag.outputs.match == 'true' && steps.check-tag.outputs.version || github.event.pull_request.head.sha || github.sha }} - docker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - uses: docker-practice/actions-setup-docker@master - - name: Build Docker image - run: make docker_build - - name: Run Docker tests - run: make docker_test + - name: Build Docker container and publish on GCR [latest] + run: make cloudbuild || true + if: ${{ steps.check-tag.outputs.match == 'true' }} + env: + GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} + CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} + VERSION: latest diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..7699f928 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,65 @@ +name: deploy + +on: + push: + branches: + - master + - deploy-* + tags: + - v*.*.* + +jobs: + cloudrun: + runs-on: ubuntu-latest + environment: prod + concurrency: prod + steps: + - uses: actions/checkout@master + - name: Check release version + id: check-tag + run: | + echo ::set-output name=version::$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-) + if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=match::true + fi + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.PROJECT_ID }} + service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} + export_default_credentials: true + + - name: Wait for backend container build workflow + uses: tomchv/wait-my-workflow@v1.1.0 + id: wait-build + with: + token: ${{ secrets.GITHUB_TOKEN }} + checkName: build + ref: ${{ github.event.pull_request.head.sha || github.sha }} + intervalSeconds: 10 + timeoutSeconds: 600 # 10m + + - name: Do something if build isn't launch + if: steps.wait-build.outputs.conclusion == 'does not exist' || steps.wait-build2.outputs.conclusion == 'does not exist' + run: echo job does not exist && true + + - name: Do something if build fail + if: steps.wait-build.outputs.conclusion == 'failure' || steps.wait-build2.outputs.conclusion == 'failure' + run: echo fail && false # fail if build fail + + - name: Do something if build timeout + if: steps.wait-build.outputs.conclusion == 'timed_out' || steps.wait-build2.outputs.conclusion == 'timed_out' + run: echo Timeout && false # fail if build time out + + - name: Deploy Docker container to Cloud Run + run: make cloudrun + env: + GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} + CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} + CLOUDRUN_PROJECT_ID: ${{ secrets.CLOUDRUN_PROJECT_ID }} + VERSION: ${{ steps.check-tag.outputs.match && steps.check-tag.outputs.version || github.sha }} + CONFIG_URL: gs://${{ secrets.CLOUDRUN_PROJECT_ID }}/config.yaml + SIGNATURE_TYPE: http + REGION: ${{ secrets.REGION }} + SERVICE_ACCOUNT: ${{ secrets.CLOUDRUN_SERVICE_ACCOUNT }} diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index b0826797..1479d0b4 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -1,8 +1,10 @@ name: release-please + on: push: branches: - master + jobs: release-pr: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ccfc6e11..b82acfd6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,10 @@ -name: release-version +name: release + on: push: tags: - v*.*.* + jobs: release-pypi: runs-on: ubuntu-latest @@ -16,9 +18,6 @@ jobs: - name: Run all tests run: make - env: - MIN_VALID_EVENTS: "10" - GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json - name: Release PyPI package run: make deploy @@ -26,51 +25,3 @@ jobs: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ - - release-gcr: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master - with: - project_id: ${{ secrets.PROJECT_ID }} - service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} - export_default_credentials: true - - - name: Check release version - id: check-tag - run: | - echo ::set-output name=version::$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-) - if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true - fi - - - name: Build Docker container and publish on GCR [v*.*.*] - run: make cloudbuild || true - env: - GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} - CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} - VERSION: ${{ steps.check-tag.outputs.version }} - - - name: Build Docker container and publish on GCR [latest] - run: make cloudbuild || true - if: ${{ steps.check-tag.outputs.match == 'true' }} - env: - GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} - CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} - VERSION: latest - - - name: Deploy Docker container to Cloud Run - run: make cloudrun - if: ${{ steps.check-tag.outputs.match == 'true' }} - env: - GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} - CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} - CLOUDRUN_PROJECT_ID: ${{ secrets.CLOUDRUN_PROJECT_ID }} - VERSION: ${{ steps.check-tag.outputs.version }} - CONFIG_URL: gs://${{ secrets.CLOUDRUN_PROJECT_ID }}/config.yaml - SIGNATURE_TYPE: http - REGION: ${{ secrets.REGION }} - SERVICE_ACCOUNT: ${{ secrets.CLOUDRUN_SERVICE_ACCOUNT }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..80eae57d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,51 @@ +name: test + +on: + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + architecture: 'x64' + + - name: Install dependencies + run: make install + + - name: Run lint test + run: make lint + + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + architecture: 'x64' + + - name: Install dependencies + run: make install + + - name: Run unittests + run: make unit + env: + MIN_VALID_EVENTS: "10" + GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json + + - name: Run coverage report + run: make coverage + + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: docker-practice/actions-setup-docker@master + - name: Build Docker image + run: make docker_build + - name: Run Docker tests + run: make docker_test diff --git a/Makefile b/Makefile index c7867090..adf48ffa 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,10 @@ TWINE=twine COVERAGE=coverage NOSE_OPTS = --with-coverage --cover-package=$(NAME) --cover-erase --nologcapture --logging-level=ERROR SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") +MIN_VALID_EVENTS ?= 10 +GOOGLE_APPLICATION_CREDENTIALS ?= tests/unit/fixtures/fake_credentials.json -VERSION := $(shell grep "version = " setup.py | cut -d\ -f3) +VERSION ?= $(shell grep "version = " setup.py | cut -d\ -f3) FLAKE8_IGNORE = E302,E203,E261 From 6c0cecf61f2fdcd6d5f60a763c7d5cb416c2f7d2 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 13:43:26 +0200 Subject: [PATCH 035/107] ci: fix container deploy workflow --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7699f928..fc1425e4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,12 +30,12 @@ jobs: service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} export_default_credentials: true - - name: Wait for backend container build workflow + - name: Wait for container image build uses: tomchv/wait-my-workflow@v1.1.0 id: wait-build with: token: ${{ secrets.GITHUB_TOKEN }} - checkName: build + checkName: cloudbuild ref: ${{ github.event.pull_request.head.sha || github.sha }} intervalSeconds: 10 timeoutSeconds: 600 # 10m From 06fc711b8b02f654fca499aab9d5ea3189a0e1cd Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 13:45:01 +0200 Subject: [PATCH 036/107] Trigger build From bf4c376fe261d5c25c33a0f1e2f28937cbab5e67 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 13:54:15 +0200 Subject: [PATCH 037/107] docs: add pointers to v1 docs and v1 to v2 migration --- README.md | 2 ++ docs/shared/migration.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c0c78bed..a7353b43 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ `slo-generator` is a tool to compute and export **Service Level Objectives** ([SLOs](https://landing.google.com/sre/sre-book/chapters/service-level-objectives/)), **Error Budgets** and **Burn Rates**, using configurations written in YAML (or JSON) format. +***IMPORTANT NOTE: the following content is the `slo-generator` v2 documentation. The v1 documentation is available [here](https://github.com/google/slo-generator/tree/v1.5.1), and instructions to migrate to v2 are available [here](https://github.com/google/slo-generator/blob/master/docs/shared/migration.md).*** + ## Table of contents - [Description](#description) - [Local usage](#local-usage) diff --git a/docs/shared/migration.md b/docs/shared/migration.md index 75cd84ec..f09122f2 100644 --- a/docs/shared/migration.md +++ b/docs/shared/migration.md @@ -5,7 +5,7 @@ Version `v2` of the slo-generator introduces some changes to the structure of the SLO configurations. -To migrate your SLO configurations from v1 to v3, please execute the following +To migrate your SLO configurations from v1 to v2, please execute the following instructions: **Upgrade `slo-generator`:** From b88ea7fc1e91a6067234039105657c044f9dd44a Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 14:08:21 +0200 Subject: [PATCH 038/107] ci: fix release workflow --- .github/workflows/release.yml | 3 +++ Makefile | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b82acfd6..150c0cfd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,9 @@ jobs: - name: Run all tests run: make + env: + MIN_VALID_EVENTS: "10" + GOOGLE_APPLICATION_CREDENTIALS: tests/unit/fixtures/fake_credentials.json - name: Release PyPI package run: make deploy diff --git a/Makefile b/Makefile index adf48ffa..a8c36570 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,6 @@ TWINE=twine COVERAGE=coverage NOSE_OPTS = --with-coverage --cover-package=$(NAME) --cover-erase --nologcapture --logging-level=ERROR SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") -MIN_VALID_EVENTS ?= 10 -GOOGLE_APPLICATION_CREDENTIALS ?= tests/unit/fixtures/fake_credentials.json VERSION ?= $(shell grep "version = " setup.py | cut -d\ -f3) From e762a6b4e840c3c766ee5ed78321f6952af901f9 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 14:12:36 +0200 Subject: [PATCH 039/107] ci: fix deploy workflow --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fc1425e4..81fab7d1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -58,7 +58,7 @@ jobs: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} CLOUDRUN_PROJECT_ID: ${{ secrets.CLOUDRUN_PROJECT_ID }} - VERSION: ${{ steps.check-tag.outputs.match && steps.check-tag.outputs.version || github.sha }} + VERSION: ${{ steps.check-tag.outputs.match == 'true' && steps.check-tag.outputs.version || github.sha }} CONFIG_URL: gs://${{ secrets.CLOUDRUN_PROJECT_ID }}/config.yaml SIGNATURE_TYPE: http REGION: ${{ secrets.REGION }} From badbf792cc206a9aeb0004f3468a55477c1efab8 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 14:19:08 +0200 Subject: [PATCH 040/107] Remove concurrency for build --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eed8d37f..6d2747cb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,6 @@ jobs: cloudbuild: runs-on: ubuntu-latest environment: prod - concurrency: prod steps: - uses: actions/checkout@master - uses: actions/setup-python@v2 From a5e00832ea07bdce127d34e66aca4eb1bf103685 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 14:44:01 +0200 Subject: [PATCH 041/107] ci: delete sample error budget policy ssm --- samples/error_budget_policy_ssm.yaml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 samples/error_budget_policy_ssm.yaml diff --git a/samples/error_budget_policy_ssm.yaml b/samples/error_budget_policy_ssm.yaml deleted file mode 100644 index 9abeef1b..00000000 --- a/samples/error_budget_policy_ssm.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- error_budget_policy_step_name: 24 hours - measurement_window_seconds: 86400 - alerting_burn_rate_threshold: 4 - urgent_notification: true - overburned_consequence_message: Page to defend the SLO - achieved_consequence_message: Last 24 hours on track - -- error_budget_policy_step_name: 48 hours - measurement_window_seconds: 172800 - alerting_burn_rate_threshold: 2 - urgent_notification: true - overburned_consequence_message: Page to defend the SLO - achieved_consequence_message: Last 48 hours on track From cbf7c0ffb985a25daf03babd276d6e527625537a Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Tue, 28 Sep 2021 15:03:41 +0200 Subject: [PATCH 042/107] chore: release 2.0.0 (#139) --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ade52e19..767c6be6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,44 @@ # Changelog +## [2.0.0](https://www.github.com/google/slo-generator/compare/v1.5.1...v2.0.0) (2021-09-28) + + +### ⚠ BREAKING CHANGES + +* Upgrade slo-generator CLI to Click library (#131) +* Support slo-generator config v2 format (core changes) (#126) +* Split dependencies by backend (#129) + +### Features + +* add Dynatrace method to query SLO ([#116](https://www.github.com/google/slo-generator/issues/116)) ([0148e99](https://www.github.com/google/slo-generator/commit/0148e99c5830081e59db6321767bef3c84bddad4)) +* Add migrator for v1 to v2 migration ([#127](https://www.github.com/google/slo-generator/issues/127)) ([796442e](https://www.github.com/google/slo-generator/commit/796442e92e35d2ceeecd12635a6e1a057791427b)) +* Add slo-generator Functions Framework API ([#130](https://www.github.com/google/slo-generator/issues/130)) ([ab1d57c](https://www.github.com/google/slo-generator/commit/ab1d57c8a1b4ad1c7183e03f1cd98db136306ef2)) +* add slo-generator JSON schemas ([#169](https://www.github.com/google/slo-generator/issues/169)) ([33e461b](https://www.github.com/google/slo-generator/commit/33e461b7aa34ce533f1e50b1f2c8ff1048f613f2)) +* Split dependencies by backend ([#129](https://www.github.com/google/slo-generator/issues/129)) ([c640a1d](https://www.github.com/google/slo-generator/commit/c640a1d9235c9cf24243beabc5609efecbcc9d62)) +* Support slo-generator config v2 format (core changes) ([#126](https://www.github.com/google/slo-generator/issues/126)) ([bf5e6b4](https://www.github.com/google/slo-generator/commit/bf5e6b4167a7081f03ca373c11e06be70da66fd5)) +* Upgrade slo-generator CLI to Click library ([#131](https://www.github.com/google/slo-generator/issues/131)) ([5b2635b](https://www.github.com/google/slo-generator/commit/5b2635b05e6d7434f54eb95fb4d3445d88ce29f0)) + + +### Bug Fixes + +* Migrate sample configurations to v2 ([#128](https://www.github.com/google/slo-generator/issues/128)) ([bafaf51](https://www.github.com/google/slo-generator/commit/bafaf5178b827ece5da7d204e8dc982916a1ad5f)) +* Migrator and dependency issues fixes ([#160](https://www.github.com/google/slo-generator/issues/160)) ([51b956b](https://www.github.com/google/slo-generator/commit/51b956b85e7769725c46be3579cc51c4b02bd333)) +* migrator glob catch .yml, .yaml and .json ext ([0d44dc6](https://www.github.com/google/slo-generator/commit/0d44dc6f64dba2e6e263699c658ff88864e367a3)) +* migrator not listing all files ([#167](https://www.github.com/google/slo-generator/issues/167)) ([c34ba68](https://www.github.com/google/slo-generator/commit/c34ba6881b38e713a3d0eb6ade7026a0ea0bd193)) +* migrator target path locations ([#171](https://www.github.com/google/slo-generator/issues/171)) ([2f7a07d](https://www.github.com/google/slo-generator/commit/2f7a07d49f40c46a6b63f82195914245aba73a6f)) +* Minor v2 fixes ([#142](https://www.github.com/google/slo-generator/issues/142)) ([2d48d61](https://www.github.com/google/slo-generator/commit/2d48d617e124f58e307d9ec64da44e67df9bb611)) +* Support JSON or text data in API ([#147](https://www.github.com/google/slo-generator/issues/147)) ([93a8c9f](https://www.github.com/google/slo-generator/commit/93a8c9f90626460dbe96567f3c3cd6920dbefd78)) +* update migrator to fail softly when invalid YAMLs are found ([#154](https://www.github.com/google/slo-generator/issues/154)) ([507302e](https://www.github.com/google/slo-generator/commit/507302e69f7065c5e114c9d9a72fa22f648cf83b)) +* v2 deployment fixes ([#143](https://www.github.com/google/slo-generator/issues/143)) ([1f03ee2](https://www.github.com/google/slo-generator/commit/1f03ee226de29249bccb854ae097708be5aed709)) + + +### Documentation + +* Add Cloudrun docs ([#165](https://www.github.com/google/slo-generator/issues/165)) ([223830b](https://www.github.com/google/slo-generator/commit/223830bb2395076f455fcd17ce1f9a9ebd1b6579)) +* add pointers to v1 docs and v1 to v2 migration ([e96c625](https://www.github.com/google/slo-generator/commit/e96c62516dad50786766db484eb3f2da9eee7dc2)) +* Update documentation for v2 ([#133](https://www.github.com/google/slo-generator/issues/133)) ([0a9cd38](https://www.github.com/google/slo-generator/commit/0a9cd38a507c9559ecb97b6d55eca3b8bc9d20bc)) + ### [1.5.1](https://www.github.com/google/slo-generator/compare/v1.5.0...v1.5.1) (2021-02-12) diff --git a/setup.py b/setup.py index 497ca4b7..8328147d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # Package metadata. name = "slo-generator" description = "SLO Generator" -version = "1.5.1" +version = "2.0.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' From 711a2a34861c94026b6fa15749023cb2fef5d7f4 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 15:33:20 +0200 Subject: [PATCH 043/107] docs: update badges --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a7353b43..cef78765 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # SLO Generator +![test](https://github.com/google/slo-generator/workflows/test/badge.svg) ![build](https://github.com/google/slo-generator/workflows/build/badge.svg) -![cloudbuild](https://github.com/google/slo-generator/workflows/cloudbuild/badge.svg) +![deploy](https://github.com/google/slo-generator/workflows/deploy/badge.svg) + [![PyPI version](https://badge.fury.io/py/slo-generator.svg)](https://badge.fury.io/py/slo-generator) `slo-generator` is a tool to compute and export **Service Level Objectives** ([SLOs](https://landing.google.com/sre/sre-book/chapters/service-level-objectives/)), From 683320e579499809aa8817fed3ef37b2caa7b77e Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 15:37:23 +0200 Subject: [PATCH 044/107] docs: add cURL example --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cef78765..3196b302 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,12 @@ slo-generator api -c where: * `` is the [Shared configuration](#shared-configuration) file path or GCS URL. -Once the API is up-and-running, you can `HTTP POST` SLO configurations to it. +Once the API is up-and-running, you can `HTTP POST` SLO configurations (YAML or JSON) to it: + +``` +curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml ${SERVICE_URL} # yaml SLO config +curl -X POST -H "Content-Type: application/json" -d @slo.json ${SERVICE_URL} # json SLO config +``` ***Notes:*** * The API responds by default to HTTP requests. An alternative mode is to From 66d2e6bb74a3ef62ba795fa6685003c8022c6081 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 28 Sep 2021 15:41:22 +0200 Subject: [PATCH 045/107] docs: update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3196b302..66f26a4a 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,8 @@ where: Once the API is up-and-running, you can `HTTP POST` SLO configurations (YAML or JSON) to it: ``` -curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml ${SERVICE_URL} # yaml SLO config -curl -X POST -H "Content-Type: application/json" -d @slo.json ${SERVICE_URL} # json SLO config +curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml localhost:8080 # yaml SLO config +curl -X POST -H "Content-Type: application/json" -d @slo.json localhost:8080 # json SLO config ``` ***Notes:*** From 79f9209bfbc53c7adac3d58e63bfa9c7cbbc5c8a Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 29 Sep 2021 11:40:28 +0200 Subject: [PATCH 046/107] fix: yaml loader security issue (#173) --- slo_generator/migrations/migrator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 7634c04e..5be17521 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -226,7 +226,7 @@ def exporters_v1tov2(exporters_paths, shared_config={}, quiet=False): exp_keys = [] for exp_path in exporters_paths: with open(exp_path, encoding='utf-8') as conf: - content = yaml.load(conf, Loader=yaml.Loader) + content = yaml.load(conf, Loader=yaml.SafeLoader) exporters = content # If exporters file has sections, concatenate all of them @@ -261,7 +261,7 @@ def ebp_v1tov2(ebp_paths, shared_config={}, quiet=False): ebp_keys = [] for ebp_path in ebp_paths: with open(ebp_path, encoding='utf-8') as conf: - error_budget_policy = yaml.load(conf, Loader=yaml.Loader) + error_budget_policy = yaml.load(conf, Loader=yaml.SafeLoader) for step in error_budget_policy: step['name'] = step.pop('error_budget_policy_step_name') step['burn_rate_threshold'] = step.pop( From 05318b1d39599b5c1819bcc453a52ca35c8a4ada Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Wed, 29 Sep 2021 17:43:35 +0200 Subject: [PATCH 047/107] chore: release 2.0.1 (#172) --- CHANGELOG.md | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 767c6be6..a3e2dbc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +### [2.0.1](https://www.github.com/google/slo-generator/compare/v2.0.0...v2.0.1) (2021-09-29) + + +### Bug Fixes + +* yaml loader security issue ([#173](https://www.github.com/google/slo-generator/issues/173)) ([36318be](https://www.github.com/google/slo-generator/commit/36318beab1b85d14bb860e45bea186b184690d5d)) + + +### Documentation + +* add cURL example ([4d6c215](https://www.github.com/google/slo-generator/commit/4d6c215c88a968d61f472c296b281495748a0f84)) +* update badges ([b63fac8](https://www.github.com/google/slo-generator/commit/b63fac866dff0e6fd85c4b961330b9201e57ea18)) +* update readme ([50ce1bf](https://www.github.com/google/slo-generator/commit/50ce1bf81d7c6a97da52cf167b1d3ee8100ddd90)) + ## [2.0.0](https://www.github.com/google/slo-generator/compare/v1.5.1...v2.0.0) (2021-09-28) diff --git a/setup.py b/setup.py index 8328147d..30a9b058 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # Package metadata. name = "slo-generator" description = "SLO Generator" -version = "2.0.0" +version = "2.0.1" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' From 83cfd1340c2b97490c596775df6ae8d32c486fbd Mon Sep 17 00:00:00 2001 From: Bruno REBOUL Date: Wed, 29 Sep 2021 17:55:03 +0200 Subject: [PATCH 048/107] docs: How to define a latency SLI-SLO from an exponential distribution metric in Cloud Monitoring (#56) --- docs/images/latency_aggregator50th.png | Bin 0 -> 51079 bytes docs/images/latency_aggregator99th.png | Bin 0 -> 48513 bytes docs/images/latency_curve.png | Bin 0 -> 34661 bytes ...ncy_slo_distrib_metric_cloud_monitoring.md | 107 ++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 docs/images/latency_aggregator50th.png create mode 100644 docs/images/latency_aggregator99th.png create mode 100644 docs/images/latency_curve.png create mode 100644 docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md diff --git a/docs/images/latency_aggregator50th.png b/docs/images/latency_aggregator50th.png new file mode 100644 index 0000000000000000000000000000000000000000..fd9e94a7dea6a37d41eaf5512b69c7ac2f1f8a57 GIT binary patch literal 51079 zcmd43byOVB);3C%5G=Td1a}Q?Nf_LMySuwv2muBNZov~A26rdHVUP?yxXa+KGklX@ zPR=>sAMaXsz4!icYav}#-CetCSMOc*Ji9ymgQ67XbJFK1C@7dR(f}0{l&97xC=U&u zJw%rLs`J1_{zG$;)^e*T~aOAaO4>-~wnoGBuF#Q+y$M58*@}oMLY=(;@y@ zfcU5~svqO~5WjW3!`uz2f)KxbMpePc1KhG$txD_Jpt><$fxtZ0wmc#S%>FGnOfEgi zQe@Oq1cW)vH!3kTHz$Xb@=xLjWl5E{5PJGI|4~j^vHsED97-c#K&4o%-t<2O_{Q@j ze{*jw`ZFK=&6tJ$FDjr-r#k#3wijkX7=Q26C|!^!WLrEoV8)eNP(r&l{;nk}X~2v$ zJM(3fQ5 zZHxdnytB4uYKjT|p1s_e=5rwb{ljOGa?Szns*xQBQ~U^YQUXv#3Hl9%yC6tL3NwKcCQsAB3isPw`?w-9bSgR6L(oY#P z-bHg5A&@K}NOdRYSn%;*Neko_ z6Xhauq+17UN_7rArqqQnS?U_~SW6-1n~c1|o`h&2fQSPfQOD{2=M&$b#W~eKkbM8z zy~R+9CcPLrj?t^ zl{f7h9&xexSrXv=>o}aVG#mlGOvP`_9~j=hDdllUu=le4^lCmjwBj%96{T7|Ek zJ67<|L$?NJy0CC+(l)<#0F0-CpQRI+hXc0VerU====n=cf9u!>*6NN022Xav6)SFs zjw%@Y@FIJFz-}$GJsWFkV1lri`uOC38W|}NJz$CA(Hgc%8Q(z*f2jnpuP5!pC|UK+ zdfm}af)N@n>7;!3(f|IPP$vVcBJWKaH(ejyA=O`z`rB(LJS732 zpf*JTMfGvTGZq8TKW$^%+QotMCP);%row&v?zMNOx_S=8 zsGc+=Jbi8Wf~jx?4Ww*WMf~KsPiFa}s)&@5FXZeR+WWnRU(Y!sdx}g=0~*FD)YDjU z!#WknmAdUgf-jf@2aH#=s|8BS9q&3d>>Zpj=-QgGw9cP)OqQC8yCYjP-hgAJJGjMW zKv$f0D62^@ST$;*Bd_(N^`q|K70^B`aJmZ7UpMuGc6qNa6X%!?q%8A1cXG&*?T3w> z!960%r=(`Cbjrr_L*+rmsTOL@`;37Dt=vd&3J*(t8dpqen{jpTwz(m{es%jA^(OI8 z8+C#e3VpcFPo*+1Y4M2MG@&o6p=~)M^&KNT9-SMHkGqA{)hnbXHKYUuH4OK)a&25^ zOYVZje@w*igLqzR)|Cye5V*&*uDYjk>DL<_UtP|*T6;R#qd``i^b4#lDYtN8^@=9RE?2+jq4`S|Z;(OX^t zkrh*Y!-Ua{%>B33ds>>EJ7|VBP25A!zXrI!z-hgfMKf!kp;{@Kx#KbF(reFuugzjf ze#`W)l=?qb7-uW|>zt1)a``GytOs2G5ftU)P=WKjw&4k1*o_z+Q(8)kB2QwC9L06s zVI6z8i>pS`fDYDqpzvzO#1}niF`~+S;Gms$>ojHrpDislmvw}3%M38kfS6gDVQ;ye zYCR!&v|s-@LBX&N>3786sc#^j(e*>{hYoy?%>V>QQ@H2Mj(Hn^upED;TnUa=O#dF z?%re6s8}41?l^j%#2eQ1Y})gC&!_Wqd(KsGO48}0uubn>k$hMPE=9q zhL=IvU`JrNXWP6E#b zFa=|1x9g4)QYZq_y1t!e!I66cWO5>5$f&Z}m|W@z$rKA#1Sn2Yd};M!h->^6HPo^J z%Vybf*g9RGAi80wcONA>>!yU5=gdB%;{dttj?cNzzTWhV1#+=>XNy8Sz;h20_ z*&%832*vw3kv9ztTiK2SjC(tQ#~(^|J}j{%`7M~o#;-8UuiJj9(*NX9mrP5A`>4IG z{bRpoKm}-Enpm}JW8)!_z1`81?iKoKP5p|i6RX68+Ko?dP0}lRr|C+hLHfO_MUbsi zi7VOt>hG&2Mu+U4(_yF}BZ2)fscH2i8jB)r^`;v#2t;apv~nZDs>?|2ZhbOfCg01i zE|fM$>&KQY4S}w~NMv2dybiY;R#mIVM%V6O+~ben&oisl(7l&|4813$!O(+BvL{qI z-}(-=$_V{63_M|_ono!zbW~K5-^Wtzu)ANV;C9fBYR~BNbW9Equi@34EQ)5E)JLppUK^O zWA!9RX{@lP$cw6O)ZQNVrJqfHD6o5Zack5~p%js#OExpuH=Sg7chtK>yOCpj8O5M- z4=!tG0O<&_a^mzTU9@d0wNJ*S`N|=Jgv?k2rRVQ8p&f9(YtK_)9)hxcibyN=GEAZg z;IfX8RSf z_MV}iOFID=L#*+5D}*BHehSL%y&^YKyES4lb}n+J!Ew!g>Ovxmd|KVN#?D;?1?@}< z3iv%C+e^J>2_5(LIZdZ7galw9FmD~$8IgD^9mo*}dnx+!)a>4|on|lz4qm-GBXiNQ3#Nq#g$GDuc|!yWf6OpjmZf?I-|Qn<;CkSNZIe1?TR} zbcYEUL`QtDe}<-~`-%lu`3-qAjuOP_3s4=cb2+uDccEpsiwcCC2wKCTr2d#eKNcx` z)s@QDr6yP;(rs)=yV}!sS+T{Mfxzap#hC?S<2pG?gFGs#-i>H~T_<$|f@S1d-Jg7&tf* zHlBx85sX9{^P0u0HHf&GMtJToHvOg#uO+(_&f6(g`h$0q(Z-L7q(i;Dl78@h?!*~8 z0>stFO6#vUeR&++EnO?8R-{fOQi&E2krVXRijy6K5aV$UkCB|3MS^qb<_$ca7xpK8fQBtD~y+juNLXaQl9P^8m%CQ3jZ zLS|!Yu%HlAt{SB&+~|3^$m?Oe;jcq0@kn%OaRK*qM%H~J7@M6c$7*~rjn@O>FE`18 zwa(x{co+o?Ny!(uK^W^$C znO28^npCiZrKNW@R~b*JHQvazL1%6&0=MDQVrQbKeTWO0l0QY5ZDL*^F=-n1K?qiQ z6l0G{HXdXsBOMiMu-!H>uu&3Ga=|5?frEpF?zj=AViA0=$vX4~D^j(XNBnd81XFUi z6+e>-L#7}#C;78Td1BJX z`B2aAdvY>x#Dw!13)qD;VZk6DM;xa^fnKXkpVcGS$VCEZ5)h`%?!?k(`-NAD!M|i? zwRZbSw7Ec(@%HjJ;B?6fW_0pdpT+B#;i47o!a~P(||CUz8aPaPUtR`X=n)2 z+*KoQ|LLdA6yVd(nIGD@)Mx^jr6Xojrx^uqvfnMV(t!fGiqoG$>De$99Q>|VL(s0}|7=L435zBN+ls!BRIim!AVq<~nE@cusu| zw#XcdZZF3!4Vr(R`-jA)P{pg4Yj#|UH45x>dUFRNGB!7m=ADtrNE5z%GwZg(v(p)D z7-Gn+mo9h~aJT*|?m}^TZlRBX9_}>4s_E!2310-K%|@LvZDAx1O!!1RZ%ny|_of6o z?YF1AL8bg^8o0Abbl7h?aSAge)EG`Z;@WvQinhPE=XcPwmF0WTG%_;cT5dL5BVA~{ z@-S66?nZ&%eOWLA8Ju??ssQ-mfKZ#o7SF@Qww8v&ix3HuBv^5)oxBd~ zF%yAYT&O1YXMpt)OM9!{a6Hm~osEy$i+!L}FV|~uSg?_Yeg=Q;HG?#$>6XkRfYN}{ z?}Pgs$39YW{nl#RC9wA|(b;|MEl{B_DwA%*o@Q3{mvPdiHK}`?k@PeX?+O-gV` zC-uVY=pvW4FJ|0W;cF0-d2wM<29#fh@Jtp28Nt(Sc4Qni-z#C~7PADkLt1FMAepHd* zS+4%F)JPR=ZRYQ;f2*eU!RydfN$h%kyklNSxG*-#iw>6ku2lo{0<~IQJ3LIi7#|-~ z3TU$8l=5EpyG71QtaO!?nn{SYbY|3?97~}FD6hUCXh?zK7aijyEWG9PlR%Q+t{hX& zdW>Px#J%oKazlj#Ra#PRri@f{A&o4L|GP9Y*Tdw(ML3m2)kI&RjOTEVp`b8sctq(p z!l#rq8@pLP7FXnh%3gIkGLV~vx}Xp8G)P}oZlKNilQrsdC-v_1JX57fk^M3eBgoTC z<@l;X_r~FV@w}YJmAPUx0q-23)D~FF7suz?g?*K-m~}NU9$0r*5_$0TvOC8wXWcjO zPIq-ydx_x{RXoVj-@n(|P>xa=(I)mmgllUek{vNwy>E8OdKwTgT?uA3+x)O}oX|xd zL>JIvj(g ztr_TW^BlsuB)>Jyzz#QHJ%eAf^RX z(AB&_oBA(X>`yjy)W!M9pVdlqzfG4eZGI2NiSl4GSTj3~p*f61Ik9x{Y6ZTU|*XQ#H|O zVkHYWo}u8rci^c(nMqpb3Xq{rf&iAUqn=Vxz)GdUFDq;Baero9hdvp^CyEhHiuoj6 zG;!k6x+%0^V6+9L8R?bXY0R+M2ipo-I-09GN(A~qDGS~_`v|IQ=Z3unHl3@>re>}C zQ89&H#%s=jmu)wz9WjKoRGW5OZk?;{*r(_1H)#}@-At3{%*;f#bi^^K<$+fsmL#Dx zJ3}RN&=%^1-BUPhd%*EHhBPt@UQjJG)^^C(*MRDSM}z1q|8Qwp7?VvGk;Q)n&2UH8 zx(+F)ua*bC|a5ZkXhJvTq^ra>YH^oeOe zCvvUgV&ALzKtuidXz;AWhy{EKpefRd06co2Y*wdJmS%6dOZF(}t8}T47fk-AGXRH? zFgFMzc_ztV%+=}a(rB0k>@3VT8K_QImJGa#NRlJVULw2T0WfAV06#y3ut~O^R7Fgh z(0njdRmHQK*UgwXW@Fb$0y%0^sBJ zoc-%GCw|F@(q8W3uM8307Os0(>_=zDZ^*ZwbC*a6gF)&l{<@YLO}F3&Yk5bvSHoz+-UynCm)D?N!DNKi}30 z$cztUc8s0Skx+>HL;$_;dxaKNe4b2X6o1Gxn?adAOYi$2pX|fQeSKz1U4er%D2sR( zw>f}D7P+U@Nj~hyKl48_+)utA#BoqA)e*kD6>AMZL{s0p6dlqKAl9o^9Szd?n8m_? z&G*s(K;ix7EyMNa1KShvwx{lk*-HnaGBjHor>E#I{b8H#J2xk;U)h`$Un*1LDCh9e z;&5KC)n)AoWFqKj&-0(saa}}>3*snqK|DDHSN4XoM10qJqqekVG&B(L3*NbQggD6F zuFY^}7Y7D%8_)9{op*J}8+QR)=L6f->F9lKyV_&IC@o!Hwn@;@&)#dS9vY?~5cg+I zzS8CW+J()+PcxBFzNSIBiHsfkX9<(ZjkUIi7z)dTS7&{6Hg!yJ2o^{Uq^(h=fh|Jb zj!Rde;S7kQGxRxsz^SIY#9RIA?eN_Mo~3F@G7wmyrC)3k7Md)#lD06{a5$q+`P#(W zUZN?27B^RXT=)T-HK$`z(_lIRFKL|Ee#A!n*BV+h3%=XeQdXZrF=C9T@QbrgaF)@U zxL?0Uz4P+f_>aQMJ#rg*%#Y?+Mqc3HRFU@;+q+l2Ga0g7(T*kyxcRL0(~8px(M-r` zO5@}(mcSRs{|Ke=Vbyb5G3gJAnh!%a_&t^rcI2qCih++G>X*LH941)PN|KdXxw(0P zW)lQdFHX`A>V2E|SpCAX>Dh*k#G4O>OT-qnIzbeo*B3cyj4h$t-NleF^sNn+ZxiL% zs%nY3;M^Q<0d${up-$wU1vw>VLVgq{mJ(Z&-JELJ%_GhdI!dD|$-(i&WUw@?6hL<+ z5`nqcM}5diWg_t1PkW;*qP)KT*VXyyrw9UZOIhIe`dEIOO^5n}ezSGg=k3l3esoBM zpDAzE+!M~uCfjPgu~mGn*@l2!XOlMwBaZE3Y7x7sEV3tUVp%!q7XCGjNA;a&pB3Wq zlpQb8*!zB^I3WoNETHbPsC=bpQ1;)9wy=uth0>1@jL-&!nT*EANC!vX*@bcXZq zosGt^rToe}ZBp1KUGUNO0&eJ)#Ng`*qqm(@;1nhPcYd<(l*)7hj$wn}-O`Y&unifE zY2o|z@llb>v+))(2`Ym!f)Qr?kJg=Ne*M-pO?R(O)e_VLekw^V9%3*Sg-?hLw=zp((_ zq%4vU^$n|V9Ocp;bI^$-a56n=z1*X=8^Fo+VaLnlUBfup*!CN{Ug_lwZ5BcrB#Ld6 z<^*#g2(i7KLKXknn>NeASO=41RPqR&A;1cG|Fd^?p(VW;zk!W))aGl3>;O<-Kc1=W z7eS#4V{?vvsS2$bcgX|4CJhpM8z58d_|2xJ#)wN+TAHunRo=a4&>{szo_?&VJRn0} z^KEu^LAts?2%vj|B}3+SW=jD)wvthQqG7H%zSwpiWW+#Rw4hH(r|vBn z9@@tzq7zw5woZL#*__L4mh!#bl7%QQJLH*E3fRk$|UpA-2$4nRf*%oJ61 zOX*Yo8uCpU%DC6o)XxAiI9FeqiBhl2W!Faj^Ftq2iN2**ACiAuSLv>Q=;i;ePALZ&AH=St$k8|`OC2XZ@(TkF2}*2~*D zfu!tN8JL&cWW8lk4D8bgxSsFLZTiWcWmfEhpDO}19`~}tO3S_}4f8QTvCV1TxsXP( zbA>yG3eXMm9wCZ8wqaF59!`>nP?yTSkjxrLs_@M~di;y7dMkM|?>Z02SQ5cQ-*(5m zLLJr)uZ^4^8ixG-`5;PvHT5-HD|D$YKx@g}RF7N=8i@^R<6~<(19n|?`^mXa%5kc@ zQR)U^sq|D+MpZ|Ixi8#mC)5R4Ez;i=(fzdIA>nb4BgV0<^(`sYpmvO^LUNQFjHhySKOLvZG7VaWZ3s)myKOLd!xHGJa=Ywnv@rY#(4IybbVZW{Pu1-T zRqF!)UsP;4S{8UQKg4tp}k9 z{loM9LE22ZLv4E>JB5)>P(QqB?_l*znh3gZYISKSBZL`=^dME8)sxuOqK2l!WTI60 zDs5*!-B#=T&V$uYyM50=ChR^Q$bNV&(91z5FI1>bj2$|!xm%sowj2S!Y_W4?JIn=p&#K_ zc{`3*y_TnS5)<)@@^*9H`J5U??p*GzAECOwliQkq)Gt`N{RHb+XmT1}+ zow?75vW6Dcb-Fb=>Vf|c{~*)2|2_1MWKib$aH{q7;^gIG=ScRu1^PU3Oo(Nh2)&HQ zc#dsJ3Vw-YUUa=Xf>(5n5?zpTVahuL2wZPo4aVMT;iV9%X)#|J3JiGLw0`(}Q{^5~ z&u-`@u@n*+V;IZLH9DlOH5IX<936nTolnET8v21|69cJ2-^D%KIouhd~cSl_{mZPE#veh-3e=3oGe@L7o zpKmvh-zjU<;K6?w(I1+>kjv{{XYxMht(rxWO}oD&tJwnJ)~>TDRmAtAH;Qbu2KWCB zKiW@6U>FNiG82l}le>^(YJWAS{3X=?-xOu_fQ<^04RoSOMlyl&Jw#dxC3aGrr2irK zplaFy?&jB%(CCL;T^WBFJtTd{fuuH=3l`AZ~u#1uh|*O zyte;j_Y90Ch>U2%P2a|-8Yx)w%H;+C*6W7+w&L=Wpe<;PLnyZJfZE))~d^nYAL$tQm!Sv*f|20J# zKNN^~_U^D;Fg?OiPD-6lv`2;^nGkaGDOzU^&zT4{gt|@HB$!_BDz;2-a#iEI&9>>w zOSsoLx7LqncbkJsqDm;e`wrhAp&IpZV1KzEg}i2HX&GAC2BLk)5@izM(;_VD2{%;bCCI z;BT%04@1MV7cD-cxVa<)t!2ObMNg}T)LRwplCsXXWbYb=i`KN0Hy*b^527$K-Wbr} zq{JmTKaHGxRapulAemC^UMtx(87xx|{V7c1Si)sn3|_nF?d^Dj+~gN?VfHK+MdCpE zLlG;e_df>Bnt$#({*D*T7!X+z*8Hlq8@^tWraC>8n2%=(P-7^xwHVZx%&-xOBfqRY zSTwB7FITRqfIq5v&uVzT|0!Evj=6p0;5p0BAC|;X96q z0{06m&&(TZ-3|ai`B>3%^fJKn=72&3Bnq25P0xDL(rrPNqvqPRM7YOacQ^s$P1SBn zn%>gP>nI1a=BeE(J5QFs9FER9Y@3|UVsmRgoG$b4h`tbQKRr9;%rQk6iL$$^et^-7i~U2t~gueciaB8*ydZ<1VN`wl6Cj+cq!yjT2pg6QLHL`_Ch%@ z@V;2JaBKlLGIK!=_r^6Xb9DHRKO2-hkpCh<6(>!z?x`*Wj~}xVNC`4}*D_F>&s$KX zrm4&G&Qbr3DPvUKY-v1<0xJc45Y5C?Qi8Dq&BkefR>Fu_ORHub8ek?_^zxGFfhSJ} zn_&wETw0}c!$+~ftgplQGZl#!s|~thk>G!C16n}rWInX?yW!* zxGe4&I_2w!k@fUFO)w8G0~gGi6TKI7_DC%l_3`zMp%5aB=|h^&d=Ln=$pR)ku(l1!r})d>#H6>XDfiUZX|8t=3s!2z#IS_3<#%~yeTk_HvGRiIh@4GPbd zI%Fi|N<~dH8+8OHRWwChq_K7~ZG=ScXx|fk4<)T8i;g+tbs)=oK+XLA*=Lr`ILsdY z+|`eLYnbZR{kv$g&l8{@C!kFv?9Ry-oeeXX_q63D0_U@1B&PI^vhd=^1JA?Y%^Y9k z6$csHj}Fbela(`ga;W+VlL$Vcp4|0-+r|k+1}o6 zhw4&{T!BzfP)zH&4tMv&512_wq26$_F5lG4w9vMN$Ha<5u2Z3+~BK(jT ziQ%7>2b7H!;<+EtH+`54U{7D-WX3ny#*UQBjwBG6^pMb8}8!Y z2&VJuwF)JDn+Tk^vk$7V{K>2$F6QnQL-L)M`j9f|?AX#Dkqa+t61X%FyGd~_e|3L; z^66|^b$U<`7pXVGp|rb)f{L_wG%cxeDMO5@?-Hfm<*!ZHzkTn9B8O13T+jPEY`j0mA%<0gxr_eeth5GW|j-KQp2Ny{fhJ(9Js?evhQ@kj#ND8^v5W`9i zwn?FsY0D7>ep0j84p65`|1lGPfB&zoXuqdAhx>Szkm-{)h!F?z7rMwU2MsxUhiNGN z=tzM`+q=FOXiwqYrh_AylJp$-w$^BvLWF0iv?iZ%PWh#z4XZEWrWYxDRr4uTW4U#{ z$POHO2#N&ASid0~2^dwDX3k7MMNZH5T>qGkgcW*+uE6kn5*V0&`wNiR+`*dd8v7Ev zeOh5EES_eRF`HX3A&Kn+zc?K|w9S*~ZVUd%u%pR;ul%7v60N{Vg1=G&9pm_C1q(^V zR4zWa`p}0sJ-Wh)xGI50&IpTj4CYijF$tO!1unu3s_nf!a@mah7>xW;EJ{(~{gn(J znfT_G7J62~OZ&#%FZAPRSjFz63AV{guG1p6ymqJ9d93*wOYNJbc^vdyEG3PO!QM57 zY3rZSy4Rd#NH6`A@e9YY_s^p-H+_){=9i~WrQki=i&Y6PXNtsb?M3L& zD%|#GRkBT&)}h=HeQUX*C;zzFkH==N6)4U}!G}RXra8o9!lK9F8Lytv)J3-!l|w0D#s36QexILjOw92yco56vi(W)O-dZ8Isg5 zo&@68VotC63zkct!x(r#&4BS@0CS$}uVM14F-(1|(Dp@njKF$1ms1~Xd zSw6}PR`4g!t&WG_@rFyo4l3?zJ&!=y<)hI`jX}aXJ1_3#h=o!G7Ynfh9LySkQ=*z^*AZX;a_@9E*sP_!YV2%TuAlQF_sSy z9LCCO*!jEu;P#=)vFNifT)EkIY;OxRl{qDp@<+l#i?F%1^!F;JKA4dul_v0lI`gdt zx!uRms$!avBL7hfsTM>k-%`L_TXCK{m~bh?kjOqBoDn_ecD39d9Z|OxE0F%)E1x;} z*H3>{(%9+oaaXKA@0;ia75KMtSM`{C)>hI;>Ij$XE31f@OyQ00JTAG^(6xL0h|I0U zhBoa;KV;E{)f3^BGTE=A@Vw&P;NEBX!lz|gX880C! z)2lCf)ka~%bdzBAFxsSnQYqM;U;emd`u(dOyu?Kt?yPSY2NQsBZmJw|c+RK$ue2hU z#;eB~R5_o8FAZLY0j)F;svGbazDvY5t9DbY>U%f@#<( z&rLA3$&B-?;DQhja~LN875%1ee^~}&ySQdzSB)-Z`jvh^lyW>@U|CNdxRq)T9w~0+ zITRzk$qFcJ@M68KIV=d6{dIel?yvR=#9*1-K^w;9Smf+%J`|@{v^j%DARNz#z}C zWha%Ot0fO<=jhWTBl9mVF8Eb1e)KBH$s!}qrh;|Jh2UYu()Y*`W?7+&(z#!HYV(Wp z%Ez7M=RIXYe2xM$^K1kuWtH|ix*A`r{A{gLCuV&bgJmga&W0l5r|i3OORN^_e|D@6 zD|Cf*M3Bs z$mch0_rK+g5s+nje@l$>BTJ^7IGk=g030B&riLwp(2y@#K1iV*Whkw1tC31ksi#cuHOeF?HDihv(& zRbPrSi`JZ^cF-Mkm3Ql<`1VQ-UFhl(%GH>lweG5Db(Hf*MW#y7Db$F3i@r_&FZQ?T zKD{003&|q+EllBpf)Tt9{xzk*yc>GHTRGl+pYMrj4fZpu3=4H}18VJ=$#zTaan?(x z+s&Eha#_X1IFmE44NS@Obp;E4J;Td0Ef~v|hu&B|4NMZ$(Ot{O{$j;$&|xm;dRzc! zPgLF=q36p()$H-tttV&mHNHour9kB?ovW7Wu(c0wubx;r?bBpoOiY7^$uakmS@1fC z$gqNgvf*O-&>Rb2gTWqNKpzr>kYy;+={Rj$qPx??7tJBT<-r4EefwJopA%R4eU zHD*d@k&5~gu>J#I!SX=<@L4|jnsDEaeHP)|FY&&&0zV1lnx00Jg!wCVlkG2l^WQd} zOtq9W9)1ZCz9<@5J=4ihjp<$`U|r*R2Po||^GqK*?uPAIfbo=w&u}W1jF)?hV}X-j zDPEoyyT)XWu9T8aVMQw zraYZoKAoX{d83I8t-4w=fyWhjI-eu6^PDU#6)Ab8;Nsh|oCt!X9)Ma2yRnCd+l#2L zYPxD^ihlU)+RF9zhOT{&ddhBLk;!{Vn-Uiw)59y}>PxF!)m~i+1k!X=GB~ z(zVq+q(3IMXKtU5zlUczXw77tKAVSYO#+_-KWTNn_sz47MS&4H?5Kd)tlvL}g(%$cHKGtK&_3U95VKZrX?jIdf(p@cB7G{u~T`yt+AM z0-rrrv9vUU#fINzNE>Ry_-Dx31?KhgD5V|Dfo3c_@^T~<45}@uHM?tX`Z$Ij`|@UR z{`|Aa{kRe<)WYu-eEw-DCen)>%TvClcc~s8vp>0b!)F$bDkPhn zCq>1HkuZ`(K;Tj9`GNpPxs`eZpD^q6;hl< z{JeN=Ih1Gkk@-0Rtszc2$NR`oSGN}1a6le z#IekBIE^PnWo!lM>xD)9CjEUIGn?20xuLs>vK2aUU%y!}%D z#pcZ>f80c~rJ-whdui;~>cQ#@|GBg^0KVfw!~XKhN>A$VqF`Sg+8nRlkzu6++k7-I zwwU{;B07gRxk$gaIh2}^r8u$T>mpYZFwUKnifdq=WFCfnU>%Lula75eH{NlnBFrX( z1eGYt+HbnVE1pRB_=g9Mz8l*eb%=y=Q*iBUaGNLP4+~+(x93>pNJ}!aF~-lLej0U? zz^fP93y7N2Yi-MIG<%NLRlM+L*@Z+N*<3SBG)h52bIN{C{(&yl;-w6=y}iAkhnQ5W z&|=%^=&t*oWVIaV)Rg6#jNfok<&2%DCrb-WFSt*i42OYvXPKT}h?P!w@(;xv*k(6M zQss1ZMhBB6FCmxrh@A6!0;K)Ws|I*3DvjdimvQbExUxw#=w??OoPwm zn9d$~mmE`(l&KkxLFNS!1PT)el}Ej{2TrFeIn~tzE|%nd8(Ul13ZwtP5PD5!;Q*>N zykfPIii(QD!Yvr6B5U(2N@l^#cO3kb=;6d0mffwn!+&W+ z1AcR!@I^3P$de8C&-+uVVVAv5Q1Iv|5*+v5n72cHX!wo=T%uN@#WFQHnY6FPrW_Iy zg4}v!nHYx}7kn3+yA)}5z%KLJ43V)&wNQY>E7&&dANR?zAN{JaFuaXds7EWmDRuZ4 z6l6R?hoWau;5xDyySTVGUZ_+^WO;aaD3B0<&nFMR z4^&J)?a=;bOf>m_Q%_&@0s=%&u!vsV;Qdp|Ml@sj=Lng7#r;3M*7pCb0-Rn8;3v-t zY_Z$D8chT54J35Q%F60{vDY+HrX>1b zUD6FW2N@2h^C2fLJ-iZ$N+TJ(R^;wEeqeobI9;_^?F}h`#PRw-0vrzaoOkZazQ5bb z%F3E^?9gXRx!+(9yk3oDFK;<$MiNsi^Ey)hEeJ@n;$UZ|xVTtGOs~@D-Ay_ky`rA` zq&kvV!VULoGpN&9SX@LDiovZ9=9}mhGqzzcCMG69x9!+h&rq?*Ia1|mX=%593I3*k zQfYC+|FUmRPX>m6CkkJ`-Y+EL!6TvFCY%j({)cUs1(e7swE<2+4Ze?QyN$75Yd^ia z*Psbo>(}_Iux+Z^nWp=MG3Kp@l;_3bGHzlw0X>I|d*3qg$D)YPLjA}1SZqM5B5%vL zhUF-fR(gn6FZ5&{ql)Wi=oo)rtOk|E9O;*tVDtFW;A>ry{GE`6v&04i`Em9o-u6}9L(9l8JDnk(J)}smo>W@+&KZ8-zEi5mQd-=fITuS)%5@Ig!7OC7O z$k_BWJ(Y-vNEkLHVcfyNflnI(sql#YFvU^DUlEcOcFoxl)EcJNZpbP2d#vgnxa=!YCWO;qrT6cE$_I z*zd;!ZG~}m`<7z2P*h4kr=2lcoE@ay3Anq`?6{dksyUMl^Pl$jI5RUmH8QbUrqT1zRiwkt z-Zndl>635{Q;cX~gH%eVuhHA14xi)1`H50S_O#!_54TRQZKunTcW2EW4ZOcyis>=A z_Cpxs|G`-=s-j1QPJ3#cv#W1Zs)QtH$ejfz6*a5D_~WkwC9_8)>oUvpOnHUPCq3_% z2(JACOiXgB_K&@UUn=EtW{+wzC54BFA0Hn}QvV~~4-GeAA=QkM@uu?Oee4z?eN+#D zjerhjWc$go`J{6{kGu~ z_c~L=c#JZ0?$K5A@VQKfP%K+S|KRM8bvNO?>3N4?qG-!VW?$`1w>RtK^9~{}WHtvC zZa7WSgz)7)!_$OfS%5&bC^wpAWaUCf0X^duu5rhKM+=W3gP`@IcDU9EE%gKa7P#cO zhc6qH^E=Xsk@fECLs+k1dD zmA(CK(I?Ns2HacpE%>(l7)9QB`Z%huW zD2^|j`&LGLom_ylAmxn_AepszC|K;7%>}`uc*yz1n4{vCEk@zcSJdTWg5f2yGRZ0m zqh|3Tk6CPQvJ2r*5|hH|=og$2jfV;lJ=UbIS?4H$(zsw47PvAk8y=uvUF?lFw3jG^ z_f53u*q4{OHy8?&3niglp|Q!YhQf8l5V{avZoRC=@cV3w)FM(w94vmJ{L|%$glNik z!IGEr=TnVf4T2;(V#z`zO&0gEnR{HFxDUe~3NTz%fgiRiceP|ccoQPOmNa_7==94WtB%n)w!-lU`L{myer3J>`?aCLmLU&ukN@z{G6>gtORd=6}t4+Q|8n@G# z&YHkaj%RjCm0jB0J@BuU{boWKr~HxJ+mN%cYBRtS?0~JbFJMq79VHlqsi=kq#}#l{ z`;n(#I_P=~HFyPU)2r_tlrc7qHgHj@&@IzXSr|5_#8fA}Vl8>8mk$*b^6Ts9mh%93!MC1^ilF?KQMvui~ifp)+fyLjwyA&vSFZ#AO#YxYRkfxn%E{?LIql zX3$tBsa*F*s)!NoIrH_hk#>wWIrin09VJBEOCwkt&Q;Ky3(PGMtn3)`TyzQ)2Ez+; z-m-OYaG<0tpQR>KriV{Qij~-a>-C=HBLfhw!!Z zTvbc%Lz?T@<(!~-{Nbic4wgQc0QgN_aC3ocr>t|>s|z8;u+A5SxF=+70LCyO0rJ3c)=B*=F{S-XKRa6wlpMIa1s)6+pG=ai>P z=5!fyo|Pg}C9%;4XO)?uSkDmg6PAh2i|=a%e$)hlh3`g6Vzh|-^%sW&u2LzyZ}TjL z=p!4kmX%mHRuAk56=&j0J&vNJgvFaya`k_hw3H)6nThW|ybf<>Z`Z$*X_phAtsQ+I zDs4x74FRQ%Ko*2=ey}tA9%EF>vzDT1kADzG;v-r#xa24{P{u+Ia7}SpMpS<;{5p126ZEDq7 zVtLRlfW`b$Z)-3YBz-<);bV~m!}?OqJs#vuH8pwoYVpkVMhgE{`3Fb$Tubuz6Z5}U zNt@m0W~-MW(kN1IH(2yyJ{DRW(#n6)7ofey`U?NOqbP((^L21Fdm$J7`@Zk9IMbU& zZ)*aNr~;%IoT%TV-Mb^G*PlT~LpWum-m)b`KWn4XG{Q7($>ySM0mKJVcXD< zu7kr`S@&5T2q~Kind5`aY=?gdFBl51a=(h>&t`FPvfD1D=4LCKkv_baYry`pd;gR{J8l#sOecs@Z7cE~?sSQ>8hvB0MqKAM?o4 zSZugg6p$G5y6y%fsU{J2N(znUD06zefvq-fpbr8LV!VxS~vUjCPqq?2%Y?2&xYj?_Gil>kn>W%on62;GAw(_ z#S-?=i{F30b5dw!=xPTBymN@;^zZJ!ZE}L&cK&X5uQ>St(4NGajS>8HiUx~JYW-J+ zO9AW2$Im~ww4GcykN%3sn?$;Z}9cF`Ng^!OsX@zUs+t0~q9Z1iTMrUZzr*~tc#5y65 z|9FDw$P|pAwWHNFi9eqHxrQ>q`5#K0?<~|V(DCv120jaqia*YI{yDy1sP!oM*8H|Z zn8I4aJ#d;KqCK%fL`cYGdr@1Z4>k?1I7h2(YTfteL{O^#-ojwbPHB*Y1c6Yqar5~5 zO-Z9KM@YSa#xNR?oIKUoC`TslJl(R{EqnZhgr6g{WT6;u6TN)BE(OlnTI65TCkIaP zjcmi@cq!npH@3!gdlBE%9!A4|)WdYxbAk})MeSLlT8sQvY-gMaCHdVl`zSB3UEl+u z;Ww`G-d-#=Z)OZQSdWz1vjSg!z9$3Z4vg-|j|UR;OeJKj^~3GtiY18&-D@5r9%b*V z-{}{ORk+#q5o0!lTCvzB-yDRC=hj5ee9Mz@qR(`9mHT9s-OuEO_jB%10_$x#xM3q7 zFZtTzu3%fgHI;~JwNhNUaeT|oYQNR#cm1>7RZd)Me*638BP(`U-Oz1j=Y`%Zm$@$O zJUl)IKpK-8-(B$7f(P*t7|cyU;}vds`US%)zLHb^e>R`E>h|f6CmRtp+XW6M2&U+G zuZW&s7Np@KQ~h*c;ARgIiAH(&wi#z=;$a)SMuRKbf4AL4X0t=+r1yc%;P#E?KoS}) z?ZfrRfZM-cynOj`nZO=jM;8p%kh3d{f#{SzNx-$1z|#HQFt)?j+LM%(V@n_MymNQ| z24It&WhsI;Rm#b%Z2H!1M%qP#gw#aoeVv@0uF4Es=B!;kb4A!JVlE@)bLo}JE%e<; zyP3dFf%#A6$1%Dp6%hD#kLwTf%CRzA`{hTQ|LGEzK_ z9}Hq@SL*Jy7ai{NAGM~U)`l)@d&7P5tWC!Q)K){daqOWD%$91tY8un}`;{$3lL_O` zwVuVfl_za~YzI7wW^TRl!K2`QZx1d4tU5Dec>=})R!Rr21%BCIt7W==86*ftr59>2oslv*$1ox(w`;^yWivYI@KFWt!V196!!^w7mEm6x%k z!k}-ZfXTTwLu!Dq*6P;SB281a!0GX+#VxP^|FPH<8Bnptn{S;XsXcvp&@S0%<1Vgo z3AgTLxBgm%zia2l!jPIufleik5V=#)l+C1KAU-`toL;>dZg;z6J@w}HYk9kyp24UI z{){H65|!Gwpbzr|TEgi?>1tp8*xn9#sHWCEQ6D6i32umSQ7teNK{h6Wm{KG?R<{RD z97g$8{8xIHQKT3HZD#}y6_OjQYssA08gHpYiJ855Q5X{(tlMbFW=@$K!=YNv|Hs$l z&|qCT=BQ38I?4Ut{E0ciZ?UbJX&2MUi2j4Z#kpuz1W&m=f1;ev9qNI%HUf#Aae=w4 z?wHk^A_L)N9QRS?MAklJ2qSs=Z$E!v&Yoecu(VS;P&iJBZaW}Y*bM0E zmf$W2DP))%$3C<(g~lhuC-UR>44>DVvn#j-O-B_a#~G2M2^1zmf3GP=l8zG9cx7GK zM9inTC%=0tG^N)wCO))Y>=f-9ekpI5wdt}OYF8)e?dY1joO(Lz4_!QNo4eF7p! zi<~UBJ!w%=ZrWq5A5yQl^n~N*(VCXP~$*Q#a zbjpXb5f=GYdRPtku-MiF_q{P%nGXs^*_=J>1L`;5*%dZf5k! z$ESdayC{-TdaqZBAMkQUHNIXx1ikh#57In+DzZu=GIXwZa#vi;;^w{^UJ%1BBu7S; z_d33VnzCG$awAXYu24LuE=1ehSy1zXR&}upo3x#d=xs$<8{)IGO~pvgLvc~*cy2EH z)Nu?aess&CY_#Mlm8uEHvL==K=+VQ6<=hFgTKRwLQIh3QhiMmAIw1XK%m}L_FAfqk zpxK!4+>bJnWa!Sd`?aOZ`C8;or1>~Sz434JKRsRNRl@{qdA1iGRLErh;H90LTa)n5 z(b6ma*q^yi8etU2kNcZlO(AG+14}_6l+`($s27*uY0KrX84(eFL!27Hrkb^(A=FhB zEoDM`OI)0BduIltnjBvceEp(Yy&LZp4m)0 z55W4KmYa)FOZ5kw=TT?vt|DoFwWJmLpf6H-@)yb!Wl8^1XYb-Umu_37R`}@n(>( zJDE|mvtX=HTS7p+>54Z;?|m?nW-5T5=#iu&zj4Zgj^+!1@+K`-dKHm#S8PaG4XK1fQd(vA)9)Zrh)!;faE;g zmKf|MYI9Dlvy;E&!|KNT1t?jft>f1za`dp#VMoWHRL0a4GavffyQ4^xVR;VSPZAU5k!1A4OTrts#4T9xAG;_6;0M5^iZ)KynRBqsATcK7OW=YAb-apa}G-z!f z^BD}5;F)(wO!;OR-5$dceo-0b6l5kq)}DkizL%$G8hUb5?x$JlHI`To5`WxS)k=O; z__!cc+s?^7QB|A*J)#sW+J~gb!P`I}(edT?bE<-ahwS+g`LuK437%3BkFp>)A{Ay| zZOW?*!rzoWn^;$OFqsPMEL|l_q-U!1junneb~4Hcw(F4ID=t7cMPR%jkymq6|ICn8@3LHp6n3VbFtqeJ#Q zc;ze&NbittvbQ^!2URME+rn?2ntXhm{c{Uv&WX&IB4uYZNIxoLPE7J24j(Rv&o$dzd^TY2CS8rufQ@tP6*wD z@6$TA_{x}&M=HUnXE&85H$FjSw|3`JO**@TRSY`Y!BdiLokt)5P*7_P2fEJC{Vec& z0EjiWGO&yaoZMNmxtrKdz4h9_X;B8>a;wj}M+BoS{OWHa6UkDj8C54L&1Tn4I~0M} zSXQkRcJESEHK?$mBF)Huf1bz*$;}NmMF@swYpGzZ6#Ifu?<-b?vABg-S>n7x$hTBP zq(?58pEN%hfyKdDuO73MYXs|Y6_GU3WlD|?U$E}c=A=VtJM9Tq27eTj0P2 z3ZgqjI?8aPB4!qWn6>*%7Gzt;`cl9RkvpAxOUOlp44n|Hf{KtyUCG_^ZBuIT0ad+Nj8^rL?hWT3iK}k!mu#(fL`Id7rcpL9g27_>O zsSJl&7iZ6RP+1^S5@^j^MQ;1eCHFkT|7@>xSI<^qqN}Q=(}X>0(prq*G~Zr$o5-|u zYb#hFIw7Hm^hT--tzok9KyG#XGjtzD;d0Du7n>%Q3jTd-#GH{9ECk+0S)0i2t{B7m z2NxjG8X5VY(E`L$VOb#`Ob1orx1W+36}+<1%AM2uRpt(5{%n%`x zGR~Zu*q`LCL31=hwF;dZUbVkEUl%j%Q4T1O!)bO|>kb)% zrB?P--)%g+e1pON`?t^ThsrCwb_W-A4GpDS=O3%7slnlc7h?K74<0F{mHE3wsM&3v zM+Y2*-ypsW7@=xoipGHq~!O?@E*RTh37D0|4QhBORuOkSSg-gHXRo z&iaf;w;{@*sIbA>&g0Y*pZhYs1@dMTU2x-tCHfU_mZJ~;7$63JEifowHB$c+W%$`+ zjATq<|7d$UN)4DzS6!*qBh#k}*OCx7+Nz#Z!$JNbE{PqpT z6Dy0m$n4y2=e}Mi5M*3Izw&GFw*hYIY>$9|-F9%P1b}4RE2vw57z(hqb&Sn#_Y}y^ zpBt$=Fg$lI;jZLJDsZPq-WJ!$!vp66-Ud)QhQ`JSHhm;GjhlxDlbGYT^=2kn&+qR# z5hH*O`quN{gqL~>1;=)yp!A>5l)$<&>9W34ofQ1BlJb8|vvr@TedmV0{wH#+?A6(L zJon^5^qYglIZ307?w4(UCA&VIjq?~_-1Y`GjHMF^5m-MqHk_-C#}`8H00Ii=xnLeX zWv;+Kg@uL5g=Y-XvH-XPi@<>JUI&Qx#;p7!c8lA4z#eeIUV#q-Zv|+cu(H$4XkN&A z+U-MJCD<#@V7Ymeb>F6JA8=jc$45WF!ER!=Pv)Bi=hL^im_}55uDdMat7bV=^Ajju<7qwcaeHxvpgJh4OgMc%KbG%NYA@2Dt)4t+PN<(IS)+aV2$L}1iHE{|^iy$`ke4*2hIh*W>o)i%S?YA60PKn`eNsUQ&ys@dUJoE%pexgWJOp&c%`}PA zPWuqB45B7=$~+YHss*zp_Obsrz5BVc36avYBh z+VdV2Sdg)VS9r#IkTewC+n?x=R7x8D>CxQW2Gz(@kFDvS-|vNWWm&SS;HeQ&Iq|q%je>3wF)4Rv zEsuvXyTc&_R!EX_&)zN}BDc^>49iJ@tZWL(FahG3d@9z9TDyJqNbeqJbwSQToy~{wWSF2NO zQ)k21^2zkHXa7iSs)QS&3?lf4lr-a3@X%$2aeE6>`fWnypZ^tNKQdke?CadD9E=N8 zRhs--oq|`D0faN&u1~nRsmYKpwta?BbW2u6*-%-W_DudlyTvS?Airqa>br6B!0 zI#5H!(A#lniJaIY`lrc~fug%%y}3DvM7&L`j~Y65aT<;m*6xU!Sy~1rCZ~{#S7tgz zG1&;3m-2QHNZ}J#l;=H*4^WSzjxYBW8~Q$O>O>36(!d=LR^Hiuo%(~eq%&1kM6~u! zeMNi5%^?`7@bh1hw$HIq)PXbq1qX98kA0p#I50GJjWwNaBr~K)R-DR7AkEU?jD8dz z9vTB?|k;M5t}3;>j0FZ zxD|#_5RE*=%`V$H1FaH?F@0!kJ0i1-g?%II5p$m{)H-j5pt$)ZhWZheljjw>_Q(5G zQ+>u{=&b1Ps8;joMu94*ge?XvW}@F z^2Uop4;?42n+FFS1%Oi?Tim3*@=`5g(Jy*4XWFn+WF8l#NVwcQ##l}9`!)^*hO@7a zzGzo!YN8OLJ1H*VYR%@Vu1RICeebSS`=f-M4%MWGyw@WlYNy1!tx9r8L|Z z4gvEIQx_@M5V_u0QYqrLLaz!$031)(5o=c#cZyOksD0X>G|`TdjL;<5zGpIpU?KE ztC8KX_4C{%RiVVhF5QGaK^J+w9zXYgL3B8$siwmug;Wood4Vp~tl5`pKKoMjpFb1& zmvhQl=H2kK-QBnR%=;W}{vg=w60q_)Pzk9!ySPi}XVKBGG!j%)hh9nv%1RobKYx3z z1bg}Xxlj&Z21`xV?EKwrK0|LY?BwUKSH%`s&jI_zaNz$7nEU3p%?U}Fn0Q72&R3S$ zs=_Q>e7u)|W5ZpD-&h)WP^(5Ml3M|K`wncGARA8(MyG~1JaHW@W?}(O*f@#Dw@6$- zK>HuMOKJ_pDn(K0u67+USdV^cUzC7T9^e7+?VdxX(FrEY)}!}W*$R9hgVAoW#J-&A ze_&dx3FB!Wp8m9QiLVe{Y^u;HCK*jbb_$>#6!z|7!6bg)_PAnq2_lAuX6hPeeE27ZneI909doz0Z6U#8O)oUc;8fy<(0njANILk zbG%&EWA*c%{YzZQvOex*({zk3J%yjUX>0qb4mbk#R zQ`C0Q(b2z*YNWc&h$>bBIwJ@e|C0at@i9|Ma!Fmv1nUVt;0b%7I1v{`tGgJ&Ud=42248W8&rf`&a zjML!M7y$nvTD#lHbiB^Q&&M}lQwj$2$Kn6#rCzIE&abVlH8($7GoLX1q&c26IhZjy znn^w0P5rU6bNRPt;CaNPcrOOY9++ln>;n76R;PnM*67j)#IF>G%6>36$ji$QlvyPV z#A)t~x^#=AV0M?W-8Jh?^pK6@HDAoirBegXxtoOlL`VR~&h=CO&9}RGZ>Qn1;|mZl zVBMwueXiqw&4m0P{_BZt>FZrUj_eb<{dZ25*9uof*xvp=nEQ_KMgFDb$+ zdlaf;YW<@V&*>}y$SsCckJ&Kfl#JzemYGNBL$OKM+#jn3%0UOTeiHO}#wT^YGO_{! zJwrR5QsJ5T=w$6*7Hy}=Q&-cubEA1vg!oCVdJFq&(`Ik zrXK;6#C}-qPZBc4EKpYa@5WyR$#jvTQrz!eZ&hAcm?Vb(M{BgLu+BejYo-V!JmSy) z;KUv!wvUdEE^2hfn)u^%d78|Mb$N7nsDb4mqgo@Uqq9>w)8$tOjg``uV6#7dQg49~ z(_dmp0yEWMJVIol7Aq4LY_Tc|N%F#P{&ZNgqq7X~l}L=pPpoyH_!5X;kv;5_4R174 zxnfWWcI-T9U%#%4Boc!6Rco@EouekUxy9BV!^m)c4j*mU#H|CfDbB0FSpJB#_y}uy zWeuc0#_%B5{c34p9jihiC^S2FaZz6tu;0GFhP3WUpD}hfY>#-wId>Oy>vJbu;-tU( zW7;S@CSOY7p0)Esq0djgE^TLP@p!u6PX!qhSL`oT7S)-lw;rfd~gY{ zXto9?MohHs$fTo~PAcwfJwBH|OeDn1k`IKoKISmhg@p2p$KT)}ydTHxGe=pPKxnNt zJfHq!cC6|?H9I->9KMG|+uTHNi=Brm67EqkK`X)3Xp#jq^D;>t*B zX;W(GE&SB_tH{XHVGV&44<%1A+-L^Ebn_Iho8oMysbe->8(mvI?esGd1ybaM_+a3^{8y*q3BzX&AOB~$9Qy0m!h7zX%m3-0q#5Xe zTT6+7o}`Yie|)^Q?~K5743LOTzFI4OA@a{&my_FAXWWo*M!%eUCw}HA1cpUiMBzVM zhOM>((qE--d0}A}dAytoLdSRY8AAVjg-Wdgoqhh}nOd+Unj9{|1GeD-_}$;)er+oD z)M)Xtw1%CX$wDC;GKbxQ3d-CMTJKmGexyL?Q;k6XjEj!-y?8@SHVHv!yql}v} zw2EG?jO0JPj#X_*LQx<#D48Q3f8Abi7%@=O{Kv8-^(y7mj8!=(0#+&z>3BF)Zx4(g zAC9Z3s2nyQ|74QOtgSsR9W>^qytY<*?EmMj13=~b@AahuIv?080rTbQ>bjMKNwI-z zgWLc0s!oCX(%!b+ZO{3znJ!-qhsyhDEtTpPm5)zf%LS0iii*W;Z}W_GoiittCX8_j zg*M8j(`VxcuCxN2)<=ki&HJJt+FojFQP- z?@8d{PIb@yY|eN}T$%yv0LXQLo7nu%qk|0DfgMJVha!o=EZ1#G!s$ywajsI-Au#h0 z#SAV81R+S?9y0>xJew|ef~(&qM`Mc}S&Tg69#9G`ER;xsJt))98yVHQm!lBjlZPx$ zzh>5UcWrl_uB<*>j8Ii5&QP07$j4-}Wi?Wp51w4nz4B2q*}i;p?3y6q{&rgUsbBcq zQ=%oDxR$TaLz&6ttY*au(P*KKVZ8B{-9=vehe-AdV7HJ+7HFUBcSX9fYT6EeYU4t% zvttoGtgg@&cq)~)U)1us6r zC#kB# za8=c;ae=6=3Zc+$d%EzE51hH(<^B)PzR|LJrCDcoAC1y{jBen<&&(H7cgrE6B|Gwm zc*DXcgI`q1)P+-hoF(AD2jYP(&y9FdAWv_KF7z#?FA7l%3`r$3{bkx8vgTFOjk=bhLUaT3)jz^u3DCk^XbQuV5N+ORXE~*5|Lic% zP_1@tCD^D;XPXM5JqDN<&Alt?KK|-=KP21|x#}oL>t5u*YZ0D1J2=qjYje` zYKPGD=H9)wGHOLp`OaoNAp29%;bZ6CtU zVtnw+|DjCDe>dN9+tvBQNj`ux$b>#=-~MvtlUxFs+(W-|j_RJ9cLEOJr2X3&3jb~< z!~ebaoq&xG?EhY(=sCN-_2v9HCAMDLdLDz&zJh(`;y9C)Z{P6?9~edLYc-M*`2Pr( zbDQ&XMR(7E4DBopy^%NFbKS|3qEP7Uy63PS|5*pQt0D(cF+vlMS(s~nT0j$%U+ov5 zgl~6hOX@!lo?lmnCUzo(qSQUS1aB+Sl=)i8q^8EjSzSajMzVuC@y|cY8|&m~v9Ymj z>>fPGI}6Xhk!^>W9DFIs?}{AhS>}m-c&qCnNawN6&9$j%v4GqmOivGF+2C3QF>{s2~R=t@7DI^vp3nDXLz-;fX=Cldp~J3jh*CmkZP{z{nnxC5L{XVgHC?4Ck+EU;|q z@*J&nHW3#t!I|H;qL%t?7|q}*Zrqjg_7NQnSO5BssI-SXN)J797F zLWPW&*v~%T)(0%N)xfa9&-`=U6zVXrlaf$;MEqUg{(H^si8h4Vb3oGBgcK!{I-@BF z-{-jn%#Om3AIL+f_~pYcNumW8kzA}>`W1W@*(6RXunt{Vc(f88cAp~8DkMIlKdTF@ z9neB2P|+nJ8{gy{8hi}0G;gi8OfWZJzS3>>+4{i)JC#Vb7Ci-yi(XuRqp1JWtE(^J z?>mVzq>R&mb_M2NeYkFc&XvoT3o90LPr`PcPgXtYk-pQQ%JmEg5p_XO4-y9%{kD1{ zA|e1$+m#@)bn-2e#9g}hzR2@-BnMKGbxX=z@8xZ(=WM4SQ}1x)WRg6}Y^TU6&9w_l zf!fAH@qfCNTfkZJ+ixwr0w3xgvvakm*OW^{fqH9uKM0d|ntwJIB&Jyw%zgz0D%1u)E?qMi z|9E)Evj4E-;eJQVAui~OcpKW%^Pn^~;|gt@3T%sf(*^hF@tlF_sHEx zqgoxMi?Ka#g<_B_91!R1ZjD>kN3V0O6J&EN-&iuMbKuarEy41ozBv~3_!63~iq#W-vWdQMud^ebfjoDuQ-V#?;A4i&{_lZz)c$(idH-KuU%d1+lW6{O z|HC5cmC0*Rl*EDOq_hU0x5Z*AQo+`Nz%erByV*$(MGe*X?R&*ZQZUo$kTh0Bx5R~V z+|?!t`Si$fjj!`h7F&Yxmgk_*U(!v|)%WR7>cuXI7P-Oh~TPqakGeX+8|b|YmTRZ|Slc16a)pgXD4 z05S-Teb*a3#=MZ&->fV_8Dk3x8&+~B*5p^T2@uuNBrNfDA(V|9Zv{yR!)VI0(W+-@ zC?k2Ljjx9Vr^4GQODuV_6|>?xN7DN*JL}`L{!)_H>Tb*0j6lD-lyyrQvnUoCJSU+e zqe(7gx%WW6^kIJsBLA^4(~ZztP+1w&O3+?WPlSL29D`Ac)Y1K8>VkQr}rGqPrGqurSODSgP z>G0%>`ug`54h-|e)Ol-!ShQNRT1U1VL_RxLuCwmDq=dvkz8=`0oDq)> zjqZMXzvs(*`5S{Wlpb3Y;e+V|fc;BBu7ygUPN5-zKv0bQ6^;&4I3t_AW?Xr!tTqX8yN+8W<}P0lL9Ql+ii1G`TQp71uKhT2ee1w=cMn-$-9Ai=UkRn{PwhFFNMV zEwJmtY5TwU)6f!|$83yX$g7#~G)GEzD4I*0xgbqywoSg^m*O)QvdQT&`&MHeRw&qG zdUiqINX;~6x{{pv;EctUI;Dp0RK-fjuaUpVFEO7L3WhpI`%(=uIE>UAt5GH;#OStR z*)CEKma+BREW<)8$#lBxMaJ%&OVk6NcnukolDw4n`eHG`v}tvZ|3+>~RmSng*ePg9 z)2Ye3%E1gPUvt0Zj(G1amhx)hkxR|56DfaG#IKRZ8SoXpfInT;CpFKEfpuT5idDwSp#q6EV$lN9>s*1Z#*+(ig~939nUoe{+M{2S!R25AU%YD`7dA{LEGa(B zqaRwsxl>O zCI{rMoH|S9%SDI3W_2w`@`hV=*I|n9QO)FcSxK202tAXHnE3_*c)Vb`8X*f5Rrzn8f6waL~v1M@%OmKx2E0tjaNjYgAMX*n5 zKhnWdkX_};cREB~@tnj(JL48ZF7A-iTSJTVxpbt+x;d$M1(c1pK1@V!vNltYbr)X? zyha4})YB{GXH%%fPHlNYM|AZ$^$N*EgWOp9n9+|b40klvEres0*srB8nyP3Uf-Zmh zB}z%2Vge-tb_BF}LHeLTSSrY6nm|b=*O&QL)#A4LwiTo>EI8$1?5ls}A2eq=Ag^Qb<4NFv5aPSLAtMW9=y5m@_=eKKWkGFKWDCj3t1m1|7PZPf6 z7>YgN#J|FHcsfd-U8-1Q@=f!9ET|#>ebb8ny?H{W=WJMpY$5bE2;rJSr4dSW!fIY3uJg8Y$nP0?E&f(>+@q3tkQgXC7F8X!Db}C zKCIMp-7LTiIQMCcT-6N>5~suf+X@o!f%-Rhl3fb^vr53u0j3jJXN}?T*lcUr!v(dB zo_Q*`Vb9xhU}v0r+_%Mjr-#0M17Vk))%${lo{Qp3Cvk+_zXlN?rZ|ZsKqyxA`MDIh zPEgA0!nm_h`D2cgjVq7Z!QAOL&CHae#ib?%>VD$$JeG&n=QASRU0rWWd%J=~?0*5-K97(c<|tXzv;JHh+tOfUlFVy^bblG2Cj%p z-Ln2^X4_ZK?BMj;?HMVJ?tJ3V%f4H>Pu_z~<~YU;k`+siX%F{NPOr@mrUp8{mgu<5 ze@&|gdC4Fn)nVBGwcw&ZBGV7iel%MD{Ps+Xpw!WDd(ZRRoE~MM^3jR@S~g-|xz($5 z_y!nc!XEng?1RM@X0h0x11=~}r+^#{yvZQit1ln-uQa`-vRhTqp3JNN@xA5JZ%%qptHjFFGHCm& zl_x0`Jxl&zt7mFHL(k1lQHP-qru#c9b0+^Rhhoe#&R$?d{TOwLJl?Q1*&lxB(iJZ> z2fVrJpuPkUhJELfJ)qp@{@KJPE>mW&C-NldVlXzl^>Q48YJJ0cv?xYwic$yb}Nc@MP7y$#XzYj^eVC^9GqRT4cs6@j0i{-$gdqBFs= zJx1jT>g+c+T8!O@11ksv|%Xw6w6gYFz1FkSYcuAOaZkz;C`X z*AD6$06}*I*g?9uf5}r~2~S7kwK;GTp32xzR?N1oLpYGo^9{T z-Q;M4=rO($3odH4?%4$j8s@XBP|cXZ2^AS~D2m0rl5@?|%}?2gNmgt6^0F|o2ga8g zI=`MS>WqsPJeeIMZ+QzFkdZkR7((cvuEbXjeOZeOxo#^-CWVP;&ulF!GD*8dbUrWM zPPwV%Ll_^#qG>wFde2y8rR6$W<+(fu555oVe9HcU|9lX@-kHl<3 zgQ-L2qczqW!W)w`(s)4ViI`8YZxGCRFKB}UyIQomcJzyAla zrd_IUckms=%00p>`{T#{1q}f|e*p(J%zzBCk25C!apn_18BvGaBHZs$rF|JJ79BdT zXs%1X=(&n|u~9*zT?vVx>f5Xz&-aEf^A^6p%bwWwmZx&j8v-%&4nGgKGuZm5XJ9Cz zOyng{T}_?0f)20;78+MqV6iSBT(57ozClVF?(|El_-L%;g2bEkkPn6~1e#i2;mil)T#W)KK@YwGp!G>@6HGGc*%_h~ z09bY}*6Lv{{^0(GEG)qo_jm`^|BwgL=vwjwo=a8vy6tVmT$emYQAQIv`muO&i=K9K z!EmE1&u#t+X8uC!1P#uaYwlTQu_f!St&-?_B!QGpsP;m!lwdj)ShSgh_J_Pe%~I#q z{H$h*i9Y`Uup0)*O(ErPwE&xoAv{|_jTD>i(K8jGj-|h1G~S3ud;8&5Kc7L`LmnP5 z4NzT`6U4nW?rcg&dsgHpu-gklHM|&DlRD?pA92!NO+>@St|Ci%zP5yMqh0yZbV`b? zgwaI_tuJ!ImVPcNHH6zhsoQl^g3S=ua32gF!a5v`q(zB!{@Vy5@svVarRW z$e0MCjJ;pwVE+6|-=kKD*{D!Bh6`Rz&3u8U7?;5m)lM?p*6HeELZG(bmXE@ z6)A*#8V0%Tv2RLoFJE-LR=Y$19y%~n$+1LB=rw{bd;%y@ zgc5b?Gk8(=!gj?4sxbB6iu=x>sM0oVXM7#Uh$JODGzfy^oSD%kC&?L$;;Nd*WlI z8$ZUX1;ym0OJ172Ju|0eeTPLC#}e6MHS{hSuu;(T@Xs)ZaAkRUIj#9;AXGoi|JVT| zD-c!8U8%;SzJ0ffyn|Gw$qf1JAb7y9?o0XQWA2uh27e+6fGZ?Y$xX-`-|ib}%r4$+ z@$)o4RW4imXP#aCW0(&3bBkOH{y+JYS8HmyxUgW3uR0@tH|RQR?EKHUZUWrkz%OWa z)>D$<9g!>6@B9A~p8m=4j|=?|pWeM5`={|PKnIR_q~@{XZ4#X#BMDty-Jn6my1x@f zd6lzg&n^w*BjrXQ%V2k_$IHb2gHHxGQf$;1XvTBkirsYu&ZNoG*$3?mPx-w8OxNT* zdjEE*$Nok@=npCkhX$ggIvpX0o>s7R?}La8RG*waoV!uVg?#1MHNC<7V`18>kNrZg zUXE{k-dbdwzd-9TKzQ-5#K1q|eE!pao23~Lmpp!P{xQU;=6xCcq%?vKX|>00Cl(Ot zNP)0IwR%z+?!$)<_54tN1IknH__#RsM32ftD)& zs8giB2PncK;ygU2vU0Y6Nx};LKD&GIhZ~oF3qaG;jp#a_%f_&nF!s{cep3FQIgbx| zb!yy-GRz?q>XLxS7~&qB*walx*IAOuaFs8uQc(e+(>$#g=qyQsE?B3W{khQu#yvnd z<_K6#0?L-Q&f?{Scym>8rr3NCf(*>o8tQhYqf&~_ISKA}C@_0MOtvtu=sOn`IWd*= zGa5fLm)a5`yh+u}t$M2sb0eZk`c|C0@I;<^ww88kgzW5*hYqdgyZC@mS2W`%)pip; z>OMxHKYs4!`1}^GM}65sHmz+1( z*F}V*Sx97YnX&6uZ07M)?-$^yjBe+s<>yQ>yNg6_<~p>-sAZU`OK=-6SoK~GQ1$n_ zB#SO;vtcomW%f}M(kzDZk`{C1#aymQm0PTYIj@#;?zpqW$;tA?3XmJHy$ayJ+;5e4 zUf?G136J-h#jko;f_x6B?n(ts{aCBFybN+H?aox$CdT0w{g=pl)ELZt^_s6_Gdjox z<}ep<^Q)Hzdx!I1^zmj;{HvkBZ9gwcec1~2x>HI?LG5wq8y_Pb(#$vh41JP<#=|k_ z9HN$}O_J$Coz%TH00&bf?cP=bl&q7JRXgTVxOH_l$OKy-%q~Zn;sh` z)wgf9{%S4l>l)LG#h7J&%d==ScoQa6$O1PJhIOr3{G`Wubhr&7X~V}eQHU5Zh`=^$ zeS*E02sKYAKP?qh+YFbQBe;J6C#XHyrCxROOX0yNZ2{bJ2O;k{0KglvD`C4tIv_oW zx^F$d#FiCBCFU`N`wqZq;Jg`tj4aAwYhHN-hm_iDcZS0kW9XYU!lgKQcuFCN^gP_j z*D5mVj2A51xMAQn0Z>QB+h4rLH9I{I@n8g?#CR{cJCO9w@bEBtI!e<3LdrzoIreKB zevjs91XKkSD|c^%Q4Kg(1{61@^C-9OAlEU>dUdQ; zNp$-Q=-gm?ByQ~AisQ4s=yMm5UMBBXyk&S%E#);ctYr~R9S%pa(+H3kve4l+^<0hqM48hDVx_NcQz5{lpcca# z_n8?PXFG=GkMN7sKri70TsjRI89aESK-mg-SML025Xpilne*}|EPe~;xKjPL>JP>d z+IG!dKO6Ta3yX^p#EizMVo=@|hidVJog2IwJCS;1XD!M4aKe=mqDP}!O zhwVUEzR|xa$?+C#m6!zjU4tG3O$}00^D>i8)joA{P~=|aF;_XtE0&N1|V zJ*OzghstdYzgaB}7LEhyV=V%7sZN2k`h9h^Zg=bnB=58+gcJZc9C2h+He#6#4GmR4 zQW6*+eNJ_Q79zjZ(6_1;1fKKk*)MhPKyn@e-;=f64oBC4kuKP|5MY2OEC=j70>VNS zyRMF{dhJ#N=XVcenKjv><47l<`|^H0#1`0x!00^&tk6j#yBDE=F66908kP#XkG3H0 zKzghITC~?$>&)-FTat8a1sxmM-^YMo zBIvRl&VS-pzQXMcc>biLX~q+Y@WKV_X4ir0KQ3JC<(Ikouy@UmCTs_I1ly3XIoR2C zW6iY(e6LUqDZQx2lgz8116Fg^t$zO zDW0cB1i}g%1*_^+v+XTdxMt5z-qvwhVMIYsMgWQ}+AKrGed2@8r$|Q^p%#m*LVNN% z_GAu=_-K1v)4*tPB#)pUY|fWc_Ic*5Y+0ONppiMSBQfXG(z_}jl-ApvNwnZ5sDjhG zhgLi(5>+zUDqYCR-h4^1Vb$zNHCQw$xegI!eAVca(5Y-+_sMjcfB=6CkR3QF9)7<( z((u^WZ56StQ?6u1EK6XZJRv*0gTh%Kp6=}K)_u3OjpHzajO#;5?U;?r_);5Y9J;Dh_I>0V_+N)4-VL>my4l7=CHdm z3KCvlp-X#SZ0*^wJFmhezfUMGuv4<>DLIQ!k=spY_cRB)0R|Bf?mlv5Os)^}fEGUx zCx)Fz!5r_4ztw&XWwS}R9Nfm2WBO`{bdYsSRTr11RWy0_=3YGz^Goo{2!#St_y(gQR8s_`^1aHsnobys z{=ou>Y_>zaZX&u4#5GS3>rRCs%>=`btF0ZikB3=)@Foxu@{>W`++@x#417(3jec>n8-H+FY- z580bjzsFC(mi^q~@_X*#Jp_OmKimdDKZ9&FjXW*zrCS?q>;$V*1A~K^OI0^N-5xO+ z?QwytDp;X4_3Rs=lU)Ed84RL6_IMkHS)}-H?d~ciiFrae8A3VHnIH_%hyA94>JUcO zDW~prpS#Hulr>WVP-cZ1Y%DiMQUCmYQ&!W|pHr7a{@~+_RKCj3DzIBUpAvKwb_!gD z3H?>~Xv2q7ipA^ms(ruYvm8)1`7D$|Z}xFuY9S zqO<3lfHx{$yY}%CwAQy(V-On%w7oz65}#l*s2LKsK+3@>L2NnBhgc{k4v&l+6K7{s zz|A58qhP3^cC80@=Cymc3a1;68>oGBD|u~1!mx$#L$&8S~ouuo|yp_YoWcjfmU0=Pu;)a%e=A)Q`qJtFUktIWFJ~EQ1JOS0)Jci{ooS znC}VXpVRPWkzIYbPLnOP^Bn>J(UXce{@Mivh%H3n^3Ts*VuR;^Jt?}?eeWczKBEHe z0Ib1b_dNn@P=v6vTSG!OfAE9b$$m75SO;iFXx%3H^@de&w|)GxXV0Kwk4%{GoQ=W6 zl1`kvp~vxL$Lf@7A_kMV9_b3tc{EV|z6^VMf-qA+Hu9OjK&T@;#xb-)vps85y%(eT zb2$wFcM7~FLwFE=ID?2i!S;cN15MBh`;sl}TA@a+^XOv={rWn}(i}-^G19ThEoqTqlCM)iArFxY{Y``jhSTVH-*=}13 ziTAWdZ@M21w*iY6qDmH2zlAfL8m|cg?A|BfrGfRZ#U75Fted{FE?z-mt;&kYz&l50 z65+Ifjd1~M3jPa}I>2U|fBhZe0zi8m9v;H*h@^r3zMtMJX{jnK-fr#NU(NP3Zr`3k zXgoGHHc?AQnMyI(MCA6Tl@&uOl?^yF%}jenyxUf`IeuY$AYT{egGgel^7FrE#Mx|A zhX5X2d<@L{$?+i~0W%o-nU_^eMz~u4{)3lOPI-dO@F6R73RzOX1xIh}d*rK*nDEIs zU;D$bIOt`9sx`i;qW$*O&D^40OC>Mg^imr$V!LlQo$jk@DAUnUR$h zF!CqA^u$O0{p9}Yqa*dJ#*%afoSJ!o{+PLlb@`U?K6c)t$Y7$em@jALb?7@8ULeq* z@==!&eb-T5mJxA^LFY?8AFazrHSQpdZ%1rREs&E|E2csrL58sudrO9=RoLDVs3p6; z-%Nnzi-BAjdOSjqG4F5AjYIGpBJL)3FrR8|Z4C{dIB4~xNCd5bT}(P&#=JJ&XM7l` zIwd6r+ZkFwj(F2(I06dl>RA7AAaj7@4D*DH)dNC4&FcIY{!|bM44hAcoEvrtip#hu zm|?8ll-_vU3(3^SylR3Dph93-84MtIB(R6nz(PYB;2@`8=fpD2hY|m|dU~>o_%R6+ zh?fD#eOdw42E_M5UV$ox&rl`n7fLuMxt9S46oR zu^y$|x2yR*&`yIlw0*K$cj`-NlXd8L-WN7)5z-19tu8lBRjaD1n(a!$LYX8h2ceMy z(RJXQ;L*%pS%MJ8>G#}Az5I`_K6J*;uUoQ5>$~8=y^8<<4EU>u9Xn8vM~uZX4BGfW z;I(!PCpS__j49P7^8IiRj&skE@S+cdFHjzF0Ra=ciI2#xf$q)oBw@?1zr)ZP@l#OS z=s*G(C0Mp+zwjiM@*KbAj(}Wp3@Ty})KiP**juwaJKyjp9kmU`*&W{6iD0(1v@FF- zil^`Wg{#nW>tBJ4!CC*Pd>Mxr)5) z5%I9rYYLnPYmLV}cEMX&G-m`H;Vck^BZZ|-)}w%WUB%_$vL$5B&v)F2#qMt_KP7Un zv+8`wbAts0h*goUP32649NXirCu1O7YaAxJgBY&&hbtqHv%o2$#CB7@Uq&X`;BYt{ zMZW7nV02Z%8LzqzR(G!O-DADB7Y=}PRzU{{H9!r!5yfTNN>Yb81T5b?f$=(1ph|Tp z3IRv(OBLUqUmDxNmDfVJi@X6y zSe&tcww@&r;egmQh^@*jt#7{RclO#X6|AaZjobP%z64^}F+f@-J@G~rLw<=ETo`J7 zVwzbec^Wn{FyJ!bbK_uSzdeR~=1ZY#+WqwGkoCYj1$&T0i#acTu@hJhi*yIib)DV7 z1M;`3j^5202qF(|oN=y8I&brOYM6eh$tqBKwB!#R_IJ)wsmg*qDi(u`TpLNk-B4Tm zII#vh#%jc7lBR;qjeOuW_-L)_R-H17{`tQ`4M(yaCH>w>MlN0(Fau8k+$5;^!N8(w^O;4B}{ay*u|8ePT|$tM_jEZndj!bp82lQQ-+B-JK|=y zR-r($2}e#(_Yh*bBgy>Rfk_?NhgVt#TVYp^oeLqE7NEB5G`&GWmNTBIg1-ETY_PMl zg9}2a0Pl7*?mq4gY4s~5iO|f1QZpe#^Xq+Xbn3-i zv)Rtf$#%?!#mva>khQi5$>H>i@{QOYdZVXbxubiG@pti9ar4B_*n~&fk(T1*iQbO5 z)305_==-8)>^j+CHzWwVLZHoel2HyxD$*-PoUqqbM1vM|TfCqvZ#_~6I_glGoqjx4 zpkMu{Gaf%USZ>@L+U2?82qo~PZ{OZxQT+o08oQgz2weC6?Jd~m6)tnb2>uw_{x!3) zmWeQ#2#OUs0rq^#-=y9{hzC>8A-vw}1owgzthz44g6%-M@1D2|e_yRzYIwz@3Mtf_ zxC$ggi8-UFW5!PuBye>PrwbSX2|2Aqym}td8mEZj0*AaeA%n1Ea3)&_w5A% z!jXf_<_X}Bk5?3xpytxSKYp|T=Z)3&u^g)|r+|RKkyRe&M~BLz`m+RkgaZz`3Y;*n zjhy=t!gU9;Q)cJ@j!XNsGi+QqfJM+|O~bT=kmT@`aY8xF2Z{khwNJ{iUEyE>#emfQ zBhFxfnixa@$dD$8P^wD$>pO}Y6^hU4vnx^lX}okj{1RIQXsWM_*PX&(1K~cQFnIK` zLQiYxy8;c$V+W6EW@79gdJX5W7sgu^>Xa=uQQF4ySr5-E4N%tMxT}<1{(8pHPn*}X zR~8MEx}s%oDfAsh!@VO_If$9RJx(;$K6$yO47q&V5*$aQ70X<@Li8bvukPUr!nvsUNl1%iUIm{c<$aPbhsLY)N3}kNzC>R2E9NQfnhClmk=}j^+XU0q>Xz1&LIG=o8#{U#2~s$KlAhLB2-aMpWTf zz(+X8X-~EOw;p);7w}KMG*`eV;N$hx`zSqhBedq~5e^gUls z8w4}&X#S1cj1K~6DH7%sJO6AwotX>$cXRRG>rni^p33+`PHZhjXbFpf`UTA8AjUp)lfu5=a8gSH|$(-FdcfAkmG zv94_OAUrhChHs6i-VK~_=dfG^Xg|;wv%& z)mmqA{3${yP$+|Rz2cmLl$f5i?6W3RcMMCRjIYptGenirm3w((F1XN?E2e!dI$O3*Ja9p-};?rJRu5`|0ZxhHXn2_emnI7SpodoyxSJrF+is}ww7>376G z9~V|QT(d@i>xmT_%(1s=l6|1Z4_-sN*TS8v=3-oQzf7;dKPxrxF8KY=_f9cQiY24YJ{5K67 zI!yyZ=ZoaUqB%&rTCY+P{E!OmFqH&Majx1<15>m$AuACldfB63IgqgHMzONOSBLjn zqWTKk(c<{{{^xs2f7NkA*}fhc;z$VqT5tZJArpp4p1zO(q^G7Dr`&^6FyrnnkoQXN z+?^Rxxlkg-d7J$`-Ph`b7tCWPD{^uI)Dp*Cx=9F?)mYQ^WH`AKgMHFt+XDSRyNnbB zlJ%g!sENWDPbte|DWvJV=4SH8j>E6u1RCb?3|T&Yz<(s_dzvqi1VM7NUm-Gk>F zry~flKudb*`*&)C-qF$HwNC_v+9i+h#ip2I)EP6Hu0Fbl>1v%Ay%F_~grjXYvCYn+ zfYb7@4qJ48k(u2N*T2AoN+Bys<1|o4P74T3k_UMQp38k{pRcnhUn5H!CE{sV^6)tu zwx;Jg*?HERCFF|HFsY?dk8P$eLsPmzWnf6W1SShkW(?$MfmSR;D@%->P}s(Se*%o! z;eIicXP_eH;_Aw8sei`qk%;m~LRI@VpW@s%;qOq4F0W7WPw!1Q6{#_kZf2V>#8qrG z-2HIff|f9QSRK?dC&TUN)or%dkzRZ~WdP87CbeI)rbrw$|EnV1qX>XOwwlX%_;=o@ zK!3Nycje))g$q*tTP-MUy)Oes1lKg64`2$*Zc#B+bMa} zZ_%IR1&tmga1QzmSvptD`hfY>5Dq> zbeC=@;_C4}#GZ4SRY$E$H!+hEg$FL;Nu@Za-2c=B{oFHW%&sHkmUB|*AoP(*Xo@dM zxCIk+p&O}g+R7ZZzdPM5N+lfV(`v`XZ`+cj`M&nH#RPb2{3*@vWPfzNxl)6T@Cs#> zjpXElN`KM^S#o!lr~s+UqOtkXwg0pi2aNB^*ANyKKHh+La)pwQ=h*mcJrFXovi*~UC=IMK zHt}z=D@m(S?gBA#eucrerhJ(T?zC=wi{+|RLu*o1GrQBrbTu`J>Cqrh*ZG|AN-ioT zNjf2bNYQjEV^RHkV9F9LVWtlG>k?(-Y$r59Wq)957GuG)U)iuaqXZA0Cy{;=3iVb~ z2a$0(ElD9=o_hsRa<_bEnGTFhn|gooex1iRtLevI-mBLt|Ee*E4SJ7(xpoq;O?{WH=cRgrl9p7#u z^k^~G-9o2_Y4)N4eL@IjY|bLcz?cv1>uIAKm`G)bWK4~G3l=0kJsE zRN!De=$jM&N4drJPu5je4cU3dRjrKJ01v)`fP`Fss-Ir!a;L>$Z{!Z|47Jbe^?2@|wVAkClqJKFW|YR|OW8gu0RlZ9TD4 zun8?CwaY2BglNfnZeZ|{cjX`d1&5cQO!o5|+(ykFudt;3W|GRIA06@Co2co{rfKX$ z>4D~U8WQZG#WTPx=2X4{%b*N~iV+-JnM|>8ZkasCIrYax)CKy?s0 zKwU}Vb&Z^5Du_^+^OEnb*e0SP4IE3P;QBJ_Ot2C35ujaY1v42fErp{&PRkvdlf8La zKNh)A&UWi-j6c9H9snK|B&+)^)X^|bpwht(hYuR8Khw{_)@cr|I9sB(0@` z1Wvw-ZZA?Jf1IFiHGRFDae9JhZM96gegvCybWK)27wgQ2e*jbeSD%I%LHF5>nyD~4 zgg~ZD7me0sNGO+?vnHnQ6R-!%Cl@9TQXQ1wD4*v=yAw@Y45Vf;!e zE0WJ0Rar-WzfVu2Awfb%)4vSbo(gV5nnMmz8>yTI?c$@7TT1kAKQn>~=({d<6Q8@2?Vk{!6=L`tzD>?T-agNwpHwtf)@(-aLykbA1p_z!4I3~! zc>^20Rs8+rbWmzccy-M!Exu?T62_ZBuckD_QvCfBE--$zVtb`X@(kDf?cI)5 zUIyxej-slGT3{_J?d#SwS^&gCT4zC>bL1#5?&DKRlS{k{zcB75y z$8||eo0-inHU_NIxf5z>e2(*bGr=|%v~pqdFrzhN_WJQvr7JgBSoxv48xk8vLTFDP z&-$uWgO*|6q%{=S0&T4^{lVj=56P-2WJzRAbFW2v2pL|<>yReRm)GJX@^N6JAQFz0 zR&STDRAEXlQ{n0u&3-0Cck8osd`N${Hrrf9A_jxv{bZt0D^O zj7E0)O3MAje!6r-oUfEE1UmQ;bN@D5q^l1V0PI1tBJ@y^5_w~$zr#`Q$J14(3p7QS z&aqSQ|2n2@aT>2x@-Mx2rDOC9t}a4>*LMz|n`adGd@&QCM;X0|)#_i&cBD|oZe={0 zwZMno)xW!RS>T$B z?{DWXJF+aFtBL&etRo8t2Z!mvOXp#k0Uq=7!?n(ZHvI+qgDyz~R(hwWe0K4L*puV9 z4;~Scv}nBB5M=?5wrlm^+PCm0#|O^8Pk3y{d|NI#Syw;qROH`dwx^QNuGGIQj>VSx zd1BSrnvzG^8K&?e+E+S{@97RI#BbeY4;?<}_U@Cz=<{PWlnuLSV%c3QmJP|axT-CL z2pS!t8j}q|&)+yucWFQ6SQTmSYhAhReQoiWjs7?;{#gs1IVa?lVaj#mmO^7hBE;L1 z2lxHDFVXTi6F06$iPH~ndyG;R9t_ePe1(-|*Vs#!>2_T7B^km2Kj#-7V35)#f<__id!=cdfBjK&bjR8r+gK z@!je;cAQLNBkgqR(YrjH!tW33@&zGe>RCL_>U)szS9#xO zdAl9WecOq>j`Sp&x(oZZhLv+}_tjriQKiWU-F7+Vxv-E&!8gMp6ERUkO_~{|H(6m2 zT0);-f0B;~&1x#Ji0#Xglo4K8h#U ztoke!^vT(kl%SPsq@$Rgn&hyCS*%>FVg*6szmFQr%IO{DQR?ksj(r_T@7--RD^XT( z?{~O%SwDwN@q*Z$uJLxmQeBp%4+3>EUB&0^PEVJQ4|fx8BI0sTl!eanll_gEgkUEJ z2L}k|w)v_ZILrgBV?5Uf-8m_&Br&$j%dH>Mn95Bl?*1NUuCpDTYc47yo=?i5MbKrd zZPjMQ-4V#4tMO^?9UDWRLeRnyd*w7AF2oi~DT=-+vQ|DJ+$`JQHWtZCn zLCTeHNE>-_uhPmE+bdt}d>VJKFt-*q{%JOBEeV^%T8$UKT6c7?O^P9q*09-~*-eBn zuikb2-D2m{%*BPl1W&2)8ieHct$Ca7OTsTA=Ep4-$F*~?_-p9nYnDHV0QH(bg_8c{ z|EW&u6p0KsKJ&l&v3zbBJw=bYMyne%I z&<{9UaSaC;804;>|9{2NBICor5W`3SKPtPX9xnNKDnXVwpIq)pxs86j|B?)=>QG~@ zza)MqZ&r6tGR&^izxX#3TIU0Nl!gAR*It!}D+VK_noSxj707twZW%Zm_LmIgbM_Kt9qi zZw68HsNi!dl*->pkB+{&uv zp<@e6$km|XTuq$$t3C=p!W4zx>p=Y~4`JBRP&?m^FR^8Oe`uAiE|giGOp-}PE}tO4 zwWr&*bIYcQ(Gc`ZH|jGQ_p0w0#iM;c0?$fh&u_@o008FHHZI?8gx|*ji^M>;8@vp~ z?yIri2USg^O=@{9-K|}lgkqNTW_OX#w&C+9tW^WGQTD>ZiP04L%q9gl;m?elCia+$ zNslx3k*@N_Fr+7XghNM;*@`=?wf1WgdsP)^W9Q$0_6@3)Q(p@0KfZF;E@U0@-ae+O``o4BL z`a6HazD#_DPT*O^>!uCwyt)p<8ZKd)kZK`LM>>MRv(Hi$%skZ!!kNj*Xlm#ejTSub zUZ=R7qT`fyXC?Tz9JhC`2Y^-$8>w^x|BU!|$NnVAPajditeS{*an>d>{z{|rF@sZG zINB89`oa??QS#znchln#t`N&VU5AxHibT*yaL7JOHD_p~zM%Blp* z*YOyZSWRnc6goqe9HvQ$D@TrEn4%Ybf7xyHV%TX}1?uNAau2;1Rt@Zi4C`vpO3Q@% zHmmiLgvg%MC)P+8^lW+Zmwa3KG!>2*&CdOvVM$oErL|vwyo$MPurfcr>6!h0x7{sf zAMl5d#J@%?E^*OJYH;3>ajHu%&c4E_ojMTwrXj}eT->d=hI}6XnVoR(gFcvjp5UN{ zxxDkZBq&s;vtqu6queDh2%&+dL^XxCD@C{XOtTgdt=MJKRhOmeWBu@5~#&HMu_Y`by<7wsg8n*Y@HDg3+`=&Ui+K!?NT z>IOx#)^EnTR`ZBG$LBbL{n(L!?GyH&L-9Kzu>?g-Qurr|5H83=54e`ul!$%t--EvO zZp7gd_YaHT<6!=m$FfDnpC(~){KriGyaHoJ;`Ofz5Q#PZ|4YHvm(Dfm-hZiN5|%&@ z0{+xz`a*5FN|b3>310m!95PL8hWk_Rf0KbNLmn8IzX$uzHhNhJUU=rxX@6syAo@pDf28qS`Df0@8YH``&+uW7ZpA1LbFOz36Efo$mk~>x8+jw}b zq`(2Fmm0*!b@jln9HJ?C#Ae8;R|`L>uL%_8^R~s^r)Wjnyt+rG5O-E`-lK!-cK6m^ zx_6%WP}tcLVl(;}Mj_=4S9YQw^2)+OOTR0wjB4u+1upY$|HGGW2iRTVX{cPofl9RwSQ;7 zUrXrl!(y9)f%lRv5Ok1sXZC9dY;pTCYK#&kCir~rc{1*_t;wH?y^muLb0I|A%tPT5Q-Tzul0Dy#OW-AP&9Q~uLgnr z2%9ph6P<9~mI^CRpD*nb!C*uC+-XxMu9Bo6=;~Z$p#b{%-l4!M+@2#zaRbo(Jak>z z>u$z+`1+g{PMJenyVvEKVXvImwJA3kAX<@k-|Li`kP0RjKE_wbxtdw< zQV?zM)YJzAXB@kYtz4^W**}#w1Rh7SJ#yPUuaG}KJ&U&`2$Nks-PERkN8_uvh$M$0ujc+{)rn7E@ zhe|bWi_HkooXj5wd!r_t&iiE~mB*y@Z>w)f#^Vq>*Z9WA)(?7Zj7*XIFB9ipmUo*XXe@e{hhqpW!&mp47MoRB zNSNBsrSTX9jGF+-bX}O%*-+uo+LjF2=4w9X6SgTscZe9w!li~mZ1RAYA6v;L+ELLI za@V6Eb*?o)&S5xU(&;uT`WnCH0fwQDPE+8>ix8Gj<*BdgC|M@ zUA8(-Tg4uGV@Lyz-JkETk8M6-C9LEt$g{0p9Iem}5Negu_?)hh1K(NA>dtl0HP|!+ zlK|pLv&11II!=B+d7{b7&TXx;S5tO%6`s@Upo?)ad~spS*g~RFeQ;ypONQp{+FKZTyv`I`f!a@BSG$?ngHq zX_ll%G8(oN{x9&x^QdXl<@&oQdrCH*NN3>e-d*xsv?gHjP|E zad1HP3xCBqR`d?viVth?6(2$+bMd$s6?GQ?sM#8iv->Pzd5+xyZxU>QQFo2s!O84K zTOpW;3MD$p=0Ivnfd1^0bKq!I>Nid3In=62fae~MO6m(%C}6{@)vVUPJy#496%jfF zIFf(Q#Wpt&HXSzRA7w%;=9oX?s6`FFDb-wFKQgU4aLc9@dUwGq&694VE^AfcE>x)G zPm!_mV)w zPA0_8bLh>70PeNHqe)mO+2$mxX0}9crFqITy5pEy$lz9V&L@r(RssST7;RvvegEeO z!KWLqBHZ&mrIOYvk|L^=ice;R$)lcEbY9O)_T;0*Q@K(zWjcV9n~ExL_op%_j3cA!y+ zTDeY! z)kAuZ_^Po*Q_16)76}%HYmJSo--*U>TS$Si?@x+(sTHT2-R5A$+>5ULeXj0($f&sQ z-LDc-Y)Wn;)~`X0V-R8Pj2N=$SR3V*Yr$E!!Vd0gpWEiUE%G^_#DKx{!|vU_@|sg& z2dwU`?u2d%--GwVcJew&%NJ(6!xo@wy*e*oY){zt}F|Qeks4)~bXtHDg;d`gq;y!Yl5sGX3n~ zBD=PWK1arX{Q)o-rHzb`k$ajO5)12AJ?VpMw5LADN}{RjR-o%7uNO=6N?53d9NiqQ z7py?T4P6WJRPV6B?*<$)%7EDlzKJAauAqKY1u}Hp61V54$NTG(rG^=7EU1`MTnsRO zs_mkVLloHu@xMdw@~!?hu81itq<6wP%GY)wZ>3!6-Wh;o!R&&;fMK4kF{AvcuHg~Q z7mU*~e8gnWBc{}67HMI{P7;xX(X1Ntxr7~!WWAo*x_X!t6^>mso8S2A%S)2o$y<73 zG1H21^{^?^EBr;m;&>AD2i5Fci4%Bup_tFtiKXc{QLDAdhkkPyLH_0?mUKtDSU{n^ z1KYrw*8~YX7RqEmU}oa<*gLbT0|qk_FU-W@Z8eaa(dC)b!E}RmnHkC(7#IeEc(n^U zR^wx7oQGNSlF@Poq)c*D%*}Vp&kw9}0=Glu*C;ZyhJf_5I}@1vu|EH&FDjzzpJFAs zMLN;4)VUT(W?~Bs$85#Fs=X+^r*evNUE!(IYf1TgDG zXYq}1uk>C(emVh9lKo5BTt)dBXB#KtcNpMr<5L!D31JRWq?xCucT*v*qHk+2p8-af z;nGxu&sIvKM}ps$YpuO+A>4{tWxGk&5l)o$#x{3oE*E=Bf*#u^+bfPg2oTcMqopRV zmvx__zB!L@UVysKAb09OKt~#y36dF=jTQj1v_1P znbjXL+E#oX1Yht2fej7yX7(!!M9Zi8m)Q>uf?C{Sl0buc*~0UcRd>+6%&y5YV?Ezf z|LD^=ecgM;=abACSvjS^T+h??H&D5UCL-upH?`=X{*)jxOyfAhV=vCxm<;h~Hj6Dj zcRO{s{`n?{)&wyiaL#e?<&?zqa+MS8Z*5(*# zOL;B1jr?|rhOOISTYsE+tkaq;@eg#g7xXWfi&&7P*zO+fa&kGw$9<5^;)~*F;yWtk zbqKn_3UTyCv@X*D`1_m80O#2@ltOHOlJeJ8lL~9v!;nXI){-hm$ov^7h(eb>n@X!C z$NT;Xs!>d_-9+K4UCk;5&r9m+d?jB+c;gwTy|*72CHQ&Je|{fQoQCOKJr5{=P+0g6 z=MbFvGG!N^-f^?{qMp6Zj!(^e3->W0klCG1Gah`2DH})&KxGE{t=8jPfoiLJGEIWO zU-0sIjUDsj8CcbaQ%7`2ad7GrUL)_*iv#18x1kcoK%m5N1rggh{ z4rbAVn&z={!YKS~t}h!{Vg9C-tg)V9brMGD%g>%tr=y6Z>KjlMR4245xV*hmqa~sp z)aHxnU_uo3i@d|9#NdikoJK_!ds#_NqeeeN#p@YIkLKH{_ zNnek^)-O40(bzdJSkS`F}{mM&<`B(~^h zpIhObcc{p=xzF!^_PU7Nfv>_cxo%mS+CiqQm4t&_9&s-o9A`Dbn+}UK#!H3M(yQEc z(EH90h^rHeDiz^g`2VdCFWAgeD}>W&`yTB~G5|34;R7m8m}s`XX5#$a2Vx4;Y`kRt zm})u&U}$%NU=X=uj_S>OoYw;Kteq8fy|PzC@o>fDfHF(S`5pFEpqG<^r0gL=8@KMsm2_8~H$HXY8x-h|ArOO|IG zA}_Pl#aEGlLSz8jMqgbiQHv^6G=Ffxl~IEBi2aU$XgE-R4Ucq(&{euoU0P57MgC4N zf>i4Wh665G{H?E zYnzmF8YsHt4Izx8pNM^~?uBo2 zvKmQUqyy|fAfQ3j~UP7{cS}VtWDirVF;H*cgVPVUNo~EsmW1!Qp79w-@$c zcQ6=zNg_%-KBWxOT2@_X6Rj^-P>;`^nEHPdbUE>65lj@9s~r%?jE6d2;jPF$3u2Y(W%GU>(QT& zoX*-FR|s9OCAKv1V-tFRutKf*Qc`h?{Hmur?HwGz=O6MS6o zCuN$c)y_-=1A7{Bsdr|lxvBtEp{ZK>P6Wuzkc@x}N1=HT_*+%&mZY(5Xai2!Vw1y8 zSxYS%AYdXkjs_P%!o#(Kp(h&RW<`<02~ndd=1@<=y!+aV@g79A%cPm?DC@(7gLrpT z2)V}%HvnX|XIzLY$y7Ys=nNa%#7LdgC#R7>$yAS5(Xh~m{gOB{#7xI~d1=So++Jex z$S%{781?;DVt5^Rj0m#|-_?h?2m@l);l>FtN_s-cHv#g+puxT5!D*n0Deh&K%=PQ2 zttDAk;X}(-rsjD8@ufN&7#ek2lVU$N{aRNiKsRCPYP?IgC}m!^iI=5hB1$--8|ef~ ze(jl<<40$Oj7itmm|1M2mYDNO149P62}eEG4-G*7*=HFZ!kgiCJ~M?DV1$FN|5*p) z05!-k6m_=EVJGuLC>ST6QAeRwe0gjfT~b%Dmll`AvFr<~IUF2m-l_3^xHOQ5@g#*) z$^-)q;xAzBbG#zFpj8Et+8!D-q2&OzoK9eD_X;)~MQ!`=CW!2LoG_Z~Zga2D47Je< zvOC1&04J9kj=Bwu@0qI~6K|79EkJy??RwF)4C#5PuPy>8LUOifu}d1*A9ssuN=J!* zp3u6_cQXiuBnGs*JJ`l6)Z_z$n%vW514;z_2C%wfl=Z&AOrCoWHz0x6f*=78vb_Bu zLBoApT&*idcN0~*O!7gMkAqw|2W$562u3#b(X&ec!Gra?%YeC;RL2=wP!;>t~yk|gz79wlo_7w*jPAaj*I>4G4Ss8Q}$9b@hHZVo5?Il zVnW}}{V6lto^DOj=d>76?P!pEK4!v5W5n+W$rev-p}+x!x=!qGackoTgX_!w_yUju z^l^f}0GI*6Kz%>rx6CA1MRtuTF9E(dE&WvAj>^7%cz3ZN^ExHphmbsf+bw&-D0o)- ziozE>mV)FrxNRDYrclyvRO?3CwJZJFUX3eR-w$=KoGnfL$I)fLq$WG7F`4c1kTI)0 z)Gp5?qpngI-7$4<@i|k^%g8D59p03*WLz4l;}_vf`HEKeljQ*9+pMizT%;Hk8bcwm zzyfJ{_imcF^KjEQ+4AQ}p+!V$MG8Rt{Ve<8n%3K-nA+&hhxk_$RiRwP)Vevlrn7Br9H@w|V5gu4mK-|zE%i9j zk85|2f%{OZDxBNOMIsr;ZA*kcjs@&LPC?qCw}XN*i#@%~V7uiJmgMtp_{vYuBf^k& zy`L06y;i7n4RW8=IzogV%0mq*XX^&30zRi7MuVy)G|DzDW=0yh)e5|uXdUChRXd2d zyzw{RHei#zPO9b*1WyWrFwn?lT{1@E&KlP9OFMr31#RSejL?e_J@?*cNLfD{KQ*_L z`j_06aj*2@)~#*Gk6)IKqW%;|U)VRxt{Y*4CP%CqBQ{TN72SskghSrckfUbp6TxdD zo%7K>p@DhFHcO@Xd9Z!ek>F5Werp?dulF%rKYjgXD-VIhxm3VdY*FHe4|*yg6UZOP zNzjoqJl>H2WV&ixXbf-FFny>fG;U4?y)mDK0#U_1N5ycn00}2nF8nh&grcEup0yS=Fzz zUlfnyT7m+DNCBcY91g~?M2>7zv#G{0Dx+$sHWfHX`^n;=VCGqVj*#x}8v3D{HqH%J z`;(h8(0%RFIYFkJQ*HOrIjw!KR*RmZ6##@*aN^j%#A0-Yr^?g z;tojw(bR12F$-2I7=^vTWEUANP(PXPU(|>9-x$QjL2R071~vz}ujv_$Boc!#ICx&f zHNE&{N!>zWq#wRqFKP}p!P9)b(`ieS+srUP700TNDxg)QFezd-WVOzaS;GDtJE1Yt zw9}H#KbNDj!R3EjX>Txpqot#q!ObkQ|=3RQttL z(lXMbQ;Bb@P6ma2_L5wBry>ffa>AmYiW+&nGC&vW1XZ2SoZdW1WLdz$gf{*G>zlO> zx$CsMRq3*)Hwef49{R&&Bfe2m>%K*^*FHlnuDa)2ZEw@6&s85+X#~bq&*}vRWUj;# zi@4{AY}aGCoDcffwt{#@$&6-3$5$M;AG165aG44nB*}u%JU}mCw3$2BDijM-$EkZG zh%)x6VfLVinLh%yG0!jml|(daj{*jlczr*i3*!_YTS!Yi3xbKl4)vL>&Ly|aj9K(z z^Ja3e(QT1NzXAPGTg2#{H|Rc_cU}$^ElOES9b#K%F<9wvud_Y+lS< z=WQ14owuhtDMC%Kk513BvlT8|Vfsb+lxYSi{4MnsiVuIsJi0mC__uLmHVHsfV#N=W3@;9(Oc)sHai5@8N2%5#3ii80{T8Kv5LpRH z0{Z_!VPR}@6S=&mXJ!)KGa>!;7fh%y3o$LiXOrKfp~oW~DR2NyM@MPw7SoqNSQsW; zlaW*sfZ>nd)ac=bUT?FquS=tTF$;MOqsQ{{`3>!n)X`76|VM`K0#WE zr)*yK4ZQxA1N0#&{2wVL?u-=pJeW>J!DMow2z`Az`XAT75C2u3%w#ieOPdW*JI{Z* z!M(#jLCowaGUEQs3IuMlSEGxXpoBB18my6{*Or>~WhO3kNsy)XJCu31+o(v9g^nmH|WSO`qh9MwofL{>So# z$8+Jkx3Bn(8Du^MY;mXWa2XE_S5Tx(9lLcHxV#SIU`JCJGH|JlwAb<1@FF>jpx$NM z=Tz;DHKGWK3S)EQlk=?ac@ADN*k*KBisbfs5BTkO#bRO`6c~xN_bjQ1Re*`5$K2&V2(kG>CiiJT_p|i z{O92-g-~0KC1H&9)`4ou!mg}`Q6%?p{$0A8oabC*0wXyfm~t2GWG=szoHku>#C0gA z?f4KSB}P-_m=Q~IUf{sF)#a)$sd;6uicu%^qI~#K#7@a7f5)Zl!@aa9av| z_dw*-#w2wP-5C=^U0kl`s!c=`-icJ!UrQ2QC#0p($3R`>G@$;Y*4D>vlYc*@a$tK` z|B{wP*QYBWrr#a zLeo(hYIn1<*azigS~^c3{EzihU$<}qKDnj!ABSCL#4<#K2fvG}x6Ebj;P$KrBbrgJ z+f6z4Nz!vXekclMotMVjKmFEHd6^(e_JxmQLk}@L83zr?W;et8QZ5vjMeyy3B1IMO zDeTyevbp0XSq@TK$mcL@DM8~R3cOWpG_Et0*-F}0@-mY*Djm>29o zF$ei9nA+l@Hj^CTarIbs5Wp`)il$Iic}&E4$m6g&RKa27P&HUSN-yQ5laMCYKW)&F z%&)Vrqa)~AoX@VZYdLC@;^B>dG5=?ZXoAr zc(7M~InL#Ia|$(~=QoUnkS9sk)z=dVdN;SUJPswX_CZh;Y-~=FSWOg{&_iU7$l)Hj zhZ#1Fe@mkgl@~=9AWEDJ{k(}L)mz~cQ^u%m#n&a%MNI;w6j~a({jc>DJjT?uxg9z0 zEC`Sj29LuB1}!k}@GLlpSgHuaRm z$yAHC0_#eY2E`Hmh4yGeT!-%Bv>r<;1p_khWXgQkVUd=6p4~sE`Vq=Sf~j^PuO+;_ z30?!)B6;tTGpdgkIw@Y-+;dzI2$-ZICgU7g9g^#et$r^nVG#&H_kvVz&e8`h2|j;z zC60QWRGd>nhSvOC0YZ^IAJA`oPXi|-ZZ-4q%E8>H#oIQBfDr%BCiz; z=4m4GDb9D39br6%;p}zRm2*U)TCUjXqy@NK+T=QU)8hChkWK+4K#HGzMrhx$8r=hRm;o%#- z5pw6hhx>yh1%-|1b%}O`$*Qk@H!U)T*t=wXa_{_NkaCHGd#Y%BUl!)g$uVy6d+4af zHE{CKb0{c3=IHBDS5YK9$}8JFe=HyDL)Sn`mAzL|^IzDOX?m*_Tr#v)UKvV9`j#0G zj~gHU2`7LgSY$kZ(~qHvvP9kG#}QKn&{!Y~wCC(DWIS;ipkMOQyKmzxz|#EO-7jm= zbez<~*?W9zM6u2*W$u#l6LcNS#E;_fhbSG&P>4qhZgPrJ#E2I%GK4=`ynAlW-!Czo zaxpB%k~AfG6d-#Nm~65mK~tDJ8tTD^g7{Ub(RW{-)rp4p&w2|s@ws1h(%xoie9LWk zzCT%fti|q1?*aASkyoi)kUP>()q;n21|-jSes}@~qTY@4qiX?IH2Ps*&(ZjQDH}EC z_rCJqKAtA;)XHEeESWp1UrE192dkEY_|yK<216Zn&o}$i)6=EjuuPDn2i4WpS(G&x zKj=dbXDchK-O*TAExxu@0AmJ@jm01%6;7M9i;Ihv)YJyL{$-Nq zo`a9)PMQnU+qqrMTTwKS?zPl#>~%q?zYs~Z8RRi2N4d=0vbmw7k&6U&)>2Ku7oL>gsH;OTRK?I)k+tYFM~WPdULty;_2Y&seENgz&*yd`QEf7DWyt7gFN_fQY|wSJ_nte zJR)4SGfa_*)syYc_l;E_Z7FtY3Zd!9cVsaRV1!WwE~hvPL&`^^0BChMj`6M=SB$n7z^IdF7cU zgZa&ZmUm0+?MAYLR>3no&J(aHPD(3V27af(Gv$m|+{n_YAF^Hf1lE;^ehI5MWvAAg zx3EM$*C<37UuM6boyi%xrBhZG=J_;qKQI9 zvlFcbQB3n5!9o;<4s73uTQlL#I$~Ctv&+3h6y?$hjwe?gj)=lHxHQfdDskd62l|hN z9?rdzNce~^2t6U$D0h5{$aHJoZZ`s_o@ZL_)2gA7B{k&Z_PArIB|?B_v7jP@rKab_ zG(5iO(Vf^Ga<8zZuu8*)J;EJcxRoSkbNX)r`DDDe5}}g8q9coBfXeFntLkIla!{jy zSVG#O8l@`1!Sc0rmb0OqL~Cq8b^X42dEZW~RLuQ;SJ9}ur}vm!a{Rj;taLFSox}!* z9f9*cQqny0GEmE1lo%m34NdZuZJwp#fkU^KRy`s0vi<21XMJ4#(Q;!s`_ac>&ikud zil~`HkZ{%jK)4ofR1=JLxHQo;Nl&j*$M+f&ZqJrED>gAm1XT zAh0Wm&)2l^pTiqx{a9bDnPa!2zDpOcxFdJ1ju_hA7sw~;5^D@v&8fVx?Z`7`W7hu8<60I~|N! z#vW+-+MrK=m6V+!UyNynjr&Gnij^k@>e)80tZt#rLgGx?Ob)ebS0xPVgttXe=IQg)u zFAYrrdaic)<5{GHsPSvjkJ^L>Q8@3HT&mMUib z(LzOG<=2YZ%2LaP^f3S7ri^9HsaYDV_>wAX4c%x#nkzw8d zQI+X$1~EQUDOv%@egqo*K*qog$9cm$CCd=(63wncPWC!{9l)YZz<5|)0sDL~jhF6NJ>fV*5uF8jZ<@1K;jJ!9Nb02ok=AQ;F z(+Y&+F+(SIEJ8ee7?vBnRQ^J{%>cQ5H5{!oy3GdQUeH8LPDfiyt#PTbRV_VXhy`#O zJKQ!_`4OmuO|M4>xqB!IuKn6FB9ksP#2(Wr_pec-q{}ADW)B^eanu1i{s*~{-N#O< zH*PKcHEfr~$#xbBTKwtYxqU|!Hanh}?svtrtZ{#truH9UmT)xXci#exa=-AlJ7WNxe z2=C)_!f-yrtT~5hE8cxIl((zRF7qtya~_eX_FL@E+D*KYL>jmgQiC(~0GI8cFLK^Msnx z+=6zt`EfY<1-eCs=e%x>(P*8%`C_G6aklHI@tpdWmf74$T8dHWJ>gPClk;4)QD+t& z{|^ht?1K<7Og!V%&>^VUs8*fL`ReA@ZEw4t!FJAv%vdkYtAE>CtwT3oe%#EPL@oYi zb1+RxzJP2^YhFBG^XWifDIFrxc&;BfN~Bv$r)zmhM9m<@IcxcVjN3N)XjM*h+N(xC znBOzMomCZfYaq%ZntistUFWCbzgKSR*i@`&g8WP^Iiu~Zpx~{tfku-=B@_1BlJ=cU z=3-MOjWm`|4DA0v*i`A$3Mj62PX_oA1N}gOo%j;)>{#i2tHI>2+GJ$d9y|aede^TF z{7@05xU9-3NN{T`dM)Fa-M$!BnXNYaW@}bY;D^427PGq@zV<#%sw7<-0+CR=O7ESk zh0dG0+zT^9m83r2QF_2aBT?I18a!OPbWd7n6J!Yz`Km(3fYtywgD>X!P7!^x~vvJg+-J(ol(P@>_5)aJK(~|la z(4t}P4@kj-#V_yi5Y$Q;M<@pvHEfpD<1^|!D?`EXAqc&ujy24*xHDX79JOAAS}2zM z#9D3MH*bqXbq&UV#wTT`R;PSOIcY}Rb5>^GUIslZB^z$8jzb6`_ZX=^fRd&#A5Z)1|YBolsWG@~&_vxfzLSg8A zy}%xAjWr`tx>|Uz$N*6`>qZ!Foho~Mw@7e(FnKX_dvr4D<~4sj>*(1QTDo(bt9Izw zk373i6;}ft7MO=(6PVHcD5Vj{?l)a;i0^66s+=d_ZBNhRPLk#r!9v~%b@**!(4)#3 zWx$oSX`sT`0uTmv5-62BG?wfKNp)@koee;zJwYKK^Q2jpl+^3oJm!o6(6w_}pstws zw0euWFu#5)@+vliF(%GZ;9Kd#B3;+C*XNb(ox!d;j!Hr58RwiN_+ENJ2?I8ssE{ZD zi{da zO&>vf_eDI0mc1H7G^Rhtk15C(i%GA4a4R(DuavACKl6Nroy6AKeu z`q&=fB(hP*yXl>4#jrfDs?HXay0&@zJ7flGF-;Iu6`-6ROWLDq_4{RNJipB@|C*Zm zzCl~Su3?+t;8^4O(${q^-l`bBmpieXcBYnFI)X6K_AlnFhljc%8QPP^9SNUy(Ssr( z093oH$}O?#ZS**9G)c>DsBpchI=wYY;Y?t&(Q%vMKX@ATFy;E_w4UR5*;cJBPAf2cZGf@bn!$Rx@x07~ke?XqvizND)w1Sz z!t~NP6oB9;o zW8ot1x|hs5?wa#8$1v*>cd)9Vp+RZyS76rPvZ6+#bVj5GylR3F##v9>bw z!Y5OD;38Y2Vfq^UREl5ZxR61g(uFukZLwVW$z`p2AwDcYaBz57r$XTZoSi~ydi|EJ9ZOPd+-Oo|L$w))&2qCT8#H8z0rmolnpA`W z0S)^_#ek?jwk`~tWf$VI1-=V=s1N)^sjk!N-qJ9S#aZ{rO+Df4Qpv(MJHJYki=9z1 zs1F4+>7Ac0h8SoazchRPj43J-rN*C|@*9(&{uv{dkg=EA z%w5pbbbmNkU06Hs;o-rUvsC`~;h|-cG(}AR`T6;X4IKUDPEEt<@Aqu|e+E$fEzv(b zEV91_5c2tTGx)ys;$JjJ#({_Ck1qfWG_dYJv3MUe#s6!>-TzhrcMSDAlR{RbBN8~2D4+A37qphb!(O^uDlqv?-nC!Qu1-^GZ$ zYD~wnPJEv4O-9q5xZb-gIjNDMqrF_a+?!}=X>nP!>xI^toSJfinkl4m$J-u{Y$)Z@ zn~xh0teekz`fI}fGh9Qa7Bgi>`};*DCDQy*5rYCcZg+e6)m2rYq8l@9PY|ViS$gen zkznZbRaIAuC_w9g%GGieYwPNsAmu)3f<8}$O8HO|9y~leGLgX0#Y3iWV5q2pg2LFB zQc@_a6R8RnS;x1}f*#jI^P=86J`YDSRmmmQAgg-#{Ze(519DFA0!zTl?Rik{N!&%@ z9A3>=`z(gi5r2_8QsQv`w^-Vr-cnWgyuCsRvVUn6Qh(RkyJw3(`JvqYnM#;5HEBRth6PUFEu(zDZE3&#lb<1 zf-={7vmOGlhYm+>Zf?0Zq!@7CwDNfTaC@=xbXBjGAyVm~c9rSi6?yHTGf2b~$g+s&A4u?T@{?a_Rrr>E;SpQGBD)~nj`;gJzg zt7p>9KX|EJO1bA2ZMQyinrze3u_>E7wEr1hA03q`R+ACWH3Z$8W0$t3_DaUKhjTff zy9wj!+gwI3vyJrPJC1MFqvPY_X}m6Fx^3Q!lMYP9qxIsqvjeFA)kcIMNL?K@s=d7( zgHqP>@j&V?>>p)z(F`5GAUoL6`aR;RLG)p%53MDI-M|4lc zZpu*63RQ~>mG?VQ1WQzlIiXG4ZHU%tKAv`>C>1K#YI>pj|2CoI&svuVAmk11eXY7v zwet`m!@72#(xEbDth^YL@2857xwBTQGT`7dV!t_E!JIO8vbV3gm_NC`e|(ZPsMWOM zNQsGw!N9;s6a7Wr-=(u*H_~TCpp!-p`gaq~nePsb20Fdfp#7633_5|tegYo)3(h-` z+Y9I_-XIDQXGj<{WI0^_9lEPkBSuIdKNwJByNngJCF3`z&x1A2E`!oDZsgIBV1OAj z_O;ARlq~f0+&?3zMY4Vque+_&jFV-dImw zsnFlk8PIzE(IU4cda}mVtyYbs=+%Vk46a4)<@t|H^~M?=gL27u4KrZ`Bl^k<1sN&T zR7u5|EdMEJu(%Vws`>w5?yaNZ%D!#kbay-JAo91 zgSf_snx3U{}{owo~;-@Es{JKnqF`|fz-p1~hAs5tuesJ-bNfa0 z?WqZjlD@)k;y7SwXr6VhP&?|rF%a;E3Dvh~%i=GGL zu=&mmol)bajmT5uM*SY4Clpm$%25L!K5{78uCq>rj{whMNwH-Ffnk`t|d zo4?6`Ps4%M-w3JFw&cWvp-(nNTrt zcA^qtY#9>b`$J`^B%=+U-VhB1$qV3KaWtXSDwNpzq|xa&?1F%8=mqYdkBc0jM9 z|8A33D7|~so<{3n-7KE@Cc?zXu-SqpjVC91H+N11K?h|tHiW&@$?j|G>HI=3&Na8; zkt7l7s|p))6CzYZD)f$w98TC&D0Q}g%c<1}64->wY+<jC$g^NO-bvGKzIc zwh(9@NoUUKbCB6~YFKtnGY^)p+7Ranr;mvkUuiUIj^5J@wy!sop*$GzLh#MNxhPh4 zE4lY8@?-o!#_i8PB0G@d`Q=4LMdA8a8BPP|8O~BwMReTPR*1x|0sCW}(C_p*Vz45$ z#D}?2geV1i8ctL0DAvYGp%ZyX2!$X~Y4#VqrRjVPx=7~nT)ukg(p`(LuWb1m;&{}mXf@viQHxTF0mHiDdwu-hhzr7D`H znEPZew#%Mbxj)5-du)Px3`~YO6tz@ufDKbP^CQKqC{{47Rw@QMi=8WZ%@JwlP$(h^ zjuJ4=Yg5)HaF+=wR-jQVbXP58XMQqhmTFN0t+zSY?c^{|%F;1poE?(RPP~!)R2j4N zNJaSDjm`O9*^3&8o7V6Nxu}DzITWqdS3K?4(n@6Qbw7put(8b zlP$U=V?S4Z30jn?>@7qp2#^JYYe20s2^E(e+6pz9M;T&DOtMsARnsnN^r?zs9ah{H zoC%%L8cYysS#0 zcpXJ%f%A%rFW-&x$lMYCU|tb*Bfm zO!yv2sSem=oXGSKD*TOkA(9gon)nX{K}=RJ&9g=%|98|24f8qurwzB8q-6OL0<*!G zhtMy4K7`dXjAx|?S*!UTW;sq-gOHmlk1=jt&a+y zr#e%w_OMa(uUl|Mv#ai)KC^7R9abBa;Z@=!xOcOGzMou^#q-`QW1@q|2GTHj+NSF5 zNI*XGIVF}S`ht51ALud2U2&a&otd~!nI*Vvr;?4jTJ`MHD$!DY9$a+XiPqCB#S z0-g=vo@v#3oh|hy3#t=ruq)PweL<$|p%2`0LQ3vVuh-fRD<*3o7wpQ7Q02Mfv)E@3 z@6=A#GAh4w*Dkl42WhKY_!MiatM0M3L_|b5Uh8_+rE{^RG!iIr>n&rfIMRrS!_c{G zSU>e~#m>oDl(dnP)Ba48e~G0(JSz*(2=TA~nj?5M$MO9SErRVBmQDJUUS@W-@5@_g zdwX913S~o@>{HAwMD5|A$ZHW9z?e833nv zyvlW_Iq-3*^<)k3p@Cy(rMfG1Oq9=FRkA2D+RC+&BvoyMkrXHh{ zUsU9HAy|u9^FMP6}c`vsYAMyG^GL+wSyBf3aSq&UBFSZ3uQMX8a zN(J}*@z79o$U#p6Ma2FXX0uimuj=C6jJ$*gX8?9kk?s2p|-x0~G zQ{@840lm;B6Qfqz`*ulBxYI7RjB0?6~F}vd`l-uQ=J5;l^ z(T3VdraU$7yHJ}QGk*7OIyqQmn`k35}H##mpnp}`TEiht%<*Wa1BD$4UI$uMli%GxIvhnZ~zPpY- z8J3u%^uCY?7HuyHnC0sRjxz6t{rTyU_t}v5xC8Ox-s+J0`Dfh*iZ(SR{=GxBb zbJ>>13CL-*QmP&|Sx^Q7-p|&Nia^`Iuq%l+Y6l27Twhg^8Z~GK!9XNv8YP!)a!56d?4Zsbv};`Fe$m@~ zvyb8i91hoRpX-W%e?3xS{wiG|6{YafKQ(D&yNd7*O{i~NPXxKl0gqX6M6%1C4Odka zZ}G{L6TfEufEq`R)#vuH8c+8fb4+YZHiD5&ps-N0pMd0XOpLk(ub4{7yKZeZ;#&dz zclb{xUK+pE4o1o)9DC)Glk5!`cO8`&yF9ZMAl=%v(luWfmxY*i$8{_ibsmXDTDv$1 zMcqDxjqGq@#&*}Vs)eT>Q^>?p@Cqq()p(j2f!QJJ*%09IM*9A^sBEQxCl9ZsO3V!Q zR`bI)`C6$Ik-hGNo^tVKkG*>+becIGo0prO!;%hjB#XAjIO+&#?b%V7<5tZA6-kgA z+DkZFGluq=6|-npILtKq1?Kgu><|B1H_!!E`rwDB=mHs(+V9Q`IS55Y!glUF2*OM+yd(8RTU>xykUCu$nbr91OuZN* z`bmW^?4GmSqIlV9OBIQ3gr-UeAIT2$>n}GV3{?;}{AgcF9{d{N`l0u$0*eZn86DBU zI3mftBQ@I4Kkao#iRKKCLmr%YhHK52=&nSZM!5x%|AfJq&4GX*B&5q)vbOjzM~4`T zhb(`GsoOei^!z1?U!Uax36<~D<~pCfBaB`(X6>0ZOQiAJaq@m1At$d~kyt*@G|Tvo zE@|0gk<&kBM)Rv}Ta$6Vz2$?6E@jWZUVUS==TyWv$K>KqQZ8S2Mq~KBp{{NQ zP7Ll(V`A=9iB`BpBzSDMt^6Kg2d*Zpg!HtwCwSWkPD z|MFJoaH z_X&87@w%9^G_iwE#8Td=SjfLDF!|1`m(ka%!eI0g-;xhG3U3zl3)kkblyB(GpC;<9 zmMt+DBEw5}|D{8M(MoxbA?_mE3exLnU2ZUB(OboxJ?+qkiohASPv9vmSYg!ri!{W) zFWWjL`F_a^?1(_Jm}pq|muYpJu#T=bIcdVQa{mgaVP^$~LHw;6?AM0edkX>d4bAPn z_T|n=jQqXd4?l@&98A3r4B88%c}VpV+We>I#=x!*#?2r{)jd)$%Mbp_(I}0fq0MhD zL&Bi3^ibBy%Sz|h5WfSens>F}KitmK)tGWuQf?0~{Z{IkZ9M!FPiS9_b?kC^iMNNI zUMKBC(g^2?sOab@N>XP(_#w2dap`lbjr;yL%+XOr>x%ZYKo4duV`#NpizVb09KiQg zrkC#dnyih+bsJNxWj&8-#(-fTrg7&Lv0t;}QQW9<%C0w2Q-sBeCsPM0wx>uko`tD& zK(H&mX5sA^X_ui=2*eh-W@ZsX+*xvX7H@>2r>NC6>bhq%$`;X3oY?D!t^0E^u%px}YPh*2 zcY?iZ^T4kY9RrOI9HA`Y)3rv>vJD)${J|!F-(f`CXv4y(5LE1N6gpL&U=Nz3a9>wx zoqwLD{2HCvT`I_rF)LZJvwX)Sw3l`<=@iubkkI0WdD=CcZLptuhgJMgZMoTOj2OE8 zi7VAGTq2)%&gBd-VZA69UPD$eD3>7AqomoX@ zL7GDYmgJ`*w;ntx10D}eoQFulE5{m^83MUKuz%H#$9Boc<}t;Y^=Pw=9pO>%yoU|XwY&( zOZ&YO2#(2o5&)r20!D&SvzW--eAo@mx@h`qE_b&&JTyP23n?CE)=!i;&vM#kN2k#z zAuNe=?5IibE5xvFgqhrAGu&06!YX^Z44_w@j)>$>e?RTIUEJwWVxaES`i79*vXWY- z6wYN)No_Tr4Nrp4@G+-z2G2Sq=wjW%yPt#ahQpIEeJ{ihe&zYz*29DhGw8HFG@))_o5 z$K$my*LsV`e|4ozP7OZ|K~q~p(dDioPbb?!l6PI(jB?x6z{;zJ-YXccls*w-cJoWa zLTuTuAqCS%~m88Sdnl5}{r?p`-kF?ceGlPhZc?WIX zz#b6#Em?l^=>hr3i&XGN)Nuv~-KdK%_%w?$L=bTdXp}4DlFG|q$)2N`qME8mDu%$K zu<#&N&o_y$L^-!O_H!b_viM2Ux}#LOefb-Q`?y6TG2>kWBQz@uZW=C$$Sq>(a7tP2 zzpm72^i6*}<=akJeOR8pV>B`((Z=MDh+VnTgF;`%%8?ObBpc z4d$lD1LM%X*|kQNMSpHe`KDNOfYPfUwKg_4iV+6n;VO>N2uYyh91`(~eCZ0iLH&g@iWQ3h`48 zUe5lo=~lR0%0zPYknih@rKRtk<@UxX|6e9&)^>Me895fPct?#~>D{tuPZ@i+5@u*0 zk*!|`=hPRm9W5(2riY{yW!AB+TMipza|Z4MlkqW8(aw0Z5@#KI7E@iklc$HX!Ff8> z4A8hm3#WnAJ@ndIYNU?aP8TmP=__1BK4u?8BQ%>H5?;Hu^=$p@P`Z)){<6^olj?ou zl!`z#t@c!eGQHo7a-iDq8(8Dx0I@g)3BdHo5s_rGUTUv6B)*An=Q`uWE=4c4E&o9Ppjo~ql^Fyg+Rpmt0a-B z_!Xb9mZMdm!lHv&ahF3Py)&#RC!e9T8_)OMj5-$2(LMR2~vg$$y4nU@-c*s~vO3E4J0KLwRRW{~}2mv1a@ZrNK$H2K>B!^El zk(6Wn5`7=X4(5ygv(QAm1ih^-kjE0Wd}=&uySJ=3*;Cx&wVRAhWG>#c1m}ug9Q6pA ze;;&PPK^Y&M&R=JIn+iLLA3tnDm%YYyYbPQ*6O?)pGC4+eY3l6Ah|#4Sn0sN(mJjPGHrU%^dq=4M=B-;TEiJ6t6}xS;2_R7*1`DDPc3T@R z;GOiaXI$-u&4mL6ns3}!fpyE59{V4Qa}cJCV7`nk_!w!E3pgx+UmUHWdYsA3 z(E`f4?Mx%planS^uaFk2%QN$Xe@_s8XT}%)AkW(2F}4W%kwyq4B>)1V7}F?=@d&~9 zB6WrDrw>W7Ru6z&%&77u{(JxFPmldaAdkWlnT@evVP;-;VpaonSc_N}7ro10Ts!Hu z2Z@GwH~_1)$1B+%x!tepUS*T_Syd8*iywfD7O?hCmFs*Up#pI5{&=OcdT}Qzvw{afS2R@QxbwW&0{Wft1g_kcuBJdQi9V z?x%|8cY^Gmk}d%Ey8xI5DTb5%^-AZ>%f5?!uBoX3c^nEb=s4^T0sx>FPLr0r^Vl!58m~ZG z@Z9O%uL02lko5#7gE$W^AzRvU74$s$t;2>wDjQv4H-8s{5h(;w0x(Gd;d{gB>E*rO zdjBEf?f_CUfB0b11!63nE3b<5^6~RscOd*hJBy+huBF*7gZx4E%YCUprf9A`oDpgV zp3IPr`+A4&`8Ig+Mau;Vp8J^k54Q-#T(=Fc=A5U6b+d7?Ud3B}1et78AosF(a%`6h z*~|kP#iyH8cx%I{7WISrZJ&?7QnRqISPeGC3LQ;YxC5L#OG!a074$g1CD!FrvGJrZ zzdE_=#p&|&IE!JmkB<*v0f0COIs#yO-5QT05RkI2-Wx2?Z{Z<-XKNdBm5_T+zUTi{ zOLOVSbWT-JtzBgu#3LF^)l@pz#uzsrs~`(}l5<|94S%1_%s-y`grKtm!&~Op2BwtP z>L8NYagS7=$}6%c#-7+HZNk0utp3xT3;phzgCf={_wBXQrCS$KV+Hn_YHEi7rB`^( z@pXHjwt&I`!AX!)9dr4t3G#^5?gtQMo{ zxY{G2Ki}*if`~guW42)EZf2qjhmp`oU$Hw#l@*|KI7Hz%vof`_rr%(6|dQq8=y6U z%Bscb$oj_gB2hP_0;wBTZayD>*=T%l9}mAgZ6Rh)jFjSc)1Dp!lp=tvF229b`t<`) z04-FD6Nri9StQn`sD-?a4La1mGOfQQ#~I|b)*UYU^(In$km`}FFavk8??>71?Z3Bn;j(ob+j_+f zqDN6;kSP8=JkIUs?9-dS)6b+ z&LS@_wb|@TmCAPKsog_bt4Ap34T&`_?krcFqY}=fVIXFeDisOLV*yrm9t<-pNH8~e zb6>tT!56CCcVN!a^qljYHS-8TD^k-;igk(P5yV`hiG^P`>Dm zP({T$>K*zCNQ6wB5EO`~s$~o#i0WXRyqQAa_qkbPJy|sgA?mThCmZGbs(N$2GBN32<`NT zZqBd#RhM71XtW4}Mcdo&%&`A@z~k$+-A0?8$J`?6;7+Jvg`Y4 zo|hA~4rKvGkHFTvLW4y^g^FZ5H^o>%{XGAjidmAczI+37Ql)0`i>^ZIoUANWi2PVq zR)tt2fKHElU5v}4?5$ZdhSN;_$%1_GO5q!hh18XUpWo6&-L+J~06QGflf*x1;g&Ma z)e{#J`d1zNDRhTZ(caQqPY*gnK#Cnm4n=X=zrL`subCi9fRN%HHg4U&-&K4oqJezU z1j^3wOz-}{qO6SWSENKUk#XAM$T`-}k3Hd)ff|@KW@biS24mFF8UJSl$uFS|>zI4S zi;Jt!^V|;ySt%#wz;W4>@M!P~ZiSm~d~-+H?{JtMo;H7$YMg&ev46tDC%7!H_P@d7 zTz(=Fka_D;Y1)c1QWgr~UXF5zM`7gihV09=pg`DAi)BdNcmHka`k}&=nKka!3F?9M zIw;c*=7_&pO)-w)l{$M0#z=J7TWkgTjpkl&Vm^cxiF-*JRVn#@5Mb8UU3ko-P0bYO7vYGIc16MmAw zA4C_ONPnY)sWc}-b4VHDZiz2;ujR!-i9JK`8LSJ5LxsRaC>?M#+1)K;|6cXEqr1ej z;oQ9-%$yHo+*SHy?2WdS zth`)2Wra-#?VQFGEG~1n(Fi$l^Bl+n>w?0># zN`q_Jt~9`@ro&EBRaTG!Mln>X&$kNqek!qe@AD(DrdXOr|KeT8rZr<7V;{=7mf`G{5fzhQ zjhLi@qB9TDqW?0mYb3aR-x$TI=!gvn8mrw0j;DS$HY#^DJ4Br&vft5@TSU-lO>NUO z5;&SB{UYFPAAXt3{HnT;iu24y0+d0n{pKztqS@WU)Ph`I@uYMK{KqV!Q%~pq`08+) zy8R~w_Ls{!&&|;chMXoOW<$>f|2J6K^zWdenMaAmAD`ieUG|FQm|o7mez`C)aG~Md znW%ht=WxWHIIZh17zp|OqFO0n-6=v`U8%Jzh3_)-Ki*P9L)OP^Arp9eA3l3iLa*yzpe+wATzTrH%%b^2O z)#@eA3039NVDdmp&;NY5z@vxFpSDsDSHPH+0kF7qUB`^A=t&0uWBhfz=A`F&irHZu zX|vo(1)o)6Pg*c@WfC1)9{PUra-%}L%7hTzxy}@9K zmZ~V0Y`P5MHD!_qks)KW4saPMI|xZ?+Q6fk8GOOMe!ggtHErP5fBb-4Acd~oh(Cti z_B_&j@+7oLmqvA>>@=jG5+<7`+N>W?YaJlUH(2+5sWEEqH{^MNAAa*7_i?k=ZU>r2 zb*ht9)qhFEC0J7BmdnM&H4ZKTY;rH!c{Y8>!)BqOms`T|G0OQ}wQQw`Zsae?tBSt? zU{(u~Hm9eHqMJF-HM0qF@g6+b=Sy`5TtbhnsfknEW>Mg9zk#WzI{rpnv(+McZVOy& zH}1u>^mH%@Yg(6Vf>{7iot>Qp@&4nvP&tsjgF+>ODLeu94CjZ0I~qf`+ZppLbek?O zHnj}%*oT9@?t*`M>|+3Vr&TdaM37sXoPshJfoDjk$d`Ahj9&b@A`>p`-;?jF#hgY}DeD1NYHfB}Y zF>l5b13-4A_Pvsnyu6-S>{jFDFRtZ_xu3VixKB(se8N%oX1;p<#d=A%yXK;nBQkmu z#0vS`c1e> ztHw`-tebf7@bQ&}{@may*bii`FONUv8i9lg?)dNe>6UXP0H%u5#O&oI`2+-{qK|ZK zXlX9J1&C|f4C*nj)#)+tFhQ(}n1~2wU}Lk@=tqFlT01#8ajxXLHhl5#EH&8r!cg!@ zLNS7!%5q{djan-8Ej1bAt$V3uGzuNQ-@TKGwx5Ck-h*3spLD^8Y$na3N#7pXAHQAM5p!BN|7aO})wSRIJ zm!CtBz#Y@o*4F0dcjw^`sS+U*-WkrOY`5ywk)k)!?i{G`|0IvEPQZPxEvDGw)xE{_ zlgG%SM4!Tl;iHT+YR0(ch^XLCA1cyF0q?HL<6~4`u?Yjgy=4tP%}Vbu3u7AiM&DYO#*=Ki%h zk&?$)b^hG((G1fR2A|<$Wb6T55FVZNMZ?owF&id{MC*OI)E&hJkwZ``D&>rnjXk9gnI3G87rlB0 zIU_ew$7kn$ejvp+WWgi$EWE8#J{ES)3{J0LoCSA*;ZQF~sQEoX_7Ln#>VVNd`;{Qrlm@*kZ`KGxQTGlF&k*WA0N zrlzbSFK<=XfayJyK5XIKx=;1zE!@A>BrJaJZQ}<0SxewHvi#FR)(xIbUYt!*o^O3Z zmtJh)U;IPYZtZ>6Nv&$GmMd611wIJ<`jaNT=2&}iwuw(6H0)PL7T65JcC@s#pbh0V ze5cOd2hKp(R*U6gee&G0TIV9+;wXXD3+wNFzLNkV{3}=&7M-e$tQ8y&!lz+6F)>l% zYcvQwIkm=(I*VMa>QYYEy-e_)C|yT8?6eZz6`Kf#mfjPyB1Vj{JSx|wv#S&u{}J+) z2pOFxW!(nrA6OKZj(for61~_Fb=Cn6!{KAV;g766otG0m+oc5@eh}umwAeraBNuIk z>u8%g>H{^6P1U5Q(4vZtlkmJWfrF>%g@dm zQmJVW%Vo4c<_3~io_yGO`wqWkP0eTB>16+m(8^|f5JOe#%~KOEs5PfjQ?|M`CnvMm zQH>4WuV{|D=+!xkADcRm&YO%9y!e2^CNPK=`V)$I6hS@l7vhu-@RNSVgYrx1)!+ zE`zB7FB=cGo2w^3);?RpJ%CUD>jRh<+fed3lQ>RT=%9P#n&63@_iFa{0O`BRl8b44 zZ#DPDLdn$abWj@rM!UrFWDu!oReI7dC%W!7g)3IUML^AUPe#B414JXg9|X3oN5Chd z2sYkZSFPC&*DLrc3mhD0F0%FFNky2B*U>z#R;J5nM@sj&&$;$wTp}j!XAOb-i0JvO zXs+i8(CGKxbf455O1(aQG{vo`5*h7@bmQ1Z0irK1jTa=oXUApz*BD z`Bhd&%GwJ$d|qG*$+21WCPGy+<&MBN9{E$WG2KFdmD8Y@0ALKzM0-ogKpuCK62_};< zoblC!ja4K{CSy(9+0Ew>_SO;qjB*H;8Ym9DwK)sB8rtFX{q_YJYqoIqC$?ry4$t+L zGt%HmIn~zQ*$SEEdtX&ykAyP_5#btKX(dl=vsWYh_{YhH-lWKP5;Ij&Ly^9-Xv)mD z>1aE$Vv8{?SJtcssw-~awRj$PY7qi4TSf$2Q~r^y`#8Q3wl7C5$dWoaHJr$~3DG=` zDAHgKDIO2E3N#mIFy3}zi}Q0Ad%_vYBr+4UJs><(FDwr-~ zJ#3W)Qq&APA?AIlW(L~R*5PmmOD*(@kYpVsT{&CHLSpV6Q6N2{h>TeRR`kz? z(p~UFsB8lhkvY?kHN|CRt@u#d-w?%a>1mF%DB1m7V8kCqbhmt6vZ+_EEnnevi$2!j zbrm3S!`GEh&>GC(5H!6r*toLrbWuP3Jd3`k$Z#PHHvwxCVg+Km$x0n3>pMF}N zdeBpvpP%2rdFl9HyB2IAF^d8$$hg?6EZa2~>HlaM|DOx`f6q?>WurINR6)Y(5Y+0O zVOuC!nV8_Mz+LeppsfM*dj|&xn@(4os$CoAse~k8!YycYIgq}|F5Fa)@B5=HG~0y6dJO1ayF{YdNoa)dV;JcEW-49 zU0(ma`~q-|QeMjPlS-**2x|L=u!MKa+m$Q5G7Egc6rX2#^xdpjfS(`J)_v;b4d~l2 zXk2R!#zIzBRwWxvtHS%^iUYDO#^KVDbKv|$~s1nBQF z`00vIosoJYngSbul?(O}*%5f?CoL`g1n1abXF;LUU{nmg`Ll-y&HQvY&A?>*%-+ z(ZPz}j;sXy)6$1=;ma_Ora%Cz#;lYI=L_NsPstU#w{MrM7__R9Q)Hz*23Eb5_%2-C zV2Ncq07ndlqfUSPzEs2;(*=Ojr!r9IiaUn0op?mdF@X^o(QdlQWRf(SH_Re1w^RM+ zJ+uf~q$Osl?<`7%OR`%QZEKF`FaP^u9$uEVcH@p9U^*q)$m}%d?b2)YBV^dWJwG@b z2gKg8^;`-pdW^lS*mNJ8ewm|r-#wSSEV3CqQ8UhZ%b45zhigHS&k@SAA*p;>z z^|Hx_K@+o9fdQOr_B{}hmw zxH`Tx#|$=%QOopm7p(j4Owo#0jL9G@_nr<0f_q7 ze=XOtTKDY&dEHNRSKR+=?&xkn(aLg5m0x}KcH>Qn7N}K=`aHv@{V?}dFi{9{S2J7H zyHr(KHJ<$Q>zkVMGj?7?D(3Trh7k<1Y$Mddipq}KQh)rOp_097<+T6y$3$F{5IkMF zCj4Y~Q1V7U1gHUVGNqo7u__Ox8fj$zqSnFnvk0ZNx)W)9XcWaQue?b#__@ulFU_Yx z0~yw(`fS*79RtMghKgea9L?+A1 zF#&!q4cL4I$#g`-xY@9gNECOmnh+f{&ZLXMh%v%abKasNe}CV|sK)*kGFFzD%#AP0 zIkUlZ7Xj0A)~-Py+>iP=OUQUCA|@v% z6J9-|(6TB4Wx_bbxdad$?s%Ipj0N~_%#BMMo<@0E*P7$a_MZ1E!*#{+oAM1&dWRE9 zd56_`-()HwWS0ySP>`KELHxJ~brRQ+f9Q4r4feY)+i4U=5-Hn@~ zzx`&#Nzy_5>f~=z{l5iCW`hFPgW|q+TPEkXiPIBEWNLYhbPZM}F<^ivfZ$QK)e>DfimvV0R;h|L|iESe<&wm-ZnYuc0^*)7H>m-fM=WeZ8s z`ET0V+E+<_*T=tJ#Utuqt}&@SWa-}VH>smV{fmEqd7gm{+Mj~Si@*OAaQ>6-zvNR! zx04l6v`deR`Hc%;a zj!EEVxT8+6(LgGpJlvz^p@2!YN~Q~fZ@!;I<|<%uEw;Sd@4?&ac`8T*Z4!?=N~6G& z6JmdlaFi1XIQNIez^%8_lE+ir3ra;t6Ac~}+acB zFE_)DDFm^CfV;@D>*gqF%R4fLAXX*KYkt&DOBc2A99ul9Q7j&S=x3jOR0nKqH!Sy zu5WM6b9tlprT%Rku8PHd>ixvE8l8`xh4`%^=-RhlhwG#&1d`lQhCOQNyVoSi9sgZO z7dis>i}yl(d-urJ|LAYU9ay8iCz`Ln32Fw*ir;jt(d?pLpN%2zGCsW8skv^n(cpE0 zUlQz**fXHH0vl*;9StdWYQmb!GUy2CJj~?78oxuBcy^|Ra-fQhcqm_FLh~43Zusi0nbP7`r?HNIfmFZ-7}Tgm`Z362u-qM=B@5P$ zpIipoy{Lc5Orr17fB3YZd4!Yv6~u z+Ds~>KN2=uPVA=Q3FOJNcIC}|W|;kekNg&VQ51pV9N~xr*%ZAa4;uAH%B5V&SQxA~ z=25&FXGD>mk8~kj);m&P4NDge<3l|FmP{48pP_J*C+fQxOfHSU=LL(hotTnv#H;No znSYIYgowGU+`={$V!(ulk`UqtQ^YJ``&_oX5rZjPd}6`rDC}q6rdRWUTTM9*6J~`G zW;A_9Nv^_|7r4gvDURx=0Q!5sf5}zf7W*k&{r}_>8ndvzx4T>9>VMU$I#P4f@|xjur$r z%969Q&68gKFzxixO2?gT8rYM#^H5&EktM8EU~(rwo%N!(ZH1%lyDQ?sxs{tWf~|4P8bu~e2tr1coZAXU zI5m+;Payt!plBR*Jtchu%_-;W)#`J@$vfio4@_`Teb@Fu1=m>!3-|`55kGK zL>$i91yLJ?H#KJHtiS&H%g_HuE-#%8vdjru;7=&mlw6_BfR=UZpih~JNL)2yiegPk zc1xQ36gg`85==$EjKmf@!2a*)rq#!N&lN&HB&&>Eocmxm9zat#9}VXtRSaHuk~u$H z5ubH^wK80b!-%ZHN)4-2pNSu3et+sCSV9LDExbTkTprqN*+C#qcF3&Qup{EeJm{VI zV2&w<_}=B{yjO{NzZ<2?qVkB*ir$SWOK`r=uP9VimKvqZtxgwyk2G6F4(Su_59cd+ z-1VsAD21gaG?2C<1eU8rzZjYoCutPdtCBY#icKQ!3TJTQr;Jo!2+u==6lAA{v1lZz zAs3h+SX?>(7quFt%txGQk#f&=&}qNK8k`;PfzaLgc4YsuL%+5DA;>}Cj;|IY2_id^ zsR+F$X-IF9e6+WmnpJmiYv%)}4~XSiQXMH<6QxZMrW__pTz zGUxy`8q+~ZYU408S}jefllHz$pouutltH<2!5DsWpWJbBRBN z6Rromi+4oV9O|VY%D=iLNy`)7@%%e2)Q^^&F*5HFmsq3hV?TD`mOmjOG^QmXTr!Wb zZ@a+rc64?QF2h8nnd)QcuKGg*i=nkvk($pU>WP%2OOwvi$~nZ9lPkr&D<$0dz@)5p zbUQj%3T;cZ!WJU0k~b2Rt%=P@3sA^}8<8j!#~oM3TdER{e~)4H{08Q{M3~23wr=&w zMx|Ui$&Wh(@2B;|DwJnQ%tko<4GE6&N>#&(4VVZqq)KOjy>*q-zjj9ir}n&Pmv}SV zW|H#d^}>4eDmqqy#)?`}=Wqp-WT5S8mXxw(`6FAy&Sv?GUcn$k1#wxc{uu=Z()3`L z()5LLZo!0hj2qN6ZW{$0L<=Bi)gV=fvEYdz*WqyAUpxNk*0bFy=Js3&Yv&dw6Lg(Y zc}7;60?Yt3#Dphd(7fD*DY{ijY3C5HCj@DC3#rDh$L-EiXL+uNCAzSEh6Pg62 zZ1B4Xrh!hSzJVtD59`zfVEO?Rp$lV2HC4?MhRXLNQW-;F1)l}xhdD!Xb8~?u38?7= z1_cNi$!Zh-7?)R?o1R7n4CrPB>gYT5Y&ePnohULqQL5RkOM^Re_^d z7V=MXY4CHnS*o3i*&4V7^HrDGvi3z>e_&I-^v^#NK`%o7OFlW4*y{@m3xnnFeVY+j z2MK-%Sf!^UmpP~BNL_AG{dGAUe8lWOTk#*R)aVWOpGDp2hMN=BtW{eto;_4AG;M8G54H9Z;!HK-fR??3bV2s+%jeoo%L;iU9K0mqcHns5DA> z4LA-}iWVb9Z97%3o)x-aXZ(UL!>ep6AQ4q_y4@i*2SWJ?0*)&nbI~KXS><#Sco=np zK8vn<7qajV=k-)42MP6v_7JLdQU;G^$ld4h0e@)v{F}e|b@I8n;5WCo4X$GtHn;x3 zCl}zd94UhMe8Q*PSQCo)77eZrvTEHH{l+=^Ha2W))gT!KSoI(h))U2jRf`n2Otx+_nC-Ejjf#l@V59ImcEm=+`)|t zA2R@W@$CA|@Nc50)A*c5on`AKqp-PApryoHgAJrt9B~um5Ks8!&y2S&Y`8Pt{@Rrw zdQnG)W`!Fql&SeD zZIMB>@gHSBE~+Z1Fsa5Zy~ni$xJ_v^GRK{HDWPK2@;NGT9-WLS93_+rbPCK1{w0FF zkWT`Tc)c{cs4|gWut5y&n39%>-i=Z$p#t`Z0H}irP-H%eOY@QCF-l{s2@Te4`$7w? z*_>0fRUE=&3=wpdj@FbW!79-}{58@@W#)owS5*rAGBR;XJE~e=YNfz{!HTRmmP&y+ zZ5FKf;#?ufCq4b#x{M)hsWb8&Tpy6Zw|ly2I8sO25{e2arnI(1ysxASB!BI(z@ zE?s4ONmxS{CydYYJ0sJ?Jt`LvgK8NIl3+2oD_-z*pphBDr(F9PWseAzYUcDAZaW~} z(PsBg4PdaMCzUb`kQ{UulKjLC^_^P1)P6E+vRS@{NbzFrd&RVeqB z*yy29yxo&^V5Y!M%nec#JMjlq`y=MK`kaAG#TJDIAbIelWqET`x6EdmP*uwoln#iV zV}tjc^U7urV*W(M3?o3s07xCuA?(H-hq|J>#d?115!wk@j4RXN z^gP}*&#Nv0H8~(tctB!44BpLqtX04J!6KN{`+9qyLV=XV+QtSDz1#;o77>RPHG(}f zK!}--jm@TsP?brmj4fvY__y(D?Z=2C9?I47=J^<-`bT<>tp3R$E$h7AwVGP443S@ zIGJRH0+k1}+1^5o^#e`YjtFLyQJ33L<%BY;>8W~2S-A4K&_ELqhH zpPAI+Dopa`GDJHKI1a~n6B0UK3j{5sb`vmOkNmkB~&5X;9d!95*O=}=j^leii7~}Csq_g>;(wB3PC9xcSL|> z>Ka^|$+cIh?+pxU1Wi+stQS#a*SkXng*p}Fz(Q@cA$}y8(ll8o93vX_bJAVkAD2$h z<=|e?wgc_Mi?f7K4#T+^R3+* zv+n^fNxCuwXK`^Gk%8R=L765NH&kF*k|EXv8V?|QFA=aye(Lt)j9MVjx)c3w}aAix7TcMcf)#wYoHF0_u{>dmr<_QPl`a9Kg!*n@vFp8)~3i=Aa`n5#Lt;CgqBBd_lP4sL-G zsbSDc@AF0PW3LXYS-~pDgj}$qiSB-XS??wSP`Iu-Qn>}-!uVu&O>KvO@2-5H*2LS{3yRv82F zE#2tvD`^nz*{L}h-?Rc%S_fx1vxZwFf$bVAQ`29y8hsue+r*N=-ilo^*C@Pac7G34 zgOb@=akR-q_ZkB?RaM4NPh6unnO$4egS)TF=20@ucPg_+XRS#`;UQU)RQ=Q%655Mh z&!Ii#m%``i6oLNm%~(Ao`(2FNYF^~8vj&GGTz1Cmwz!3KH<^+!1TaNdDUZK@N|Q^V z2-_U5JOD|uRZ!ZssoTi+*k1#86PJPZuxL87>HT{vqG%x5VCoK^P#|C;wlWm$Zm`jn}b?4 zP&ReaNGl?L!9)dBQr-jB@D!*|vIAcQ>^WpLod49t`JvfyNHB%)G4P4p<4XaKpW=wN z90m1Un;RQC`ETpt_fWX2M0y`Zl5!e3I5`0`fF0Dd#wqFmrnJbZ?iSOHh9HoP0Ts0$ zyxm%C|GxMi#eHX3R9Up8-L^RvNEXmSQArX7iLDkvD4@w1$vH^QU>mT=k~1o~3MeE; z!9b3dP~;?0l4J=YynV6z&AdM|-+XVr8GaFqTlb!`_uBicwf6VT%~2o-0Wd*mLeOad z6!S^wyun062*ki`xFQE|V$|i@1V)Fufx#uFFyNz-2Z7E3D_|dd-rihw_|{i&PmSE^ z`@IP52)e=@+#{HF0!c|B@m4!`FkHTHXc0s|s}jPzr{+^w6M&yii`$&ib7~iWe(+hs zFsyrlUR}zlC5Y_EEk8n|3#m#F-a;jD5R?oM)tSV-g|kHy-x8THeIMcS$t!V zk$UB~niK6REw?nAV8`i!EH#8KupyJQ+Q6&Z)iYe4r3WIksn`cKu=9Z38%j&+ryW67 zU{F2Sn?hBEyr~>wljxZkzkvEl({r-_#YBvbE!peohR-mjskgA!S)Fk_X)Ds@pm&UY;9KEvXbBKQ%~t(QeHXGC zq!*8eSZv&j=-37@R5CvJALMi*z&u>$fjp>8$#86hxNs&RfveQL-MN59J6#W zuZEpkIOQWhw%~Rr!EduU-{G1rZM??Sqvk{Rq{BKt$&XA?wm*NZbCM%)~B z8wD!>%p{-(&#f=df9&iajig$5!42}UW5==-lIbrl>AUuuLhMI7g5m0Zm&DI5cSSAG zcbX&k3KGjK5YjSmV7$waU%eX0&v5(=Bp8F=JHWt7!ey=-gsvUQBAatL`TDT_!1VJX z_MA-sJF(oTGer!^d;q?23{=#Ooq3{-ammQ!ATt?uVEAX$z>P;dp?`P zh-0gu?Lar-I8r@9O?za~x+_!u&Ym4aoc)al`z7F!b%x>AE^H)9RIket^!xx+z1`ag zrLg*IZ#?LxtE=nLYq!iN;BtJyHg@ydu-E$|G)R%WC5X`Jp&A+8Dmu;e1Da@GSi;i# zI~}VNC;iVbFnKK$G%|VFfLAqo6*&0fK$vD^WMpZ$4w%cqmP200Z6c*qTz~BEV@&Sw z%Fla0(T)Jh`y28kJgObYELj5emJ9TQt8hD(-foV-a)++uF=p?Fb*n)BiuG3%X2>e- zvRKg5?5ka!oc$n$5%OI$T^X;30Q(pRu3fNek3K_W=Rl4dg)s7jdH2%S2U6Sex0ucq zzd<4K=guL-QTB}B5I00xkQu`=IC6@bfA4si?V$Sg3$S+~96bojHV2B_`~JOu_<@iD z9Oj&VPni*O9ReVt+l!1qg>B{K<@NkrmtC-xE`wax*@6O>u+PK%M=wT|5vopf^~ErF zy{s%{+=Hr44gkLOa%EcD1Yo~YN5l44V{3Kcal?89UNdvI*JR}_=Je!#u58N}6WzT@ zQ}9VPab6r?0pd!0^4VPJ!o_@WV{`#K$apYHETueGj*%Vwu4$4YZ^Sk~DhhggL;hI<=?Wx0b)` z$h_$2=m3A=z^3J##)%oELV?+19*~9hfB5Y$eU*c>QC_sxNBRKO)ckBP{I=WMc|Ayz z)?|T?5Iy~-*dO#m`T?)zttOT*4sZkjrl37Fmu|M@08y_>-|y!Jbc}ILpp%AbV`Jx2 zQ6>r@&?5X?sGJG`_#lkLQ#4$nKUx?52KF+j*8#*^BT>XT9Spvt(?~jY6{Zm0M}gP- z62IH5U=RWb!c;4=76nFjp(na5ClUA%7M9xO zU~S0Yja_>#$Y>kH~m=$4kkOYyhK@~aSm3p6<>0&)SXx_cOR(e^fcc7EPd-uLO# zCz{%=`mq`X;{4fB{H-%fi24p92cQ&K<%qBRN>HsGw7Z<9+iovauc@l}fcEqpxCB7a zbnaZCI2@oi3a$Y8xDny<+LcP)7%BF0VF5S~yuSk!?XfZ^5@u_3#mq=0|% zd?FWJ53=Jd)ZF-_z%jr^^g4Y3mf`F^$cIF|Ilctb@p+F*K;}j0z4`k)`M{}9mhkh{ z(CEeW3{O8P<{LaBa7MWyEB|(@-d|mRPuAP}AW}0D@9upGp+TPn{!etzYi|0(|i;s%W2}n&0z}`YT zepch+OIFO}JHfJeqmr(3Dq=vhhrZNTtorW9Y=sOhu|p4cxBx$(;>CSD-)5cW+9dDi zGSJiS09$LAL~?+Ez~ob4+px`GUtREQ1xdybRv|Gr0#4b3>#HkWY-IM^Re8D74dK5G z5({i5e*GSXs~_A&YYad7Lv!n#&)0kAuC1?cS?q*!wi0HfxvBuAa8+S3nb>%W(+-FQ z`jc&r-z1elWOTmz+tQM~;yB0oU0>Lg)Vns5TT59h->(9>V54aM@UgSq_J&Sf@<;_Q z4?U?)sBUy9w%PNm9pPCG|Kgq-zvw+>mMOY5_5@jW0Bv>8ZA78;9qOsxe;I&PZr+|i z?(Oa6UvW4}&8fW(Vfs~0PEJaI_`|V3m%|3(jf3UG(b_eDk}b^->@~B|pGQac0eO^6 ze*K%&s!9yg!;u?K@ZpR62bA*Krc z)C#qZyvXCBf{_xXPyLxt`c0FAwf%ds;r@_3%$85T@t>W5zfXQ9Kwta45Gr!WOArT+)JlzR!h4|#?x}!#IBzWY!8cMq?3ngfkd+Mq z2nX;eY}yh9W5dPaRzXB$-foh<1)Y*{X?H-`2AxijTlvpk9)g{KIIly_c5?rgKb5Rt zP&X-mVst+k_bam~C2Vx3E%?#bHc-0?rqPZdnFS?RIdK7}q+VvZ*9b zPzZx3vz=~E@-a(*W+zWMCha5?^}{L`kjV?g8PZM&Pgbb0k|yPqjE*!G$UU=pcxr$7 zZatP}k)VRhv8o=sVAL2o8RFdE5lTyuiOF?-ZKIK+Q1OC4lry`Y_gUeDDq~fYB0UOq z8_%S)?oQM^F`^jXsA7<%IbYYK`1G3k$cdCcwEalAOoxQ5VKO9?$w;A%nB$MAK9L_HlA%~VS1qtJg)SQW z*1+b^u}a;y{s8sbQp7~uo4NtG%x!+Qs$J_4u?L6*Dngc&)T>~0&^h|X@ZQ58Utl|7 z0l@>1L`r+`{|9NGzrR2I%Rk-Bp!4DfCq77f3L-3|IVtY1vw-TLAuzgsZh!gQ5_F{I zX2|fWAI~+>v*fC>ofF?OIBzKSmP`RPQ4;itDHTN>ppAFrDintqwu2yP=&N+Gwzs$M z>zev-mnt0ThN<=T0mAtJEeHtpM>d(EzWIVEVWBQ+cxejJ6>cw-6$8D!g5!@$W;+Yp zutWYrkTr~;t{LfS;wL!|L#=YSc3wzydB4mrPOJ>w=hQhcI&HYw)cC@wvV zZ=?WE)~XYHshomh*9q2xUq+$aSQ&eA4hFI?K(`3cek&+l_D6hwEJ7%PTM7ZaaOub9 zh}RP2HR#C@orCtn+!CEbA3^@%Ac2Vhi5Q z%n;_=<+vSi^Y=>0vK*KZ->fib*3SEmv~r-!SvwV@|i zboGqb((K~mPq0L)hAIGjXx2MKmXHD(&Mpz1js$>B4Luh!)RWK+x)6YIT^iA2ix!8r z*KqE(|YZ<}W-B)?lTTz4> zejZS7^z5#ofx(zRgB`7JV*$05O>5bZV=MHhT>&#(4_qaM`8uyxHNhyYZ{ zge9vBcMHHkZdep--4=84wfN^up4s3a47Eazm9btIrR++!#;U;Ww|8Z$MzS8@8L9S zSK2T735@OWU#ENI7C1(oXzLYvaNa$(_;+*Sp^+PNVF3p(Vz5B_-(cR%*{pr{^z%-S)&gwR;qFG z7yaqbr3HEd{(mRqEG*rx{E)o!OrOS&WRW zJ8in);-P?Fm@CNkrJqOM|2wVaBqP%I3oJ3*VacRFX82HiOoHe$K~=7oV{;P1-Bsxx?7X(vL*uIUb4};GBmQpSx-@2ih8DN7R#pM+;~$Pd0nCV zb#5msbZ==88QHU`i~4nmn$gYiOd?cpO8IVbDyCIofE$BJa`Z4c+UC+i)ksCgNs+BC z!we+Z3gs)A!lsQH=)k4sAspK{K^udq%kQ6(C99~i(AzpTA3O~055ZdD%FXybySplq zglJiLOo@>{4)-80YB3?p!mLf$BO%zDMmChT;3@v7ljB^Gy?^pv()S815E%B%g_D`& znwKT+!^K?5uu9NikA1uwufkRpm?>#hGA2LTe3inZkp>l>w7tElE*E+eUE)|{!O^`| zVZ<#Z|D!uO5VurOUS^bfd+dr-2dDziLvFtVF(Sa<$VerSPG--(PP&_l@$_&uB z5oMJ{u$-J#2lt1$neCEb*K zofzdCG>gR#;I2h?zilZfv#DaEAo164sH#!oYpSOn-9V>iYhzBV8kwP~%F<&x18_~& zp(>O-M$K{f9a-%bV$WU}!fqI2oaozacg&d6xQJ>Gq(`S-E0p|LFbrKgNsMS`B*`Av z z_e{Qb8Xg`74%yt|BBTE2;o+Z5)TOjWqRv z34t0n)J5xqwF57XY8_Sl9j6(u42l=3&v+XykHwYfgeut}oJ>1DKqfB^r%iIs7}i6g z*CRIy7HYNnGFxd(&{4%R%Pr58<%`|l~6fLignogxr9oh7lzbe00uSG$DGo$*SCRf+cU4oRB%?f%{mNfS89 z>bHr}N;<^q4+OFdp^))KkdQSKYO8QKrlCE7sY%j$yp zMSuzKct8^#j<2-$3Vt)NxxjA5Mq$$FPm>sbm&?vdpNtQ*%UGqGA8=RbW4YC8r&X!+;I@>^y-5n8S2R!c ztjkf;lN9qk+da9SHGJf*;oE9TA(#=94$X02sp&uIkpJELc5g6`{KsvU*n@d@#Fn^nE5UNmog7P|8AKUshu z(a59I^SwR!d_0GtBHx`QvdwERdZuYWZ+FS zLZLbsXlFsmqJn>!^oJ_Do{;ajvfFs^Rf^8&@iS!pl)d8KKf=VF7Hx==@k(+VvK5b+ zJ57xJGp(dJBKfT|BAje^elZDCl60Nrwo{qgCn@|h36;?;nWwW}OO zFpM)-A(?qJ8xW?CvuhB$`8ne%=#De}qQSuWeicCLqpmP_?VEmPLi1g&*8!3Ff!2*1 zAC|9L*IcZk)9qS_oxr8=%-iGr@m{7`u;mU&ggZJdw9*@SngvGibRiHkD?w(9Zzh4nZ(H|7AeK|7 zXLwj_+{`>7GgnOY^lXbqKl=>yfCZDpV4y*RUFZ9cao-C8)~9C*@s}1*I{4h_n+P+tS1eQGc!9u}JzDhfe7N zG{p3Nen~_}w3|s@NfviFRR!nHy)Qo;aEcB+a%x=`?d+5DPaKmzb~;V7;;wjRl!{Vo zFzYE|bbCmQ2qj9X&4$O`qu!*u*l1SviV{b9=&%|mMf-G@C(!dGDdPjFg9!3HOLKGD zAU^pzs9E7?3N0`?A2hPJ4;@Nc6NX~;;|R2#guS;O`uXi6G$cBeL?BJ(YnNDU(H6gh z^RIvD!~rMYU3Xm^5P%{b8m!P9FTw*1HGlv3NM7Ktp(v%8e{vISTqwMNd5cc996*S{ zoH`BVd;Oqew z<;DZiZaEaekN^3d|G|VK_mIr75x4b*KksO{gy&YusQdj@xs>?zQNwc$Sz3GxM*83#*dkHG2pnK|mo2kbWUHts+N2#z((~lmH5>P-cJU~8J)WcdMz0g1;KEP ztBx?y#NI|*MSC|J8XSJ4{*|j~-umw7uzvWr50>5Pt&O=DUPC%kF3Y$`y=oi3GhYJ7 zA0l~He;H`5IOC}~nB~(WRy1J;;Kx^zUoTw<G^+rNuyfd0wpxvVIp?7mQ)vZAb z+WZK4jQmILw1Db~AY5cfgJY?Ntad=^U^?Lit!S1n9h~Fi9d99=H!8*6#$5TWmHo;4FV&oQ*yHoNhX`Ir={W1LQ@jp8`31Gw zXgXaTQHOB2rs)s;T-=Z63C*c03Jf*L_o<>(oAEJS;dh0Xa5_zs)tF}d!0nuv_CGWn zbRQ^E$(dH2-*k-kmdEVM3abUDwH_dw@V z{P;8X5e!e)b~>usm9dd8ckEp6h7~-I!qgSjmC2lM?HZtQa*?IdsB>y5jq54EgwD+4 z@8XSUQhm9!g45-uWUpq4^c!cLqx9Py?cbC4{}d@Hk^zrlAY}B34o=5ZfLQLDHyywv zcdbOW#zI5yfq+WCt}cy8ViR44GRz&zEsP0PZB=Ne6iaE%I$h6Mmlpk0566ZcH_1zn zZI$M2HX?5>mQSV2ZEM!$nRN9>=ux%@n>*g%`@ENbD6S0zb^8ciXjTVKFy77+s|GH1 zrgt%<7@S=2-CJkt&eeQak)?b+XQePB5Jwee#^=h(q=c~&(7>{2xJ9_|&Ux>KM#AhD}>0Hs`(`ij*7o8izBpkga!-R;;envotE0JF5isd8*Q&Bn1WXF9ds7 z2^m^eHS_J{-gjLyA!=3_RT$AsD9dCdTM$(X7OS;dj~=G+cd!aKDbOtlI&s6~WJzWT z-E2g1$pv_qlg20qU_g@F)CpuqDfTShpItb`Qwe6gNQfrE+;+mDraO6?wW;;y9bEwCLVer7v%UbXx_q?~_tF>VYqe!qKYP5rz*zctsZt-K zAsKSLMrREKzLi88rH}sP9f76ElR?@ar9KK(sOt09N6?|l;&8WystWJ#mUwrC%~lcp zaeXvZOKLe6^u(Cj5|l1T(i1ZUbH}4>ITHjx zMsa0an`rT4rSC@B(8CX(_a^r5^Y1LI2V*5usDKPr7I)nL7HxsIS~W>Z8%n^BYe2kH zaMT;Op*j9Kj;BbyX&?pEg_m`5WlLUQP1uMy3MH1tAK3|drDCYjW*VDeUKX_a5P{06 z^ctFMM9i&AHPpl2dnSL2zF|{zbkk?lEm^t3)7vli`gj$wS9I@Fm{yxe4Q^&q`6IIi zchA8Be&_BW*IaBnwYu@QIT{plaUD@s=yA1^Q^sY2Ye|6<&+<}dwM%pfJ7{J0Xaa#4 zVv3%=BD5!A-esP3pH84T6N5%^YWZ~&R9sH-050ICF==l~4k^BVS%c%0&t{-$;dKXp z0qN8V-R^d&Gs8Hui|wE4WoC@@Z)MJVax69Sy?+)fQ^(1Q9hOn=c4mDYfHN|6;t^yZ zyRc~We6CnN#UDa-%Z@)dz9p`t*fcJnIzMrzpduc{ZdO6ZuAd!JLXhE46i6<{Gmx<4 z7Sq(0H)Rd$2SO^^YYO5z`2LDG-{AOCklIZvTivky>d8MZq_$dhr6RZz`6TA0*9f{5d{XpOj~?cL!3g?X)GpHqI0GF@P*; z0|eqfc+~T5S4AWVru>M~7LQryom!_;I4>q3V$13;HY!!%=qfH8Gmk+c)WZ&70{ghC z=Q=lFO*zCEc~WCMz58C#h2`MOwOXT2t=L>nZccC0mA^2IH7T6zIFv+OL$AgMI@6dq zed&85WX=F9vaK;$9&Oe`G`l8zFkvn3e54&T@5*w@_P-XL;QyA^iTtavJ!Ig#$OKmvdD8O0zxmzQ p^37io)kEh4{~f{o-}-EB|CsR4FIsx#rD@3jyDzDLA>K83{2zuaMzsI{ literal 0 HcmV?d00001 diff --git a/docs/images/latency_curve.png b/docs/images/latency_curve.png new file mode 100644 index 0000000000000000000000000000000000000000..265db0ed1b93f094851b32328122fbf93815c28d GIT binary patch literal 34661 zcmeEuXIPU-6fRp_QB*)_Qe+7zJxH&@iUgz=2@oKl^j@TQS48P7AiXJ_KtKqHbd}y0 zLN5VCdhZ?XBw=rP?%(_K=20Wzn|9`u_q=l^AsT85H?PxQCm|uZsrXb*i-hENX%doO z%}9R*pSW+jbAq>PPEQdoBqX<*@PB`ahxqTFL`&(zh)Q5Qpn_=Tg{b&oJ= z$**ZtF!KfUxhNSNbc>DV6pZ~w{{}X}9xt!uy z-jwmcg(FQDCNRUoZ$9Y9nQO)m=`R>k!TMHzF`d`JOa9eSl8fuzTkqg-UOv76@#?OZ zozEYNNlGgB3n|7p5GSp8EJbS9v<+$NslR2thO8q)scs7 zY;05*ABSlkqHdd}x)gGAcXD&5FfgpS8@$2c_uFZyi@n;@cx_`t#&>fwD@yvMrt9AF z;NT#fk~8$j&)wZXa_Pg$fRjogRo&sCmRV;+d`e2na0&9{s5ij>9zk^s1ssK{-%!Ys zMls9a#>f3Oj{@uJ>M$5gTwEN*_tn2+V`HZ-&JO}-rdwssxBC10k6Yw5sl3u19c9Kw zC+l21zm%5`*j4-x^N%(E4a_OAMwAIzdoUh#@9g}bA>j0ItG!(@t544e;kEyDbaeD8 zv+t+U(sPf10}n8ll@$hvKvMX{p7F)r)~pN@Gqa-b5-NmSY)7jazH4mKE zLXUx?y@Oq?NqIgBHN29Rn#y?Zp4x+0$0I0LOw(3(isI(f{WyPeVP8S==nWcRXM;9^(yIJRRP;A_Q zL+XcSZfLC8QMw@FVVY70e~*4Fo2+}Zakj@GT5M;3l)%J`-z;bxd)*I@jGJnI432&| zmYFg>O7Te(r+HSgs72N6BsMWVe)2O`eOt=d*r3L3XC8D8w#V-2JT=0W0Wq}ZE-ET& zfF5$mxQ1KsI`_0(*xhv%NYch(>TiI~56k)nHW=rw<~2Jv9B&z{9pU>GV7w)BvWig$ z`ZSZJurb+i0rbhl+PYxSL2{uUdm^?lBq|agAMb#x-pk}aL-AZ~oatF#Z*Om3$!7wc zezDSK`@Nw3Vx~mdRTI8(27d95+5d3*+k%3E!n=3xYHDhHPEIg=r(%&{BdRBJbPD(S zL#S-m>{3Q6e{U?~h+%85R~EdA?@C+yz7_Ho``~?Gh#h58Vq%ATB^MX$>bE+|4)+}W zcV-lrrF}k3uAeSkoO*23`&|YAZaBp_|K|%75>0CF-Ac$A&+6&=%f=b{%S%PH2Mn;9#R;_$HKpE76hl?;nxbh zVoHppQxw{@G7SUG!KK1t{4Ia;gJ(#L^H9?Hu_QMDkz=YhG4>`+9mmUp-ao5KoXJxYJEHV@5jc z=u0yT!Zr;=p*%}i@`Hn;>BFmuFv9ls|`r)9tHvGz#6=B+gUGc2rRq9K3h4wwAE!1KzH z8<#&*$~6aLOjaoAmo@PQ_DZ5}2SGc#dO8g0z0{S!o?GFACcN7>h^kpH2e)X4!Q>W+ z3oS*s^9zUmHy}&!?I?B3sCVV#PL|f=g%XmPb*SEFpFTxuO zlmlLXnhr#T@(W8m6scSkxEGfqJelgZK1 zzYD-wo!5#^-lQ7F4tfRefD`023Tm~DU;~NtlPlZuS)D}%T5x#!f{%|6095BY85cY) zwI}Omqrbu75%O7qfq}g;7rsLD)aKT5Kg1j=$A`uYDjn`yb3ufc^ze(N?Opwvc!1&> z>JbU=u6aP|wS!Ec_zr+mUtizF$;QQbp8|6bMX^zxcfEkVnGhogxts0h{^<)I8#up> zx`X30|Awi6!^Iigxe;d6(8!49CEoHj12yA1t}<)Og;TC%KNy{hkWVP%$i?_~?(bU( zz3d61_>P@`K(hK4`?3_W`mFr?&gQ#QHfnczmPSXnwze!Y{604{1h~74q^ZRUx%Jew zmx0N?>U;w4Y2Agd9{pYFHOvNa^lTillEO_*U0|6n4o?7X+)tCd*d)JjS{tk2llhnV zhV=2_Y$i{@F*XBN<)R=rwzhvfre8Q%)^h8QK!7yMIB;g&V76a#Zfh%-!2ba*C0_&s z1qfBoc9uim2vOy|Hr5T01F`{UCt`V_1cwXv?iiKm!Uq8?&BIcfW^yP2j2+7nntS9b zMz4cwAi%O$%5$AaT2>Rjg}0?Tn7ayn%gZ(hxHIF;KN|VSCko%9Pyigm;fgVGKyxPP z?zAeVHhHB-$+K=0iXsHTgq#ZUl@J9oQx6~X44IFdM#$5y7$)Z;;zOb{Io>r!``)h( zUbZ`gSq@kEZZYfIqHnTj$B{qIIKUat)Gy9q#LYS`={+}{bM z4~GLhy5+h1dhxa_M@Jb-%pdPseILV7s7Qtu+-@C|&qU7RTeG;wJvHqNCiWecLyA-v z=a||b@Beu_U5Mi8q-CNbW6?IhYlM!#tG?lFltj%i3>F@Y$$c=I1MM?^&{@{q9l@rU zF6C8L{rPikIhD3$AouLnmr=XoqM|1C)`JEZ6>Nop0TFL+=Rm`!m%CbMBXy5lA!}`I zZTL(abovTLt*yitr>n^?pBn4UmT)XbYx+~F+O9>hwxPkX$huPB93Ig+yW|SilQp{P zUQ==#nd;I>c!?UFsvxM&{4a(Z&V>nK)>9q^^b^F%n()qOV;gJhrSr3cdyl+#JB`n` z*C%Vfy<2OIIyg9Jo!uHN)J=$wFB7T|K58e}p{52JrQrpq^s#Q;ZD`RSw-b7O(1)!s zMeD5WuZ`CkZM4U-M_B9}&j9!sfcXUbon2&9%nT(vrQ-$kwlJ)CS7OTd9xM zo#4f9?X=G`_(sKcCYOA;a6^Iqpp<4ZhEalt*{8!Ypo-qqo)~641z6U zrgT_BJm)(Ws?5*ts)k;|6#f3Z*<<-8`r>Rn(9;n0u4g-)`&3iy{z7I{+i+DVACNY*z_x!t`BHU5&51iF9JZ8(a+zL){y*HS_r9BhX+bQL%=KN8u0x!EnZn)+nfRYijHZK`+f{DF$<%g2re!Y0~H~-4A z3(I}tZia{INWVAQ10c`2SmYZK;yr)xdwZwGdm}iSU(pWIN(9-$St!`uZLKF1w)aEq z(=eedh# zb_m+^HMIcMB@*t}=SFF%nifMw1V#|ZkHBw|5%V66|7ve$hz4i}cqX%X-EeT7Z-Q{md}`89oOinSTtc4b?q@R0GgjqIwCS z|AN5-moaxtTj*L70fPxZ_O-`nqPN16B+O|E;&U1;c&c&paYHJy=rUQbc&$WgeA*ZM zi4kx0e|0a~HI5wRhoy8c8Wi-|m_=LR@CznM6ijnNYQqQVoqObTY($o22_k;A2pp1J zk#Iy)ZV0AA0!jqiua?25+Rcde+X)Zw5K@Vc2qB-6F2mi1+!XEUhtW$!?fmmcpmzys zIlTRLN{gE^5ok~MlX1avyza_UEU+ot#p@XqvRv-HCMP#<2Zc78A7K*v#o|(<{>-N_ zPwW9cQRrC;KM_+qXlLh_a|q{;0njiq#7j(6UUjO$Lj`^|dkyDAkMR^3yet^4gQ8HV z#+gsUD*%tpIB`IR#^&bc94c$d%io)i*4pAN9!d}AZ5?&WB(*(-qI+YOj0U^#wYuY)_L?ZJyu_OF;1` z%x-=A_H7{}0B0rS4=`bv6{!T>4e8U(Mzy?Hb40wkrgjMdPX*-)j6ci;UQ^L*Hs7H= z$D%KXBQ(~?(Zk~at1ffKcQLndaT0LR@Z{eu&r`mOZ9Zn-&2J!b>FVfYazF@px}-0` z5?Lf{Z8zISfvk@}d2d8KWo(TO3#YSj*aooBkh6rP0^%ygpIUab^hN5WTZJoe^ z#4idRmZ?GLMzD6$Ba(9gB>FO(Qi0IX)h)5JuNXtiW0q2VmQM07sj;wIiEwjq`iKsc z_armED?K*U?bq5y+yUpI{1C00Rjrc6n1!RQtv(>>nZB8pM0z5;|QM3i8#vrnX)dMMix&eQm{tRWIG$_cv#rnA+!Iuc5rsH`X=| zyxAET7z_*y_?TRyN4pVKo| z?tIu@N4{L-N?(i(PAz=#vH;v^fVZ%|b5#Gl`MB2t$=8&=LibX&#GfOU&S)=aLS9pGx*&yGQsw$vUHW=Pd6bKEC6 z=}6H z*uo}Zd`n5P*mAhJrv|@Z5|=ElSg9NAI51;-%i=hM>t{1!6B)o6|0q9glPXt8FgL%{m1w0zvilejM!!u`Tl)5Y-;MBqEDFL0;Ja!??K6 z#I~e1gu(13QiB%M4$L$qNb9jG+F_f<=obR%6YyI<{Z5F}H()hPDI!j1UgD;1ZzqT4 zW#4cei5|bXL3*r;eaa<+7OKSVm0v5tz4mtHb7tG1=$`|Xc&>m%Cco@D7IOHZh3*n) z6_oW`Q`XX2=A>0Q#1b3(bCq~ezD8kkqsJ_kZG4SXE&v6`LE@B%tG9Tg85CB8J-il zj)b0UQp-jg{oh$*T0ZlKLHCHE^NZ`YKA>hqf<1A&H3D8 z8-2O%96{di?Vpn4A2&&@Rd?0oCKUMnjbEctrS;v&s|&~ko>Fu;5pXW&<$r&^8n*1K z(vT#6x$-nrzv4P~Nz2ptF?SkRg$%#W!7;vhtvkz8O>0tXXreBg65IuiDjeq5&!)@SnOb#wl0 z@T;%J(ENGx#i%nOa!5?m=cKCq{RGXP%gMcPziB=v{JSP4r6L5|VW>i&RW+DV`d)kL zE@9xq@ux)?KmcCOIOv0xH`-T&n$;wNt<*mIlIfNA{kQf;oGxVqEA}ap#=tTH5&rW# z0TNHY+jQK7}b@nDQX0MbX zeBWmfULxD5{;r9^!Sc8&FWNmOB7Y8*fK#@2{=1S8=uEARX)6Al#e>LJYk>^n551>- zI>Vei_0uD`;MQyoew>J+MIyfYGX47LW^%#{F?23WgWC3fWLIVpEFF=Zh{uG}K7h=& z(hO3(7peqnLWHnlG}?(E-_TWhk&PS+`TbEqGkvUF3sn1(bfq;UmEiR?`U4J_e72Vl zUb>fNk`b7|uU8I=Rx-It1ra8gCV3s21vOe?dg3&5^yln^dKiUv>YRI z4vJp#PE|7*uOJa1e!u?sffwBZ?y7K$+>nGVjeW(4IMJ+L9DQgEVOFgrw}}Wza~lff zG5b$^@#GT8$9)4Qe=3^AOKM7DE?1yTD$sr=cEDoAm^2q}E6hkRtiS4f3sy1_CZh4N zb2{a(#+&7qKQnr{o2rq1M4_d8Gi1bPENkQt97)4kX`_U`?V_jta7dCVS8Ml zn1Upq0S>2R{?2xQh0w7onvF+rDAVlL>UC!-BjN$RnrVF3a6DMDj;lLSzt~lm&n`ku zozII(N=E7G>ZXPP8uT`7%}W=r&jV=(`tK!ro@fTKl_5;B+P(zU#grNK-RBE6Dj8i%>mATT+bkW?%Owjm=sfkQH!;Oc)V}Jm;ci|@n=296VjM$kqC5!h4TU*fTu09 zTeE3QaKDSQ14mq~HJaM=huHa6)Kr7V#+rOqbPgifd#6l&djhBMQ|~h>6J& z30upItO|ZQnW?NvRcFMf7!wf@ah3d@@9|O|93IjJKq8pQ*7=EyLuJxSyxJS}TxAMB zTl&ORI9+6I4pm;Nu9e1}EV)YDi}JW9Lu2YNSv^;GfJ-p1!eDCZ>jAK;99PBe=4=Sw zi(-#$w-Vy&+^*`?h6~f*Itl!C34q1P9?L(INj&T*bjeMt0tR?Y@~kdX8E8?5QHh7A z)21k>D!5o{I5S2`q|G;+&harzdH%cR4*LmmAwbD_*Wu33&u3es-ZjpIrGsLG#l^+F z0Q_O>bKL7?r{$}&sO{wYpT|STZd!r~18$2WRANmHGdiJXyy0GhiZO2<8(RhnxL%@I%JGC9U3Cg^XyYgK3k=d)g#wH*f5r1hwdOb+8sU(%0UofO|! zPC&2o0ksFyv%{b7UC1|FUQ)$q%w}BV*D^Y(R5u`NZj&pI*Pacq$^l9fOWJBwEO zmt18@&-?={EnDfHc};B`JRxNOWh9Oho=e{N^U~@5WYwzO<2jsjbi&7RSKCR?I)>Ih zNCx$t=xP|Pt$hp|fWuMeM;RAYC?(BnD9G_az{LR~ezMAiT@|aFM1Z7Fc4m-O8okZD zfl6+1H{~=k2sKr3^+Fec;)#gv?zy-)P?KV`A@=5ZI0A`GnUuQ=JP4jC$&cF_$hd)0 zTo1H~A5jz3w^2xw{}~LGd4t$6_wuS5wCRr5VpqHkqdX%JnB!}Eq3Ry3?13E!OhwGZ zn9BIg$`M^nX*hNf0_myZ>6}O6&r3ehy+-{ZF2T*bu735-Mk>Ns2}ni$2aONs_A|F| z0_{`Jy-`uLyIJJvD#oj1pY2}9V2n`3UYQCx?YeS=pV`;@x;hexhkZ>^J0CKw?P}2Z zMHyU^JgiHu5-`fED|uXyGv~d{cC&}Ysc{i= zheH-a*77~cVGlp@L5N5_Y{pFE+{?#Tpr}Y#RJ82Cp(1{~FQ}cD7{>KnbB1e4EwfOf z96JJHw$*5>3I_JAE6Y_xY(Rt5TprDrVA9EJL~%U%sZzP5Bk4?!%%9p{BJc=d_Ngb6 zv!|}wAx~>&bZVTrU7~BAGF=AfPhYg*axGN>FZFj@7y8Uim!tp~tsW*GX7YUcAVH@) zeia-0m=hv8guR$W@7u7admu}g2{B4fTiOZ{^sR|Gl_smeLuV)tg#w6w{YcAGltxH&I0XkvLPo%~^eaVH1iSera0Y_Ma*ylc=g1l{D8A8|- z8`GF~lUb)8wV zmZ>Mb2kcQ%+nEDA6$ac}S}TwR8X}t*#=+P4&Dv{%kGmwI35igx>29Mlg*td&{+6Au zF&6vdi}CS?wYN2;(P-Bn0KPSv5a?q)HWn+*w|xUq59*7ALhSPH+VeE8wFEK!n{nB= z3B>KQ=Re<>8tZ%A$^tf{kL1 zQn)kKwVK8qv9e;#hzy;$w1FUv_b-IDTF7{?pMa^UnrbtR%P^3cZY%Q0l&ch!7L zj3aaFH&I-gW)kT9N!N9)t|}sj{VI9KD2!{Bd4W7#ilr;QYAD*3AHl7)2>Wo=`sZjGhu$kT0nTI~>gb{Mv zzwFa)bh5KI??3kU)Te=4nXfB|CLa_F>qz$HNN|ZfAtW6(%ZufwlkAN^>lDreBD$3% z&&Pez4jd|*+WxnPEQR*+8n^pqc4uPSpKHN-CTfO!PxMVI7&>!0FSGQ&BJxqe;kkqB z&h#a}?5fL|t9BjpdV%l)dOIJ*W#<0OgXE3(9Q??S(8pXP^TVZmY>>v%$q|V=v*=0DKnaudP|uS;Q(_) zwig%YfFOet?jbsq#D))J%cf;#Kg#>;49-VDX?!Qc#fDkF?c}2LK@4tNm z%;|CD87c?fUYbD;SfsJKqn()ncZkb=zXSD&03oD4w1nN=j|>Akw-#vasDbJjtk`Wh z-`hTZd0SlBE{SM)E_bx?DLd+&I^J+Bv$!Hx_vQ|Jbllx|?Q?tdUw0Bg{XoY2jZglFlY;lto4REo@j79FBe^C{N;VAc{YLdHRZ-**W$zynpOgaR-T1 zEiS&_xi;dgy;?|=hVOf$ccvG8vs>??K8K-C`{Q-*yfY%1&O1stx-1 z`HJ~{k`Wi<+*V2{nABHXN7_z|JjIUmI~_eBF7ayIu=>5IfP+ydckQC_(2^3%1IG%H z_}eU>rN{|0@^sOLYr9A0O12(lHChK4-eu)pJFFW=*XS~!yP^$@nurwB^h*tDIDG5z zfp>ETbHssqsPp=4Tanw3$`avqJK?f{yQW`WZC4Sw562#><=(f0``>nFzPSo*A?&8A zaCl`z-(fGExt-wnm~gh>dFOTL0ocs(ci1{3XICB6;!DZuy12=@{zbs)AdY#bUK3Kf z%?Iz})Lvd3o&O9ttD0hXA1&xnEVORa4s5P7P0U@0+UPYQl$?IjhQs8aHB5C@l5d0E zDiuU`;q=rujO>fuV&y^7H+xwdEyDvZc)h;2eX*B^sWW1M5S(dMX*v?AjCteV$xYmM z89F#+63vX6-a4ZyGm}cIS8&=W5U`QG9h4p2#gE$#C2ke@imSH&=JUfNoSE3(P@+y_ zom<70=)oA82U1yEU9FISLdlY%@W)G-$Y-yBMd;DNa)6$Dq{{dTJd)ssn^!PQPP7%5 z%qU&07L060S=2*@jN^}XW&~1R3i&;)sglTn0(a1?qy}UzO4V!lofbj zXSs$s$wFsWO3G+*nbS>~^H~{(sd~SYA+%cM_?r6yk`lDvd>J>4eF3H$P$noY#Vz|! zMb4qj%;U)33_;HX3(TQR*J-8S9Z( zXStl+&Bc?^vS8Y~2K5>G^0btc;-H*#%`u<`gAu$3N`8|)@TC0L!t2tzt%J<>!lfru z-oy1aT0B0NN$Qr_f_TvP@+f_o4%va?+CBxG{A`qrdk;XDd6N#kH6TqVnZBWx!sc zagOzI)Wu;`ospv$<4Z;g(81n9owTn2-ZGaHaJFW6m%Sp1mNbYEa{n3MpaIegI2_c8 zxNEPT9ZccT`F_pRnS&6$-|&IvAPQA`fIFf1zSE$Kh?LKY;lyX&r{=FwsQ2b{3;rig z$4j_u8I+t#&5Vfd57230;kEI9E?$E38IVO|dZig|{NY%)H&kIeoZM}}*g7>!JOaTm zng1W>AI3U($`o3IXH2D}yAC3E0DP!vJrW~Fchz0!oUGY)52{hQT>Mf#Zi_weO9i%t{@e9w0Mwl5>Eb+ zClpk&{N{^ag`pt|a$8RJ-ws=&<>!WJw>o?GA4HMwoW+_v+=^%dJeVWWeqyX6Imsn* zdYgU#Un9FMB?C01k@2Gf5ZalO|ISvP;|RusfVE$d$i!nq-VvQ20yg3iy$DW-xwGgWmL<76e&6P zV0{uCgWQQ`i)^N&JObK((%#B!iTGSzdx|?d0m)Wd*=o+le5%3Jh@QTFwb#l}W%v^L z+eUD-1q!EzPY_^qCl>2}%lfgJdxsAL2*E@k^|1CvjxObrP-dLkE^cjT4Q*G&R4I_q z^fP|A0NO_==B*qzuJL;bDxXK~L_ls^Gr8n$yyeh=pisi@8l*$RAc`?OsuyM7f#Y!c z*zI6@O@a=7Rw{EhzV30w6nA>+&zjaMa(V^#Dkd~vb0^v=hpE9W$Jv}$5yBw;veQ;4 zf#8V;bP#XfUBv>xlCh?${)hQo#kRgqa*lfeb)84y9=2gw=ypPjNQnKaUf^k9zK<)y zz`$n(T?xw1EA&DAJ#G%H41@ABr#Z*{Pa8!X*PUcBCSKg4t)l^>K4@^a$yM;R*kO;nbm7?aMKvzrK4 zC3)@%dvp7TWGa>rmCCy%?my<-%j@5RaA3f#$n9cn5mfI)7~1a;hl9u^{D=m+@uAYY+eoxaU?43`}yfNI&dG3t$a=o ztt8qT#Q@%;)@CRe)lrU_2}*qnTRk76n-a z543-?*p&;(eyX&+RpHCVtH|g7Z&*g%sp1*$s?QLpyW<(pY)X3c;i^gyYV3)cGxN(0 z*}nq^Zr3niFY+u(HQ+bb9rF7QSp^`=P@a^{(}DWLDspBhbNymAg?^{z$IoQK(80jT zQ-!HQchNF*DT{2dLmJwx*NKvsFIc6MkFeXFnAidEQXPYOUEN3OuV} zm_IuiJWI1Pv_Z4QD;t>;q($QNTZPCyMmP1y)rH=%*s2s>KkOA+F;St8!KkTX9FFhv z0}K5E#T=I>q5mL?C5T5!U-q?f?#uU${Oju6_a-(-QfxUreWdjr5({Jg(q-jiBGT&Z zVu!0SmLBP5kK+=6a7ODFU%700Eis5}YjfZp<9Khz=U%n7_6cDTiC+0m&Bx^KVayZU z%&QhB1_%cZY<{_+;N=vNIVQ^3=9U(0l_c^ok9~5c>(QgqFbO?o7gmov3zs4f(XumD z7f1G~^yCO7MQ#0Jmdn9<^ip}xcAlNQd$-rPu-5*47)jcdY?g#Qx7ZJ?nKL+Pmh=df zVnqkD4JjMxUJjO)trSK+AucU}>5W{QNs>&jv25LUok2$>MH|f8*=8PyV$upmC)+-^ zRCB%8x@{U&B78ZTMIvnB2ZHJR@YDIlxs$ujNp+YkDJSfOm1*`=gK@o2iSNZ@acBia zqKEGj;_bPwdlbKVKx@U^m}LqqJ3rYq&UB>>;=Fa(?yb@Qe^JO_Q{=CfYARW-2MUyL5D_>+^%^4o&N-(#u92 zAtuK2Ue9X#LjbOPW!(Jk$^<^ zwcA$FzIxPJ*1{V0nUCvyys8z_j3!I#(MbD-ojJMt4mxC%ZV8enz`AN(1;1Qee^Z;Z z(}Kxl(CGJu#Hzg3$;O3AAQj>IhuRk9{6ph+cAm30)TB*KwaW(y{PA&4gqZogQl9;d zFB#ah|9QEW_rxRdvWSSZu&z|wYE0bZWY1K925zzgB}*-DF3*)6KYpEPp*SiFmfyq5 zvx6*^13vF6SgtpBNlV*B^`34`*xv69cNgi}*V}Ng@L`zuIceT+;)s!JB)FJI&iOJNru^=<#XUaGq=`239{y?vr-%7Ul`QiU;VNJ}uUA;>Asp+PnN^ zQNBW$JQo>*+;O45il7)LVhST#j{(<)vk# zLR7MhG%6#_6_azok4~pbF+F=;>3QLQRJjp}QZd)(x_?E0%_f9agR{)d+#Nm335^0COfqz+UF*cUl6m&0v*?@e9`x| zn#_$h!9H>Le6j11OsN|~V)Iawih_3lc2xxY zn&==3#;o8j7AM=;tt3grmqtO4>fJn-fa={=7g~rQnW})`OOM8hscvAnU`KP~dTt=M zORg4^6K8!$_An34Q@Zw}j7*i*qVtp0sohe7y;b!{^~jhNvOLm0m_JmrI5)1oA$G&j z+=t--wGk@yy==~v^ItE$OdqNk@;li#oX61Lx4FT;i+nC~OXfRfhukIL}J3pRIzEnuZ?-03L(a7nc7TByh z?Rj%o8uWDOJE9kd0hHCcU3q$16q|XjxkEnJK+Dq&$HeXsTymm77UWs15*(a9us2PY zTyCZ84OJfKr-O;A=g3t?1Lm}aYBHd&uJh!Pz*RH#BF$Ld{HtVh-{pz9S_-Rud6y9N z-3acANBjqsJ8K2F$%jgmAIX^OiAR$pi~M1u-GyunGCfKmF@7) z(XcdSYp0m*)YYB##E&wjaOuj$Jhi$nsOlx36dy`jaK-2vLD;Xx@81--6)nji3A}>~z=DENiVL73Hj_w$|?%smn3Z!0oNSy-Qs%ZbZ+k*l{jI$HlvOOk@Hk z^jPFpjy4CFx+nXR9w86CnBwjAgg(0TUAgtQ-*j6SPrukKIz$l@E~?o&A zkgNeMV>&JuVWtAb(RcB?cG0n#Y&rjsayu0e7g$`^HZk%kzp)QQ3E-l@KkXB|$qyZ6 zxlq~NFZ9ZabE;8c@ubT z5Y3jJTK-|MHj_*QRbTYA-0q`|u0_`^z!-Eazv?p>(59KLn3|e`dEC*b4Wdel*KgPS zh{rBPmF&4)@TAx(JvUB4t&!W;hA9ZH@oU^a^qLTtF(Vnq8)0c-z=eZQ8q0dJ2pg|n z^lfkmdiu|MIUpD%dexwh=n(%;DN|G3z+APfmWSnjH4GrGLRWUd1ltVo-C`G0?_}mR zX{Y19u`gEN%cknT%aUf9S^Nabmq^ku{9i4Apl}U?g`rJWwoX4slBE!rHauVRErw5hM`_fLq z-`~+?lK%bdsd-A~^5JF0Fm!UY6|L`+`rfN)gIwnVs@E@;USgs4jS}xlAYB6`C)TL=&lx zr5mmOV%1m{vGbsd=G@5wxL(h73EDJEX0a%9KTEquE>y@M-?G1(+_`hIIshI2 z+QlE*29duNopgmP+B(Tv9J-g?PwtL6>t5~A5rMvY+%lm4=mF>Z@751r$^1ruD$@+v z4`>#uDxCY?j`j;9?W(7L_33q^jklMQaLFT+F{th-rI3y05fCrfZF#CI4!>dzlW*6y z8^t3?9ODw?nOx&OHHqhC#GZN$vz(pj@;|CQo1HyA1WI2=@fz{zxkdD6Mfr8(^t)B! zB2(ks{)ajhl`r;1BmOxnqvpJ#_?RuHX`cF3!)=1;O}KOV^CW+=`(C>R{Ix0b$W;fs z#NipM_4JGpa|V5eAPP1ZJfBM!#53`^x8?CbOZ&cP7(8>a6wG*!>WZVFBl}~sgMhlf z+PCt^PR}Mi68mf{T=EbxOb%!3e@f&%TQprY-(65}KSJ%!=0tCkyS5O~(>23{W z{Qkt`7tcY$woiW!j`o`#dh2y3UKP0K($b33C0X0q%xhsHUAD#^EUH2lH0HT2IrP*v z`UQ}z^7i^RE5l{Yg4{T$OUKQ-gC66or6je|y8PAI#bRo|kp@x7QfO8Oz7+w9yyxAe zPSW|^Kr9opspTwUEhD3PR`9^YV!U#9cwJR z8TMcRA7d)~N^>guygo~pQNZMZ0aXEY=Nib?h8N4tE2dkTM%vgo~O8^XAiI=>iw zL#3@Y?~i*^eRJr*je4QVh5w~~oh(c~OIDQ&SdZi#!GeK&jU<*Vb1rnY0h&^ur1+mg zeEm4M{iL2DeFnZAAhc{^ax?@`HBpiH7r5RfnWhH5J|ccD!7m4lw>iJ-_enNb8yWK5 z+g)0pR&tp6OyB*5(aCI;jI;}%n*C(+5RCKpXd!+2`_IG_(fj7b58}ocQ_@04r-Wja zbc3o_jqj73-qq!<=85mbU*oLTTLoFxNsnZ}L2SlDjLW)Az(EbT;t6i<2M>TRx*&Y# zh!a`RSC%q2@P!pe1(KDS3F*=khOLk2~H8)2` z0IO|QUKtHYhZD(B*X%GAPFam#zQh;nxZaIw1-D7lcckAuq>s>viq(>%da{Wt@!keQ zO#eX!Fdn(({>%%IGD?fD`cf5KnNo01e|ATi)X#xm$w7VBz~?9py?U;e&d?#md7t{` z8uwdQ{kmUChOvX8(3Zq@*O)Rz`k)E7NfT+AlxuO02p`m#G`Ir)X9i|br1NyZn-`h64BN5I$K*Q{;n!_979%xHd{nYRkXn!7fZ?U zGz*`U2UyPhBAs#8G7>`{AibxYan(+WVMlmrMS=T&=ch;N=Rg^~7Sd2C=n7#?ai;SG z`0Qi`7G||h8{ufTYPf2+n~t2O7`bY}l1&A|pVH)b&w&U)g-P$L+X^0*wY}Hrx6%R_ zyL9;8^nBIJJ{T!WVi1M>@Q~ZiRx5MBRx@qPBK{)mIS%J4F?if->_YmncqP{wEVy#7 z#d9Meta;S;TqyuwTUmKekc<-2%HcbGu3#p^_K*LbQou2{cQ5c&DNX_&mYq&ICoS?T zteAha%#uZ}$K6e0P_q2m5=0+oVJwc)u%N$De7#XF` z@O(0U8hJ7&qf@8dQ&OHQXcWpQcbY*~Oj_Ficy|e0Vx$RcuB>a(_wn>RMDa0#Z5}e+ zPnV=!5fKpqMNK0bm8BIQmrNE+R!l6#?JG9y78%DW#6#O&Cm~2JG|hxK2j@pt9TGuU zNlrwUrpH%za0<$?;a&A#n)gra=PPbXKQ#TVw?6kAknu;COSm1-{Y*TvL7I zx%mNw8X(!Zz45m0U}#L+^QdLGi4I+XJJIz#0>Tl!O&+#v`jg`AYsFP=E|B%~dAYN= z^^zO>`J2J;#%sg1pmOPbnfsr1{3zdMw+Wf+&%Z^<#@&Snpt?u7fAQ`0**RC;ho*a>M3mF zjp+@Xq-|Qo)ZuHoEDvbQAr}*6nSOV#~&OetH}HWX0smbq)Wf#`C9nFBjOE9t zc;BQLPeKh&%Pb~8^Q@r*wS4>7By49D;7hG6sc1INkT*I4 zy&R(27Vj;!Ewn+*;KKG>30QkNqpE+Rx(c}7YbIX*gM`+wbg|zJe5FYS(B*)$;efN# z1(}OxK7W4tsLrw*{Llnth?bHy`eL<|KWAW`@G_V>r?*@Kmmz@C8Q?+TmuQ`oe<5T> zD=af@U5uHeT`Z)`;&136d0tI;N+U5j&yX-+;d;cXE{Hg z>pXK|#@#=^fv%@_o!ok1X$*yex_Z3a+;ItrZ*S})fX@In{Eo6J%8HjLUL4){fy4@_ z^`cCzqqm30_o`nWvdi@=uh~FDmu@E6vq?8{BIs~gqY(j$QN5o8L)9?xRTXA^g41vQ z9lk6S5~dd3z|3Mp6ZRJF;Bmq#zc_d3n`0%8o9B9~ z1l_Cd4ghb=x)>8TacdEkn{TCwrq}X2)9T;~MrC5+MCqtqNszI(+JT^Sco`o0NL@1@ z^c;Tew)Qhh1|1o$b(X(}JBB+kEDMsurtcdbNd+zjEFe(@Jffx8=Q3UW_ZLSGY{R`0 zqJqC(MrQZvrWR@6TdYat$w9@j<083G@7}MP4yfCtp^rb?_Edd0ra#buOjgwy4`9So z8&6LT#R8VE76Ou~+(fjx!{)3A=mecioY89t`0diDNv<7UafOo{;OsazC@sR`z#<27T4{pU zkGN^Xg1iOWw1Fh?&|*McRENu{QisZPaACHgX^gar0;{a1t=U&nGndP6~T4o zYRr6_i528|5e+4`m`u+_Tn=`6YhYcr@VsTMz2c$3P320-Wv?Zx2V5vb?s8iQ4_c~hdlE<{lec5 zf1GO>vAJ|sjxdq0ZCd93^GA?Q4<*JG1Zd&ADlyTn+Uip`!&Lf_WJI}|QHSjU*~2b% zxH~^D!R=jP2*v9a_UI+*WNME+AdR3t7dn<#Gdx?q2YH_U_gIp1Omj4@I_GUZ^g3&Z z>x+!nrfHl@mdOV*futL3*-WxCR+sN&%nWz-hc(M}JR|oTNaeA|74q_>T9ZBLd$83$ zxuJPe{i4p&cfi<->RKnP{Jvp#7k?e{{C^m+b9@!i+-@ILSmYrooF?sZ)ZLaf6jZ&I zuOU^c?2xnlHpue8poK}iBeO~w(`Y=;tJBrUjx+8S&zV8p4+KK`+sJBSSHQ=8eg#XfKF)})T@ImoJL|E%sKZfe6TE#j(sZdm zSlvUhBXP6Rva+v5+*vw@gHsb-6yzPrQGjRXzT5r13U~1EVEo+z2jR7Uu~x_#NUOk0 z3RLa#&9_MHbl8s+aQc1h<^0qaM^lq(kSSEi3XX}Ewl}VCV${a92OTY?qFJ<1s?<~P zs_N2%S?+C@T<+U-aq;LFbpUABej~;)Guor)e?H2?52FPv2Ge|nuAw-#dd@*iXedMN zL-d(Lq8uKgWw;Ynuytl`Dfd|q9}^haP|A@y>ZoIW4)Lgy2aFgjxIhRYcD!-5;2Or* zUPvc{O9*F963DU}G`2CO@2)H}>uC-|(u-3;S}_!?_xY-|=bn8^?T}J6b8D5c9lg=o zM}9@_q&%wP7*EE>ENPOJ!jd~j2?k`Vr#ah80n~9Ft_+I-FS_3dRB92M)Z+T01*sG3 z`0p^M&FIBxr|LLcJM$8FM-rYU0BiuAFmW3gf>yn7%3zVMQPEkoXiQx2g3FS!Cd*@W zyw#rq!0c&wX8gCTjmz4hUo52Ot^st1*f-6j1EUB2?2R^BK%DX**K+%WCEss#TvunA zyY@<=LnDrLWh+L2D~vhs#Xr_y$vgaQR2vR(5b1fOHU)R2&RB=~Nx;odiZT>4u&! zv+M)RyTQ4szBfE5CRu8T>s1Q!(>n4oqOX9H`(wn5bNR>(N7D}ge{9eGXTEv`@$153 zzc*gxy4cR`&zAD9WeY?wdTK^airQQhBwFv@3_6DDF8}xUx_Vk++NnI~f^n|9!}(RO zE#K#Et-e5OUl7^?fh6Tne5+8RxJnvlV_|9lM#~_ALp;1S4hdaCRK^076K#t+lLv|g z8RYF{UNkiEGLJg55Eb#^+#r7k^*gmZajNYBXL2aTUOsS}Seo!R+Ng%ayn6;;);N@8c8;+=~fvl zj?o~TDCX7E3WXKL#1xz*x;&B*!T-kyjBr0?P<}%D5D{M-^b+0_@(%1LNE|$OYerVSB#r-E@#~BzcsXr*oajK-WtKda-8-+1A|x^eh!Xsuy7K?@ly`lhB6RfdICbHkIrJWcWd~o_O{%y?l!RN_8>1C*+*JSIx^gGgqL;(>*c`m=YQ*0m4OIGnL|3H}C+8^+iM`%HNTVGx#< zV6Qt^tDA#}(MYu8aKv4LJZGVttere-6Wcx(f1+j0QtR$Eip>LtazS|5$=1=&rQ0+D z#>MY%7a!C|4$xyj;9t8AhJ12(>YY`PkW0ju(F@7D0Y!`q|AU~I^i z+lo%obsxzpphEuJA-OSN(Th;fW#_YOmdpYsQWm3O^&nA_ zkn|FkCef~5*4{hm-!I;DbCOd}TTC2$L@Lj+10^BjV&^^(k)>+ee6f}3J7M0*qVApF z7G3J4=4?qwKmdn9rw2azY<2jg+rAXtW$^#B5^k71#qf5ng4L-CM-JfgX#@dRVFcv`I-d32blQG zi&x`ssnR+aFFP@)(1(GUem-&rJch7N&+Xy+Ht#o=W`BJ039XI#@tH4)ci~b1 z9V!v6**$trQ^z;sVYu_Js;0Yr+mH9X)WEFOa)3?a9B*MEU%K%x+qwRnv%5L<2j!~JB~E{m zR8?^9SA|3!{jSP@i21L&)TloUY}j#8!Kd;rv8%n}4lI9=g|7I6SHng2DE`bh`iQH1 z+?5=;p*#gx#rW^A(s0w5f;TuXx!D_k(jq^&WZeIUl7Gm8tyJqdm|7AvYI%gUG8O)~ z{jRDJR3kfvvGp;Uz%odJ%95>veIMlE?@8b^Px9?uFudzjgl{J2`)u3p(zGsUGNZgu%#|s|k<5lRBSOVSLUK z5?;(x;^NE3Oj7nB$E+>|E+r@7EqlKev{~93y_)ATJS86P&S1yCDNQ5H%l2i7$%^=a z1UozIEnKCG7!i*R@~#_R(mM7nuEJ@YY8|vcnH8wX%UTJe2ac;%2(-euy6=4%?)05W z)=U9A43sFYYZ@w`Yb-{CFsFwCJYip`^<*^F75MGYh1vx?V3Ycx7Y07QE&HMd{Z6S4 zOo{+}*|l5xZqj)(mdky^T~vQ)2zF`Lvlt)fH)!e;o1|6f^!4Zn!XIdvSDtzWK6Z{n z&sm_33ZLL^fEeufWkMlq<-~VIT@CCdZd9CEs$D^gnWwfm5&u2?n8G0ToCWNj$KSM;(99J-YR{!0c4{#c_hQ^M2SKXH@sP#grCM0|7$u~s1{@Gf4GS{^Phs~B6$iD zb=C2Qz`F7_MPmy-O;4Yn`koUK6AkXVcEDcWy1X%j?iS@Vd`qBOF(2|SD^=hgzIeF; zRsa!S5_&bf8>O!Aukj%&BgT&7BWQ9d>y=~gluRV<#3FNc7(7i zOGj<9Od08YZN29F^t2c9R>=JxI9Y=!JhiPIUNVH+^Em++Eyh-#PEb0Lw-TqWfy5`y zRj9Dt{sIzu`#=k;SIxEcD!X9`x=1<|^p*Q{-#es6unq{gUm^_=5t*0UG8-rEX9itUAJ9mMy=HD7QJ;BTYl@^T6)oU z`}Lh&a&0B9=No@0-iDqdF7^qAM{_Rw1KSX??%mHXf;Xqd6NXz$%Qeku-<9?)$(F{y zVc3|ic%KCB)mjW9qSDc^iMDOcg?V%NBZ2ZKaKz*nz8E|!2i6dG>s3L90{&N2(ZyX} z{WG2SS4KqgX0LV)Z`bhE*R~G3k2#8eFkf9&6JxECHJ=k^7+8=V9$Q}ffD26b_p_I7 z#tdgi4Tz5<h-pr2Hrl0r9?|Kul5+Liy)3qX0|$%p@XypV!-BU#*|{U>cQD=O~!HIVM!0LcRpy28at#ZwX}HI2%5dm#!Gd)N&9ZRk5IoC7}eMp z-FF2P_wE4eB~B^vwR2whlNKF!e0FyzKxlyWoEB#_w&Wn{E0^7WqJ9*{A;^hwgz(zP z%+&j#{U@aD@o%&{nk;If8Je|h{ zPfZ0168i4ujy9{KG!y)W4U8}fHPi##%jksEcWv+c9>sP4d=!`a(ky|6e^XcJU31m> zDGE8mxgznwj)k#rKJY>$cM;er=txbcOI_D%j9=r^Y)4ZsdD;Y5b2y{L+ zUf7r*{28X--hg%9YXz7*D`(d`(ema_?SSG7e^^j_4BbDekGXtF7HFG-?$AMBp`1V8 z1hNh$I}UiR4iV;YWM9>{1itHxjB+5VfV6Ih%{|M&yMB9kNinOm10Jz?*zQx3!sD^N zpI(&liZ65zX8L5665JeoBbBumUR7$W`Mw?WP6r944?%xbc{OA^SxS!lZjr4vC|Vtb zt&Z0?l=mU%yq_6cG3)LOSce zj<_%?*0Z}cK|-nLcP!pV?rjdV?o6jl_J{SCZIe&ke5cG86>bIA?2kc7%>|}c?YO_b zQLaqF9^CRSKJ3yN#WU&i24VhnjoXg1*)E|B8)O2alB`rpvh%SpS^b#H^(?xrOVP)W zFzhBi1ng_{|OUUv(I?A9bXl*5%?lZ%^vZ$>X(brD3ubgR$BLZz>kv1lFEV(wWm~Xoc9s$-aJY? zy`KwH?d8zHl=PT1ZS2zieH-Z)DC8rcvp#-)8{tYv>)?>t;atvZJOkjO`$anb9S*n& zDeQw4jYypigb#@D<`^T%jh5WsPOgRkpf9cvF zHB_cM7fhaH$SEiU9?cD$9uZEXRda$*$iXN1!3X)lM_s`l{ij{v?@#f!Jx#-5!iWDG z+DXj@_=+(H7&dq{+JWb< zbUIubt(ZI>cWG&AXwXnm=PTasoShw>oefY_@;Rt<81HR<@b_{bBB<-R^=D*cgu#H< zE-X~1Rp;B37l~|>H5jS-|H)0CkCW}I<}a--=B4u9z4dw|{FPm?X3~%*5%9jR6Pnp< zb|%6>IgGU9xBBy0oyWwTimGZeVa6zAT0ba_ftUGxw2iNGbX~&f0T|{!R^=x)xeNoo zNVT3yUYt2k4HVwE+tt+tPTYx@HSSeUq`(m45`fLHzb>4X?ec8+>suCk;*<|+l^Rt4 z<9h;#he=J|tIvcd(I-Wu!q>bZE&3;Fv|o}RW7w=rgp&a!3MeVR&B_RzaX>^5k9Am1 zH7Uu;|7J38^m!-3cdyOiQxg-XDRsN6vS=66;X@iw+4re?NayPGAvhwJiz6Vyg?Sc+ zu+~@E+nadM7bk;d&s|D>Jed=af$7xE892$1ivX+TT|!S3f^Wzx!`*qic6UN9U+spK z*#^}&CM)PCnWx3aos4)uPyJ*z&(5Y|)Uxc&-onnsS7H}ei>pjW^l|{fUveOFEiw#N zUlIj}!YjpmGh2RpOHI~Ch)HS!no1SpMM1*4LPR>&x6H88R7xu=cm4j}tc7!fbDg2J zuUR5D-IPPIgAk#7-TK+9&7e|!4>9A1x8s>+mbN?`+;u=HJ!RC(bi|XVS2bs#cPWP} z8?dAk=K+_Et;9j+Na*BvwVbr{BlYE#<>i$Xny$e~>|A|G$>Pk+OvkgjiU$8xT(bhC z^r7hpn3g&4AmxI9Ub?Jw2OL0C^xeQO$`2rZ3_9|$ zvf@efG(4H&_1D&n~<=VboBKT zt1=X{ex06dO-}yup4i_m4OYm|DstJ+6gQy)y=cjM1$*{LaKL$IXGiO41!$lGgfCAC znKI;sdPR9z`*APX3^3kJeKkKLcnO>Ggf7rvuo|fS8>wwuTvJoCs^s%u0>YyXRR|dU zW4m5g4TF=El+&(%pGLg^7$(7tqn~4A^}(l#&D_E5Z!Yq`WSz1%$k)T?^xfGifY5I8zJYpq>@?$H1nz!x)?%Wkt9-_pf1hl>==%2oq8#ZMPPE~ zJ3DQ4fnoe>+FSqB8V_(+rTz9TiAWeWO5)3{M9*~B3Tot*3=jS93KsWk?a$HtLOx*f z>y{k;`kvpMf16~kT^>Nv2p=LQtKQJFO|xi~fQw#?AZdbeW%kFaI5a&9zV@l9 z4F4u{$agK$q~n~G3pI+r*IqW2$!hKP!YmwTyqL{97`|67a>=Zz@>79eFYN3I14&Iq z_qVT}9!5JwuswWFl|RZ-Hq|CI?(CV_**NqU!CugoXOsjLUHkQGVvZx1D;q3Z|J@Zm zjPGdjxzby4J)4&9Rx_Z3k8^3I=~d-0aXukc8JaTy1*=fr?&&5t)b1mGBSux-!f5JB zWnI-aSo!O=(MoMnxP)@%81TlLswzx0{D=>`sv6`oEqurq-{?`n+)xiPTBI!jivOKk7L9)@rt}P*O_jmGdF|2TP}ncR;|;YA5r1 z8Cu%+nouc5(DmQlV0*-GOBecXQFb*jS{n+_?JE}~xN{gcT`b9C%BdQbVMM60HLLJpq6dth8eO+~z-CE) z3qzV6o0}U{2|D^%QVh1s~+F9Y1nVR)L3(I_?Yr9g3y>&HVu3P~MFrN(UV2sPt5i zAxvzxT!i%m$T2<53Oe{bIoT1#s7d^Ai}cq>oB(~P=G~^%yP6e-f`GM6Nj~W4hrdho z=&xT*{^WjO66Kg*i>yyiaSwSaD9KTzsc144tA^V^kq6exs`MS2Y8nn-?oNzPjHNj{PA-lK2zNdj*#;J? zJ^uE?>rqH+;Yf;6qwsa{#G{fh^BC~${kG-?rUnL{m5Fu&xwYf3uY((b9L0emLu;1T z<7|zC#g7GxK&p-+rR5D;=#CF}Nx`QIxw=q!B{~1~>5ler+AO7D=i+M!SRC3OsHK4> zkNx^(iquW`69776D+n~1w?MF| z`w@ng(9{u0PETi9^0TtOO-I=b`(kbXx^e`x_s_=K+S(>TNl8)BEKnoUDgg~z zb7@s`%Bs8+vy{M?^@_AOj!A~H=T@5LCq%-dyQ~5XPWY{6D@I4BrlyiGJas1DoOI?2lL3)gIyP2C6?e*DS|2 z%LZ&5CGMYbUmy)D`pEi8;J9lZXCu|Q&d|#}(t97+`&QYQ)uOy{Q)MYy&6t^3+rV1^F|md7cJOOs7*Qx>WHAWw(1s|3{fKpXG2 z=|dbBQd{L7AT0n9_3`XL4q)Mat#o2Ytd!8ql=lTBde%rSoX^vz1~`1W0@dtxvz%gc zFzAy#Ibl0lyQDc%ApvMOKu7$;hg#FpE33vhHuKMJ?_)cUV}i~OzeA^Uvd==eBiXhb$hzWf3LOsbaU32=ntE%$SD z2q@e)^Kj9VwX4c6(Azs1b{75|b`L5|gGbS;4me6S0BN|@MfR5R?_lPxn3>$IuHb+H~{Z-WB|!rIJwxXfke^)#K6)S zS})mC2;>lUHJl(3ZnNL`HHSAcn|F(}m8~cvV-GQ%9?LsaPk7ITscp zT}^=O%C=vXu~U=TQzJk(VqbMRfbA@M+4;|TZ5Wvht?-u?`9we!0c7{m@?>I_pALGM zK8*itR189eX|4fXPluvktGhS5J~4r=T9+oX!)Ei+5juq9aF}lsNn?&4JJr$A;UU|> z)UvSxBnJZ&y-j!98@xgAei!7H>gA0l!XPa$m4co{>KVmk^p4-U&{qGaUkYF zL>97t<_yJ2c;39-EB<*i$!^%;?&nQdTTn19JV=!3OZmoz_#GSK`Fo;SU(es0+x7$O z;G_2I(C)~isc`RX(Vk6K#9X~IjwcB-g(9U7wF&;&Gi((<@^vw35?tU@0dF#u|iq+USG_$#QTrpdIb zsQknCMPBIHd$4l(*E7;!)e4DGZ@N#Md<#>ADiGw9lpapk)>Opj%>&7WUAC(sZZ|sI z(lGTU2K?9V!PrAX*duVdNth&z8!Oe4@t9iZ8~I+Gv=-50`4Gfc_h1aZG;}yjfqs#b zlP8YrRdxi710}I3>;{2`JSx!DFVbV}vS2!?EupKBX%huROVbd8V!at&X6lP}X<#)* zh5%hCI~(6wU$dk?Sc|!OEkTN}46*-D8rCt=N%o7X^Gi zo|+mJg4FHOvD_$-gyFc16_6v_UJVMjUK%4Uk7JkmOW%a%er|_dBE0JsV$v0x@fmJGREj~2z%u8cEgp>H)P*vH`cDgyk9?W zh?d6V?U=BrG~w$}A}vR_=ITM&Xz!Ju`%4vsY3bM*COE#ZkR9sLf92kL{zyTTZxY66 zM_rrKCg;P*VpJ3rQCz53R!+%(Z*3}|td(hXWoZc{Xs4eJStm_KG>P8}bxTC_j1zQV zq0+H|B*Mm9SQ|(3S4PzVYkA=f6QV?~gb-0hp6xb@p$ic4;#8zI21RX;`_c*!0y z9YOXbil%?fE41^_*mYFAbBTwCi<&dKj29{*vJ?tVLy2`FjbI`Fbd|MYat6@Pz+nth zkU;MSI41GKR?;_<3}FvVKx$h?!=}xRFTaN2do@Z9)kM=@@tn!w?b&=w_8qT|m$(x0 ze5RHiS}xJy5z&jwp`B(_riM&jcv144ceH2X%0ny!3mGMiEN(w56Wn-v zQPd;i3;T3brFV1f2O=kG~C$CjkcsIDGO)4o}SX2WJ5A z#cti+upgNnG2M93srp%0tkul~E;s-mnM&fui zJI~R4Q;3xE+v|JC3HWngvP&x{c(RfK2)nTDSFB$osG&!MKsNV(?jBFM!jN=`xkRc& z*jq-KsR*t}c?a62PcX#cVGRQ^Q|qa3rn#w^IG^{Y_nZQ>M`X0awcM%jgtNec`lac-!RVzg(J0-YVl3?N1L9- z!$Yl&^!4<#6a1P=?3laURER)YIa@SfZF?k}eQEm;@?4%^ZiYsP*smQgQ8~C*&zH;Z z4lhchq+_qDtk;`}-#54d4`mnZ1yR^YT9Zb0!>%h=+5v5mRJe6nhTpAx{x_FeK+lT(e`iJ0vSx!;W zfNdxY(2Zj)46pS{Trqy~@_qy-p7(M<6C3u8VDCouzw#mNWvhEu4B60OpF+OlM*p}YmFSeWxT4|m1d3ZYV zNtj>to(>YsG9~FYH@?*4 z=h`jEY8hS6!>B6>E=0hxUrMKH@^`!wLgl(=n;Q>7~{D^oinodb=G~%N1|68Vv$J(%hp%8 z_gOC5PJ-EuwIP#y7?gy8Jf~02B!eiqTo5QZAuS$$zH)zLYnnbiZ+A681p>K9MycG> zL5_%)g342V9s3Qyqbe!@P;Gnvi) zKGM=iU;BFOj0E#nKp4rmY$Dp(*lJ^%V*Us0MITBmk(PHiT{A5g#Pq$Ji8HQ{O5s%uc$+399E)@bv%ezs3tE-IcOUP1) znn@w=1CV8J-3kga+NqWP5~uNMr^R~38x+(4zvxHbMY_q>h5EVdu5iq` zh8q<*y|=U!&8af>TFV zO>Fs!wy=K{AMyOB>+V8KvO$y^>uo5a$qdBu@&x}(e~=0l=vo`q*GjWdUj$O=Y~Eg8 z9y?|Y1?-_|B>*t*w;_VKIU_4;_b{;a1#e`3KVf9Cs$}u7##ztFzs@8xGZP4ra-jr! zF9)33OPqzgPqRtkiPDm~0mpyYWQS%d2KoWb#Cb0`q+--j{zlChNQZ=apEdkyb|?2+ zi~AmxcEMVCshhU_#8V@(ZJF{cQU*102{&fRjep)nSuk~KG_c!}w^eg2m_UJH&O^3x z)fy-X*~#g+c7?SOMn;8R+w)>~a(9#FkKmCQqm;;B%6|;_UA1NXPnn`7M+NW|Gfb3SnOx zagcQzv#)wU)YaVv7=?%I%^20@Rk_%!$T*BqwNp`H&$9>V(Me@km~8A!y>OQstZm)r zEqQ-u$IHJ+2!*aPR*;rckXI;6P2HLcF4UD3@!qTyM5l&H7(ITez)6Y&WB}q)-E#}} zWrnpS>3F{T5xkLUB~lC9S~W$K{1tF;mKpXvy+hbkkERPv%hPihGIQyYphhJb`DR~{ zVvYd-&a=9$z!u)ikQaqut2zN*d~r=wk^%vszhHVK(+7<7O-JSxQ;nNw;MU5GQ#KVq zMfb*i=}dBse-mKkKc;hf#u)m^Rs!G~y?;0e&mMftn@^0aYWn3MnSPeQne_69E+vWI zwy(+#b03tr0AbIR(rm9@*2*7$KN3|QJ+pVT0(iO3!>yS_w3Mc#cG1lVWP}^f5180q-I}g{QN8!enz4D3kUJd-?Ks$AA zR!f^fs8^htQwvwl3kj`uBHp+|ELYLSFZMW&gZNQUBNm3#21T9vU(4#bZ8q;o(E5@Q z`@7L%LQ;5M`xX*uVxrn9J@!Ar-YVm394S0}hzRoj$pm<1+8GJP@2c`@P{Y4n$+0x@ zm{>aCOsiu&{|;TChD+f3=h(kyoFaET$7W<(-wg|uh{C_;Cv2iviDy}Ut?V?c-m*`N z1%c*2Ay1gFC5s*rnncH%@k^B!j76@^qYy}B_P6Cc*z;^bq_*STbWkrEA0N*HnKKRmLdBAmb)yNS;pgL8&lf_VH8n1_8M43^9Qo;Fv4@U9vws{ zfFqD1RZEC5$I6S&+6)#jYhNuW-T8ZZdTC)7M2yGi5N)D(ZruFYi^k09P3w(xb5K=n zV74E|vI{w9a-o>8wSxb26%_Jo&|4}WVZUs?O)`8ASqoX4!Mxp)dV?`y!eX=&n1}}4 zlDRt<*+XY)af(lg5-56_=Zp2^5n9W!bxy*jD0Oc+KDA)6JZmY}Ytte+URtX{uvfg* zEt?CKz1LHZUPRcA4?>rkO^z?_s91fh)(|Zt3`?w ggv#{);y;eqv`;!6bidpWp%gbzNG+8T#FLQ!4?FLD1poj5 literal 0 HcmV?d00001 diff --git a/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md b/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md new file mode 100644 index 00000000..912ea470 --- /dev/null +++ b/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md @@ -0,0 +1,107 @@ +# How to define a latency SLI-SLO from an exponential distribution metric in Cloud Monitoring + +## A - Define a latency SLI-SLO homogeneously with the any other SLIs-SLOs + +SLI = Service Level Indicator +SLO = Service Level Objective + +**Best practice:** define ALL SLI-SLOs on the same model: **SLI (%) = good events count / valid events count**, for a given timeframe. + +Implementing this best practice means defining a latency SLI as the proportion (%) of good events (i.e served faster than xx ms) over valid events. + +Example: + +* Authentication response 100ms = 50% +* Authentication response 900ms = 99% +* Authentication response 4s = 99.99% + +**It is therefore key to indicate the latency threshold (in ms) in the title of the SLI-SLO, since the value of a proportion (%) is always non-dimensional.** + +Having a 50% goal (aka SLO) is OK for very small latencies (the median actually). Not all latency SLOs are necessarily expressed with 9s. + +That being said, some monitoring systems may represent the latency in the other direction (see [here](../providers/datadog.md#datadog-api-considerations). In this case, the question is: "For a given percentile (median, 95th, 99th) what is the max value of the threshold?" + +It is indeed the same curve, but in one case we give X the abscissa, i.e. the percentile and the system answers Y the ordinate (the duration), in the other case we give Y l 'ordinate and the system answers X the abscissa. + +![latency curve](../images/latency_curve.png) + +In the example above: + +* SLI latency 10 seconds = 99% +* SLI latency 500 ms = 50% + +Alternative approach: + +* Latency at 99th = 10 seconds +* the median latency = 500ms + + With SLO Generator we set the bucket threshold and the system responds with a %. Doing this ensures the latency SLIs are homogeneous with the other SLIs. + +So in this perspective your question "how to do a latency SLO at the 99th percentile?" will become "What is the threshold in `ms` which results in a proportion close to 99%?" + +## B - Use distribution-type metrics for latency + +It is often too expensive for the monitoring system to record the latency of each response served (full distribution). + +In order to limit the number of data points we use in distributions, monitoring backends usually keep an aggregate counting how many requests have been served with a latency between 2 values ​​(lower and upper limit). + +We lose precision, but we gain a stable storage of time series regardless of the volume of qps (query per sec). + +## C - Find the typical bucket of the distribution metric and the associated border values + +There are different ways of doing distributions in Cloud Operation Monitoring: **linear, custom or exponential**. These are the [Bucket Options](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TypedValue#bucketoptions) of the DISTRIBUTION type metrics. + +Olivier Cervello's [gmon](https://github.com/GoogleCloudPlatform/professional-services/tree/master/tools/gmon)'s tool helps to inspect some data points of the last time series recorded on a metric + +Example: `gmon metrics inspect loadbalancing.googleapis.com/https/backend_latencies -p --window 240` + +Adapt the window size until you get data points, just one is enough. + +The response contains the latency distribution [Bucket Options](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TypedValue#bucketoptions). (Note: this information is not returned by the GET on the metric descriptor, so we need an inspect the timeseries data) + +This looks like: + +```json +{"bucketOptions": {"exponentialBuckets": {"growthFactor": 1.4142135623730951, + "numFiniteBuckets": 64, + "scale": 1.0}}} +\``` + +Once we have the type of buckets (Explicit, Linear, Exponential) we can calculate the min and max values ​​of each bucket of the distribution type metric. + +As an example let’s use AppEngine latency (see above): exponential type, grow factor is square root of 2, 64 finite buckets, so 66 in total (+ and - l infinity) + + This [Google Sheet](https://docs.google.com/spreadsheets/d/1pvGC_BW1l0D1D8GJY8I3H4QL76xVQ8t0QF_dIQ5lg5I/edit?usp=sharing) allows to calculate the border values ​​of each bucket of an exponential distribution knowing these parameters (growfactor, numFiniteBuckets and scale) + +**The SLI-SLO threshold MUST be aligned with one of the border values, since we are blind between 2 borders** (see the trade off distributive explained above). + + In SLO Generator we directly indicate the number of the bucket corresponding to the value of the top fontiere of the bucket as in this [yaml example](https://github.com/google/slo-generator/blob/master/tools/slo-generator/samples/stackdriver/slo_gae_app_latency.yaml) + +## D - Choose a border value for each latency SLO + +Console / Monitoring / metric explorer + +Select the service and the metric, with our example it is GAE / Response latency + +Take a Stacker Bar performance + +Duration of the graph = 6 weeks + +Aggregator 50th: + +![aggregator 50th](../images/latency_aggregator50th.png) + +The value of 6 ms returned by the graph is close in the [Google Sheet](https://docs.google.com/spreadsheets/d/1pvGC_BW1l0D1D8GJY8I3H4QL76xVQ8t0QF_dIQ5lg5I/edit?usp=sharing) to the 8 ms high border of bucket number 6. + +Aggregator 99th: + +![aggregator 99th](../images/latency_aggregator99th.png) + +The value of 125 ms returned by the graph is close in the [Google Sheet](https://docs.google.com/spreadsheets/d/1pvGC_BW1l0D1D8GJY8I3H4QL76xVQ8t0QF_dIQ5lg5I/edit?usp=sharing) to the 128ms high border of bucket number 14. + +We can define the following 2 latency SLOs by specifying the threshold_bucket value in the yaml config + +* **Latency 8 ms** (threshold_bucket **6**) SLO=50% +* **Latency 128 ms** (threshold_bucket **14**) SLO=99% + +This will make 2 SLI-SLO definitions using two [yaml files](https://github.com/google/slo-generator/blob/master/tools/slo-generator/samples/stackdriver/slo_gae_app_latency.yaml) in SLO Generator. From 74b454170556595461c8b79b3c9dec12a98c8cc0 Mon Sep 17 00:00:00 2001 From: Bruno REBOUL Date: Wed, 29 Sep 2021 17:55:21 +0200 Subject: [PATCH 049/107] docs: improve datastudio md (#54) --- docs/deploy/datastudio_slo_report.md | 74 +++++++++++++++++++++++---- docs/images/config_has_changed.png | Bin 0 -> 12290 bytes docs/images/copy_button.png | Bin 0 -> 2827 bytes docs/images/copy_this_report.png | Bin 0 -> 22141 bytes 4 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 docs/images/config_has_changed.png create mode 100644 docs/images/copy_button.png create mode 100644 docs/images/copy_this_report.png diff --git a/docs/deploy/datastudio_slo_report.md b/docs/deploy/datastudio_slo_report.md index 644bd7e4..f7c9f92d 100644 --- a/docs/deploy/datastudio_slo_report.md +++ b/docs/deploy/datastudio_slo_report.md @@ -1,10 +1,22 @@ -# Build an SLO achievements report using BigQuery and DataStudio. +# Build an SLO achievements report using BigQuery and DataStudio + +This template provides a basic dashboard with 3 views: + +* `Morning snapshot`: last error budget and SLI achievement, support decisions agreed in the Error Budget Policy. + +* `Trends`: SLI vs SLO by service/feature over a period of time. + +* `Alerting on burnrate`: visualize when alerting engage and fade off by sliding window sizes + +## Prerequisites + +### Setup the BigQuery exporter -## Setup the BigQuery exporter In order to setup a DataStudio report, make sure `slo-generator` is configured to export to a BigQuery dataset (see [instructions here](../providers/bigquery.md)). -## Create a BigQuery view +### Create a BigQuery view + Replace the variables `PROJECT_ID`, `DATASET_ID` and `TABLE_ID` in the content below by the values configured in your BigQuery exporter, and put it into a file `create_view.sql`: @@ -48,18 +60,60 @@ ORDER BY Run it with the BigQuery CLI using: - bq query `cat create_view.sql` +```sh +bq query `cat create_view.sql` +``` Alternatively, you can create the view above with Terraform. -### Setup DataStudio SLO achievements report +## Setup DataStudio SLO achievements report -Duplicate the [SLO achievements report template (public)](https://datastudio.google.com/reporting/964e185c-6ca0-4ed8-809d-425e22568aa0) following instructions on the report page named README. +1. The user that would own this report has sufficient access to the BQ data set where the aforementioned +view and table resides. This would require provisioning BQ access to the report owner, at minimum as BQ User and +BQ Data Viewer roles. +2. You also need the fully qualified name of view or table in this format `..` +3. Table `report` in SLO Generator Bigquery dataset +4. View `last_report` in SLO Generator Bigquery dataset. -This template provides a basic dashboard with 3 views: +### 1. Copy the data sources -* `Morning snapshot`: last error budget and SLI achievement, support decisions agreed in the Error Budget Policy. +#### 1.a. `report` data source -* `Trends`: SLI vs SLO by service/feature over a period of time. +##### Step 1 -* `Alerting on burnrate`: visualize when alerting engage and fade off by sliding window sizes +Make a copy of the following data source (copy & paste the below link in a browser) + +and clicking on the "Make a Copy" button just left of the "Create Report" button. +![copy this report](../images/copy_button.png) + +##### Step 2 + +From the BQ connector settings page use "My projects" search and select your SLO-generator project ID, then your dataset, usually `slo_reports`, then table "reports". + +##### Step 3 + +Hit the RECONNECT button top right.The message `Configuration has changed. Do you want to apply the changes?` appears indicating no fields changed. Hit APPLY. +![copy this report](../images/config_has_changed.png) + +##### Step 4 + +Change the data source name at the top left corner from "Copy of..." to something appropriate. +You can close this browser window now. + +#### 1.b. `last_report` data source + +Repeat the previous steps for the second data source starting by using this URL: + +and selecting the table "last_report" from your project, your dataset. + +### 2. Copy the report template + +#### Step A + +Open the report template from this URL: check you are in view mode, and click on the "Make a Copy" button between the fullscreen button and the refresh button on top right of the screen. + +#### Step B + +Point the "New Data source" to the two newly created data sources from the above steps and click `Copy Report`. + +![copy this report](../images/copy_this_report.png) diff --git a/docs/images/config_has_changed.png b/docs/images/config_has_changed.png new file mode 100644 index 0000000000000000000000000000000000000000..e569c4550593628b281bfdcd2a938b932ded4512 GIT binary patch literal 12290 zcmdsdWmr^ExGtiIA}A#tiU>GJNH+pQcZY-sLx*&OqQKDIDKIc}cPS|)(hVXxba&i^ z=REg5=ic-4-amI99%j$%*=z5$zIT1!`@U-hDk(_fV3A>=p`qbOOF>o8(9mtban}QM z@SjCON*27`br6+Sd+^}F^n&6a@R8I}T+30_*2K}p&>n_nYGZ2+V{$OEhrw(d%xoQZ z?zRY_p*=&BhKi`Urfki+xT=kwv+TOWCrZC~`1S3BJNQ!1Xy5)3#Z{6)`UcQEkbJ$% zP80MAD^oH?iWl>c0?Hz;bWhY)bT>Dijwko&^Wy_S&oK@H?>F6stpli-5qm8hvxhTcHhRFnq@EL{VjD>zUG~#?cKt4e0 z#U1^WiM|(Wd6rC;u)SUx_mj@OtH=3q2wjU5KG~W+Q-^$x2t09RLh(5U!S&TQHR{vN zb>l-}&&Q^I-mzdgr>~ost%b{Ktgrln;*WuI@BUHJt z>Brp>l{J??YMBf=Lik?JGJ4G%eMu_lG58i|ZK`u0S4?e%%Re@E^bi>o_~-?FC6Gr` z$GLi(qR7#_ZtiP?o^@Bs-!R+lXw|x#bufsAB?#tL>E>{=TguA9!cwQZ{f7p`ZNvl; z3lSE+nL65V*)bta-GL!qK_hkU-8ekb|XYX`wrJ-J~=wt8n3Yp{)BeLONhG1 zEDIg_Y_6qsu^xhlhL*wg;E|=R|~7j@}^JC@J(XncL4V9?%5BY*8Pas3ZS<&hj9VaPf_6J-uLwZAj$%MVd;4;2zOgsTF(W9z5HxD_0;~Q}R{i*- zri;LBFB7ENgKB+$O4T#C;M>y8 z#O%Q7FmSZl`WN!*U;74xTJF=zM}FX;l^WKfO(q=>Jj|d&NZhuOKuZMpt@A=~a1cqT z3hvkSuzTQMMEK%q(G9HFVwsDo=7xp~#d;0DQg{z`7sL|f#>&-u?_6PGQwby(l5`cX zU0+?0b6Yn$AL;B3|9xnyz~i=WtfMoPDISb|_bz1vO+bR6N407jxzGn5y#5qk_I!gF z_BaD&P9~~pbq;%T_rTy*gmg6BfZ)jtQaMXNP%!l=&4|F``9B4&^xRv|dT=&1@ zq6YUf3J4PmcT2BECpRa*^WO@06Vo!4Q7 zVf9zhbT!q*!9>BL0gf5fpAb&I_rsn?4;$U z$-~%%=6wF<$^f%?!|s<6N=$u4IaERww&6PkhfP*nPhUIyK)*4$KEpVBS7TGO4M$tY zb8$H-JvLl4pq#IiRt;N}2%&h6qbMuyFhrJWujc-@gaA6yHMlqDH% zZ*PB?b$z-+jd@HhC9C)%lZZ(Nn^XYmBdE@2DeSU6voT&w&F}mgrXer94svIIkeBT!zcvzw`%5P z9*wXF@bR7Zq1d}$buV^?MSHX7WM^+qlr0{oPW$=!d3$?*x&N4q&k-p{$Bx|{x4W5l zFPg)mf2BV;oPswhK7PDVyP^f>W;)`B*x~9ZQvj24eoV{t`9@h*7nOkPpM?dy5x%vm zy%G#e%w*kEouaVdV1D<_wKmKr%UEt*k0}j&7LPQT-tp2>y|egke<3qWalZJwx#?nP zczD^~O-kzL{LS^r?(RpguZn2`leKn>DE3@1FTeQu&UqflJ2+YL6BEY|W=WQudUt@g zd9M?_HnQs9fBlJ=bzMQfR<8SfP8Tq}Qf!;gax{K7v!hb+UPx%^C5>87D_L1t&7{?a zQSveCR7#QU&$`T03fc3V7p=t`cx#jGcaqyK5ibp-ykUpP)V$?S9x gVoo{!8Mk> z_03!8;(&l&LATabq(}u51EUFaI+fq0+7A5%7gst{N9D|ypHnN=?Rk`O`S}An6Ovb8 zPO!bU_114b{4qN_tD8E}{VMP9xM&(mNb#X@nv&=JpYM0>;sg|)8;&-6UBKZmEt-0} z#m-Q2?kLqL;e_kY-bp*nSH*v%?{w|y6w$qSVRW=UN>(MYKEce)?5m-ar#@uru!kI7 zPhc^!nW+_lQuNL?J9rFcic@r{SL&8+d=SS!A}+qUJQx6jA>lra(Pf{PmnU3whMKCf ztS}vjqbq$Qz|Z)@5m{EIh?OoioQufyy16=mAY|6-HQMNCL~vIIlG8tOiX_op*f}3B zZXWEdww(SA-j}?|WMI~<-M#cXfVSItg6x<#oXf0Nf0%xADd}Swo4Th}eCOSi%W{6~ z#mP=Q9m8u#qburf?ANbu4y!W!B2_w+1OyHn$4q$-01h1}9jTg)y27B@YOi)?>V~VW z-C zwkodN_;C*%FEv=0Ufl`gLT?cvJg4;$>7`b#Xa|G4iHJ&U_p4H^_)b0PFqy;Md8*3Ls5oai?<-2n}YV zZotQ$hjYHZ=rsFQ@0U9}PETF)$mLR4?Z%7sgu!rseMkUb`~6L9$j>EE^xNsl_{|&v zagEJZJxF|kYrm-jvXzB*F5RXM&CdtG!Q@iVV=4iLs>zs>oXxo=kBj3imd9SoMar6$ zy4Tm&z40vb1U0Lpng%a3k2c1GX^%Gt9WJW38Wx?w6iI|Dr12#t#aVc`9&7qa7Z(=? z-&CB&3wn$d=@AnWLZK};*9{y!786j%ttn1(@>DJ-9bMfS|08lyvFS&dr2ugI8+`Ul zKhGYm!WymSX53mfCo4S9_B(|_)8hnYOAWtd{VmhIt1*3Yh#CgLG$lR$f}*|zpqj3+ zrT*>nNu%IwA%HrYJC=gywjgSroU1m{Zsfo!r-l~t)66OYL4a_egOf!u8As$Mz0GlyxDpuMi!QEI6eVE za8NLuftNQCj*gD*>FKy!-g0JzqNB^rGn=%=uF|eBnFr}WaVBlKH;#MHI&;)wtWZz& z+rz`c+ld*YwI-s$ZTmuf*s#0XYpl~N=Gg<;o1G!mTtz#Io529;?q&OECYW$;wo0Ca z)~}w{lkHhqsNq(dzT5KCL^-ml!P&b&|`+~PLQ3{TpS4ZQPV}*%tjQkrK z_`B@e{2xct_4f5udL=64f7R@2AtNJ`fu5swW*g{3%U^D0X40VA90mlt%B`{_Lgzeo zW&;W69+R+UT-M)~3g`$Z9DtMHD}(k?;~Q0$;}lQBm&r<}nulzs0gRrI-4eyRhB`?} zNrJ)I!Ul75bExBGyB*E?Ox;1mb1|N{gWcT(KBs)Y^{Q0S;s~J_2HETD`(K(F(|y!N3&$pWQH%keOHR8$mvP4|+QX8-tj%#0(RMUO5sdx@4NXid%e zo{7(ax{i3SjS;BcKE?nqb2~rZxvwW{4BtDPRB31?F#)cxcF*eR#_`d2c5T6p|38+9hE(|78 z?@>z->4@I$j6$KHBM4dIE8It#q)e>tba{YFy?XU(W30Hr>jLC0tE(m{3nz>)msGWf z{@>hPT}c#dWhW^4ovb#8c`odZyQqCAo<1E#$j1LZ{su_Oo4R8DnjPH8kv&?Lmdo~41Ne@$hJ{%4&7%$du@%CzmuPIbVq(-BpQ~#;bg%xpwGE1Dlx?-`v^O9M zlz@j$)Mb5^`~vD~2q}B#+BU5hUiS4$w(IMPsb;_-pxI-{-tN=U0i%rPN6=w zv9v7Q;hSbA(YktiAm3o^Kcc9WqnWC=E_jCZ{Q2_=p|s_l9mRpUPo`Orf_sAGZO7(I z?4Y^;Kp3=mU(M7xU_PdpD0k|;x;PO&U(4DT2SqQ2%WJj6==12YIs>7iY+9HXtk99E zNPwHWrZuyb9zzjQssB#5**+0YXfM7oR)l!|)Y>j=(Oq!f5)gdTUulsF<)mp;j;F`B zHIZAhCS@k&F6QG0nZKFJlSPZyM)HOy9ZY?0u3U;V2=MVgxRPg!xr;2TRc^*U;1r(k zNln+FFP-sxZ0K~rrLsjCM=@yH?eC@b{{5f6MD{==R@%CAPt{PGpa;IPr)Q%#e{>3u z9SsZ1Ku3r8xiVc7`U;@d{=+H!E(*du$~x5+(okX+{gEe;bH3kzI0!@{OOaOSTkHf0 zF;3jU@cTFvwAPBT9^rPe>b_}AhAe=7cSpShm&9}%EI9 z#HJgZw`ekd;SJ0Xu1DxA@jfk*CKr2J32QrgWII^yw?#lw_WQq#ll7z|r}+?VzJZ!r z3$jOVumCr=FvE7fx&JD;fOB0umr2V+oFeEF2bTyVc_mNRbilA+(F%k? z(CsT295@T^@!^3f*B7~Wh?0fIGaH;Q#~aAV$bi${T1Q_N?p^m^1+M+s!K4;&%@{Pp z$HxaV&Bm#$xw#oiNKIX$1_5w93=bo{J{rGIg_He6vqV33`!k0MELQ1T>5g!%&*8+N zQ`ts#*U|p|KC?ly9Fz)2?s5XuW;DE1UN4Dt5|BB_xvXj(&s4fqlNBx|@WWvm)0+89 zAt50k=dCO)!7~$LVh%fq4f+gN4fwQ2YHST?>K*rnY+J5!)F8KF?L!AvR+ZIUBQG7D zgipm6nC~-MI=asG_L&Bsf!<-+^(FrwYRHbZwpMa(o5E2GI8rvA8HbFc)XO|}q8eeW zt)(`BG|b|%p2Af&Hl`Jmv(ab9BNaR{^f`#cnQdR<_!p2_%9?leq@0|bPJjbJM~bZ* za^#bygg)av>esBxXZ`TMU<|_Zl!)k8BD-mC9Me;HVR?$ZuO@T657$LSLPEku;CQZF zB8{LXZRq-J!-*_Q6Rp?wU{+U#$d|{Ie3eeqW7gBv$|wU`>hr%!UCs4oKO>$8JfT*o zi!67Zotl`?QYH-b&c^n#Tg#``0F_I@-86~=>a&Qz0LeqQ&SDwNx~@=i`k+iH8nFbrjPMTmzxeQWiB6>dYGTi6#1?bRJrTNf-!_e+tNs?=#wwl^V znNb(Be*Nk449k)A*2?Fz@IQue5Dk$!`7k;ZR(3Kq^3@8=%Jo633Wn#H=23iSyUAI^ee88!3OEl`@(H8m&k zZw$zU&XfX)Oi6MScZBRb{=$(bkD72Kwe@sfR&Ocb)tgE*IB!5^3+I4{{uA*B#PBC< z|9ESfT*2zU1gO8Kp4;o(@ydi{5eBf~>_^A?;oBKR^QpwYdfwC)_2Rsc=>qxn>(^dQ zX8M;eZ&gxtmj>@IzD*pfr?ue07TP3FIggA;J7@a^1zul%c_PH>9)ERcKAfwVOofl1 zsl-5tB@T4p?Kg6Xe<$9%rWusJ|50k#*4Wg<8}{h&!!&l&FpeN2UK%nktBC+=ZxRv` zWCSBPiwOW{)QMdXOv&fCcCph!jf?yJz%!VL$;3;bK>i~+4O^qs=I~F`u|n-TckW=- zwazq97v$hW2%SMjjIxafMqSA_7cm+OzzW#d_6x)faHs@oFVVH*u0H%CE(!Hv%5X#L zDyO2NViCA$1s5bVEY{%fX_-`c1)70+PwjnbXx5-aOiY{@@&pSjX@4$eKu+B$(5mqv)!#oVK}cbQ?NjY}%e}xZv1-bJ9#N7^|bMqCykhjs6ZJ42=Ks31e@% zRUtIkWn(N9zQpEsvKkWobs3@lVcKT2JhrBWJ2q`LR*Ji)+ID_!ty|zCIoxXFmJ%z; zPm!$O$fZmO=eHCD7wFPG!Lb(hx%SL5c%nhiFd%xMo(buxXmlgbJAN!1wpLNSndSTC zix=6MR*^%%`y9`!7g-t>;)W}ZT>?^??6EO1B+Y_LfYuKoP$fiOqriMv4}L_E^{ZwF zB)4jt=_ydi2TXnRXUj&kHEx=<5I1O~NhOU_bEcZbIvr&cZDp_DptHD~cT6N@AZyK5 zy4a13j2s*s#FgdbWn+uKx_P*5%rD=s(ww!K2iy`6r0e5pTQ&FCM;3S->C#Rm1AH4h zv11Ip;STl=4vWX_9b19J=?<$r-s>ZIzdXVT)e`Gxo6jXjrmL(q@7{zw)ACh7y76q!tS-*RK5o$C-4|k zeY0*aXm}P2JjMx(|K{f^z>55&7jfI%rtd#{l{owV;w5&T5P2B|GNBKrd8)Y~{zx^* zG_gHu2oWxPeHzL=m+;0xLd8C6*-6mrd}L`Ct7veYtsrXjbnDgoqw+EsB{M(2$No}x zCIg>y8KBqG}zPs+CeltRcjdpbH0>b)9|!*TM1 z+umv5-ixa`TFnqGF7*jTLtEx$jJ*Xv$o_yu0UL;w459YQ=W?M7I4=x@L56Mqq13{2 zX1=8CrnrL>c1D3_dw)C9J@>n}^%|D#>ok7b%$6Vh`SHb~svg**JBnWOAB!{pX2N9S zQ-yuz#>Z<+2hz?LxNYZji5s@Z`rq06H9A07*VfkjamdN;SGXUo1A5Qa?ijqub(Y#a zv`gTVI?-f#0+g0cjg25+d*8pmrRn+k6Rt;{9FssBHIZkO3$(`B`8nN7dav{D+;s1A z6!6MIb68%#)~&Xh^m{~{#9__mxb}OrKr@*UE?Ue=+_H>*`|JH~dO)Qeot#E}s!EBj zhWx`xn7v32dJ=98J&UZE5{4&Fo&4za0WOY$oUj6adX$Ff3 z2GLs|)FjNEhK3+y&NwqVJ+@S6N}(xEzAn2FKjv_n099A^9s!n8i9t*A`yakfgiimA z8~6wwc0JE}_+h0TcW|9rv#U6mlcL#tF{fhnm-k)=@grhpxdEy9I-}f$i7LynoSd9r z$-+B}+SGvO>XB*b>QWF83_2k?XXr`b4BvTs+}acJ5w{P?pn(sjKT#hu6h%YAO>g zt*)}@H-e-+B-%g{t6Y+4_^$X~epnF(ngyXLLM6^HYd&ZWh%kUEoPYwg{JR%VE(hM- z5CF+dO3G_7f;8~C8a2~bWKpEq-`g{vsV(!)P(jQz*zN#NrqXs^h=<4ZvS1NZV-+rq zJOTSdDMv@gfZfMGtbOU_5}x4<^!75ohHTEafR~S}>uGMfr-keHT(wtEsCaNGDJiFz zIe2(@fbpvAT+Ok5Fqt0U?|;7wVuQJ>|B_}8QIlpi1u53BUd8o-lH*>`LJE8*qKmmsPy{ozuC&?=pxQM04qQ8NyZ{$rcV zQ*6wENtGoLmUOgNsF2Dh4uwJ`Bp%;Pf4pMbnQtjkDFEdKQ9{|u|N45sCwp23c%g{T zz`?W#)yWzJsrmH$yf|Dh9jLwd_}oFWN}W*eRV{D#tO%a83Z;m`!q?Gz6W=TEZ&952dbrD^SWhl)MAWdzU(cbM=n>-XCyRx4 zH5usO;2^t6-vb;RPVPHm*m#rP2FpO#G&MCH89c(m%6>ua3GQKo3`vZPR${L~qoY%<|#+8^+_9yY%8mu2VuoSvQ@pF19w z$`2&l88A(Nc6wdX@iEcH#s=!R+!G5-H60rOF?Ccqal%v`M~+$m zXVpu{d2=GE%96`^My=`js2q|(^d;H|gi*8TuU6QCRvgLac>XJ7rO>-NdqhA$7qBd; zz+$1(#@Jlbg_;YLlMoW5^boCzG)yLS4a6Rxqrnjs6&&1qz|zQS*Lu$neJAhDou}f+ zd!KYV){um^lq};zh{`J}rMqC2_|gAkKK7p^@&Bj07|a6h`ql|wd~j*?{EHz;bh>o_ zAhJq6ftd^l#YiZ=lUP1)PJRb#2t!aiu2_h-lsNG%(^XOoynjz5qTF0i4xvH?)7D6b zGA7C2ZV_02(MIbo6np$~HymqYXO_L0sgk0=w#)!1_j#NCRl=GmCjc4 zy<(wNWg2(2pWCv3Ds_sUJvi?v^E!6db3sEpN)(D6I}m`lsr_76H5Xs)c&Lx|>0F9d zj|x<(QA%0)Z-Zte4|~^kM9VR}1Ho<$*?Nx59+Iz<|U;#d8B)nb6xBV8O< zS;g@|QU1FAI5+3_s6+=fmgwz?aP0K2dD~hRK{DeHK~&#fy-Zy!+Geo8nw^{t`giF# zBwk=WTSgP({|tIO#9h42V*;iFcyrKI%V1ppd?PIH*WA!G?v{>slm7-yCPDX9zfT@qO4v@n zto9(}+&w1_-kMZl9Qk^2Y0r@axXB(BvYH@%jn^7vid<` zNmU8vD~}SJ55kJ)_Jl9za}h24by8z$H`8G{oF*Rk;t`2l^1b@3cs}oBp~O381{=~8 zmIA>N8DFut224k=m2Y;-eV3jF-|S?%%ejzh6Zl9PB`o}!bz6z0R#ZRi*I$mtFLU8f zd(M|us9mmk)*(eaK7=zZNgI?V8pCr?ptF@mT+{X>=TC7Q0G0!6{)pN zi?QK=kyE0RRX8506j%;;YAr0L^Tkv{m3lZa848`-9;u=9;z(Gz5`9U=n006$j5=g; z7y9b!iTd?|CewQ>@HyW{1OyAFeUv+tqc5kH5Xj1buBR~0wP#F41P^qnqSX1gY)Xpz zYv8PI$+f(bH5Px+r^M2|YB|a69iJD|BEwml(yrQ}BMgrGlb2wx7-5&q?6uQHtvNcW z*#5*IrsiZ7&hYw^p8MOQF3zJ}X0`&u1V%!%k&l=JBWE3dh5Z?QJ+)~p=2I?AO7?f1 z`g>=MjqO^XD<=YR@zPF~;Xf(}2Ysmsl~ffi=0l3-n8=S<)5GXCgy!9#JL1X%@A;yP zVKLvnXy%!+9@#2dQb!hqxL1DT4`(=QuIM$^>u=I<=TyB=6EAU$uj@h0SP$=1eIT_! zwHXV$tsKK{{0kX|s#Ps|du>+-*sxO-N14tpDVf@o_;1Rw>SCBV_KdRyWT0DDb#!Tc z%NztH7;i}Lqm4*BMCpJ_V-Da*dDUvu2dj`_N;;hh7VU`xEqxP8g1HpkUt3GD$_QJ0@)hW|k0mZ(;9 zOfrj($_v7;SXNq)vItZLM^AcOqVlh69AC}fU$aB{y(7*x2>!P*T>PQ_zvc&Xgo*Bd zC6E7LtU2k&Kc#0nnyRB!Ke>8yZ$J4)Wtk`#>bUW3xAR~><^tyRw&-PBpA6InO9J+B zdrp*u?h$~0y2LYL_O9T#6Io;L0VBi2D87oW2#1j}4887Wp>K(YU$MdY?)^p_4wVUT zI=-OUcl@S2D}{@)_dimW*j`ZL`BHqSGCrw9vn#De?BesfJ)%hVFy|7#Y?u9)v{7K>C{OhN9<=9ih1GP zHFpFd&x+p8qW7wpuf~nkA+`@xpIY-ulk$#9#=88$a>B~M#BhAxFTIQ`T~|cRZOfe* zb)8O=(M&l>G;M^~1j|W{LpZbl1j=wXNMig2sihf5%P&v#@WFxtZACzdWfC`s2mW-M zpXlYp>9me=c6GnjlX%lkuDuV`OTraL!Vp|9!QMEN=h)GH=%12G?C#QCbhC?9r>k7J zv)yfV0}mb8Eg0?xlQG!nqsn@Ri#Ir7!%>}VBNc$AvYS1Ei-!=%5H7|GRNIZ&CSJQQ z^Ew2wqrioytmBO1zF>G{z{permDgndiYt7adTo!Ik5FjTk<}b>3J4i8;eM9#2OVUCr0_^TraI>5|s=E zZ{D?{eVRfTJz{3ml=3N@ww-ZjppYL~A_~+vk;RYppmOLf%x6=_BTj2g9uCEDX*DA@}K5!BRLo~1wPgl`s zIc58Y>RvWn(c&quYIL?h?mI`MHs|SiX^qmyn3C{C7Md^p!d^4``>kx>Fos3#ZC280 z0R~zK_WIBCW`y2#!}?h@g{-51_y#0bgz{9@T!;KbcKJkLB`z=h`wUgvj7^0}keM`F zn8Iv!Fk)3NdS~!E>pBs1FE7@hu!S23rF48$Lav6aq?gM@)~h8Yz7RNIFM7GL(25Th zt|H>}xI8x<%G+NviH-dDb2^}>@LYa!Be7L=l)cw#5bjls<8q$kEJZ&!(A$pTzo2Kl zYGau$zLz#90P-A=gQ*Lb(@dL2I-lqbwzVRN9NKgFH&Y^M}j4L0+gGTfG=3( zo6GpZ>9TE{BU7pXD zZb=zWQh0J_sRD$9<(3;YWywCpmbOg_#bUNv%CF{9)f@Y%PWe<9>9JG1?KX&IJUrBb z{pGGF+9!9v!PMDho?=x8b+hOIvQUYvLAWnN-u7vVg3D<)o4~+<084x~uR<|@mnG** zp=6FTdoj?bZ6mm$cHAP5*WV?rutgp{2*N!=CXXZ5UZBg4Eiq)(Eo@OMu4*|b&v3o4 zzC{ykX63|cTPx2y)Gip^M>lYn)3H@CH-8zkJQze_z=kb`{pFUY?yOb#9n$k6LRt?R zpFKFD4VTHbf*=%Mh0CNfy079H)boHXqmtMac`>|DeCh1-kFg_4s4C@7)${;?FqL6C2n@@4->gq=E@B#U@R*7RKo#kC9 zS99MRTDAy$c)L~gja>YYtVdm%m_&J|6!!Acl91*_h$$ zb4B_VRWfvKoJOA@e%_d)h|T-@ZNwyRUiYrqBMyr~@lbwh4WH5dCs>L$wP z`924C>4VzYe7~0$K7(V^vt=>_6*9n7m37|WMGCcyk?Qy_Yw5aTBEhz7C2Q=o2crme ze;COtZnk<;dF{Am$2@hJnEFA)^lL>Zc5hS$jN}8U#(Sv$bLEr!*r+=%BBcm1l#QV@ zzjnX9P^QJ-m>L_f*STVOjD{ow4W`OazZ{*9>&0y=h2_P-v>}42y=;XxAL2#_`UxWA zmc{)EpvuPgJ}-*{=PmNyXUCgpN7kMg6Xl%4MrI>d@0wpJ>oA_E?tK0SpWN?*4e1FB zFWwHhNC^8sr-_6cYJGqMaVlY~9clP=XyA358R@G9HtfM0%w<|PVyx#HbYK|(O!gBu$%?SKIB{Hm(X`VP|aR(V~O4 zJOnB`ELBDnE7N5xD7ys#!N-ELAPUMNC=cDD3y#3h5uIV7l!t;8Y=>c2>Z>NKwzLX0 zY0@M&O>T1ckA%>srFpbT+UxoJ(Mj(8-Ftq2eD2A~`JIzeJkNtb5DKDrCqN+dCq!t1 zpddmM1O*YAASj5?1VKTBCI|{5G(k`hp$US52u%IEfu@y8xkjB7??fVj8Z2u$9FC5r=ATkDYNbLhmG<-;avXmA5!u->Pi^4Rp#R}_ z+S|gtisLx5#maHq@L@xukyTU~G(!%XolViG)yb+JZ030!JP4f*4?YODJ095y3W-FL zqDht|NX=$*ymJYg0W7r1gezD4@7k@lM5UsKu#+b-ZXAk>@!fZzC^R<4E<-`1R4VOu zd%UvZbkx@3&>_^;dXMn{McoLvas~6}==@guwerl8czrUq1R*8Dlkm!GbIc_BA>R?>-E*u z)gcCFXJ;1{76y;A+c9ev&YZb<#%tkW#51QeU@cIjMwY+YPC8iC#Sa{hXb=`JgTY`h^d@vi2j8fAke?q} z&V*kBeXSIRG>`xc27}RPyya=96UD_ia^#i)G~M#X*3_~^;U`G=buj?G6uMlPHxD0w z+*^#(DK_2uc_0vO5bw1d9EZmqLuF;qK+V>F;IB&|mq*aZ{#y2`$zx_c85;52943jO zf?q?3GAmcE3^-CyP~iW-Hy8{%ckT=b*tl_H(A&mUXtx_BCD^||lvv5NpRsKlR;&oC z1vASp+|yR0zgZyI{h9sUlH}28@qUsC(IJc>$BrF4R;*ZYyvOft5#aHlv=n>yhH=S; z4KSI)YUD2;n!Ps9&5Nm}<%;)OLsy-pv!><9=xP|I$vvetYbw$NpJ+ul3Jj?b1Zn85sGanIjvs{6qe z?o9~&MLa%vc^=O^gI&9VN76J#jl!5Q7(4dr&=Kj=3NdtO=jbSdXcKapqA2{Y)$QOQlVG??wEfX8>*7Tep8u%*P#r6kP(^=`y)ZK|K>`4u z{Kv85wEdeFmqI4GCrkSL%*4zTv6PAcUb*4A`)otUcrE=wsTzQX$F`HUedpP0 z7WZvK#3gx(1rsAI^_yA#tLC1!M~q672;l2-Y=!1|tXzpLTYPV&Qhx{R`#Wq{KejeH zO_Ht_T{U?Y?rB>#MX6#$yU*CJnmhoejZW~!%zeAdp!`4`d!Wu&{=B+rc+e&onkt*D z*nZNMxBhx@E>oP#WDV)ksebqco1zdsI-WVFcUGLWr>aDY?^m!+e&?rlUAkmbv8vVT zSyO45zuQ#3+_#$kpJ(lvL&PO{a*e|G)U7&X`QQROeRM*GCgF=KuGN(mhl^iyUnCPg zNes2W)_$-qaKtq?N4DjWm~K%iC@2UxqSb2suhnXG_kd7{tSt6!G&j6H6-EzHD^eF+u`hh*ET9X^Jwz z*IDYXx%ZuC7u_c>$&&+kA`fMI%{xxpA`=>bSKQ7{$KGZHE6J(V>JV#o0mN45B}=em zNh}+jsHKmWr@VN$y}sFHVR>DeWZ6`ue@Kl^-SC{ZjOPum@1sB+y+TjE zGr?~&fb#|?fSsS(cYfMsxjpOT89KrdLT9JrVsrEm{&Z}HMqE*n9HpB(-8g#Laqk@o z$qY3zRr1b)MJ-T3EjRs>|ZwCunH^6See|(Fy%hTlx2w zj051d@^sfn*zH~{m;`)Zb)ySIJHIhj%b@1C{Yla=h<;_%jKTW%(u zo7q>th4xlSnoNOnbKW9Ljo!Jr z%Bqr4rS~cpPLg}AT02{1VLch@@W$3N$H?Bxw!d}C_H~PE>+i6{z#5+#)cGc(f&hrMB#ldf`vhJ^Cy_N9DI&85B|5~Wt zQf)hYiT%96nKe}M>Yr4Pjt?@YmYXHrFh{lRq_uR9>Golg@+AJ;eJJq^!%UhqDYCZ0qS5FE z5OlI4iJ>0OWhQFr@#|ZgOzzH=7s8+>@*&591q-}eQG!OQ)oPti$HZ_SYmLG6dR1=& z(*PPxL}-D~r!llvtLrG*1rLP&#cH+6Bl;DT_%djO)~J&$?bcRfo6AN1V^DPAIJePcc67Ls zk_O*W=|S^PgFyZZR;-|DsZt?N(L`%kL{u3}LK6r<88kx<0wENL&;&t2geC|IA~Zoz d5TOZz;{Pq%>c_L0VG#fT002ovPDHLkV1kuEi|zmb literal 0 HcmV?d00001 diff --git a/docs/images/copy_this_report.png b/docs/images/copy_this_report.png new file mode 100644 index 0000000000000000000000000000000000000000..0f6722a3d0fa39dc0eb19c12940034d27bac5718 GIT binary patch literal 22141 zcmd43cRbep-#30V&Ju-$%1B7EWoPFUBC`c4D_`A6b^JHolh=?R46g!A%ewcTQ-``rx>b#2LP z9g2`uw>&2=t3jnLORL|(`IhiV_n_WnjmxfcS*I3BUpv|uIhx$axSZW<5Sf{+LKAMo zXGudt6R3VJOjVw8$KS_g=Qplb#+pcn+8saGTp8yxMLX_*XXY{wne3=afvue-8cFh3{A7Jc&PCcu0ZoclQ0>fq(d!rHPN7{J0n2-}(NZ z{-eS}Okt6cIYwojgM)+b-zO>t9~-U-lDIYfT${5)dC1@dCCToc?_UU7gs~lw@h&eb zyKQc6u2qcLNJwdED}OlN6n`;JvLrY6DP>Yxnx4MC`^vBWGS}IGw>Pd&b!Ib=D=u)- zu?q?=w5F)|K3!WFFZfbD{QL3a$Ii~u&l)&`CWvE@w6?S>?~#deq%qIqobN}PztXk zD-=xeHC7_>#f#6wzn5nRXD24=h8%kftpr_v8Oq4W$kpB5wYRXPjzZJUZbl~4m_S&4 zZb`>v_}1|*<&W7}2YY+oGRfn|-@SX6kei$P_U&_HxA~FJEy;=!XNX@x3}-Z#kN!3J z`Ho5?Z)3d7U}IzU?Ui5GYaX9hQ%mjq8YQf9oKpSt-qD%9H%4W*Xh?SNr8uVD&3{)` z)~F_dEKM)q(+|~Xk+s!TF)^|IM+HV|nS4xz{$AeUH#s?pKR9qy;A2Zm-o<$7#Kgp( zKY!L+QufxAQsTH-3D4c#xHWfx*zoXnQHgCnYsi;3h|(GF zwXrgsB5rr0GGpYPD(UP3c zHu3W(qnP8nzP`&d4+>_**F0#J@2pD^m*Cpc{rr3ZIi+Y3J8gCqW#uI4p`_$wK>>lS zKMPHMWQVEbo-oO%Ny#ey{8qA%q7s>(pU-RfwmI={|I*UZa4tP6xj%pY7!+BbBPqE# zN*X@jTV&%)dGX>!zxO+sM=QB^NWti22e4-WnMrACsMpHE6oe$uzx?YG?3`bUpQC;h6bs-8S~Qc|+kW#Yrg z$S7t0<%pS|zrTN9PtQTQbUu6aiKy^5{oPEAd1C*!;u92}gHk)h35;jzT&;*^Lo zX4hBTk-M3Bl%76b+RLNbcb~&xWq;oF+QTNEL!+ZbUK_V2GxyPtj*j|L=3O#myKj)f zoqo;MjkS~<)A)B(&EEDa>rFD-rra`K>(02t+so7X1_sI`r%#_|_EpcZVq;?)nLSu8 zDV~>?cbZxL;zfsv=0xA8Tzaq7f1e$;=M~b%_Sefca6wsRReW+GUb>>R)MH_+5e3NX zMQrWxK{@lLxXlY{>grFcxgQ^Jknci8wNz?bJ%0T79m@H^?XAe)J8lhC`#pO`#mdTR z{c!fj4>`6_)xW*IXl0#hAKnbZm}v2((=Uw7-ub>V=+pG|YiCy%mS^sRAU&>=Cr=6p z%=>I_Fhw@Zw52uWn>EmKUy|M0(q*Sdoo6Smvev=ybU5%+JLL;XO$Oi%k*MTSFUG5}MT^~FVIJwszKjZeV+?_E-s5 z66R3$i>WGo1r`)b?Z)QjW6Lvr`t&~hyE{fulc`WzTDm>^0L{eEQugIH zoLpS=c`okm?iNjPMg``LXd&lFI5;>;ZVc~P+jvb%A9_-i#aGg6T}WIUJMI!^_&}Mf zL+i`)g63bIR#YWi`=Ea-RxWAQj$s>!t*-ta#+COM+tD-ouHPE`7;wnUN9j9S+~5A>DL#AdG>q08e;#)f!=DY7 zx$-facIRHT?(3|$?Ye8%{ivv@eTPm`%B9bxoV|Aa`u+R&1qB8D_aBve^N3Q;$6r1B zAgjX5moH<)Zv6Oy?)x<+al@kSDYGGOq+05KywY}4tv&?uUJ?$gJQkB>=}>SXEe*|P_mY0zsW z7F*u-9-1VT-2VOh|7>n{h`d6rzg%paS5zb{BxL(7XK!y`U;G)Tlc!ES^f=aBU0vPP zwSqqO2R*9$FclM1m3_assHi9w8*k*)_wTvaK0WgF-Q6?Goj%|wVwCoDjM%8RFGOv5 zCl&L1zg0hKU7UF5&K$FMy zpQ!!+=W$qbtCY`{A!m3!AMuNa$8LQYKu_ju+o!!~8awYMe9lbf!d%%L(#t&e;>8Qh zmIrFmpG-6>tF}}tDwgszsifSdo>`LY>3s7u-2{U4Nfef~wKbq4Vp&8LuBzGwTC%aR zDQZPKw6gk%ZB?XWphf$d=31T@3k!?1^gws_CGYH@AXOzLr3)AKUw3a9e7)yt)fuLq z2V4sjJr@X;x9C3Yr3x^`rs(YKjN~=)Se+XhsJM-n?|P-5oLv(h7FJ&Mrh3KTtpMpP z^(m*xb`A*%2~4%wzM98J7Z!iENx1&1L@$l((P*WM0+cV?ZVO-_ie@*ldE*dZY z>C-3dQS3SX)2CZnT80+Zuu)xr0+?T>rkan|haWq348`|RP*8S8Mo442qn(+VnW^c^ ztgNi)=)P|yPAJKY)YS4MHyj<6)YKvxNqP{GjxG|BwrhLX+&dMr=>oSCU`Ul?0o_vmTp zU39rQdG5l6#{~ro*jGoQ^))q2V6-dl&{8f8K&nt`f<<~o|)&^$BfaL?#POK^JWFzPN!XMme0as z9Ls*`SN~;~2(yL=jE;s4`-SJZc6}m~OHY~q<~1rkaXDeyJCcu`i>sld!)0@Q3A-_T z#>cOgMklPOXesyVhlfd_I|;I7LXRl8xw(O7QdOgG{cL4RR97Owp50uZ%Gt+!&kVhpLX#1jysb zXt$+4+gtDM?mEn4@X@m48Jmi)+)$BC7e++2NLjk!<@r!kGCd=6clWhFYYSc3mtD8F zyf6Xj>8GsM03U(Os16+pM*05PngSreHI%XO zxOk=|f%#)c2XQ7Bs~*o>Yg5Y5_P`b~@m?Dp`rzOGL_Kdd%q7e-p6l|zAK#*)B2;{4 zW@dCxZ4C+4Q`C<>{TLZB?a<$||A^zln5tY|-y4UvwzeGO3S|<^wyK&MZow;T#??

Fvckoaq06)yde6hBKwBZMKOTDpF-J1YQoryR(q6e~23KBMq! z7V5cBGNKb&l}zjmXEWc~Q#Z3soBcR$D_vd1947pgxsG~uJc1F8!F1{!r>e&?adfbv zxb1l#&SH^4)!{%Za{ds@iT~-OR|6D-t4-#~iV&2UH3=dL@r3Y-*xLkGtDEVD5p>^_ zghdM+VnErK9%V55nr}iUm!p3XbYrU$TIt~V>oS`RmwGBh?5Ak=e#Ac@;DK;Kp6E@S zawZ85-$?qrc3AGAu7?xR&l!hrW3c~ljbo%Zh;>$ZSKtZJ30{YCti|IU*ih_ z8ecgzfFpiu*M9gHO7nM;QBH`VkdUgk^|uy45<6WpQeF_}J?R67lLymS#TM`3aUiBY=l2pyu9H_y_U;~%5mX&zmgwpO!sD2g}TP6w3UZ9 zU+M7J%OvC8P4hgV|5MV}5D?^5iJ1(u^l@uNX`~-Ss$_eW4+=daxcxNktO`f|1xWSg zJstnj1wY#d5?l?ug7=U}$Ww@h2tnX{0LXla&U@)zHJq})XkhZ!+=ZuCG=qW#pCnsh z=@CX@4zQRqx2*iNrzDbRYZBb)`k)wuRX_npyZw6Q048=pXh(Paa2~Q<|E{>w%+3V# z5>EeQ9kc5P;AkAm^7ZI&2*9Fz{Nz8c{#xZntTOTaiA*s*!Vq2bA4MmG2EcTmeI0xi z1|*Bg51Zck6-y7j3!Sni{bCI4?byNML@#$TyL1Iz_KZntKlRKywi%Xr@9Gp|t^6=9 zx?bmgf&7)j8`M}*=le+h3`;T{_j^ztg1v1sw%)|SD$FAIJorOU99$5b?FUU?WQ}9>kWoGEDQ!h7Zt9JLbi6Uaf-e9){W7rW8@Zu4lDu4{=|px`XlLEeAA2UFRu@W zwvtdd6+JEFe4s?V?~fC0CkBm>?!S>>P{)`&S_w($LYMP)=}(n=ax0E>2?wmKsSY@R zOp|h8wc~o5i2b8^JxY3GW0o*Od0HE~^bZ~gsH`!By#vzSBCMh@u<-JNRX%zq7Si!sDxJdd6fIP zBKz~%>Vnqmbe4 z{uy5X!h+Btj88FGFX$?b?}iP8qJ?>^JzOKX4@vNJn8bEH-(OUHaLOIvP3sSROpj3} zW}oiO7+h#Pn>T2gDli;51fzA9n6 zryW|7HkVZ08vD|{Q+{VEc$M$$$TiF11h~l0PQck|t2g)fZzA6x`}#j`BP=k!Oay|3 zU$1m9>h~q{vGpYJFfr@ZaEFuuS%>u-%-I-3=Ui3G`@5mK#gDxJab`zyI^IHnu(>PH zlUf7r4T^R9xvtmUTaHOuzvJfSh|f2&?bmbD-W|_|zcpUe-%B-^a=DG&liCS?;lj?r z&o)1E2X9cOOKv8YO73Kb;tVS%6};z$4k6>9P@SG!X&pu_;xPuGsd|!_Fa3YqWBw%^ z+6y`)kc1-pcp&GfHaH3tYNvEm*Z@HFvgs$fLx#jcDA7L@u)=J{B_5_*Y$#DKx?=@> zD9L51EiWuXyDEMqeXejzc!IvVrbcis4G>y{S6XY`&px;WlGv!3 z3%M>RM+^&IPXORmsZ=brN1Flp7R(8j9#$t3`XVUiq@e*u?4|w?@&9q{K)6^?qwA8J z+^w6?l_>IBjUD*rafX7x81eYn0twCvTb;P$>cP1vc#DV_q7F%bP>dI75eBG%mH>k@ zR%!$btUM0whX?6$8+qu<%@C%o-7zzM+Ef$}R3ZZBR{kW&GxljKXcW8x7ck)fD$z}L z!0rQuu*t@ceH)FCyk1%zhDkeb$_o9?AHi~=rnObQ%!%W0A%dXEWUbxLxR6mN2>)B4 z6r>S7eXRArVJARPO>z6U=TkM)#ub4RQuG*)L+HICsDya6*IBj88SjvDKF~%kEx8p9 zNQ;8Qa?`xr-_V64E&$tNIf)9*BK;EUL$e|y(AsZaZGTWHbsp(hb--p~TT3%>e9xCv zmLCEHDJ9m5dv4lOf}WRWvXBDpl~kJocRj+Ui7)Tw{E^<*%$gN|bWM ziY;Va4Jux#TQf8ZLHRR;O^5Y$WuryE`NgLL07KIv!00~Tx7Ymg#}51F+krJ41a~m* zOJ-4kI=2KXOewK#72zhS*X_n_Jp`~xc_Nq2Tj`KNQuKR9kf@k)I2wJ5aeByk(p_X` zYKoc>Pk6f%(j+o|Zdqv6ekEJ(gKXVwxuqJBuY+>Zut}S5Bv_?_Qj8tp_vuJ1rEf z=LX)02nMPtQ5rnI{@O+7wdYq>PncnC6NZcbzSSg9wdk& zb!pk{6X?Zd8tzTzlMB_C)?ikD6p09m3~G#t3Q11=_5Lmeh9xlk<+4Y{)|Cr$joNO+xqy;C^B zmJffH#7h`0JAEF2vAe~zaUG-(I?=MGbl{sIKIQa3iA>&!Ax#`0=}NZ+t;bfzZ|UP5 z@FRd5Uh6=74V1g>m}n@{J3lb*OK}E-&hs4N)cq=Md{*2Z zWSjn!HP!Omc&<`Fzz2YD`V8kbe%lOFW5`3e!9=7xiWpuUkF`r*_?nxTz)PUi=ad#pt&!VR==#3R_o;Y#HrfX* zsi$u9(C5*3b0BF?yv^{5nHQIbU8vrnkCYU^b5CFVs{02W^PeN?uNR+R;vm5A0v~K9 zB{dBD5<%{DeaFqjc>Es(9m&zuQT)n`ecfl`MZ{l~)3*Y;TAsPCcEy&2!v*OTa-nP_ zgK!EHf$$G0x6c9&N1RY643H!tPT=_u{;yry=N@$6xzWj!SAx5RC|NDUIGaW*FiMxs z6xVKM8Ah=QJhw+Zz*seJ@O5~Z#+=-99d%6zqYMBm$d_o2!v3$Hj}GBS!vMDD>ssK1 zDmLp)Dt_~02ocC>n_Y5aDrPvNl;*zq#pvq%;9w4&C^rk(Vz5!gn%|ullYi73|Fs(b zS(Z&vN`|Op1JpKk1`Isi0iD8;yWK43o~-^3!uSw% z;a~VhN&dWW3rviHD>liq635`G%zX(J!z8sC`8vw$Vo2Yg)pP44Z zH9W=M1+Z`7bJSSMKqU&g*11lf`qxhYE?7Jnx==V*3E;Q7#6u>Gvp{xF(PS;0{+F(DpiXG(2?R%q#btarV4m=1dqUNC-)OsBQ{U~OFpz|!oYX>ok)U|cHO@OzuwNpS zvI>v2|2&4Zr9aonlo0>V<9Wdwrw7RD>HxiSym37jy>r0m@g5Ll#Kj1kQ1lX)_?=4` zI576tv>D%YQeidz{udzPjJx-CU8n^(L&$l72HhT2AwHSyWa-0@U{n`?LUH5d!u=CL zLE~{YTR`&MzyohFG8;ps$3v+xhSfeJY8kk!sWR**pzOw=V!NDB)75J|1E} zOfbRde=08tbVQng2{7U~0ElK;`nl7uD}Sh}`)3Ri_>2RA?iC);@aH0JqI>8aKtEQ- zq){-DPg$S_d4GAjQ#*cAdDSAb5XxG^m;wy6sG{uxMQy$&Ig>uALq4z`WKK*XMMy}P z;RLJ==Sxi$+zZZ^J9h6p5F;RYkdbn7+u+$6L_DuGTL70F4pOnxSVHw<6jDwVms?ZV zn5h#uO`o|gQ_PbQcn=G|&e-~&tr2l9)XKzq0C2tk{b79kBn-F4z8OuGTN+&)|1er; z@&Od?ZNb0J*2S<-zkCvV=S9?b=>MWX!=a)U=w=Q8Az)z{Y|&%qVydg9MP5`E|LT5m z&_ild`XJ-%PP>Stq$Dz8I-*4&+ucpi$bc&99}77=^|P0oa~n$r&jSg3P+ZmSVUjDd z{dWAd>k)=+z!$EaoN&slWqYiw%-xmC&HN(ScqBk`47OhM5f5I7Xyt6Qq?hYY6)~X> zK*}Nv)fHxc?wS(Pc)59i39(%!vKB=hzL5SLAlYrf)mRzS?JM>}|9NOVfH&1;cbI*> zf+@e}aY0fwflSw3`Vhe%W5DQIdp$`@h5duS!|&?q=l8qZHaXvfGq-=pQ(@g9Bj~1M zIa$}rZhWe|aNlYDz715QKPo=zjPm=(bst`yDrLsdnu|uN!_P`LSA+QfF1b-TqDlym zPsbY5*wloeXIa7OGEz<z;Y-H1>5q5*ocdx>D0M8AOH#G=8eJ37Ss?OruhtvUYPzX zmD7l-5#ZoA__`8Ex_gZS$%5p!zNqEO8d1260RjsX?MaFC#*-eT2Ofa=s;YEW?{Zss zs1rHQs}a%FD88j1fRrHBjWtw2lHQnliDgMZqt2u~!bTyG@j?vn{Btlc)+H)i=QQ3& zY3(BB4*{X;HT$kdxzd^~;CNst$zb_}sSp88acCnC|9MV&dM`&0HOO4f_#UFGB>Hm& zZhlU0n1k+f0N5W%y}UrVkcLqTaE1)L18zzt|1)YhodIBii8^c|m&}tI23!(QdWA-S z#u%M6#YqOcyq9kK*OVKl@BFZ0e9!`_j}djNpX*ngtF2q=kNCxwK)%~GtZ$>^ z*1g}l-M|UN8yB%CsZm{PGA=U(=K1cy|FLyH3jj{}=RclSNd0@W|J+O1q?2UO9PW!`a!9}990 z>Vb=h?Kj<0{h)Z%9L_CC>qZN%JrDmJ1V3AeR+f!ASQq={mc)FATA2U#JZd1AsIJ7i zl{4nG;YX~)1@Sklv;%D@$=ec91dY-y8_9DS?zhK-axH)^bmp=1%oz!UbTO*_10+l2 zgY(bB)kVOaL*e(Z$G1obE$Wtf#JJ|v7kjl(%n`5{p4B8C-GY1yxV2r2n@J7x|8)k8 z;+?v`Va`NJ=SzQ&H^1YP7JvnArFJ!&`*m|BxAoG$|HZak)2XxA@qpAKU?KX2A=EsT zd`j{YB;e7lxu8L6`^H`g*KVD;rTEN*U)gM)kM#XWGs@3?00V&y_JFExn;_l!yHzFw z93Z0kmiJzO)&w%d2z<&sC22}?c{8|njMLG^8knJm9B8-?3Gz^<9q(8$8?qg3_viB$Eie^8 zIe{|Xe=h1f2b_JG7C{}qHKVl)jP=42G?;12OWHZcz6Rxw4BuK>MNJRxSND6e-VRJV zM>1rvhcM}Hh7$ki<(sFp1#~(mNJxDk1tlY^f@8g?De;Vn+w@(MBPdfZn&KBS1Z^J^ zYEYO$3FXl58Ac^QcU~*ABDYo|CjuUDulSYXt)IWv_!@CgF4W7bs6{o{OthuSTnGuY z#Kth5qV8`wk~(kgiuJl_CT$>Kae$~oI8n}MvcPx=`km4X7jDgtRt#gov2r;7ukOYx z`IkNu?|_u6{X)$gshWygU{P|v?eQtqo~vBBYqrucW?W@)mSVA@jNvZ*$01GjY#U}M zEsLZ`e~+HaEO8p7>vK_`=;ikL+x?kRY_73>_!zYLxfd_3F};QfhZ2?jU^u(t{DJU* zEJpvM{zmLnz0I5N_+ISV2mi0VFO7$?eg8%(QrSWwL{ZsNA?s*U5=zPR4XKhrhrXF@lg&~B!UCRtc+wgEA$aY{@et<_&o5hyb?k z3^y*Zo<#JuIglhb@rgSyf+D2N#!}>R`MSdK9eHfwmfb>#J31PN^G-FOI0Ickv(wER zk3=~?GJMweLgjuL+}U}1QuW|bSmr&((6G>$(08H4Lgu%d$v*Q>1f@AMOAj=X22C#d zP(KQNcjPtHwZeY8)#C-mgzA?n1nLE!Pwd}N zB6?|+0tISI?que1f!K+(bx7Q{0&kVS5TJ|~7)#@Qap-gO1H~&0L!s@89{NT?K~b=| zrl7-)p$kra1S3mWt+Hfi&hqZq(d9sxzhcgo{keb5 zSHA2?+Hkc?H|Xxnf}G{S)hMe`)6C-#(g6tEjvq0-BmSh+bFqQz%6Zn$t>VUcuNw?W zMxzwn12`7JK|H$QF8vug2{%@Wy!Xp0hM8snGNsM&jV&`!Ny42{-DYQxS6?6F=wIHl|}IxfL2bwJ%{9AGo*wRiQW`Qiii zp+6weFGxK^Q~pO+#WeH0Q%ysgCkcXLi(?M}BmSkhhSKQAJ-`F*JATr~G)ciDg8VMB zQj8R|gKhy*a&ICsylP_%-C(F$W}Wi@q;(G~7!`$jdAPnxJP_h^^W+^r^4xo2(+{xm zk3NR1%*`)wz47E&6dK}%JIqsg^b*DH?DvkipDmKuE9`+`|rUrc`RA=gcgaCNRpmMxan zPZ|$a?4&MIYh2mSb9gj_2I?OzZ+PEi+djBcYiJ!XOq>*cE8Z*1!&08tdpS47m;UCt zn@;rh&G?!NaRmB<0@ex&m#S`GC7-Rw3n3hkK^oH0|VZw6k@VL?)JDL6P zW+wgHKIM7dAnUn*WqES0zY&x`IozcY>Ji=5hQhdVjR_-_J~3pDS6E7t?6BRBk3 zrSyHQ-xhUA5+p8pkdx}S|!xII_qu9iMsEqi~I zZOvZT%7 z;ULGet6xJaLnqtX6rQaDk^(U(e)oCObIO`_$9yfLB9fs=A3fFAKmcqxTW#fW9?wu+ zc>2;sqCmoKvPx*vzO$>ofi%3%&)UvRX7?L-)&$XuC5XJXS`siicix}Pa40tZ>hI1& z=YED$E92&3AFJc$-qiJ7Vm7xr(y4^ufUr!l_#MIVGnI9?u;sp_F|n+TBqBZvww6UIq9bQXfe4E9l|=L=cXf~AQWbm1nyD^# zh|i-ttKC#7QgY;3{_XZGdxI?uaZZg-E4dvL)h|>zWxJ_dop8Ng zdZXPdiqq>*UB@9IPJC(|>a5k+x~JJ1aZ`?~xlXT(3uQL#Z5XmzfmiAZKIa$vOiLlJ z`LKe!{ef?4c$baD`~}}lzUuHN1U{IL#NC*rXK|ZtLAx^>SUp|b+RoG`4~dnItpT+d z-l$w=63C2kPyDGzjV180xcDCN8Fqfi=-gwZ9ogrpb%)kwgsQDh(jK0IK%t77VgE=j zQGx8E$?$h?C_OSTJrrxlU{inII8njF@xX_H-Q*?Yc7@23ML$Uc)7T}`c$c`1uqQ?g zUz%PYI~n`@*i{)`wD+{w(ZgxXw9^_$muXb5iHcxdF2ek36r0g_U@2Whu3FCzw-#bi?@Q~|z6N*?uB z@;2bO>{c-c81=3qsoE(TH=B(Yk-kS0fd-xnu7D}~ZX`?7XJr&57$WwhfRIn3h3l(M<|Dyn6GY-7KC0X?+1YFcCxLm+iq)-4lkdNVmR4E{E1F;X9lT=zUe- zX=<@;Kg2Wz;F#yaYDzx?QTDq)B-cd0)Y(EGl;H|{>w`~I4+(X%vQ%0;_DtOK1thY= zFUYy%v+vHCn`y!B%C;SPoUx9~R7EO!xq1RR(;Zedu$F^w0j~VJ1r*!gpl2qkV< zGB#@1Uf4N#*74M8$YKwOKGZq&ogvgw!m~YIN~i*`2wBVx72Q0l8ZhB3+sqIxGGLmn zuPnGwR+L73753)w^HzBGk+V^|xLUN77cV_{re3Lt$W%E48}>ecE}yl%zV0#V=W*aW zOFe6X91FW`lm8~#zroLlHEb{>eoRm>D3R$~eR_6i0khKb4Ny)k9g(SGm)G~w|8!%1 zydgY=BcFJ97H$_>MzjH?^IFWvN-+eA-f8SAw9C?EFF*c~)zXh}5;_!zJtv-70G{nD z8>acC*;}nP%IrIDB-5^qOT0q7!-i&VOuh1{O=hOQ#jF??n#r&+i)}%)FE4#&`~1BH zN>CWiwF88L@)a@e1ytViAVj0pS`pfmhfB`+nRiFL+!={Wow~LdZf# zL=o-DN(y@l7~=J(1%-ubM}V@AMH-`xY<%gHV^ZU$>JP*HXY1qvAO7*uBPRv-Or&`a zm%7}1bGg+IBGT1DkfQYKq4w*?dk^L7xiDnUrab-9q$DQIN=F20u?w>W9jdg3YwVhei}g^NTso}Z zM`w$H^7&3+MoyKyG7JWo*-}&@JHwc8$x1$t`-;uYW0knXtjAtefMnwM88!C2PpM90 zEpV}P)>}CSM-k@}GA@hIe#AyhCfeH)Sc40OQS6swJ{8 zT;LgqCm!%g^L}0K$a!u9C}>5H$3~g5n+@m#JGEf!j}3W-vECJLnEx8~0x%6N6cE2* z@1y_gPXddt&`9m2-SZ+QU~DKlH;g5CdO9|LHbeoabjI^Z?Gsh4HxfJKRK2-!3Us`e zYGD8xoChnnL`80|BU4lku@8nY6}&%x9Tbt{8@0LK9(+sTs$RSNG@|@|{r!4HF5=b5 z0Ckn4nwRxv9wwL$g&j{qm1cvK8w6fGkO)Mbh*yrx19_xSLEd*uKy2+`Llci<>06g- z-R3ub5Rp_ZBtjPAs4x@s?qdF z{PJXYu9{CS>6pN>cfh{!Z@ZNj0zJ2&%1!WvOIJMwT7~Xep4e!RL`WIJ%lq>2fgEDA zFQjnw(#<&TJ2JG6gXb1ir*ruwe#*^^|csiCiisY?ye@eRQO-xJQKeL~KQ1)~Ba?$TSqW_5t-e|C@tX){B zec2_>`k zs~8%haNk%U$4`d

u|=HCdPO`Y-%9M|P0YB>{YBHvX~A>=E|n zZ&P%#{b+{GUWFFS(r0TPa@0r>ROz7YT4 zY7US$z9*;VYb-3J8^uQ(LHZy>CI=k6=9epj8x%=0GID+PP~NSrC*79S<;f1_lIIO! zQESnclWuiI78V|MR}ja_6r4(X)*Jay{zkIV;amL8WEHtWMw!rH{Sm4bVGnU#s<)bF&$+>y%^vd5Td<<;utib068~(;^{QUkIZ6CJ5l+rdneNiuJ?hE) z-OrU<5-2|YJvoZ(&YCT69Gz3Q67%HKwO9fTLRH>i2bC95H+Pe2DrIfG_>MoMOctH6 z9*BT(wj#N7vkkaZpDLWV4OdaganL*4e8F$p(2=SdhZ#?B?<1{X^(m;dDnExLN$Stk)a>l4pcw*s6qHYsgyW8!d}`9?W6M zUD|$L<itC)H(K_^y^G<##-2i&rUGU|FNzy5=gU6QBxqI^fW3iJ};%uVIHk~7X73leaVHkN)GPs@_fVm65q(KY}`(K z^oTm4!H>eZ%l7KTd-SHloJ7C-xU`7okZ||imQ;kT^1>Cx%VOf3+9Qs0$$Kywi)nl# zqrm{FW2+qfuygo8T%qG++nJ}K!?zRm^j$>%J)|1D3K(G4uFi<}>HIZ&F%~Ku5U(S4 z7LF_oEoLxCWaVI8ea-%b=kvaB{!1F&M_#%Vo;{nM?B#`GN$|+&ly1I%ZC3O)e|^L4 z=VG^2^9T}0Im2&Fo4)l-u#OqHk7+5J`Z98 z%=GTQ3x9n&75Y7FH5?Muc(&Z@4o{?o`fE#1tI{;?lW5w~-{#Crbo`q`_1ixS<>*{@ zW2aD;IR4i4#1dG^n7d5`t@RVhX;48cJ28Qv%2iSZV5obczDO!&Kfn*7UW}B}Z~pyv ze;A;DL#_nAJQ}dzGiVhTtS(V@bWaUKd0WvG@R#ilg23}2D4|_{s3R5fJ5=p2qgMIO z_ME{O<(Corx9N>$2;2eC!{MyY)`WXUYwS0Jo8o)E0d<(h2(KKLq#&k&^ zg#YrqfAx)_AJx1SzM-ebb^reT3l0trV+u{}NeUOeyu7A?Y_eEYe*Wp(Iy%Am`T31A z3rT0|FZbdQP){R~i+aJZc@Thru7e(QR(bc?QoXH)?l?-NNx6Hl1EzQ<{-u@K%$yz- zJkB1_rEDDkZ2o^3NdUv_kzBwlxd6(r!H3RRKLVo2mo2zh*}jzkAnsexIrt4Y!utVz z>`2I9bBt)+YSn7x0ze8pK5;AU5tBseo52h7C^(@Os=Ay4t&`5*g1H`x=2YYl3X+sL zY!0!Hlrlup1kf_Uph+?W)IcQ@0qfJvyQ8CuF`#+)%>z)clSrz%t^V^} z{Q6?-4SNln2Z)0q>bGxI)OjYe%^5w*Lr_sRMfF+OYMK_3F9++|BHm@_31mRtY9W=M z!OyEy0s7BzVxDEwV-WCE4_-Ka2M_gh<=*f6)LHGn?F{@a*(sl+i1Rd{#oY~xkq7dv z)DMEDu7gV655g0IcFNc%5+J;5(Sz_a+H_zOzl3tzN6U4d{c`90j6y;8#6=p`wU6n^ zdmy#jLU|=ZWW@62|Ihe&R*4tcW%T}fozgsoYAxPAVWcxb55 z$#X{Oji28&7{qIWkl9QxA4IqZ@MPqovvd2hJ9@pl&B zJbA;nZj(qvc^qEBCc1T0M#?6-joK8g&Q_u;VAOyLs*l|qS9Pf`;Pw@V!scecKV7j; z>xj{Pk>gu0<0|eRxEc|2DS#%Bfq{<7$0DE-KtoGM&+zvP_MQ2DF1lIPm~Q~`P?@dR zc7G)Eo<^rsQ|46-mbDKl5htB|FqmfG!Y}57Rqa|*j&-6M?!in#tX~8gISZ1^zI6uar;6ID3C7sF@&{}KqBq0ZVwIU)Xg!VEFr3A>~dKg zhKs({>;P`*3aHp~J|Q_!P$rGA(#S|xJj2+wBnXr z#;2$_)GNz*DX+4)jlKBz)txquw&egDG9RMnK)TfISQu?v9t9UVi;SU{EaYXl(t*EBNiD)F3#nnxD<^Ww*b6EmxNIv&)I|Ulp+5HF1nuo)`Ar znI{TB*sx7kpa|Vxg9F7W9Y?nWOmsCsvq?hSzs(>B)@5Vx1A>>Pe9Lck_XWU%>Hq`; z-IA_M{)p0W3(7D^rOw*G%KN`kBQY~r_iY7_dET9Hh`79W%yGynb=fvONtbUi^hz`U zj1)w~)Q}*CRuYJh0CEcyfY$-xzDfCd6JRAKVTp*62y(rMavm?JR!K7?l|lN?ABZZN zEL4b4+)rc)=$h&CZLc+<-}q2``!Y;*9+pLV_og%hdSIeXMj-pL2F)4>7Bh?<*kI-- zai)BL92@O%#zxv@+W=>PuR5LHdd96?0eL2eUGv;qMF4pavq^MOm0KCGBqSyT3xPR3 z2IkbLqt5={=CqaJE*bwegiqekC+PbuIsC0jbkbl zJ%6&EbPW|Ga58s^6==ZQjl4nm&QwoM#5_@_gptMULD%JsnaQN`4!?P~mZ;?A_oVV_ zYI#ab0pnwlu>|E&V2oq5e8<_dFqDYN3Zg77&e&CL-;lMH^9VAruSlFGaTjB;FinyG zg@5ZCf4{P}r0C1)r6giBv+vjr>8n&wY0r%<;%N97rPBZbr8m{gWzJLIbAH(blC}D5 ziCX}7VAk|87~50=bZEJA+c`(H{AO+us0r)`oJ3>U<0CE)rEb5k=vas1aY6@j<0~>6 zw)PNq+~81hOgpp$MZk3Vbksln_vzw&(0QJWt%7v=TAGws@?1d-ons70-;ar2EjWG7 zXotv+#Ap?p;xoy?(HuEy247zXEbri32q1yTrOfeRVqY7K(iJ)LX%x~%7{*C>oB#z? z<9!FW2a%fw`@RT&c}?RGl$+x>XGb_Wkh!=0{=j>BTiDX8j%a|aS%=5i7k`FCc1`w; zFXe1ga^S~EKrlCzB7p;lv8rsAcFryjFrt?75puRGINV~>mF8^WaED?;%_RCe?B?yf z?1=z87*6;!DhuSX;YKr=dm>!sPhiw-Fw?zXKTXAU%eZYrF}ZvQ#EOmEDd~CMA#RJE zK}$t;HF^Dg4rt6M%GPf``F+?;mlnbhNaP^M87<o;0o;vkl36i-du%O4mTl*J z=6q{Kf()$S`DtP--%@;2Z`La<028A)Vlt!K!lvvWLRtc)jvl_C=oEJaOfhc1N9jeZ zwj7)dm}y0$GXM;+4acgM?F?iWYYQ8AkFJ=K#<@iL81D-$kzyC3{kI}y=XmEvDU3TJSUubi0hY%|2n3Jf!mXl-j`RtysVkYxdY(f6oMSjW+ zwXwCxt?9^_VNKZPGB=GD6)bV0`xX$pH^wKs+=)^yaFshhYdi{YNI4l%827FdS}Nx< zUb3qo0R8Mx+Qij>nx~8mbR7dbqz?b$9Qtt(sM8Iy$zv2@vlh%58CIjKTdQ$ylfc(m!I4DWT2r?Qb3LU9`JZmZc?KE_IT`cjPX-`d4n_OFuO!p*2Z=#eJ}b1=X zGJ$3sU|y*_JH@ZWtaPctu6hqxg!FOL)V+0siCd*Q>YiD>l}*JQE>SK>a@A!MX~2uX z*40d;69w)gUgn>cRaG;%oEXz6uCb5K!_PVs&>bm~zEFCY8;cQ?jHD=Zc0u!Uao|r zHcRUY9liAxb8>Z?wT-FugDs-t(T6t6Ljg+WO0ry!9w++v?3YZMcKssS{ zWWGbdf@$D!Zr0dg*Wv&N5|vCPH{u{^?&APH?v)kmv5f$^wG84OO{^~Ld{1Vs#@Jcn zyvCS^(232Xo!&s~w$Fn$-Os-TZ(#-ut52IxkQ>PMvV(m;Y%|rP#ds14pqHDO-pMg_ z23>xD-8%+o#?0ZwoxIeuR^Lc0ASqt559k^kDtO=m4`~8lv4Ac7q{*D^-?nfSeSVk> zv7qhKsEE>cH5{mz`L1IC2F(>HcEmsi9FCg<=K^xm{nlV3DpifPMQuF`yD%^#LIb+q z^diC}LFwuxQtbrhGV{aSsF{@|c7}EZ{{q3XXIM%8&|wL6Z4`H^v6xRqR#mqU&nI^PRWgot%-8833o_G@QI|RX(%dk-U1urYH~W znpwywjGU1Y`uANk*tiDju@AgJ)x%&%l9p#J`?a7XOtC|G*M=iJ!>EtrU6ugt*S6Rp zC0tS&)SFRR37K)l$s^1sJWxGJD{FJYo|wkOey_I#qg33DsWrl{gQ&6hjSbm_i4VV7Goedo6TC3G zi5-oBj!}4#6WxYahs%gdq6Z(ovSzikNl8_-vIh@9flU;|gnL2MNYQZfVMdk#e`vrR zBVYb)z-c?#Xuz6W;v0bCN8E-Z4*yjo>9*z!g01(u`KiyXgzWNckmiO-no+vrW;P*@ zj5jOVL3V)d1&lh_ao_xGXvnu28k)mj)GnRB?O=4HsPC&3Isc@z5o?LeCTY`agf3%{ zPC+zqNplh^7vNgVwp=T#*4vy+cT7qBa9cQN{9 z9!Dwf5)EO$CB>W!A_%*x7HE6EJ_l45?U zNAjFjxd$T%GKe|EhwVy5Ivrjj3pU`@4$Xtu5`Qmd;l1rw5f$@ouw{ubA0hVNgGzh4 zzy!Fzn~C6$AT#0e*c-skjG(PZlo|Emm2GizNNI-3r=(y>-?y5gnBB06LmkI|xJIW6+oUnO@SC-2{FKg+87)yBk5CT{K16x>Ee;Yc@dAgB!l#59JGyMfEo-|u z0KWUQvDO63K993dIfs5_RP3gS0;qVy)nl~yTbgpI*@f*hBXIz(Ce7jpPS5o0e9ao= z6YG(l^+MN1>=Ff>bp_M0w*ZKk`$I>8X6I@kv#H_SKEuKX>Zx*D57v3^Wux#AA?KXdI3*&y1ZYKBdrUJS+)?)vX0J7ff6p zd}T)HXIIV3;$%d`v6BwR@dkCn(19@4NEgD7@bq)Cr>kZ5`f{-u5=!HK(ArxL9{~Ebh1u8IioY71NLV4L zF{V|`IXGp|_O;p>U)uu6&3;O?k3$h5*K}k%0dX2RY@}|`mZ0KHH)6n%=&{{6F=UsM z1&1m%FN;MIoiE6h868flu~%m-wr*Iyg_?_Z%VB(!~Nyc3^j$Ty;jL9}atOBPJ=$Nq8w`G>m@+Cw=vFedNhpro(sO_jf z!XjQzIsnc%p|`!NgY)4syDhQp9~ZN|PTtO>#9j#Ur_D?QMQ9 zF$L`iW4X!Bizu^BVq(a5VxF8tbgJu@@pjjR@$C8Q@;B&7nI2-|u>{npm!p}yGXnpL z=4OyHm){Z(E;eKn=}IxtRJg%cMjgIL$9Pm&KcTOZ@i8ArW=J2b2tgq+j@ea|8SZ;+ zFiS>X5gG;Y1}Wk#O#vC&evp^Zu*)~XF-{40CfcA!lq!-Vv|$}`ORer*gM!9_xsvoG zY|FLPB%_iTOD*{#9@q8CUh17$gM*fN3HANKsre2rR|dNe%Th+ud$#A!FnQSS|GD(& zt*@Hn+f8s|^UU}XCEN-wo?TN+0R@ZZJy$J}_;$C&N$8`wHak&Q7Y<=c;u7E)rKG6s z%HL+Q=4fDMW82BjOh=8HGl653FY#SeefChfCkEvBq$_ zX))x`v{_T)TdEYt59{4Yj~e~vQGHOtit2=l$;DE+;g0rXRQS?un`b$5~JOa3R z_-Tt*C;u&(_S;?3fQ8m#*Vg=fwtji; zzgubMQG3M1#T~j8+>D@QeTqoO#Ke?J-PR@vIp~93QFoX7&jb6pq|MzJW{1lP%2@-) z+7{1ZcY;kL6IU}ne&=iC!G(DxbXGa~`oAO-;`IVEY1;Q>L}@C2G+r9ji9W5~N8;jlBeS32=e$JT*P4a^SB z)_uRWohlsqLq)fX?XiaY<<|rRe;6Kf;hpaAZ6nYPp8YBjBc^qY#WNNeQO5ey=Kbly z+-c7PjFPq8G(hprx7}RU`0f48S_Wnwjs3^`{nf6SM}g{#)63o8W&ZmIS!mCjkM8)F zsN+Az+g=dZM2+QB*ggN4)xT7Bis$)0UobG7F_gG|uQEAb4K9o;d7Bnf;=MUBU--QQ z`Jk$Vl5}?D{f6z*RWJ3ZWCKy;1AEKDiQJJwDd8d>bcdl^VsFR$jpb_H!BUdN<~Oo) zJ5S~OYTVNM;iXgl@p_{Bl_?IpE)!6*B}8ON;{?fYP&Hx`LBZRRUwEt8`SHiviTOul zRQK!K#gM5c#C7M5et(7R5@gYgLt)OdeqyB3$GP_#2#gwjvv_k!jV$}JT7+unw=a^A zq-9q!LgC3O(cM0wet&zo5H(k7jKaf2$gmUsS%+pd6N)Gaw~KQc4@Fd`H<>v3Ps|im zxbwD4qsxtCvUAkDq8j6)VIPr*Ee+cQgyS`8p$0b)KuIj$s-n70Bh!)J;DmCsV%;Rf zru)cDiDe6MRZfwJw#82jX^9$TXxL#QJ|^_~#^pG!4c{}1qNqh?<+$6Y?9DOSGHfbT zGqkI~am#KcLbJ!Snrq{>(pO42WTTxlDWX9FY%cdUaUw$Y!=u_~3D@ zk(+78D8gXd*C}i^!(~V*Q+mYEDr>WJqSSq}Vr#v0MchfdQf<9wDAm^xLtgHhxh`+L z_sP$9S|iLhkR`BiXN%xt@s@gq{Al|^@^T2KrYPT?_l{<>6ZzNgHeOfx<0t|_NI6Wk zl0!z1&S;35IaR7{UT+>vD%(fpHZ3WepU)c|EnGsZmoBEoMB4Rq5TF+-|K}H)H{zWY z=|XP8W$9GM%(c6m(=B&w!hUxfuuk!FC`9;Is5yw>Rxz6L#F9Cr)~D(h8%l#_1UpY- zqOV~&xqtLtWf6rxwus<9Jgra2f_>apTel;ZD>(GeIB5@7aoe2RO6K#Z+Ik|yhtq#-joh{c?<~}2e&Q--r6YuFP4;$b^U6X{tu~h`Z3}Z6ZC7*qJ?&zO zRwg1QhR914uwqIo)X2g(ExS09tpmE}O2(P*03Na*n^ST0kB#gmnfn$*f7=ogVlksb z`2|ed!VHQ>QpcyOtWJ&`W_PpN zH#M@WpcU@4KV_pot<548dv7>~#M4Jyh0R)~<`k-N`-!-9H5#1Y@e?t+Nyz9s9EZ%e z{rIdW16>`mHLSXBg4JmoGS;<{(EDVNRUzAWvZO!zun&1AKFhDVB8fRN%K=u6D?$vx z72M%Qw6^!PlO??rN3V$?xK##fX6EU2y_v{nBLk9N_m?}jKgJa9sU*Z~Js!pN z-Wx4gMr*gU(A%w6x)>$tH1)e%CH7mNZMvQPN;JzGpCWH!Y9~9Ra7gXp;uQz&ehN3Q zQPMXljr$4S-G0mVhPs4;VhwA(bn=<|m=5)%k+r!8XNfmePr9E#t3}Ocm{s>3?kAFr zio9pBO`pn&H{{1i@tf3ZD+7ej!-h*;Z^SLF*ixj75DICEwgaQrwyrI8x!psP&soo3 zEZ3PRZfp^XNH(Z$pRUNjG#&i7>Pi}B%@1Vz zG`Sllsd81K=!znsU0ja^Sn|BUErKTNBx3ts-kCEi=& z)y2w4-SoB-=%NCi2uq?Y+%Tr!NkM`Olo_s>Dh%J7{T!b)@@b+uKBZzysc=1K`OJ*J zm1Y*z8iq>O?Sw__(wDRK!=EM_8h$S)59VKiQ|7b>8t>ud+~KNj248df3^7}tC$whb za22CYS-CIcmJnWgPPgF$P1nk0;$D~$EK@7^RFj6E56+<9S&VK{dfIO#7j|DEczE)E z>?Yp7Jz|&8^V*xE$E;vERPh*r5;qVXHLHu^J+QSpy^yLj&3w6)!eiLG3K@lyT)tcen zSB~yFm}RQq&Y}1EY1(j@V`5j#vMBnzw^!D;QWqnvt_@CpHMZBDx@y>TKcc0tpDZ_H zIUJwzIkud;ugxl>`gFQs3Jf9Y1b3T_L+`>iU8~gb%Zcu*&^_kRM1FYm{&1|&o1;Z793t7Kntk;Y$J9^yd7IynqvPwg0O!&yVc{OXkgrS z``vyT^}l*8KONNHFHVKiF$(@qAIen&v8n9WhYxuE`?rM#2;}}d800r+N9F~H5{13v z9fN-!gTK58s;S!X-|seX3CihCTcCc+3j8tT$kbq5LCnSP{6;w+m)#}V45eXD6>YdQ_0C21%`8(7PLXrml6xBu z$C9b0*ul--)+ap6>8M3qGCisEwT#Xhd zI2A3&-%mu_;U^-pkcC=a-MKlFMa6Q&bPGiJ{AypjQF*_AM#6@>@_an0Kdo;LUnG;2 ziB+%Gs20PkZMwI}Wx%L7IjxyhEa~xjnWFWT@rXv+Pj{xpAkE4lqm({ZU9aqd~e9nCn`sa?uk*_A6pScmdJi`&P3Wl%tk zbuunRA&#?!T+Z1NF)y1hTJ4ln`Q$b0Z^wpJH}`rKA-$O=C)QFOPjqR$e^gM)ouJM3p*f52QXEcRIA!uzdvQ>k=#<% zSAxHvnLiKDzh1LBqWe%R{2f9n71p#^R;ud>4H!jCy{(ttl|mo+bCUoa2)KRNza7XSbN literal 0 HcmV?d00001 diff --git a/docs/providers/cloudevent.md b/docs/providers/cloudevent.md new file mode 100644 index 00000000..40d51b1b --- /dev/null +++ b/docs/providers/cloudevent.md @@ -0,0 +1,29 @@ +# Cloudevent + +## Exporter + +The Cloudevent exporter will make a POST request to a CloudEvent receiver +service. + +This allows to send SLO Reports to another service that can process them, or +export them to other destinations, such as an export-only slo-generator service. + +**Config example:** + +```yaml +exporters: + cloudevent: + service_url: + # auth: + # token: # a token for the service + # auth: + # google_service_account_auth: true # enable Google service account authentication +``` + +Optional fields: +* `auth` section allows to specify authentication tokens if needed. +Tokens are added as a header + * `token` is used to pass an authentication token in the request headers. + * `google_service_account_auth: true` is used to enable Google service account + authentication. Use this if the target service is hosted on GCP (Cloud Run, + Cloud Functions, Google Kubernetes Engine ...). diff --git a/docs/shared/api.md b/docs/shared/api.md new file mode 100644 index 00000000..5fdc3837 --- /dev/null +++ b/docs/shared/api.md @@ -0,0 +1,105 @@ +# SLO Generator API + +## Description + +The **SLO Generator API** is based on the [Functions Framework](https://cloud.google.com/functions/docs/functions-framework) +allowing deployments to hosted services easier, such as +[Kubernetes](./../../../deploy/kubernetes.md), [CloudRun](./../deploy/cloudrun.md) +or [Cloud Functions](./../deploy/cloudfunctions.md). + +## Standard mode API + +In the standard mode, the `slo-generator` API takes SLO configs as inputs and +export SLO reports using the `default_exporters` in the shared config, and the +`exporters` section in the SLO config: + +``` +slo-generator api --config +``` +where: + * `CONFIG_PATH` is the [Shared configuration](../../README.md#shared-configuration) file path or a Google Cloud Storage URL. + + +The API has two modes of functioning that can be controlled using the +`--signature-type` CLI argument: + +* `--signature-type=http`: can receive HTTP POST requests containing an +**SLO config**, an **SLO config path**, or a **SLO config GCS URL** in the +request body: +``` +curl -X POST --data-binary /path/to/slo_config.yaml # SLO config (YAML) +curl -X POST -d @/path/to/slo_config.json # SLO config (JSON) +curl -X POST -d "/path/to/slo_config.yaml" # SLO config path on disk (service needs to be able to load the path on the target machine). +curl -X POST -d "gs:///slo.yaml" # SLO config GCS URL. +curl -X POST -d "gs:///slo.yaml;gs://GCS_BUCKET_NAME/slo2.yaml" /?batch=true # SLO configs GCS URLs (batch mode). +``` +***Note:*** The last request (batch mode) allows to send multiple file URLs that will be split and re-send to the service individually, or send to PubSub if a section `batch_pubsub_handler` is found in the slo-generator config. This section is populated the same as a [pubsub exporter](../providers/pubsub.md). + +* `--signature-type=cloudevent`: can receive HTTP POST requests wrapped in a +[CloudEvent message](https://github.com/cloudevents/spec/blob/v1.0.1/spec.md#example), +with the actual SLO configs put directly under the `data` key, or in +base64-encoded string under the `data.message.data` enveloppe. + +### Export-only mode API + +Some use cases require to have a distinct service for the export part. +For instance, some SRE teams might want to let application teams compute their +SLOs, but require them to export the SLO reports to a shared destination like +BigQuery: + +![arch](../images/export_service_split.png) + +It is possible to run the `slo-generator` API in `export` mode only, by setting +the `--target` argument to `run_export`: + +``` +slo-generator api --config /path/to/config.yaml --target run_export +``` + +In this mode, the API accepts an +[SLO report](../../tests/unit/fixtures/slo_report_v2.json) in the POST request, +and exports that data to the required exporters. + +The exporters which are used for the export are configured using the +`default_exporters` property in the `slo-generator` configuration. + +Similarly to the standard-mode API, it can be run as an `http` endpoint (the +SLO report is sent directly in the request body) or as `cloudevent` endpoint ( +the SLO report is wrapped in a CloudEvent enveloppe). + +### Sending data from a standard API to an export-only API + +There are two ways you can forward SLO reports from the standard `slo-generator` +API to an export-only API: + +* If the export-only API is configured with `--signature-type=cloudevent`, then +you can: + + * Set up a `Cloudevent` exporter in the standard API shared config to send the + events directly to the export-only API. See cloudevent exporter [documentation](../providers/cloudevent.md#exporter). + + * Set up a `Pubsub` exporter in the standard API shared config to send the + events to a Pubsub topic, and set up an [**EventArc**](https://cloud.google.com/eventarc/) + to convert the Pubsub messages to CloudEvent format. + +* If the export-only API is configured with `--signature-type=http`, then +you can: + + * Set up a `Pubsub` exporter in the standard API shared config to send the + SLO reports to a Pubsub topic, and set up a Pubsub push subscription + configured with the export-only API service URL. + + * [***not implemented yet***] Set up an `HTTP` exporter in the standard API shared + config to send the reports. + +**Notes:** + +* Using a queue like Pubsub as an intermediary between the standard service and +the export-only service is a good way to spread the load over time if you are +dealing with lots of SLO computations. + +* You can develop your own Cloudevent receivers to use with the `Cloudevent` +exporter if you want to do additional processing / analysis of SLO reports. +An example code for a receiver is given [here](https://cloud.google.com/eventarc/docs/run/event-receivers), +but you can also check out the [Functions Framework](https://cloud.google.com/functions/docs/functions-framework) +that does the marshalling / unmarshalling of Cloudevents out-of-the-box. diff --git a/samples/.env.sample b/samples/.env.sample index bb90163d..75475e4c 100644 --- a/samples/.env.sample +++ b/samples/.env.sample @@ -19,3 +19,5 @@ export DATADOG_APP_KEY= export DYNATRACE_API_URL= export DYNATRACE_API_TOKEN= export BIGQUERY_PROJECT_ID= +export BIGQUERY_DATASET_ID= +export BIGQUERY_TABLE_ID= diff --git a/samples/config.yaml b/samples/config.yaml index a594c45b..5b35d1d6 100644 --- a/samples/config.yaml +++ b/samples/config.yaml @@ -1,4 +1,5 @@ --- +default_exporters: [cloudevent] backends: cloud_monitoring: @@ -18,6 +19,8 @@ backends: url: ${PROMETHEUS_URL} exporters: + cloudevent: + service_url: "http://localhost:8081" cloud_monitoring: project_id: ${STACKDRIVER_HOST_PROJECT_ID} cloud_monitoring/test: diff --git a/samples/config_export.yaml b/samples/config_export.yaml new file mode 100644 index 00000000..fa24721b --- /dev/null +++ b/samples/config_export.yaml @@ -0,0 +1,9 @@ +--- +default_exporters: [bigquery] +exporters: + bigquery: + project_id: ${BIGQUERY_PROJECT_ID} + dataset_id: ${BIGQUERY_DATASET_ID} + table_id: ${BIGQUERY_TABLE_ID} + cloud_monitoring: + project_id: ${STACKDRIVER_HOST_PROJECT_ID} diff --git a/samples/test/config.yaml b/samples/test/config.yaml new file mode 100644 index 00000000..d4dedfaf --- /dev/null +++ b/samples/test/config.yaml @@ -0,0 +1,19 @@ +default_exporters: [pubsub] +backends: + cloud_monitoring: + project_id: rnm-slo-cloudevent-test +exporters: + cloudevent: + service_url: https://slo-generator-export-757by753yq-ew.a.run.app + pubsub: + project_id: rnm-slo-cloudevent-test + topic_name: slos +error_budget_policies: + default: + steps: + - name: 1 hour + burn_rate_threshold: 9 + alert: true + message_alert: Page to defend the SLO + message_ok: Last hour on track + window: 3600 diff --git a/samples/test/config_export.yaml b/samples/test/config_export.yaml new file mode 100644 index 00000000..61ce2a0c --- /dev/null +++ b/samples/test/config_export.yaml @@ -0,0 +1,6 @@ +default_exporters: [bigquery] +exporters: + bigquery: + project_id: rnm-slo-cloudevent-test + dataset_id: slos + table_id: reports diff --git a/samples/test/test1.yaml b/samples/test/test1.yaml new file mode 100644 index 00000000..e8f4540a --- /dev/null +++ b/samples/test/test1.yaml @@ -0,0 +1,22 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: pubsub-subscription-throughput + labels: + service_name: pubsub + feature_name: subscription + slo_name: throughput +spec: + description: Throughput of Pub/Sub subscription + backend: cloud_monitoring + method: good_bad_ratio + service_level_indicator: + filter_good: > + project="rnm-slo-cloudevent-test" + metric.type="pubsub.googleapis.com/subscription/ack_message_count" + resource.type="pubsub_subscription" + filter_bad: > + project="rnm-slo-cloudevent-test" + metric.type="pubsub.googleapis.com/subscription/num_outstanding_messages" + resource.type="pubsub_subscription" + goal: 0.95 diff --git a/samples/test/test2.yaml b/samples/test/test2.yaml new file mode 100644 index 00000000..0921fe7c --- /dev/null +++ b/samples/test/test2.yaml @@ -0,0 +1,24 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: slo-generator-availability + labels: + service_name: slo + feature_name: generator + slo_name: availability +spec: + description: Availability of slo-generator + backend: cloud_monitoring + method: good_bad_ratio + exporters: [] + service_level_indicator: + filter_good: > + metric.type="run.googleapis.com/request_count" + project="rnm-slo-cloudevent-test" + metric.labels.response_code_class="2xx" + resource.labels.service_name="slo-generator" + filter_valid: > + metric.type="run.googleapis.com/request_count" + project="rnm-slo-cloudevent-test" + resource.labels.service_name="slo-generator" + goal: 0.95 diff --git a/setup.py b/setup.py index 6f93f9e1..d7dbef61 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,9 @@ release_status = "Development Status :: 3 - Alpha" dependencies = ['pyyaml', 'ruamel.yaml', 'python-dateutil', 'click < 8.0'] extras = { - 'api': ['Flask', 'gunicorn', 'cloudevents', 'functions-framework'], + 'api': [ + 'Flask', 'gunicorn', 'cloudevents', 'functions-framework', 'requests' + ], 'prometheus': ['prometheus-client', 'prometheus-http-client'], 'datadog': ['datadog', 'retrying==1.3.3'], 'dynatrace': ['requests'], @@ -52,6 +54,7 @@ 'cloud_storage': ['google-api-python-client <2', 'google-cloud-storage'], 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], 'elasticsearch': ['elasticsearch'], + 'cloudevent': ['cloudevents'], 'dev': ['wheel', 'flake8', 'mock', 'coverage', 'nose', 'pylint'] } diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py index d614c27b..58200b55 100644 --- a/slo_generator/api/main.py +++ b/slo_generator/api/main.py @@ -19,25 +19,22 @@ """ import base64 import os +import json import logging import pprint +import requests -from datetime import datetime -from flask import jsonify - -import yaml +from flask import jsonify, make_response from slo_generator.compute import compute, export from slo_generator.utils import setup_logging, load_config, get_exporters CONFIG_PATH = os.environ['CONFIG_PATH'] -EXPORTERS_PATH = os.environ.get('EXPORTERS_PATH', None) LOGGER = logging.getLogger(__name__) TIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' API_SIGNATURE_TYPE = os.environ['GOOGLE_FUNCTION_SIGNATURE_TYPE'] setup_logging() - def run_compute(request): """Run slo-generator compute function. Can be configured to export data as well, using the `exporters` key of the SLO config. @@ -48,28 +45,28 @@ def run_compute(request): Returns: list: List of SLO reports. """ - # Get SLO config - if API_SIGNATURE_TYPE == 'cloudevent': - timestamp = int( - datetime.strptime(request["time"], TIME_FORMAT).timestamp()) - data = base64.b64decode(request.data).decode('utf-8') - LOGGER.info(f'Loading SLO config from Cloud Event "{request["id"]}"') - elif API_SIGNATURE_TYPE == 'http': - timestamp = None - data = str(request.get_data().decode('utf-8')) - LOGGER.info('Loading SLO config from Flask request') - slo_config = load_config(data) - # Get slo-generator config LOGGER.info(f'Loading slo-generator config from {CONFIG_PATH}') config = load_config(CONFIG_PATH) + # Process request + data = process_req(request) + batch_mode = request.args.get('batch', False) + if batch_mode: + if not API_SIGNATURE_TYPE == 'http': + raise ValueError( + 'Batch mode works only when --signature-type is set to "http".') + process_batch_req(request, data, config) + return jsonify([]) + + # Load SLO config + slo_config = load_config(data) + # Compute SLO report LOGGER.debug(f'Config: {pprint.pformat(config)}') LOGGER.debug(f'SLO Config: {pprint.pformat(slo_config)}') reports = compute(slo_config, config, - timestamp=timestamp, client=None, do_export=True) if API_SIGNATURE_TYPE == 'http': @@ -88,24 +85,120 @@ def run_export(request): list: List of SLO reports. """ # Get export data - if API_SIGNATURE_TYPE == 'http': - slo_report = request.get_json() - elif API_SIGNATURE_TYPE == 'cloudevent': - slo_report = yaml.safe_load(base64.b64decode(request.data)) + data = process_req(request) + slo_report = load_config(data) + if not slo_report: + return make_response({ + "error": "SLO report is empty." + }) # Get SLO config - LOGGER.info(f'Downloading SLO config from {CONFIG_PATH}') + LOGGER.info(f'Loading slo-generator config from {CONFIG_PATH}') config = load_config(CONFIG_PATH) - # Build exporters list - if EXPORTERS_PATH: - LOGGER.info(f'Loading exporters from {EXPORTERS_PATH}') - exporters = load_config(EXPORTERS_PATH) + # Construct exporters block + spec = {} + default_exporters = config.get('default_exporters', []) + cli_exporters = os.environ.get('EXPORTERS', None) + if cli_exporters: + cli_exporters = cli_exporters.split(',') + if not default_exporters and not cli_exporters: + error = ( + 'No default exporters set for `default_exporters` in shared config ' + f'at {CONFIG_PATH}; and --exporters was not passed to the CLI.' + ) + return make_response({ + 'error': error + }, 500) + if cli_exporters: + spec = {'exporters': cli_exporters} else: - LOGGER.info(f'Loading exporters from SLO report data {EXPORTERS_PATH}') - exporters = slo_report['exporters'] - spec = {"exporters": exporters} + spec = {'exporters': default_exporters} exporters = get_exporters(config, spec) # Export data - export(slo_report, exporters) + errors = export(slo_report, exporters) + if API_SIGNATURE_TYPE == 'http': + return jsonify({ + "errors": errors + }) + + return errors + +def process_req(request): + """Process incoming request. + + Args: + request (cloudevent.CloudEvent, flask.Request): Request object. + + Returns: + str: Message content. + """ + if API_SIGNATURE_TYPE == 'cloudevent': + LOGGER.info(f'Loading config from Cloud Event "{request["id"]}"') + if 'message' in request.data: # PubSub enveloppe + LOGGER.info('Unwrapping Pubsub enveloppe') + content = base64.b64decode(request.data['message']['data']) + data = str(content.decode('utf-8')).strip() + else: + data = str(request.data) + elif API_SIGNATURE_TYPE == 'http': + data = str(request.get_data().decode('utf-8')) + LOGGER.info('Loading config from HTTP request') + json_data = convert_json(data) + if json_data and 'message' in json_data: # PubSub enveloppe + LOGGER.info('Unwrapping Pubsub enveloppe') + content = base64.b64decode(json_data['message']['data']) + data = str(content.decode('utf-8')).strip() + LOGGER.debug(data) + return data + +def convert_json(data): + """Convert string to JSON if possible or return None otherwise. + + Args: + data (str): Data. + + Returns: + dict: Loaded dict. + """ + try: + return json.loads(data) + except ValueError: + return None + +def process_batch_req(request, data, config): + """Process batch request. Split list of ;-delimited URLs and make one + request per URL. + + Args: + request (cloudevent.CloudEvent, flask.Request): Request object. + data (str): Incoming data. + config (dict): SLO generator config. + + Returns: + list: List of API responses. + """ + LOGGER.info( + 'Batch request detected. Splitting body and sending individual ' + 'requests separately.') + urls = data.split(';') + service_url = request.base_url + headers = {'User-Agent': 'slo-generator'} + if 'Authorization' in request.headers: + headers['Authorization'] = request.headers['Authorization'] + service_url = service_url.replace('http:', 'https:') # force HTTPS auth + for url in urls: + if 'pubsub_batch_handler' in config: + LOGGER.info(f'Sending {url} to pubsub batch handler.') + from google.cloud import pubsub_v1 # pylint: disable=C0415 + exporter_conf = config.get('pubsub_batch_handler') + client = pubsub_v1.PublisherClient() + project_id = exporter_conf['project_id'] + topic_name = exporter_conf['topic_name'] + topic_path = client.topic_path(project_id, topic_name) + data = url.encode('utf-8') + client.publish(topic_path, data=data).result() + else: # http + LOGGER.info(f'Sending {url} to HTTP batch handler.') + requests.post(service_url, headers=headers, data=url) diff --git a/slo_generator/cli.py b/slo_generator/cli.py index e65dc474..e22c3afa 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -57,7 +57,7 @@ def main(ctx, version): help='SLO config path') @click.option('--config', '-c', - type=click.Path(exists=True), + type=click.Path(), default='config.yaml', show_default=True, help='slo-generator config path') @@ -118,9 +118,16 @@ def compute(slo_config, config, export, delete, timestamp): @main.command() @click.pass_context @click.option('--config', + '-c', envvar='CONFIG_PATH', required=True, help='slo-generator configuration file path.') +@click.option('--exporters', + '-e', + envvar='EXPORTERS', + required=False, + default='', + help='List of exporters to send data to') @click.option('--signature-type', envvar='GOOGLE_FUNCTION_SIGNATURE_TYPE', default='http', @@ -130,17 +137,23 @@ def compute(slo_config, config, export, delete, timestamp): envvar='GOOGLE_FUNCTION_SIGNATURE_TYPE', default='run_compute', help='Target function name') -def api(ctx, config, signature_type, target): +@click.option('--port', + '-p', + default=8080, + help='HTTP port') +def api(ctx, config, exporters, signature_type, target, port): """Run an API that can receive requests (supports both 'http' and 'cloudevents' signature types).""" from functions_framework._cli import _cli + os.environ['EXPORTERS'] = exporters os.environ['CONFIG_PATH'] = config os.environ['GOOGLE_FUNCTION_SIGNATURE_TYPE'] = signature_type os.environ['GOOGLE_FUNCTION_TARGET'] = target ctx.invoke(_cli, target=target, source=Path(__file__).parent / 'api' / 'main.py', - signature_type=signature_type) + signature_type=signature_type, + port=port) @main.command() diff --git a/slo_generator/compute.py b/slo_generator/compute.py index c70bc6f2..fa31103b 100644 --- a/slo_generator/compute.py +++ b/slo_generator/compute.py @@ -20,6 +20,7 @@ import pprint import time +from slo_generator import constants from slo_generator import utils from slo_generator.report import SLOReport from slo_generator.migrations.migrator import report_v2tov1 @@ -55,6 +56,11 @@ def compute(slo_config, # Get exporters, backend and error budget policy spec = slo_config['spec'] exporters = utils.get_exporters(config, spec) + default_exporters_spec = { + "exporters": config.get('default_exporters', []) + } + default_exporters = utils.get_exporters(config, default_exporters_spec) + exporters.extend(x for x in default_exporters if x not in exporters) error_budget_policy = utils.get_error_budget_policy(config, spec) backend = utils.get_backend(config, spec) reports = [] @@ -104,27 +110,40 @@ def export(data, exporters, raise_on_error=False): info = f'{name :<32} | {ebp_step :<8}' errors = [] - # Convert data to export from v1 to v2 for backwards-compatible exports - data = report_v2tov1(data) - # Passing one exporter as a dict will work for convenience if isinstance(exporters, dict): exporters = [exporters] + if not exporters: + error = 'No exporters were found.' + LOGGER.error(f'{info} | {error}') + errors.append(error) for exporter in exporters: try: cls = exporter.get('class') + name = exporter.get('name') instance = utils.get_exporter_cls(cls) if not instance: - raise ImportError(f'Exporter {cls} not found.') + raise ImportError('Exporter not found in shared config.') LOGGER.debug(f'Exporter config: {pprint.pformat(exporter)}') - instance().export(data, **exporter) - LOGGER.info(f'{info} | SLO Report sent to {cls} successfully.') + + # Convert data to export from v1 to v2 for backwards-compatible + # exporters such as BigQuery. + json_data = data + if cls not in constants.V2_EXPORTERS: + LOGGER.debug(f'{info} | Converting SLO report to v1.') + json_data = report_v2tov1(data) + LOGGER.debug(f'{info} | SLO report: {json_data}') + response = instance().export(json_data, **exporter) + LOGGER.info( + f'{info} | SLO report sent to "{name}" exporter successfully.') + LOGGER.debug(f'{info} | {response}') except Exception as exc: # pylint: disable=broad-except if raise_on_error: raise exc tbk = utils.fmt_traceback(exc) - error = f'{cls}Exporter failed. | {tbk}' + error = f'{cls}Exporter "{name}" failed. | {tbk}' LOGGER.error(f'{info} | {error}') + LOGGER.exception(exc) errors.append(error) return errors diff --git a/slo_generator/constants.py b/slo_generator/constants.py index 8404100d..341592b3 100644 --- a/slo_generator/constants.py +++ b/slo_generator/constants.py @@ -27,6 +27,9 @@ DRY_RUN = bool(int(os.environ.get("DRY_RUN", "0"))) DEBUG = int(os.environ.get("DEBUG", "0")) +# Exporters supporting v2 SLO report format +V2_EXPORTERS = ('Pubsub', 'Cloudevent') + # Config skeletons CONFIG_SCHEMA = { 'backends': {}, diff --git a/slo_generator/exporters/bigquery.py b/slo_generator/exporters/bigquery.py index 7cb98468..cea6d3c4 100644 --- a/slo_generator/exporters/bigquery.py +++ b/slo_generator/exporters/bigquery.py @@ -88,12 +88,8 @@ def export(self, data, **config): table, json_rows=[json_data], retry=google.api_core.retry.Retry(deadline=30)) - status = f' Export data to {str(table_ref)}' if results: - status = constants.FAIL + status raise BigQueryError(results) - status = constants.SUCCESS + status - LOGGER.info(status) return results @staticmethod @@ -161,7 +157,8 @@ def update_schema(self, table_ref, keep=[]): iostream = io.StringIO('') self.client.schema_to_json(table.schema, iostream) existing_schema = json.loads(iostream.getvalue()) - LOGGER.debug(f'Existing schema: {pprint.pformat(existing_schema)}') + existing_fields = [field['name'] for field in existing_schema] + LOGGER.debug(f'Existing fields: {existing_fields}') # Fields in TABLE_SCHEMA to add / remove updated_fields = [ @@ -213,136 +210,106 @@ def _format(errors): TABLE_SCHEMA = [{ - 'description': None, 'name': 'service_name', 'type': 'STRING', 'mode': 'REQUIRED' }, { - 'description': None, 'name': 'feature_name', 'type': 'STRING', 'mode': 'REQUIRED' }, { - 'description': None, 'name': 'slo_name', 'type': 'STRING', 'mode': 'REQUIRED' }, { - 'description': None, 'name': 'slo_target', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'slo_description', 'type': 'STRING', 'mode': 'REQUIRED' }, { - 'description': None, 'name': 'error_budget_policy_step_name', 'type': 'STRING', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'error_budget_remaining_minutes', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'consequence_message', 'type': 'STRING', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'error_budget_minutes', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'error_minutes', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'error_budget_target', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'timestamp_human', 'type': 'TIMESTAMP', 'mode': 'REQUIRED' }, { - 'description': None, 'name': 'timestamp', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'cadence', 'type': 'STRING', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'window', 'type': 'INTEGER', 'mode': 'REQUIRED' }, { - 'description': None, 'name': 'bad_events_count', 'type': 'INTEGER', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'good_events_count', 'type': 'INTEGER', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'sli_measurement', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'gap', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'error_budget_measurement', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'error_budget_burn_rate', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'alerting_burn_rate_threshold', 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'alert', 'type': 'BOOLEAN', 'mode': 'NULLABLE' }, { - 'name': - 'metadata', - 'description': - None, - 'type': - 'RECORD', - 'mode': - 'REPEATED', + 'name': 'metadata', + 'type': 'RECORD', + 'mode': 'REPEATED', 'fields': [{ - 'description': None, 'name': 'key', 'type': 'STRING', 'mode': 'NULLABLE' }, { - 'description': None, 'name': 'value', 'type': 'STRING', 'mode': 'NULLABLE' diff --git a/slo_generator/exporters/cloudevent.py b/slo_generator/exporters/cloudevent.py new file mode 100644 index 00000000..a6358375 --- /dev/null +++ b/slo_generator/exporters/cloudevent.py @@ -0,0 +1,66 @@ +# Copyright 2021 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +`cloudevents.py` +CloudEvents exporter class. +""" +import logging +import requests + +import google.auth.transport.requests + +from cloudevents.http import CloudEvent, to_structured +from google.oauth2.id_token import fetch_id_token + +LOGGER = logging.getLogger(__name__) + +# pylint: disable=too-few-public-methods +class CloudeventExporter: + """Cloudevent exporter class. + + Args: + client (obj, optional): Existing Datadog client to pass. + service_url (str): Cloudevent receiver service URL. + """ + REQUIRED_FIELDS = ['service_url'] + OPTIONAL_FIELDS = ['auth'] + + # pylint: disable=R0201 + def export(self, data, **config): + """Export data as CloudEvent to an HTTP service receiving cloud events. + + Args: + data (dict): Metric data. + config (dict): Exporter config. + """ + attributes = { + "source": "https://github.com/cloudevents/spec/pull", + "type": "com.google.slo_generator.slo_report" + } + event = CloudEvent(attributes, data) + headers, data = to_structured(event) + service_url = config['service_url'] + if 'auth' in config: + auth = config['auth'] + id_token = None + if 'token' in auth: + id_token = auth['token'] + elif auth.get('google_service_account_auth', False): # Google oauth + auth = google.auth.transport.requests.Request() + id_token = fetch_id_token(auth, service_url) + if id_token: + headers["Authorization"] = f'Bearer {id_token}' + resp = requests.post(service_url, headers=headers, data=data) + resp.raise_for_status() + return resp diff --git a/slo_generator/exporters/dynatrace.py b/slo_generator/exporters/dynatrace.py index bb6445e9..646a397d 100644 --- a/slo_generator/exporters/dynatrace.py +++ b/slo_generator/exporters/dynatrace.py @@ -55,9 +55,7 @@ def export_metric(self, data): if code == 404: LOGGER.warning("Custom metric doesn't exist. Creating it.") metric = self.create_custom_metric(data) - LOGGER.debug(f'Custom metric: {metric}') response = self.create_timeseries(data) - LOGGER.debug(f'API Response: {response}') return response def create_timeseries(self, data): diff --git a/slo_generator/exporters/pubsub.py b/slo_generator/exporters/pubsub.py index f353298b..481241db 100644 --- a/slo_generator/exporters/pubsub.py +++ b/slo_generator/exporters/pubsub.py @@ -20,8 +20,6 @@ from google.cloud import pubsub_v1 -from slo_generator import constants - LOGGER = logging.getLogger(__name__) @@ -46,13 +44,5 @@ def export(self, data, **config): project_id = config['project_id'] topic_name = config['topic_name'] topic_path = self.publisher.topic_path(project_id, topic_name) - data = json.dumps(data, indent=4) - data = data.encode('utf-8') - res = self.publisher.publish(topic_path, data=data).result() - status = f' Export data to {topic_path}' - if not isinstance(res, str): - status = constants.FAIL + status - else: - status = constants.SUCCESS + status - LOGGER.info(status) - return res + data = json.dumps(data, indent=4).encode('utf-8') + return self.publisher.publish(topic_path, data=data).result() diff --git a/slo_generator/report.py b/slo_generator/report.py index ec4dcc00..c5d95d96 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -49,7 +49,7 @@ class SLOReport: description: str goal: str backend: str - exporters: list + exporters: list = field(default_factory=list) error_budget_policy: str = 'default' # SLI @@ -92,6 +92,7 @@ def __init__(self, # Init dataclass fields from SLO config and Error Budget Policy spec = config['spec'] + self.exporters = [] self.__set_fields(**spec, **step, lambdas={ @@ -328,7 +329,7 @@ def _validate(self, data): return False # Tuple should not have NO_DATA everywhere - if (good + bad) == (NO_DATA, NO_DATA): + if (good, bad) == (NO_DATA, NO_DATA): error = ( f'Backend method returned a valid tuple {data} but the ' 'good and bad count is NO_DATA (-1).') @@ -339,8 +340,8 @@ def _validate(self, data): # minimum valid events threshold if (good + bad) < MIN_VALID_EVENTS: error = ( - f'Not enough valid events found. Minimum valid events: ' - f'{MIN_VALID_EVENTS}') + f'Not enough valid events ({good + bad}) found. Minimum ' + f'valid events: {MIN_VALID_EVENTS}.') self.errors.append(error) return False diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 8739c755..8b081e1a 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -26,9 +26,11 @@ import re import sys import warnings + from pathlib import Path from dateutil import tz + import yaml from slo_generator.constants import DEBUG @@ -85,7 +87,7 @@ def load_config(path, ctx=os.environ, kind=None): elif abspath.is_file(): config = parse_config(path=str(abspath.resolve()), ctx=ctx) else: - LOGGER.warning(f'Path {path} not found. Trying to load from string') + LOGGER.debug(f'Path {path} not found. Trying to load from string') config = parse_config(content=str(path), ctx=ctx) # Filter on 'kind' @@ -144,6 +146,11 @@ def replace_env_vars(content, ctx): if ctx: content = replace_env_vars(content, ctx) data = yaml.safe_load(content) + if isinstance(data, str): + error = ( + 'Error serializing config into dict. This might be due to a syntax ' + 'error in the YAML / JSON config file.') + LOGGER.error(error) LOGGER.debug(pprint.pformat(data)) return data @@ -220,8 +227,8 @@ def get_exporters(config, spec): exporters = [] for exporter in spec_exporters: if exporter not in all_exporters.keys(): - LOGGER.warning( - f'Exporter "{exporter}" not found in config. Skipping.') + LOGGER.error( + f'Exporter "{exporter}" not found in config.') continue exporter_data = all_exporters[exporter] exporter_data['name'] = exporter @@ -516,4 +523,4 @@ def fmt_traceback(exc): Returns: str: Formatted exception. """ - return exc.__class__.__name__ + str(exc).replace("\n", " ") + return exc.__class__.__name__ + ": " + str(exc).replace("\n", " ") diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index 83cf0d2d..5a6fe093 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -186,7 +186,6 @@ def test_export_multiple_error(self, *mocks): errors = export(SLO_REPORT, exporters) self.assertEqual(len(errors), 1) self.assertIn('BigQueryError', errors[0]) - self.assertIn('BigqueryExporter failed', errors[0]) @patch("google.api_core.grpc_helpers.create_channel", return_value=mock_sd(STEPS)) From 1279899c456179d6b05cf1dc5aa3ee3f423cc546 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 2 Feb 2022 12:54:46 +0100 Subject: [PATCH 068/107] docs: clarify API export-only docs (#206) --- docs/shared/api.md | 65 +++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/docs/shared/api.md b/docs/shared/api.md index 5fdc3837..a9223d64 100644 --- a/docs/shared/api.md +++ b/docs/shared/api.md @@ -20,10 +20,7 @@ where: * `CONFIG_PATH` is the [Shared configuration](../../README.md#shared-configuration) file path or a Google Cloud Storage URL. -The API has two modes of functioning that can be controlled using the -`--signature-type` CLI argument: - -* `--signature-type=http`: can receive HTTP POST requests containing an +The standard-mode API can receive HTTP POST requests containing an **SLO config**, an **SLO config path**, or a **SLO config GCS URL** in the request body: ``` @@ -31,14 +28,24 @@ curl -X POST --data-binary /path/to/slo_config.yaml # SLO config ( curl -X POST -d @/path/to/slo_config.json # SLO config (JSON) curl -X POST -d "/path/to/slo_config.yaml" # SLO config path on disk (service needs to be able to load the path on the target machine). curl -X POST -d "gs:///slo.yaml" # SLO config GCS URL. +``` + +The API also has a batch mode that can be used by calling the endpoint with `?batch=true`: + +``` curl -X POST -d "gs:///slo.yaml;gs://GCS_BUCKET_NAME/slo2.yaml" /?batch=true # SLO configs GCS URLs (batch mode). ``` -***Note:*** The last request (batch mode) allows to send multiple file URLs that will be split and re-send to the service individually, or send to PubSub if a section `batch_pubsub_handler` is found in the slo-generator config. This section is populated the same as a [pubsub exporter](../providers/pubsub.md). +Batch mode allows to send multiple file URLs that will be split and: + +* Send each URL as individual HTTP requests (one per URL) to the service (default). + +**or** + +* Send each URL to PubSub if the section `pubsub_batch_handler` exists in the +`slo-generator` config. This section is populated the same as a [pubsub exporter](../providers/pubsub.md). +This setup is useful to smooth big loads (many SLOs), setting up a Pubsub push +subscription back to the service. -* `--signature-type=cloudevent`: can receive HTTP POST requests wrapped in a -[CloudEvent message](https://github.com/cloudevents/spec/blob/v1.0.1/spec.md#example), -with the actual SLO configs put directly under the `data` key, or in -base64-encoded string under the `data.message.data` enveloppe. ### Export-only mode API @@ -50,29 +57,39 @@ BigQuery: ![arch](../images/export_service_split.png) It is possible to run the `slo-generator` API in `export` mode only, by setting -the `--target` argument to `run_export`: +the `--target run_export`: ``` slo-generator api --config /path/to/config.yaml --target run_export ``` In this mode, the API accepts an -[SLO report](../../tests/unit/fixtures/slo_report_v2.json) in the POST request, -and exports that data to the required exporters. +[SLO report](../../tests/unit/fixtures/slo_report_v2.json) in the POST request +or in a base64-encoded string under the `message.data` enveloppe., and exports that +data to the required exporters. + +If `--signature-type cloudevent` is passed, the POST request data needs to be +wrapped into a [CloudEvent message](https://github.com/cloudevents/spec/blob/v1.0.1/spec.md#example) +under the `data` key. See the next section to see in which setups this is used. The exporters which are used for the export are configured using the `default_exporters` property in the `slo-generator` configuration. -Similarly to the standard-mode API, it can be run as an `http` endpoint (the -SLO report is sent directly in the request body) or as `cloudevent` endpoint ( -the SLO report is wrapped in a CloudEvent enveloppe). - ### Sending data from a standard API to an export-only API There are two ways you can forward SLO reports from the standard `slo-generator` API to an export-only API: -* If the export-only API is configured with `--signature-type=cloudevent`, then +* If the export-only API is configured with `--signature-type http` (default), +then you can: + + * Set up a `Pubsub` exporter in the standard API shared config to send the + SLO reports to a Pubsub topic, and set up a Pubsub push subscription + configured with the export-only API service URL. See official [documentation](https://cloud.google.com/run/docs/triggering/pubsub-push). + * [***not implemented yet***] Set up an `HTTP` exporter in the standard API shared + config to send the reports. + +* If the export-only API is configured with `--signature-type cloudevent`, then you can: * Set up a `Cloudevent` exporter in the standard API shared config to send the @@ -80,17 +97,7 @@ you can: * Set up a `Pubsub` exporter in the standard API shared config to send the events to a Pubsub topic, and set up an [**EventArc**](https://cloud.google.com/eventarc/) - to convert the Pubsub messages to CloudEvent format. - -* If the export-only API is configured with `--signature-type=http`, then -you can: - - * Set up a `Pubsub` exporter in the standard API shared config to send the - SLO reports to a Pubsub topic, and set up a Pubsub push subscription - configured with the export-only API service URL. - - * [***not implemented yet***] Set up an `HTTP` exporter in the standard API shared - config to send the reports. + to convert the Pubsub messages to CloudEvent format. See official [documentation](https://cloud.google.com/eventarc/docs/creating-triggers). **Notes:** @@ -100,6 +107,6 @@ dealing with lots of SLO computations. * You can develop your own Cloudevent receivers to use with the `Cloudevent` exporter if you want to do additional processing / analysis of SLO reports. -An example code for a receiver is given [here](https://cloud.google.com/eventarc/docs/run/event-receivers), +An example code for a Cloudevent receiver is given [here](https://cloud.google.com/eventarc/docs/run/event-receivers), but you can also check out the [Functions Framework](https://cloud.google.com/functions/docs/functions-framework) that does the marshalling / unmarshalling of Cloudevents out-of-the-box. From 75df96de406fa59abae693a7fc1c699041119182 Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Wed, 2 Feb 2022 14:19:41 +0100 Subject: [PATCH 069/107] chore: release 2.2.0 (#205) --- CHANGELOG.md | 20 ++++++++++++++++++++ setup.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab4b368c..168d62f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [2.2.0](https://www.github.com/google/slo-generator/compare/v2.1.0...v2.2.0) (2022-02-02) + + +### Features + +* add batch mode, better error reporting, cloud run docs ([#204](https://www.github.com/google/slo-generator/issues/204)) ([d305a63](https://www.github.com/google/slo-generator/commit/d305a63d2b3566815b6be6a04605a4d2beddf197)) + + +### Bug Fixes + +* alerting burn rate threshold null in BQ ([#201](https://www.github.com/google/slo-generator/issues/201)) ([d25f0f3](https://www.github.com/google/slo-generator/commit/d25f0f397fbe79f6fd265a5905952743f9a7a9ff)) +* custom backend path for integration tests ([#203](https://www.github.com/google/slo-generator/issues/203)) ([7268dc1](https://www.github.com/google/slo-generator/commit/7268dc1d843d3cf8bf3388f42590e6e6fba4ed86)) +* dynatrace slo import ([#198](https://www.github.com/google/slo-generator/issues/198)) ([df86234](https://www.github.com/google/slo-generator/commit/df86234db3dc14e91c7ebc31c29974e9d312834d)) +* remove row_ids to solve de-duplication issues ([#200](https://www.github.com/google/slo-generator/issues/200)) ([56d9b9b](https://www.github.com/google/slo-generator/commit/56d9b9bc551e1e37f6070ed1fe61cbaab1620f39)) + + +### Documentation + +* clarify API export-only docs ([#206](https://www.github.com/google/slo-generator/issues/206)) ([7c449c3](https://www.github.com/google/slo-generator/commit/7c449c32321f0fff3690c6177e3a85340afff2c8)) + ## [2.1.0](https://www.github.com/google/slo-generator/compare/v2.0.1...v2.1.0) (2022-01-19) diff --git a/setup.py b/setup.py index d7dbef61..266147c1 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # Package metadata. name = "slo-generator" description = "SLO Generator" -version = "2.1.0" +version = "2.2.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' From 64290c61263cc03327c6fceb294b1ebe530afc74 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Thu, 24 Mar 2022 14:47:57 +0100 Subject: [PATCH 070/107] fix: prevent gcloud crash with python 3.10 during release workflow Co-authored-by: lvaylet --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 150c0cfd..b5ac68a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/setup-python@master with: - python-version: '3.x' + python-version: '3.9' architecture: 'x64' - name: Run all tests From 67583b24c54e221073c307d409f551457eac20d6 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Sat, 26 Mar 2022 14:17:53 +0100 Subject: [PATCH 071/107] fix invalid target name in Makefile (#221) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b663587..6722352f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ To run all the tests, run `make` in the base directory. You can also select which test to run, and do other things: ``` -make unittests # run unittests only +make unit # run unit tests only make flake8 pylint # run linting tests only make docker_test # build Docker image and run tests within Docker container make docker_build # build Docker image only From ccbfb5ce250585446a6852328a4fb0fb549eb6f6 Mon Sep 17 00:00:00 2001 From: florianmartineau <67061306+florianmartineau@users.noreply.github.com> Date: Sat, 26 Mar 2022 14:18:40 +0100 Subject: [PATCH 072/107] fix: add timeFrame to retrieve_slo dynatrace (#212) --- slo_generator/backends/dynatrace.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/slo_generator/backends/dynatrace.py b/slo_generator/backends/dynatrace.py index 3d788407..87172bf7 100644 --- a/slo_generator/backends/dynatrace.py +++ b/slo_generator/backends/dynatrace.py @@ -161,7 +161,8 @@ def retrieve_slo(self, """ params = { 'from': start, - 'to': end + 'to': end, + 'timeFrame' : 'GTF' } endpoint = 'slo/' + slo_id return self.client.request('get', From d10fc429ca49e8c662c712d8e9349beafbbc6e1c Mon Sep 17 00:00:00 2001 From: David Jetelina <6594163+djetelina@users.noreply.github.com> Date: Sat, 26 Mar 2022 14:22:01 +0100 Subject: [PATCH 073/107] feat: add Prometheus Self exporter for API mode (#209) --- Makefile | 2 +- README.md | 2 +- docs/providers/prometheus.md | 12 ++++ samples/config.yaml | 1 + slo_generator/exporters/prometheus_self.py | 67 ++++++++++++++++++++++ tests/unit/fixtures/exporters.yaml | 2 + tests/unit/test_compute.py | 3 + 7 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 slo_generator/exporters/prometheus_self.py diff --git a/Makefile b/Makefile index 29cab558..4de52396 100644 --- a/Makefile +++ b/Makefile @@ -105,7 +105,7 @@ int_prom: # Run API locally run_api: - slo-generator api --target=run_compute --signature-type=http + slo-generator api --target=run_compute --signature-type=http -c samples/config.yaml # Local Docker build / push docker_build: diff --git a/README.md b/README.md index d0124e95..72b3f428 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ alerting directly on metrics or on **SLI > SLO** thresholds. ### Requirements -* `python3.7` and above +* `python3.9` and above * `pip3` ### Installation diff --git a/docs/providers/prometheus.md b/docs/providers/prometheus.md index 1120b777..c0da1ab9 100644 --- a/docs/providers/prometheus.md +++ b/docs/providers/prometheus.md @@ -154,6 +154,18 @@ Optional fields: **→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_availability_query_sli.yaml)** +## Self Exporter (API mode) + +When running slo-generator as an API, you can enable `prometheus_self` exporter, which will +expose all metrics on a standard `/metrics` endpoint, instead of pushing them to a gateway. + +```yaml +exporters: + prometheus_self: { } +``` + +***Note:*** The metrics endpoint will be available after a first successful SLO request. +Before that, it's going to act as if it was endpoint of the generator API. ### Examples diff --git a/samples/config.yaml b/samples/config.yaml index 5b35d1d6..9ef03fc8 100644 --- a/samples/config.yaml +++ b/samples/config.yaml @@ -42,6 +42,7 @@ exporters: pubsub: project_id: ${PUBSUB_PROJECT_ID} topic_name: ${PUBSUB_TOPIC_NAME} + prometheus_self: { } error_budget_policies: default: diff --git a/slo_generator/exporters/prometheus_self.py b/slo_generator/exporters/prometheus_self.py new file mode 100644 index 00000000..74830000 --- /dev/null +++ b/slo_generator/exporters/prometheus_self.py @@ -0,0 +1,67 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +`prometheus_self.py` +Prometheus Self exporter class. +""" +import logging + +from flask import current_app, make_response +from prometheus_client import Gauge, generate_latest + +from .base import MetricsExporter + +LOGGER = logging.getLogger(__name__) + +class PrometheusSelfExporter(MetricsExporter): + """Prometheus exporter class which uses + the API mode of itself to export the metrics.""" + REGISTERED_URL = False + REGISTERED_METRICS = {} + + def __init__(self): + if not self.REGISTERED_URL: + current_app.add_url_rule('/metrics', view_func=self.serve_metrics) + PrometheusSelfExporter.REGISTERED_URL = True + + @staticmethod + def serve_metrics(): + """Serves prometheus metrics + + Returns: + object: Flask HTTP Response + """ + resp = make_response(generate_latest(), 200) + resp.mimetype = 'text/plain' + return resp + + def export_metric(self, data): + """Export data to Prometheus global registry. + + Args: + data (dict): Metric data. + """ + name = data['name'] + description = data['description'] + value = data['value'] + + # Write timeseries w/ metric labels. + labels = data['labels'] + gauge = self.REGISTERED_METRICS.get(name) + if gauge is None: + gauge = Gauge(name, + description, + labelnames=labels.keys()) + PrometheusSelfExporter.REGISTERED_METRICS[name] = gauge + gauge.labels(*labels.values()).set(value) diff --git a/tests/unit/fixtures/exporters.yaml b/tests/unit/fixtures/exporters.yaml index 8d5d4140..8ff67708 100644 --- a/tests/unit/fixtures/exporters.yaml +++ b/tests/unit/fixtures/exporters.yaml @@ -43,3 +43,5 @@ # data labels - good_events_count - bad_events_count + + - class: PrometheusSelf diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index 5a6fe093..48537dd2 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -147,6 +147,9 @@ def test_export_bigquery_error(self, *mocks): def test_export_prometheus(self, mock): export(SLO_REPORT, EXPORTERS[3]) + def test_export_prometheus_self(self): + export(SLO_REPORT, EXPORTERS[7]) + @patch.object(Metric, 'send', mock_dd_metric_send) def test_export_datadog(self): export(SLO_REPORT, EXPORTERS[4]) From 6b44f534e7eababa7dbae4ddbe221daf298a1f42 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Sat, 26 Mar 2022 14:22:49 +0100 Subject: [PATCH 074/107] docs: add missing 'method' field in readme (#213) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 72b3f428..27059be7 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ is composed of the following fields: * `description`: [**required**] *string* - Description of this SLO. * `goal`: [**required**] *string* - SLO goal (or target) (**MUST** be between 0 and 1). * `backend`: [**required**] *string* - Backend name (**MUST** exist in SLO Generator Configuration). + * `method`: [**required**] *string* - Backend method to use (**MUST** exist in backend class definition). * `service_level_indicator`: [**required**] *map* - SLI configuration. The content of this section is specific to each provider, see [`docs/providers`](./docs/providers). * `error_budget_policy`: [*optional*] *string* - Error budget policy name From 7e1aec94d958caa5cfc3da5e02832540f45bb7d3 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Sun, 27 Mar 2022 12:05:44 +0200 Subject: [PATCH 075/107] docs: add Python 3.9 classifier (#226) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 266147c1..4b6d593a 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], keywords='slo sli generator gcp', install_requires=dependencies, From 393960d365babd5199d30eaaa0882f91a05ffc29 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Mon, 28 Mar 2022 13:50:26 +0200 Subject: [PATCH 076/107] fix: make unit tests pass again with elasticsearch 8.x client (#223) * use new convention from elasticsearch 8.x client * handle multiple connection setups * typo * update docs --- docs/providers/elasticsearch.md | 20 ++++++++++++++++++-- slo_generator/backends/elasticsearch.py | 22 +++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/docs/providers/elasticsearch.md b/docs/providers/elasticsearch.md index c30b1dbb..1f4e08b8 100644 --- a/docs/providers/elasticsearch.md +++ b/docs/providers/elasticsearch.md @@ -8,8 +8,24 @@ Elasticsearch to create an SLO. ```yaml backends: elasticsearch: - api_token: ${DYNATRACE_API_TOKEN} - api_url: ${DYNATRACE_API_URL} + url: ${ELASTICSEARCH_URL} +``` + +Note that `url` can be either a single string (when connecting to a single node) +or a list of strings (when connecting to multiple nodes): + +```yaml +backends: + elasticsearch: + url: https://localhost:9200 +``` + +```yaml +backends: + elasticsearch: + url: + - https://localhost:9200 + - https://localhost:9201 ``` The following methods are available to compute SLOs with the `elasticsearch` diff --git a/slo_generator/backends/elasticsearch.py b/slo_generator/backends/elasticsearch.py index 2b5d4b0e..36c6142c 100644 --- a/slo_generator/backends/elasticsearch.py +++ b/slo_generator/backends/elasticsearch.py @@ -16,6 +16,7 @@ ElasticSearch backend implementation. """ +import copy import logging from elasticsearch import Elasticsearch @@ -38,7 +39,26 @@ class ElasticsearchBackend: def __init__(self, client=None, **es_config): self.client = client if self.client is None: - self.client = Elasticsearch(**es_config) + # Copy the given client configuration and process it to address + # multiple connection setups (such as Elastic Cloud, basic auth, + # multiple nodes, API token...) before actually instantiating the + # client. + # Note: `es_config.copy()` and `dict(es_config)` only make *shallow* + # copies. We require a full nested copy of the configuration to + # work on. + conf = copy.deepcopy(es_config) + url = conf.pop('url', None) + basic_auth = conf.pop('basic_auth', None) + api_key = conf.pop('api_key', None) + if url: + conf['hosts'] = url + if basic_auth: + conf['basic_auth'] = ( + basic_auth['username'], basic_auth['password']) + if api_key: + conf['api_key'] = (api_key['id'], api_key['value']) + # Note: Either `hosts` or `cloud_id` must be specified in v8.x.x + self.client = Elasticsearch(**conf) # pylint: disable=unused-argument def good_bad_ratio(self, timestamp, window, slo_config): From aeae96ec76e7a18585af3ea2713b0d08033184a0 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Sat, 2 Apr 2022 15:10:48 +0200 Subject: [PATCH 077/107] ci: prevent setup-gcloud actions from failing after master branch gets renamed (#229) --- .github/workflows/build.yml | 2 +- .github/workflows/deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f7fb090..d2992a8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: fi - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: project_id: ${{ secrets.PROJECT_ID }} service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 81fab7d1..2ec59d53 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,7 +24,7 @@ jobs: fi - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: project_id: ${{ secrets.PROJECT_ID }} service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} From 391e44015d8d4425c9d5413d74adb0a9243b2e84 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Tue, 28 Jun 2022 15:58:40 +0200 Subject: [PATCH 078/107] remove deprecated option value for 'disable' (#237) Co-authored-by: lvaylet --- slo_generator/exporters/cloudevent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/slo_generator/exporters/cloudevent.py b/slo_generator/exporters/cloudevent.py index a6358375..0c6c00ac 100644 --- a/slo_generator/exporters/cloudevent.py +++ b/slo_generator/exporters/cloudevent.py @@ -36,7 +36,6 @@ class CloudeventExporter: REQUIRED_FIELDS = ['service_url'] OPTIONAL_FIELDS = ['auth'] - # pylint: disable=R0201 def export(self, data, **config): """Export data as CloudEvent to an HTTP service receiving cloud events. From 28582c0f98f7dc95977de0bc124c0284cc51b9f6 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Sat, 1 Oct 2022 08:28:06 +0200 Subject: [PATCH 079/107] fix: remove useless and unknown Pylint options (#247) * remove useless and unknown options * add default, arbitrary timeout to POST requests to fix Pylint check W3101 * split long line + reformat file --- .pylintrc | 80 +-------------------------- slo_generator/api/main.py | 2 +- slo_generator/exporters/cloudevent.py | 6 +- 3 files changed, 6 insertions(+), 82 deletions(-) diff --git a/.pylintrc b/.pylintrc index cc4f6171..0ff24061 100644 --- a/.pylintrc +++ b/.pylintrc @@ -55,17 +55,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, +disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, @@ -73,68 +63,7 @@ disable=print-statement, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, redefined-builtin, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, arguments-differ, dangerous-default-value, logging-fstring-interpolation @@ -326,13 +255,6 @@ max-line-length=100 # Maximum number of lines in a module. max-module-lines=1000 -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py index 58200b55..cfe0798b 100644 --- a/slo_generator/api/main.py +++ b/slo_generator/api/main.py @@ -201,4 +201,4 @@ def process_batch_req(request, data, config): client.publish(topic_path, data=data).result() else: # http LOGGER.info(f'Sending {url} to HTTP batch handler.') - requests.post(service_url, headers=headers, data=url) + requests.post(service_url, headers=headers, data=url, timeout=10) diff --git a/slo_generator/exporters/cloudevent.py b/slo_generator/exporters/cloudevent.py index 0c6c00ac..70313459 100644 --- a/slo_generator/exporters/cloudevent.py +++ b/slo_generator/exporters/cloudevent.py @@ -25,6 +25,7 @@ LOGGER = logging.getLogger(__name__) + # pylint: disable=too-few-public-methods class CloudeventExporter: """Cloudevent exporter class. @@ -55,11 +56,12 @@ def export(self, data, **config): id_token = None if 'token' in auth: id_token = auth['token'] - elif auth.get('google_service_account_auth', False): # Google oauth + elif auth.get('google_service_account_auth', False): # Google oauth auth = google.auth.transport.requests.Request() id_token = fetch_id_token(auth, service_url) if id_token: headers["Authorization"] = f'Bearer {id_token}' - resp = requests.post(service_url, headers=headers, data=data) + resp = requests.post(service_url, headers=headers, data=data, + timeout=10) resp.raise_for_status() return resp From 7eaaf8c5414850f75e80b21c1423eeb881239a8b Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Mon, 10 Oct 2022 19:32:39 +0000 Subject: [PATCH 080/107] feat: add pytype linting (#249) * add pytype linting and fix all reported errors * revert metaclass fix to make tests pass --- Makefile | 5 ++++- setup.cfg | 3 +++ setup.py | 2 +- slo_generator/backends/cloud_service_monitoring.py | 7 ++++++- slo_generator/backends/datadog.py | 6 +++--- slo_generator/exporters/base.py | 4 ++-- slo_generator/migrations/migrator.py | 1 + slo_generator/utils.py | 2 +- 8 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 4de52396..988f35ab 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ unit: clean coverage: $(COVERAGE) report --rcfile=".coveragerc" -lint: flake8 pylint +lint: flake8 pylint pytype flake8: flake8 --ignore=$(FLAKE8_IGNORE) $(NAME)/ --max-line-length=80 @@ -80,6 +80,9 @@ flake8: pylint: find ./$(NAME) ./tests -name \*.py | xargs pylint --rcfile .pylintrc --ignore-patterns=test_.*?py +pytype: + pytype + integration: int_cm int_csm int_custom int_dd int_dt int_es int_prom int_cm: diff --git a/setup.cfg b/setup.cfg index c3a2b39f..0d1debba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,3 +17,6 @@ # Generated by synthtool. DO NOT EDIT! [bdist_wheel] universal = 1 + +[pytype] +inputs = slo_generator diff --git a/setup.py b/setup.py index 4b6d593a..92a5f404 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], 'elasticsearch': ['elasticsearch'], 'cloudevent': ['cloudevents'], - 'dev': ['wheel', 'flake8', 'mock', 'coverage', 'nose', 'pylint'] + 'dev': ['wheel', 'flake8', 'mock', 'coverage', 'nose', 'pylint', 'pytype'] } # Get the long description from the README file diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 6e4a6dba..5385d6cc 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -20,15 +20,19 @@ import logging import os import warnings +from typing import Union, Sequence import google.api_core.exceptions from google.cloud.monitoring_v3 import ServiceMonitoringServiceClient +# pytype: disable=pyi-error from google.protobuf.json_format import MessageToJson +# pytype: enable=pyi-error from slo_generator.backends.cloud_monitoring import CloudMonitoringBackend from slo_generator.constants import NO_DATA from slo_generator.utils import dict_snake_to_caml + LOGGER = logging.getLogger(__name__) SID_GAE = 'gae:{project_id}_{module_id}' @@ -591,7 +595,8 @@ def compare_slo(slo1, slo2): return local_json == remote_json @staticmethod - def string_diff(string1, string2): + def string_diff(string1: Union[str, Sequence[str]], + string2: Union[str, Sequence[str]]): """Diff 2 strings. Used to print comparison of JSONs for debugging. Args: diff --git a/slo_generator/backends/datadog.py b/slo_generator/backends/datadog.py index b9ee6a6c..d5fd6f31 100644 --- a/slo_generator/backends/datadog.py +++ b/slo_generator/backends/datadog.py @@ -112,10 +112,10 @@ def query_slo(self, timestamp, window, slo_config): from_ts = timestamp - window slo_data = self.client.ServiceLevelObjective.get(id=slo_id) LOGGER.debug(f"SLO data: {slo_id} | Result: {pprint.pformat(slo_data)}") + data = self.client.ServiceLevelObjective.history(id=slo_id, + from_ts=from_ts, + to_ts=timestamp) try: - data = self.client.ServiceLevelObjective.history(id=slo_id, - from_ts=from_ts, - to_ts=timestamp) LOGGER.debug( f"Timeseries data: {slo_id} | Result: {pprint.pformat(data)}") good_event_count = data['data']['series']['numerator']['sum'] diff --git a/slo_generator/exporters/base.py b/slo_generator/exporters/base.py index 0364710a..bc2de873 100644 --- a/slo_generator/exporters/base.py +++ b/slo_generator/exporters/base.py @@ -60,10 +60,10 @@ ] -class MetricsExporter: +class MetricsExporter: # pytype: disable=ignored-metaclass """Abstract class to export metrics to different backends. Common format for YAML configuration to configure which metrics should be exported.""" - __metaclass__ = ABCMeta + __metaclass__ = ABCMeta # pytype: disable=ignored-metaclass def export(self, data, **config): """Export metric data. Loops through metrics config and calls the child diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 91ef5a93..8fe1a2e1 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -17,6 +17,7 @@ """ # pylint: disable=line-too-long, too-many-statements, too-many-ancestors, too-many-locals, too-many-nested-blocks, unused-argument # flake8: noqa +# pytype: skip-file import copy import itertools import pprint diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 8b081e1a..848cc312 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -36,7 +36,7 @@ from slo_generator.constants import DEBUG try: - from google.cloud import storage + from google.cloud import storage # pytype: disable=import-error GCS_ENABLED = True except ImportError: GCS_ENABLED = False From 0ad7ff2f196bb84d1e2723c0f5a6be43f4d87457 Mon Sep 17 00:00:00 2001 From: k1rnt Date: Thu, 13 Oct 2022 15:54:54 +0900 Subject: [PATCH 081/107] ci: replace deprecated `set-output` command with `GITHUB_OUTPUT` environment variable (#252) * Fix use GITHUB_OUTPUT from deprecated set-output * set shell * set shell * fix: double quotes around * revert set shell --- .github/workflows/build.yml | 4 ++-- .github/workflows/deploy.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2992a8e..b303f0ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,9 +20,9 @@ jobs: - name: Check release version id: check-tag run: | - echo ::set-output name=version::$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-) + echo "version=$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-)" >> $GITHUB_OUTPUT if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true + echo "match=true" >> $GITHUB_OUTPUT fi - name: Set up Cloud SDK diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2ec59d53..08ee3d01 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,9 +18,9 @@ jobs: - name: Check release version id: check-tag run: | - echo ::set-output name=version::$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-) + echo "version=$(echo ${{ github.event.ref }} | cut -d / -f 3 | cut -c2-)" >> $GITHUB_OUTPUT if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true + echo "match=true" >> $GITHUB_OUTPUT fi - name: Set up Cloud SDK From 98a25a79a5732af0b7e2dc2ef16c92a0f10a634f Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Fri, 14 Oct 2022 15:17:24 +0200 Subject: [PATCH 082/107] tests: replace `nose` (in maintenance mode) with `pytest` (#254) * make pytest succeed on existing tests * run tests with pytest instead of nose --- Makefile | 3 +-- setup.py | 2 +- tests/unit/test_compute.py | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 988f35ab..61b59079 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,6 @@ PIP=pip3 PYTHON=python3 TWINE=twine COVERAGE=coverage -NOSE_OPTS = --with-coverage --cover-package=$(NAME) --cover-erase --nologcapture --logging-level=ERROR SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") VERSION ?= $(shell grep "version = " setup.py | cut -d\ -f3) @@ -66,7 +65,7 @@ install: clean test: install unit lint unit: clean - nosetests $(NOSE_OPTS) tests/unit/* -v + pytest --cov=$(NAME) tests -p no:warnings coverage: $(COVERAGE) report --rcfile=".coveragerc" diff --git a/setup.py b/setup.py index 92a5f404..d3250d17 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], 'elasticsearch': ['elasticsearch'], 'cloudevent': ['cloudevents'], - 'dev': ['wheel', 'flake8', 'mock', 'coverage', 'nose', 'pylint', 'pytype'] + 'dev': ['wheel', 'flake8', 'mock', 'pytest', 'pytest-cov', 'pylint', 'pytype'] } # Get the long description from the README file diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index 48537dd2..1a4abaf1 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -48,7 +48,8 @@ # Pub/Sub methods to patch PUBSUB_MOCKS = [ "google.cloud.pubsub_v1.gapic.publisher_client.PublisherClient.publish", - "google.cloud.pubsub_v1.publisher.futures.Future.result" + "google.cloud.pubsub_v1.publisher.futures.Future.result", + "google.api_core.grpc_helpers.create_channel", ] # Service Monitoring method to patch @@ -119,7 +120,8 @@ def test_compute_dynatrace(self, mock): @patch(PUBSUB_MOCKS[0]) @patch(PUBSUB_MOCKS[1]) - def test_export_pubsub(self, mock_pubsub, mock_pubsub_res): + @patch(PUBSUB_MOCKS[2]) + def test_export_pubsub(self, *mocks): export(SLO_REPORT, EXPORTERS[0]) @patch("google.api_core.grpc_helpers.create_channel", From fca2409b71c314c30b24329f87ae63501dccd188 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Fri, 14 Oct 2022 15:55:47 +0200 Subject: [PATCH 083/107] ci: run linting and unit tests on multiple python versions (#256) * test matrix strategy on `build` job * fix type hint for python < 3.9 * disable pylint warning for python 3.6 * remove python 3.6 as pylint only supports python >= 3.7.2 * fix import for python 3.10 * expand matrix strategy to `unit` step * use the latest major version (v0) of GitHub Actions * fix the versions of GitHub Actions when v0 is not available --- .github/workflows/test.yml | 32 +++++++++++++++++++++----------- slo_generator/report.py | 3 ++- slo_generator/utils.py | 10 ++++------ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f51be87..20947dde 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,13 +5,18 @@ on: jobs: lint: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest] + architecture: ['x64'] + python-version: ['3.7', '3.8', '3.9', '3.10'] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: - python-version: '3.9' - architecture: 'x64' + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} - name: Install dependencies run: make install @@ -20,13 +25,18 @@ jobs: run: make lint unit: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest] + architecture: ['x64'] + python-version: ['3.7', '3.8', '3.9', '3.10'] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: - python-version: '3.9' - architecture: 'x64' + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} - name: Install dependencies run: make install @@ -43,7 +53,7 @@ jobs: docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - uses: docker-practice/actions-setup-docker@master - name: Build Docker image run: make docker_build diff --git a/slo_generator/report.py b/slo_generator/report.py index c5d95d96..e9cde1d3 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -18,6 +18,7 @@ import logging from dataclasses import asdict, dataclass, fields, field +from typing import List from slo_generator import utils from slo_generator.constants import (COLORED_OUTPUT, MIN_VALID_EVENTS, NO_DATA, @@ -80,7 +81,7 @@ class SLOReport: # Data validation valid: bool - errors: list[str] = field(default_factory=list) + errors: List[str] = field(default_factory=list) def __init__(self, config, diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 848cc312..d221e702 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -15,9 +15,7 @@ `utils.py` Utility functions. """ -from datetime import datetime import argparse -import collections import errno import importlib import logging @@ -26,12 +24,12 @@ import re import sys import warnings - +from collections.abc import Mapping +from datetime import datetime from pathlib import Path -from dateutil import tz - import yaml +from dateutil import tz from slo_generator.constants import DEBUG @@ -419,7 +417,7 @@ def apply_func_dict(data, func): Returns: dict: Output dictionary. """ - if isinstance(data, collections.Mapping): + if isinstance(data, Mapping): return {func(k): apply_func_dict(v, func) for k, v in data.items()} return data From ecca999abbe5ff02be3bac0f908f405477e1da40 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Sun, 16 Oct 2022 10:04:31 +0200 Subject: [PATCH 084/107] ci: let all `lint`and `unit` jobs complete even if one fails (#261) --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20947dde..01eb1acb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,7 @@ on: jobs: lint: strategy: + fail-fast: false matrix: os: [ubuntu-latest] architecture: ['x64'] @@ -26,6 +27,7 @@ jobs: unit: strategy: + fail-fast: false matrix: os: [ubuntu-latest] architecture: ['x64'] From 9baa33e2b7bc7b1231eda13b3c136094ba1c54dc Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Mon, 17 Oct 2022 18:01:35 +0200 Subject: [PATCH 085/107] feat: add Cloud Monitoring MQL backend (#245) * ignore virtual environments like venv3.9.12 too * bump google-cloud-monitoring to v2 and migrate code * remove redundant parentheses * fix typo * remove unused argument in docstring * fix mutable default arguments * add `make uninstall` target to uninstall all pip packages * fix wrong migration of field names * disable Pylint warnings for fields that do exist * add type hints * add Cloud Monitoring MQL samples * reformat for flake8 * aggregate (sum) time series before computing their ratio * annotate and clean up distribution_cut() * add private method to format MQL query + tests * disable warnings for known unused arguments * fix syntax error * disable flake8 checks in tests (for trailing whitespaces for example) * fix type hints for dict.get() operations * make type hints and pylint warnings compatible with python < 3.9 --- .gitignore | 4 +- Makefile | 3 + .../slo_gae_app_availability.yaml | 37 +++ .../slo_gae_app_availability_ratio.yaml | 36 +++ .../slo_gae_app_latency.yaml | 39 +++ .../slo_lb_request_availability.yaml | 28 ++ .../slo_lb_request_availability_ratio.yaml | 27 ++ .../slo_lb_request_latency.yaml | 25 ++ .../slo_pubsub_subscription_throughput.yaml | 24 ++ setup.py | 6 +- slo_generator/backends/cloud_monitoring.py | 64 +++-- .../backends/cloud_monitoring_mql.py | 263 ++++++++++++++++++ slo_generator/constants.py | 31 ++- slo_generator/exporters/cloud_monitoring.py | 68 +++-- .../backends/test_cloud_monitoring_mql.py | 68 +++++ tests/unit/test_stubs.py | 8 +- 16 files changed, 658 insertions(+), 73 deletions(-) create mode 100644 samples/cloud_monitoring_mql/slo_gae_app_availability.yaml create mode 100644 samples/cloud_monitoring_mql/slo_gae_app_availability_ratio.yaml create mode 100644 samples/cloud_monitoring_mql/slo_gae_app_latency.yaml create mode 100644 samples/cloud_monitoring_mql/slo_lb_request_availability.yaml create mode 100644 samples/cloud_monitoring_mql/slo_lb_request_availability_ratio.yaml create mode 100644 samples/cloud_monitoring_mql/slo_lb_request_latency.yaml create mode 100644 samples/cloud_monitoring_mql/slo_pubsub_subscription_throughput.yaml create mode 100644 slo_generator/backends/cloud_monitoring_mql.py create mode 100644 tests/unit/backends/test_cloud_monitoring_mql.py diff --git a/.gitignore b/.gitignore index 05b77b9d..5bb99d99 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,6 @@ htmlcov/ *.tfstate.*.backup .vscode .env -venv/ -.venv/ +venv*/ +.venv*/ reports/ diff --git a/Makefile b/Makefile index 61b59079..c1c15e3a 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,9 @@ develop: install: clean $(PIP) install -e ."[api, datadog, prometheus, elasticsearch, pubsub, cloud_monitoring, bigquery, dev]" +uninstall: clean + $(PIP) freeze --exclude-editable | xargs $(PIP) uninstall -y + test: install unit lint unit: clean diff --git a/samples/cloud_monitoring_mql/slo_gae_app_availability.yaml b/samples/cloud_monitoring_mql/slo_gae_app_availability.yaml new file mode 100644 index 00000000..694bb1d8 --- /dev/null +++ b/samples/cloud_monitoring_mql/slo_gae_app_availability.yaml @@ -0,0 +1,37 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-availability + labels: + service_name: gae + feature_name: app + slo_name: availability +spec: + description: Availability of App Engine app + backend: cloud_monitoring_mql + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | filter + metric.response_code == 429 + || metric.response_code == 200 + || metric.response_code == 201 + || metric.response_code == 202 + || metric.response_code == 203 + || metric.response_code == 204 + || metric.response_code == 205 + || metric.response_code == 206 + || metric.response_code == 207 + || metric.response_code == 208 + || metric.response_code == 226 + || metric.response_code == 304 + filter_valid: > + fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + goal: 0.95 diff --git a/samples/cloud_monitoring_mql/slo_gae_app_availability_ratio.yaml b/samples/cloud_monitoring_mql/slo_gae_app_availability_ratio.yaml new file mode 100644 index 00000000..8e029ba8 --- /dev/null +++ b/samples/cloud_monitoring_mql/slo_gae_app_availability_ratio.yaml @@ -0,0 +1,36 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-availability + labels: + service_name: gae + feature_name: app + slo_name: availability +spec: + description: Availability of App Engine app + backend: cloud_monitoring_mql + method: query_sli + exporters: + - cloud_monitoring + service_level_indicator: + query: > + fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | { filter + metric.response_code == 429 + || metric.response_code == 200 + || metric.response_code == 201 + || metric.response_code == 202 + || metric.response_code == 203 + || metric.response_code == 204 + || metric.response_code == 205 + || metric.response_code == 206 + || metric.response_code == 207 + || metric.response_code == 208 + || metric.response_code == 226 + || metric.response_code == 304 + ; ident } + | sum + | ratio + goal: 0.95 diff --git a/samples/cloud_monitoring_mql/slo_gae_app_latency.yaml b/samples/cloud_monitoring_mql/slo_gae_app_latency.yaml new file mode 100644 index 00000000..89d8bad0 --- /dev/null +++ b/samples/cloud_monitoring_mql/slo_gae_app_latency.yaml @@ -0,0 +1,39 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-latency724ms + labels: + service_name: gae + feature_name: app + slo_name: latency724ms +spec: + description: Latency of App Engine app requests < 724ms + backend: cloud_monitoring_mql + method: distribution_cut + exporters: + - cloud_monitoring + service_level_indicator: + filter_valid: > + fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_latencies' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | filter + metric.response_code >= 200 + && metric.response_code < 500 + good_below_threshold: true + threshold_bucket: 19 + goal: 0.999 diff --git a/samples/cloud_monitoring_mql/slo_lb_request_availability.yaml b/samples/cloud_monitoring_mql/slo_lb_request_availability.yaml new file mode 100644 index 00000000..98e8ba46 --- /dev/null +++ b/samples/cloud_monitoring_mql/slo_lb_request_availability.yaml @@ -0,0 +1,28 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: lb-request-availability + labels: + service_name: lb + feature_name: request + slo_name: availability +spec: + description: Availability of HTTP Load Balancer + backend: cloud_monitoring_mql + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + fetch 'https_lb_rule' + | metric 'loadbalancing.googleapis.com/https/request_count' + | filter resource.project_id == '${LB_PROJECT_ID}' + | filter + metric.response_code_class="200" + || metric.response_code_class="300" + || metric.response_code_class="400" + filter_valid: > + fetch 'https_lb_rule' + | metric 'loadbalancing.googleapis.com/https/request_count' + | filter resource.project_id == '${LB_PROJECT_ID}' + goal: 0.98 diff --git a/samples/cloud_monitoring_mql/slo_lb_request_availability_ratio.yaml b/samples/cloud_monitoring_mql/slo_lb_request_availability_ratio.yaml new file mode 100644 index 00000000..3a6e7cd4 --- /dev/null +++ b/samples/cloud_monitoring_mql/slo_lb_request_availability_ratio.yaml @@ -0,0 +1,27 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: lb-request-availability + labels: + service_name: lb + feature_name: request + slo_name: availability +spec: + description: Availability of HTTP Load Balancer + backend: cloud_monitoring_mql + method: query_sli + exporters: + - cloud_monitoring + service_level_indicator: + query: > + fetch 'https_lb_rule' + | metric 'loadbalancing.googleapis.com/https/request_count' + | filter resource.project_id == '${LB_PROJECT_ID}' + | { filter + metric.response_code_class="200" + || metric.response_code_class="300" + || metric.response_code_class="400" + ; ident } + | sum + | ratio + goal: 0.98 diff --git a/samples/cloud_monitoring_mql/slo_lb_request_latency.yaml b/samples/cloud_monitoring_mql/slo_lb_request_latency.yaml new file mode 100644 index 00000000..92cebc06 --- /dev/null +++ b/samples/cloud_monitoring_mql/slo_lb_request_latency.yaml @@ -0,0 +1,25 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: lb-request-latency724ms + labels: + service_name: lb + feature_name: request + slo_name: latency724ms +spec: + description: Latency of HTTP Load Balancer < 724ms + backend: cloud_monitoring_mql + method: distribution_cut + exporters: + - cloud_monitoring + service_level_indicator: + filter_valid: > + fetch https_lb_rule + | metric 'loadbalancing.googleapis.com/https/total_latencies' + | filter resource.project_id == '${LB_PROJECT_ID}' + | filter metric.label.response_code_class = "200" + || metric.response_code_class = "300" + || metric.response_code_class = "400" + good_below_threshold: true + threshold_bucket: 19 + goal: 0.98 diff --git a/samples/cloud_monitoring_mql/slo_pubsub_subscription_throughput.yaml b/samples/cloud_monitoring_mql/slo_pubsub_subscription_throughput.yaml new file mode 100644 index 00000000..bc7b5eef --- /dev/null +++ b/samples/cloud_monitoring_mql/slo_pubsub_subscription_throughput.yaml @@ -0,0 +1,24 @@ +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: pubsub-subscription-throughput + labels: + service_name: pubsub + feature_name: subscription + slo_name: throughput +spec: + description: Throughput of Pub/Sub subscription + backend: cloud_monitoring_mql + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + fetch 'pubsub_subscription' + | metric 'pubsub.googleapis.com/subscription/ack_message_count' + | filter resource.project_id == '${PUBSUB_PROJECT_ID}' + filter_bad: > + fetch 'pubsub_subscription' + | metric 'pubsub.googleapis.com/subscription/num_outstanding_messages' + | filter resource.project_id == '${PUBSUB_PROJECT_ID}' + goal: 0.95 diff --git a/setup.py b/setup.py index d3250d17..5f512569 100644 --- a/setup.py +++ b/setup.py @@ -46,10 +46,10 @@ 'dynatrace': ['requests'], 'bigquery': ['google-api-python-client <2', 'google-cloud-bigquery <3'], 'cloud_monitoring': [ - 'google-api-python-client <2', 'google-cloud-monitoring ==1.1.0' + 'google-api-python-client <2', 'google-cloud-monitoring <3' ], 'cloud_service_monitoring': [ - 'google-api-python-client <2', 'google-cloud-monitoring ==1.1.0' + 'google-api-python-client <2', 'google-cloud-monitoring <3' ], 'cloud_storage': ['google-api-python-client <2', 'google-cloud-storage'], 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], @@ -89,4 +89,4 @@ entry_points={ 'console_scripts': ['slo-generator=slo_generator.cli:main'], }, - python_requires='>=3.4') + python_requires='>=3.6') diff --git a/slo_generator/backends/cloud_monitoring.py b/slo_generator/backends/cloud_monitoring.py index 86df3e01..2ba77850 100644 --- a/slo_generator/backends/cloud_monitoring.py +++ b/slo_generator/backends/cloud_monitoring.py @@ -41,7 +41,7 @@ def __init__(self, project_id, client=None): self.client = client if client is None: self.client = monitoring_v3.MetricServiceClient() - self.parent = self.client.project_path(project_id) + self.parent = self.client.common_project_path(project_id) def good_bad_ratio(self, timestamp, window, slo_config): """Query two timeseries, one containing 'good' events, one containing @@ -87,10 +87,10 @@ def good_bad_ratio(self, timestamp, window, slo_config): LOGGER.debug(f'Good events: {good_event_count} | ' f'Bad events: {bad_event_count}') - return (good_event_count, bad_event_count) + return good_event_count, bad_event_count def distribution_cut(self, timestamp, window, slo_config): - """Query one timeserie of type 'exponential'. + """Query one timeseries of type 'exponential'. Args: timestamp (int): UNIX timestamp. @@ -112,7 +112,7 @@ def distribution_cut(self, timestamp, window, slo_config): series = list(series) if not series: - return (NO_DATA, NO_DATA) # no timeseries + return NO_DATA, NO_DATA # no timeseries distribution_value = series[0].points[0].value.distribution_value # bucket_options = distribution_value.bucket_options @@ -149,7 +149,7 @@ def distribution_cut(self, timestamp, window, slo_config): good_event_count = upper_events_count bad_event_count = lower_events_count - return (good_event_count, bad_event_count) + return good_event_count, bad_event_count def exponential_distribution_cut(self, *args, **kwargs): """Alias for `distribution_cut` method to allow for backwards @@ -166,7 +166,7 @@ def query(self, filter, aligner='ALIGN_SUM', reducer='REDUCE_SUM', - group_by=[]): + group_by=None): """Query timeseries from Cloud Monitoring. Args: @@ -180,15 +180,20 @@ def query(self, Returns: list: List of timeseries objects. """ + if group_by is None: + group_by = [] measurement_window = CM.get_window(timestamp, window) aggregation = CM.get_aggregation(window, aligner=aligner, reducer=reducer, group_by=group_by) - timeseries = self.client.list_time_series( - self.parent, filter, measurement_window, - monitoring_v3.enums.ListTimeSeriesRequest.TimeSeriesView.FULL, - aggregation) + request = monitoring_v3.ListTimeSeriesRequest() + request.name = self.parent + request.filter = filter + request.interval = measurement_window + request.view = monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + request.aggregation = aggregation + timeseries = self.client.list_time_series(request) LOGGER.debug(pprint.pformat(timeseries)) return timeseries @@ -220,12 +225,20 @@ def get_window(timestamp, window): Returns: :obj:`monitoring_v3.types.TimeInterval`: Measurement window object. """ - measurement_window = monitoring_v3.types.TimeInterval() - measurement_window.end_time.seconds = int(timestamp) - measurement_window.end_time.nanos = int( - (timestamp - measurement_window.end_time.seconds) * 10**9) - measurement_window.start_time.seconds = int(timestamp - window) - measurement_window.start_time.nanos = measurement_window.end_time.nanos + end_time_seconds = int(timestamp) + end_time_nanos = int((timestamp - end_time_seconds) * 10 ** 9) + start_time_seconds = int(timestamp - window) + start_time_nanos = end_time_nanos + measurement_window = monitoring_v3.TimeInterval({ + "end_time": { + "seconds": end_time_seconds, + "nanos": end_time_nanos + }, + "start_time": { + "seconds": start_time_seconds, + "nanos": start_time_nanos + } + }) LOGGER.debug(pprint.pformat(measurement_window)) return measurement_window @@ -233,7 +246,7 @@ def get_window(timestamp, window): def get_aggregation(window, aligner='ALIGN_SUM', reducer='REDUCE_SUM', - group_by=[]): + group_by=None): """Helper for aggregation object. Default aggregation is `ALIGN_SUM`. @@ -248,13 +261,16 @@ def get_aggregation(window, Returns: :obj:`monitoring_v3.types.Aggregation`: Aggregation object. """ - aggregation = monitoring_v3.types.Aggregation() - aggregation.alignment_period.seconds = window - aggregation.per_series_aligner = (getattr( - monitoring_v3.enums.Aggregation.Aligner, aligner)) - aggregation.cross_series_reducer = (getattr( - monitoring_v3.enums.Aggregation.Reducer, reducer)) - aggregation.group_by_fields.extend(group_by) + if group_by is None: + group_by = [] + aggregation = monitoring_v3.Aggregation({ + "alignment_period": {"seconds": window}, + "per_series_aligner": + getattr(monitoring_v3.Aggregation.Aligner, aligner), + "cross_series_reducer": + getattr(monitoring_v3.Aggregation.Reducer, reducer), + "group_by_fields": group_by, + }) LOGGER.debug(pprint.pformat(aggregation)) return aggregation diff --git a/slo_generator/backends/cloud_monitoring_mql.py b/slo_generator/backends/cloud_monitoring_mql.py new file mode 100644 index 00000000..2cf96bd5 --- /dev/null +++ b/slo_generator/backends/cloud_monitoring_mql.py @@ -0,0 +1,263 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +`cloud_monitoring_mql.py` +Cloud Monitoring backend implementation with MQL (Monitoring Query Language). +""" +import logging +import pprint +import re +import typing +import warnings +from collections import OrderedDict +from typing import List, Tuple + +from google.api.distribution_pb2 import Distribution +from google.cloud.monitoring_v3.services.query_service import QueryServiceClient +from google.cloud.monitoring_v3.services.query_service.pagers import \ + QueryTimeSeriesPager +from google.cloud.monitoring_v3.types import metric_service +from google.cloud.monitoring_v3.types.metric import TimeSeriesData + +from slo_generator.constants import NO_DATA + +LOGGER = logging.getLogger(__name__) + + +class CloudMonitoringMqlBackend: + """Backend for querying metrics from Cloud Monitoring with MQL. + + Args: + project_id (str): Cloud Monitoring host project id. + client (google.cloud.monitoring_v3.QueryServiceClient, optional): + Existing Cloud Monitoring Query client. Initialize a new client + if omitted. + """ + + def __init__(self, project_id: str, client: QueryServiceClient = None): + self.client = client + if client is None: + self.client = QueryServiceClient() + self.parent = self.client.common_project_path(project_id) + + def good_bad_ratio(self, + timestamp: int, # pylint: disable=unused-argument + window: int, + slo_config: dict) -> Tuple[int, int]: + """Query two timeseries, one containing 'good' events, one containing + 'bad' events. + + Args: + timestamp (int): UNIX timestamp. + window (int): Window size (in seconds). + slo_config (dict): SLO configuration. + + Returns: + tuple: A tuple (good_event_count, bad_event_count) + """ + measurement: dict = slo_config['spec']['service_level_indicator'] + filter_good: str = measurement['filter_good'] + filter_bad: typing.Optional[str] = measurement.get('filter_bad') + filter_valid: typing.Optional[str] = measurement.get('filter_valid') + + # Query 'good events' timeseries + good_ts: List[TimeSeriesData] = self.query( + query=filter_good, window=window) + good_event_count: int = CM.count(good_ts) + + # Query 'bad events' timeseries + if filter_bad: + bad_ts: List[TimeSeriesData] = self.query( + query=filter_bad, window=window) + bad_event_count: int = CM.count(bad_ts) + elif filter_valid: + valid_ts: List[TimeSeriesData] = self.query( + query=filter_valid, window=window) + bad_event_count: int = CM.count(valid_ts) - good_event_count + else: + raise Exception( + "One of `filter_bad` or `filter_valid` is required.") + + LOGGER.debug(f'Good events: {good_event_count} | ' + f'Bad events: {bad_event_count}') + + return good_event_count, bad_event_count + + def distribution_cut(self, + timestamp: int, # pylint: disable=unused-argument + window: int, + slo_config: dict) -> Tuple[int, int]: + """Query one timeseries of type 'exponential'. + + Args: + timestamp (int): UNIX timestamp. + window (int): Window size (in seconds). + slo_config (dict): SLO configuration. + + Returns: + tuple: A tuple (good_event_count, bad_event_count). + """ + measurement: dict = slo_config['spec']['service_level_indicator'] + filter_valid: str = measurement['filter_valid'] + threshold_bucket: int = int(measurement['threshold_bucket']) + good_below_threshold: typing.Optional[bool] = measurement.get( + 'good_below_threshold', + True) + + # Query 'valid' events + series = self.query(query=filter_valid, window=window) + + if not series: + return NO_DATA, NO_DATA # no timeseries + + distribution_value: Distribution = series[0].point_data[0].values[ + 0].distribution_value + bucket_counts: list = distribution_value.bucket_counts + valid_events_count: int = distribution_value.count + + # Explicit the exponential distribution result + count_sum: int = 0 + distribution = OrderedDict() + for i, bucket_count in enumerate(bucket_counts): + count_sum += bucket_count + distribution[i] = { + 'count_sum': count_sum + } + LOGGER.debug(pprint.pformat(distribution)) + + if len(distribution) - 1 < threshold_bucket: + # maximum measured metric is below the cut after bucket number + lower_events_count: int = valid_events_count + upper_events_count: int = 0 + else: + lower_events_count: int = distribution[threshold_bucket][ + 'count_sum'] + upper_events_count: int = valid_events_count - lower_events_count + + if good_below_threshold: + good_event_count: int = lower_events_count + bad_event_count: int = upper_events_count + else: + good_event_count: int = upper_events_count + bad_event_count: int = lower_events_count + + return good_event_count, bad_event_count + + def exponential_distribution_cut(self, *args, **kwargs) -> Tuple[int, int]: + """Alias for `distribution_cut` method to allow for backwards + compatibility. + """ + warnings.warn( + 'exponential_distribution_cut will be deprecated in version 2.0, ' + 'please use distribution_cut instead', FutureWarning) + return self.distribution_cut(*args, **kwargs) + + def query_sli(self, + timestamp: int, # pylint: disable=unused-argument + window: int, slo_config: dict) -> float: + """Query SLI value from a given MQL query. + + Args: + timestamp (int): UNIX timestamp. + window (int): Window (in seconds). + slo_config (dict): SLO configuration. + + Returns: + float: SLI value. + """ + measurement: dict = slo_config['spec']['service_level_indicator'] + query: str = measurement['query'] + series: List[TimeSeriesData] = self.query(query=query, window=window) + sli_value: float = series[0].point_data[0].values[0].double_value + LOGGER.debug(f"SLI value: {sli_value}") + return sli_value + + def query(self, query: str, window: int) -> List[TimeSeriesData]: + """Query timeseries from Cloud Monitoring using MQL. + + Args: + query (str): MQL query. + window (int): Window size (in seconds). + + Returns: + list: List of timeseries objects. + """ + # Enrich query to aggregate and reduce the time series over the + # desired window. + formatted_query: str = self._fmt_query(query, window) + request = metric_service.QueryTimeSeriesRequest({ + 'name': self.parent, + 'query': formatted_query + }) + timeseries_pager: QueryTimeSeriesPager = self.client.query_time_series( + request) + timeseries: list = list(timeseries_pager) # convert pager to flat list + LOGGER.debug(pprint.pformat(timeseries)) + return timeseries + + @staticmethod + def count(timeseries: List[TimeSeriesData]) -> int: + """Count events in time series assuming it was aligned with ALIGN_SUM + and reduced with REDUCE_SUM (default). + + Args: + :obj:`monitoring_v3.TimeSeries`: Timeseries object. + + Returns: + int: Event count. + """ + try: + return timeseries[0].point_data[0].values[0].int64_value + except (IndexError, AttributeError) as exception: + LOGGER.debug(exception, exc_info=True) + return NO_DATA # no events in timeseries + + @staticmethod + def _fmt_query(query: str, window: int) -> str: + """Format MQL query: + + * If the MQL expression has a `window` placeholder, replace it by the + current window. Otherwise, append it to the expression. + + * If the MQL expression has a `every` placeholder, replace it by the + current window. Otherwise, append it to the expression. + + * If the MQL expression has a `group_by` placeholder, replace it. + Otherwise, append it to the expression. + + Args: + query (str): Original query in YAMLconfig. + window (int): Query window (in seconds). + + Returns: + str: Formatted query. + """ + formatted_query: str = query.strip() + if 'group_by' in formatted_query: + formatted_query = re.sub(r'\|\s+group_by\s+\[.*\]\s*', + '| group_by [] ', formatted_query) + else: + formatted_query += '| group_by [] ' + for mql_time_interval_keyword in ['within', 'every']: + if mql_time_interval_keyword in formatted_query: + formatted_query = re.sub( + fr'\|\s+{mql_time_interval_keyword}\s+\w+\s*', + f'| {mql_time_interval_keyword} {window}s ', + formatted_query) + else: + formatted_query += f'| {mql_time_interval_keyword} {window}s ' + return formatted_query.strip() + + +CM = CloudMonitoringMqlBackend diff --git a/slo_generator/constants.py b/slo_generator/constants.py index 341592b3..46a75b70 100644 --- a/slo_generator/constants.py +++ b/slo_generator/constants.py @@ -16,27 +16,28 @@ Constants and environment variables used in `slo-generator`. """ import os +from typing import Tuple, List, Dict # Compute -NO_DATA = -1 -MIN_VALID_EVENTS = int(os.environ.get("MIN_VALID_EVENTS", "1")) +NO_DATA: int = -1 +MIN_VALID_EVENTS: int = int(os.environ.get("MIN_VALID_EVENTS", "1")) # Global -LATEST_MAJOR_VERSION = 'v2' -COLORED_OUTPUT = int(os.environ.get("COLORED_OUTPUT", "0")) -DRY_RUN = bool(int(os.environ.get("DRY_RUN", "0"))) -DEBUG = int(os.environ.get("DEBUG", "0")) +LATEST_MAJOR_VERSION: str = 'v2' +COLORED_OUTPUT: int = int(os.environ.get("COLORED_OUTPUT", "0")) +DRY_RUN: bool = bool(int(os.environ.get("DRY_RUN", "0"))) +DEBUG: int = int(os.environ.get("DEBUG", "0")) # Exporters supporting v2 SLO report format -V2_EXPORTERS = ('Pubsub', 'Cloudevent') +V2_EXPORTERS: Tuple[str, ...] = ('Pubsub', 'Cloudevent') # Config skeletons -CONFIG_SCHEMA = { +CONFIG_SCHEMA: dict = { 'backends': {}, 'exporters': {}, 'error_budget_policies': {}, } -SLO_CONFIG_SCHEMA = { +SLO_CONFIG_SCHEMA: dict = { 'apiVersion': '', 'kind': '', 'metadata': {}, @@ -51,7 +52,7 @@ # Providers that have changed with v2 YAML config format. This mapping helps # migrate them to their updated names. -PROVIDERS_COMPAT = { +PROVIDERS_COMPAT: Dict[str, str] = { 'Stackdriver': 'CloudMonitoring', 'StackdriverServiceMonitoring': 'CloudServiceMonitoring' } @@ -59,7 +60,7 @@ # Fields that have changed name with v2 YAML config format. This mapping helps # migrate them back to their former name, so that exporters are backward- # compatible with v1. -METRIC_LABELS_COMPAT = { +METRIC_LABELS_COMPAT: Dict[str, str] = { 'goal': 'slo_target', 'description': 'slo_description', 'error_budget_burn_rate_threshold': 'alerting_burn_rate_threshold' @@ -67,8 +68,12 @@ # Fields that used to be specified in top-level of YAML config are now specified # in metadata fields. This mapping helps migrate them back to the top level when -# exporting reports, so that so that exporters are backward-compatible with v1. -METRIC_METADATA_LABELS_TOP_COMPAT = ['service_name', 'feature_name', 'slo_name'] +# exporting reports, so that exporters are backward-compatible with v1. +METRIC_METADATA_LABELS_TOP_COMPAT: List[str] = [ + 'service_name', + 'feature_name', + 'slo_name' +] # Colors / Status diff --git a/slo_generator/exporters/cloud_monitoring.py b/slo_generator/exporters/cloud_monitoring.py index de4285ad..17f54fa6 100644 --- a/slo_generator/exporters/cloud_monitoring.py +++ b/slo_generator/exporters/cloud_monitoring.py @@ -18,6 +18,7 @@ import logging import google.api_core.exceptions +from google.api import metric_pb2 as ga_metric from google.cloud import monitoring_v3 from .base import MetricsExporter @@ -39,7 +40,6 @@ def export_metric(self, data): Args: data (dict): Data to send to Cloud Monitoring. - project_id (str): Cloud Monitoring project id. Returns: object: Cloud Monitoring API result. @@ -57,33 +57,44 @@ def create_timeseries(self, data): Returns: object: Metric descriptor. """ - labels = data['labels'] - series = monitoring_v3.types.TimeSeries() + series = monitoring_v3.TimeSeries() series.metric.type = data['name'] - for key, value in labels.items(): - series.metric.labels[key] = value series.resource.type = 'global' - - # Create a new data point. - point = series.points.add() + labels = data['labels'] + for key, value in labels.items(): + series.metric.labels[key] = value # pylint: disable=E1101 # Define end point timestamp. timestamp = data['timestamp'] - point.interval.end_time.seconds = int(timestamp) - point.interval.end_time.nanos = int( - (timestamp - point.interval.end_time.seconds) * 10**9) - - # Set the metric value. - point.value.double_value = data['value'] + seconds = int(timestamp) + nanos = int((timestamp - seconds) * 10 ** 9) + interval = monitoring_v3.TimeInterval({ + "end_time": { + "seconds": seconds, + "nanos": nanos + } + }) + + # Create a new data point and set the metric value. + point = monitoring_v3.Point({ + "interval": interval, + "value": { + "double_value": data['value'] + } + }) + series.points = [point] # Record the timeseries to Cloud Monitoring. - project = self.client.project_path(data['project_id']) - self.client.create_time_series(project, [series]) + project = self.client.common_project_path(data['project_id']) + self.client.create_time_series(name=project, time_series=[series]) + # pylint: disable=E1101 labels = series.metric.labels LOGGER.debug( - f"timestamp: {timestamp} value: {point.value.double_value}" + f"timestamp: {timestamp}" + f"value: {point.value.double_value}" f"{labels['service_name']}-{labels['feature_name']}-" f"{labels['slo_name']}-{labels['error_budget_policy_step_name']}") + # pylint: enable=E1101 def get_metric_descriptor(self, data): """Get Cloud Monitoring metric descriptor. @@ -94,10 +105,12 @@ def get_metric_descriptor(self, data): Returns: object: Metric descriptor (or None if not found). """ - descriptor = self.client.metric_descriptor_path(data['project_id'], - data['name']) + project_id = data['project_id'] + metric_id = data['name'] + request = monitoring_v3.GetMetricDescriptorRequest( + name=f"projects/{project_id}/metricDescriptors/{metric_id}") try: - return self.client.get_metric_descriptor(descriptor) + return self.client.get_metric_descriptor(request) except google.api_core.exceptions.NotFound: return None @@ -110,13 +123,14 @@ def create_metric_descriptor(self, data): Returns: object: Metric descriptor. """ - project = self.client.project_path(data['project_id']) - descriptor = monitoring_v3.types.MetricDescriptor() + project = self.client.common_project_path(data['project_id']) + descriptor = ga_metric.MetricDescriptor() descriptor.type = data['name'] - descriptor.metric_kind = ( - monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE) - descriptor.value_type = ( - monitoring_v3.enums.MetricDescriptor.ValueType.DOUBLE) + # pylint: disable=E1101 + descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE + descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE + # pylint: enable=E1101 descriptor.description = data['description'] - self.client.create_metric_descriptor(project, descriptor) + descriptor = self.client.create_metric_descriptor( + name=project, metric_descriptor=descriptor) return descriptor diff --git a/tests/unit/backends/test_cloud_monitoring_mql.py b/tests/unit/backends/test_cloud_monitoring_mql.py new file mode 100644 index 00000000..b215ef17 --- /dev/null +++ b/tests/unit/backends/test_cloud_monitoring_mql.py @@ -0,0 +1,68 @@ +# Copyright 2022 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# flake8: noqa + +import unittest + +from slo_generator.backends.cloud_monitoring_mql import \ + CloudMonitoringMqlBackend + + +class TestCloudMonitoringMqlBackend(unittest.TestCase): + + def test_fmt_query(self): + # pylint: disable=trailing-whitespace + queries = [ + ''' fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | filter + metric.response_code == 429 + || metric.response_code == 200 + | group_by [metric.response_code] | within 1h ''', + + ''' fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | filter + metric.response_code == 429 + || metric.response_code == 200 + | group_by [metric.response_code, response_code_class] + | within 1h + | every 1h ''', + + ''' fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | filter + metric.response_code == 429 + || metric.response_code == 200 + | group_by [metric.response_code,response_code_class] + | within 1h + | every 1h ''', + ] + # pylint: enable=trailing-whitespace + + formatted_query = '''fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | filter + metric.response_code == 429 + || metric.response_code == 200 + | group_by [] | within 3600s | every 3600s''' + + for query in queries: + assert CloudMonitoringMqlBackend._fmt_query(query, + 3600) == formatted_query diff --git a/tests/unit/test_stubs.py b/tests/unit/test_stubs.py index 77e74953..cad83efc 100644 --- a/tests/unit/test_stubs.py +++ b/tests/unit/test_stubs.py @@ -21,7 +21,7 @@ import time from types import ModuleType -from google.cloud.monitoring_v3.proto import metric_service_pb2 +from google.cloud import monitoring_v3 from slo_generator.utils import load_configs, load_config TEST_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -164,11 +164,11 @@ def mock_sd(nresp=1): Returns: ChannelStub: Mocked gRPC channel stub. """ - timeserie = load_fixture('time_series_proto.json') - response = {"next_page_token": "", "time_series": [timeserie]} + timeseries = load_fixture('time_series_proto.json') + response = {"next_page_token": "", "time_series": [timeseries]} return mock_grpc_stub( response=response, - proto_method=metric_service_pb2.ListTimeSeriesResponse, + proto_method=monitoring_v3.types.ListTimeSeriesResponse, nresp=nresp) From b045eb95f0418b00e6fefa12abb3dcb41556396f Mon Sep 17 00:00:00 2001 From: Pradeep Mishra Date: Wed, 19 Oct 2022 16:58:21 +0200 Subject: [PATCH 086/107] fix: support custom exporters (#235) Custom exporters are failing to load due to `capitalize`. This PR propagates the `get_backend()` logic to `get_exporters()`. --- slo_generator/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/slo_generator/utils.py b/slo_generator/utils.py index d221e702..c896d8db 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -230,8 +230,11 @@ def get_exporters(config, spec): continue exporter_data = all_exporters[exporter] exporter_data['name'] = exporter - exporter_data['class'] = capitalize( - snake_to_caml(exporter.split('/')[0])) + if '.' in exporter: # support custom exporter + exporter_data['class'] = exporter + else: # core exporter + exporter_data['class'] = capitalize( + snake_to_caml(exporter.split('/')[0])) exporters.append(exporter_data) return exporters From 874844570b3e030d7a6fef095498d7ac781b1ed0 Mon Sep 17 00:00:00 2001 From: Yoshi Yamaguchi Date: Fri, 21 Oct 2022 19:42:34 +0900 Subject: [PATCH 087/107] docs: document how to write and configure filters in Cloud Monitoring provider (#266) --- docs/providers/cloud_monitoring.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/providers/cloud_monitoring.md b/docs/providers/cloud_monitoring.md index 26054ad2..d01b7aef 100644 --- a/docs/providers/cloud_monitoring.md +++ b/docs/providers/cloud_monitoring.md @@ -2,7 +2,7 @@ ## Backend -Using the `cloud_monitoring` backend class, you can query any metrics available +Using the `cloud_monitoring` backend class, you can query any metrics available in `Cloud Monitoring` to create an SLO. ```yaml @@ -17,13 +17,14 @@ backend: * `good_bad_ratio` for metrics of type `DELTA`, `GAUGE`, or `CUMULATIVE`. * `distribution_cut` for metrics of type `DELTA` and unit `DISTRIBUTION`. +The syntax of the filters for SLI definition follows [Cloud Monitoring v3 APIs' definitions](https://cloud.google.com/monitoring/api/v3/filters). ### Good / bad ratio The `good_bad_ratio` method is used to compute the ratio between two metrics: -- **Good events**, i.e events we consider as 'good' from the user perspective. -- **Bad or valid events**, i.e events we consider either as 'bad' from the user +* **Good events**, i.e events we consider as 'good' from the user perspective. +* **Bad or valid events**, i.e events we consider either as 'bad' from the user perspective, or all events we consider as 'valid' for the computation of the SLO. @@ -31,6 +32,7 @@ This method is often used for availability SLOs, but can be used for other purposes as well (see examples). **SLO config blob:** + ```yaml backend: cloud_monitoring method: good_bad_ratio @@ -52,7 +54,7 @@ the `filter_valid` field which identifies all valid events. ### Distribution cut -The `distribution_cut` method is used for Cloud Monitoring distribution-type +The `distribution_cut` method is used for Cloud Monitoring distribution-type metrics, which are usually used for latency metrics. A distribution metric records the **statistical distribution of the extracted @@ -62,12 +64,14 @@ along with the `count`, `mean`, and `sum` of squared deviation of the values. In Cloud Monitoring, there are three different ways to specify bucket boundaries: + * **Linear:** Every bucket has the same width. * **Exponential:** Bucket widths increases for higher values, using an exponential growth factor. * **Explicit:** Bucket boundaries are set for each bucket using a bounds array. **SLO config blob:** + ```yaml backend: cloud_monitoring method: exponential_distribution_cut @@ -80,6 +84,7 @@ service_level_indicator: good_below_threshold: true threshold_bucket: 19 ``` + **→ [Full SLO config](../../samples/cloud_monitoring/slo_gae_app_latency.yaml)** The `threshold_bucket` number to reach our 724ms target latency will depend on @@ -96,7 +101,8 @@ backends: ``` Optional fields: - * `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). + +* `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). **→ [Full SLO config](../../samples/cloud_monitoring/slo_lb_request_availability.yaml)** @@ -130,7 +136,7 @@ Consider the following error budget policy step config: message_ok: Last hour on track ``` -Using Cloud Monitoring UI, let's set up an alert when our error budget burn rate +Using Cloud Monitoring UI, let's set up an alert when our error budget burn rate is burning **9X faster** than it should in the last hour: * Open `Cloud Monitoring` and click on `Alerting > Create Policy` From 75b4d7109fea27203d46550d9e0584d7264fd2de Mon Sep 17 00:00:00 2001 From: Faissal Wahabali Date: Sun, 23 Oct 2022 08:14:45 +0100 Subject: [PATCH 088/107] test: add type hints and `mypy` linting (#239) * add type hints * add more type hints * fix lint * cleanup * integrate mypy test to tests workflow * fix types test workflow * disable more pylint options * add new line to mypy.ini * disable warnings locally * change Pylint annotation position * update .pylintrc * fix misc errors * feat: add pytype linting (#249) * add pytype linting and fix all reported errors * revert metaclass fix to make tests pass * refactor Makefile * fix mypy errors * more specific types in prometheus.py * remove Optional type on arguments with default * update test workflow * fix lint * fix unsubscriptable-object errors * refactor variables declarations and some typings * add new line on .gitignore file * remove clean from mypy make cmd * keep line before cmd * bring back Optional type * bring back Optional type on utils.py * reverse last commit * add Optional type to args in utils.py * wrong arg type * fix annotation-type-mismatch in compute.py * fix attribute-error on main.py * fix line too long * fix indentation * fix return types * add Optional type * fix already defined variables * fix None has no attribute * fix line too long * update Makefile mypy script * reverse last changes * reverse cloud_monitoring_mql init refactoring * ignore lint duplicate code warnings * skip pytype attribute-error error to fix later * ignore attr-defined type errors * ignore union-attr type errors * ignore fixme pylint warning * add missing types dependencies Co-authored-by: Laurent Vaylet --- .gitignore | 1 + .pylintrc | 2 +- Makefile | 5 +- mypy.ini | 2 + setup.py | 5 +- slo_generator/api/main.py | 9 ++ slo_generator/backends/cloud_monitoring.py | 2 + .../backends/cloud_monitoring_mql.py | 37 +++++--- .../backends/cloud_service_monitoring.py | 84 +++++++++++-------- slo_generator/backends/elasticsearch.py | 2 +- slo_generator/backends/prometheus.py | 24 +++++- slo_generator/compute.py | 16 ++-- slo_generator/constants.py | 32 +++---- slo_generator/exporters/bigquery.py | 2 +- slo_generator/exporters/cloud_monitoring.py | 8 +- slo_generator/exporters/cloudevent.py | 1 - slo_generator/exporters/prometheus_self.py | 4 +- slo_generator/exporters/pubsub.py | 2 +- slo_generator/migrations/migrator.py | 53 ++++++------ slo_generator/report.py | 36 ++++---- slo_generator/utils.py | 58 +++++++------ 21 files changed, 231 insertions(+), 154 deletions(-) create mode 100644 mypy.ini diff --git a/.gitignore b/.gitignore index 5bb99d99..292b7198 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ htmlcov/ venv*/ .venv*/ reports/ +.mypy_cache diff --git a/.pylintrc b/.pylintrc index 0ff24061..a1b47771 100644 --- a/.pylintrc +++ b/.pylintrc @@ -66,7 +66,7 @@ disable=raw-checker-failed, redefined-builtin, arguments-differ, dangerous-default-value, - logging-fstring-interpolation + logging-fstring-interpolation, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/Makefile b/Makefile index c1c15e3a..c870f876 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ unit: clean coverage: $(COVERAGE) report --rcfile=".coveragerc" -lint: flake8 pylint pytype +lint: flake8 pylint pytype mypy flake8: flake8 --ignore=$(FLAKE8_IGNORE) $(NAME)/ --max-line-length=80 @@ -85,6 +85,9 @@ pylint: pytype: pytype +mypy: + mypy --show-error-codes $(NAME) + integration: int_cm int_csm int_custom int_dd int_dt int_es int_prom int_cm: diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..976ba029 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/setup.py b/setup.py index 5f512569..e54c497c 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,10 @@ 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], 'elasticsearch': ['elasticsearch'], 'cloudevent': ['cloudevents'], - 'dev': ['wheel', 'flake8', 'mock', 'pytest', 'pytest-cov', 'pylint', 'pytype'] + 'dev': [ + 'wheel', 'flake8', 'mock', 'pytest', 'pytest-cov', 'pylint', 'pytype', 'mypy', + 'types-PyYAML', 'types-python-dateutil', 'types-setuptools', 'types-requests', 'types-protobuf' + ] } # Get the long description from the README file diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py index cfe0798b..228f0c04 100644 --- a/slo_generator/api/main.py +++ b/slo_generator/api/main.py @@ -17,6 +17,8 @@ See https://github.com/GoogleCloudPlatform/functions-framework-python for details on the Functions Framework. """ +# pylint: disable=fixme + import base64 import os import json @@ -98,7 +100,10 @@ def run_export(request): # Construct exporters block spec = {} + # pytype: disable=attribute-error + # FIXME `load_config()` returns `Optional[dict]` so `config` can be `None` default_exporters = config.get('default_exporters', []) + # pytype: enable=attribute-error cli_exporters = os.environ.get('EXPORTERS', None) if cli_exporters: cli_exporters = cli_exporters.split(',') @@ -192,7 +197,11 @@ def process_batch_req(request, data, config): if 'pubsub_batch_handler' in config: LOGGER.info(f'Sending {url} to pubsub batch handler.') from google.cloud import pubsub_v1 # pylint: disable=C0415 + # pytype: disable=attribute-error + # FIXME `load_config()` returns `Optional[dict]` + # so `config` can be `None` exporter_conf = config.get('pubsub_batch_handler') + # pytype: enable=attribute-error client = pubsub_v1.PublisherClient() project_id = exporter_conf['project_id'] topic_name = exporter_conf['topic_name'] diff --git a/slo_generator/backends/cloud_monitoring.py b/slo_generator/backends/cloud_monitoring.py index 2ba77850..27500578 100644 --- a/slo_generator/backends/cloud_monitoring.py +++ b/slo_generator/backends/cloud_monitoring.py @@ -43,6 +43,7 @@ def __init__(self, project_id, client=None): self.client = monitoring_v3.MetricServiceClient() self.parent = self.client.common_project_path(project_id) + # pylint: disable=duplicate-code def good_bad_ratio(self, timestamp, window, slo_config): """Query two timeseries, one containing 'good' events, one containing 'bad' events. @@ -89,6 +90,7 @@ def good_bad_ratio(self, timestamp, window, slo_config): return good_event_count, bad_event_count + # pylint: disable=duplicate-code def distribution_cut(self, timestamp, window, slo_config): """Query one timeseries of type 'exponential'. diff --git a/slo_generator/backends/cloud_monitoring_mql.py b/slo_generator/backends/cloud_monitoring_mql.py index 2cf96bd5..f5bb7938 100644 --- a/slo_generator/backends/cloud_monitoring_mql.py +++ b/slo_generator/backends/cloud_monitoring_mql.py @@ -49,7 +49,11 @@ def __init__(self, project_id: str, client: QueryServiceClient = None): self.client = client if client is None: self.client = QueryServiceClient() - self.parent = self.client.common_project_path(project_id) + self.parent = ( + self.client.common_project_path( # type: ignore[union-attr] + project_id + ) + ) def good_bad_ratio(self, timestamp: int, # pylint: disable=unused-argument @@ -77,14 +81,15 @@ def good_bad_ratio(self, good_event_count: int = CM.count(good_ts) # Query 'bad events' timeseries + bad_event_count: int if filter_bad: bad_ts: List[TimeSeriesData] = self.query( query=filter_bad, window=window) - bad_event_count: int = CM.count(bad_ts) + bad_event_count = CM.count(bad_ts) elif filter_valid: valid_ts: List[TimeSeriesData] = self.query( query=filter_valid, window=window) - bad_event_count: int = CM.count(valid_ts) - good_event_count + bad_event_count = CM.count(valid_ts) - good_event_count else: raise Exception( "One of `filter_bad` or `filter_valid` is required.") @@ -136,21 +141,25 @@ def distribution_cut(self, } LOGGER.debug(pprint.pformat(distribution)) + lower_events_count: int + upper_events_count: int if len(distribution) - 1 < threshold_bucket: # maximum measured metric is below the cut after bucket number - lower_events_count: int = valid_events_count - upper_events_count: int = 0 + lower_events_count = valid_events_count + upper_events_count = 0 else: - lower_events_count: int = distribution[threshold_bucket][ + lower_events_count = distribution[threshold_bucket][ 'count_sum'] - upper_events_count: int = valid_events_count - lower_events_count + upper_events_count = valid_events_count - lower_events_count + good_event_count: int + bad_event_count: int if good_below_threshold: - good_event_count: int = lower_events_count - bad_event_count: int = upper_events_count + good_event_count = lower_events_count + bad_event_count = upper_events_count else: - good_event_count: int = upper_events_count - bad_event_count: int = lower_events_count + good_event_count = upper_events_count + bad_event_count = lower_events_count return good_event_count, bad_event_count @@ -200,8 +209,10 @@ def query(self, query: str, window: int) -> List[TimeSeriesData]: 'name': self.parent, 'query': formatted_query }) - timeseries_pager: QueryTimeSeriesPager = self.client.query_time_series( - request) + + timeseries_pager: QueryTimeSeriesPager = ( + self.client.query_time_series(request) # type: ignore[union-attr] + ) timeseries: list = list(timeseries_pager) # convert pager to flat list LOGGER.debug(pprint.pformat(timeseries)) return timeseries diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 5385d6cc..4668b43f 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -20,7 +20,7 @@ import logging import os import warnings -from typing import Union, Sequence +from typing import Optional, Union, Sequence import google.api_core.exceptions from google.cloud.monitoring_v3 import ServiceMonitoringServiceClient @@ -35,12 +35,12 @@ LOGGER = logging.getLogger(__name__) -SID_GAE = 'gae:{project_id}_{module_id}' -SID_CLOUD_ENDPOINT = 'ist:{project_id}-{service}' -SID_CLUSTER_ISTIO = ( +SID_GAE: str = 'gae:{project_id}_{module_id}' +SID_CLOUD_ENDPOINT: str = 'ist:{project_id}-{service}' +SID_CLUSTER_ISTIO: str = ( 'ist:{project_id}-{suffix}-{location}-{cluster_name}-{service_namespace}-' '{service_name}') -SID_MESH_ISTIO = ('ist:{mesh_uid}-{service_namespace}-{service_name}') +SID_MESH_ISTIO: str = ('ist:{mesh_uid}-{service_namespace}-{service_name}') class CloudServiceMonitoringBackend: @@ -53,7 +53,7 @@ class CloudServiceMonitoringBackend: omitted. """ - def __init__(self, project_id, client=None): + def __init__(self, project_id: str, client=None): self.project_id = project_id self.client = client if client is None: @@ -62,7 +62,10 @@ def __init__(self, project_id, client=None): self.workspace_path = f'workspaces/{project_id}' self.project_path = f'projects/{project_id}' - def good_bad_ratio(self, timestamp, window, slo_config): + def good_bad_ratio(self, + timestamp: int, + window: int, + slo_config: dict) -> tuple: """Good bad ratio method. Args: @@ -71,11 +74,14 @@ def good_bad_ratio(self, timestamp, window, slo_config): slo_config (dict): SLO configuration. Returns: - dict: SLO config. + tuple: SLO config. """ return self.retrieve_slo(timestamp, window, slo_config) - def distribution_cut(self, timestamp, window, slo_config): + def distribution_cut(self, + timestamp: int, + window: int, + slo_config: dict) -> tuple: """Distribution cut method. Args: @@ -84,11 +90,11 @@ def distribution_cut(self, timestamp, window, slo_config): slo_config (dict): SLO configuration. Returns: - dict: SLO config. + tuple: SLO config. """ return self.retrieve_slo(timestamp, window, slo_config) - def basic(self, timestamp, window, slo_config): + def basic(self, timestamp: int, window: int, slo_config: dict) -> tuple: """Basic method (automatic SLOs for GAE / GKE (Istio) and Cloud Endpoints). @@ -98,11 +104,11 @@ def basic(self, timestamp, window, slo_config): slo_config (dict): SLO configuration. Returns: - dict: SLO config. + tuple: SLO config. """ return self.retrieve_slo(timestamp, window, slo_config) - def window(self, timestamp, window, slo_config): + def window(self, timestamp: int, window: int, slo_config: dict) -> tuple: """Window-based SLI method. Args: @@ -111,12 +117,15 @@ def window(self, timestamp, window, slo_config): slo_config (dict): SLO configuration. Returns: - dict: SLO config. + tuple: SLO config. """ return self.retrieve_slo(timestamp, window, slo_config) # pylint: disable=unused-argument - def delete(self, timestamp, window, slo_config): + def delete(self, + timestamp: int, + window: int, + slo_config: dict) -> Optional[dict]: """Delete method. Args: @@ -129,7 +138,7 @@ def delete(self, timestamp, window, slo_config): """ return self.delete_slo(window, slo_config) - def retrieve_slo(self, timestamp, window, slo_config): + def retrieve_slo(self, timestamp: int, window: int, slo_config: dict): """Get SLI value from Cloud Monitoring API. Args: @@ -171,7 +180,7 @@ def retrieve_slo(self, timestamp, window, slo_config): return (good_event_count, bad_event_count) @staticmethod - def count(timeseries): + def count(timeseries: list): """Extract good_count, bad_count tuple from Cloud Monitoring API response. @@ -191,7 +200,8 @@ def count(timeseries): good_event_count = value return good_event_count, bad_event_count - def create_service(self, slo_config): + def create_service(self, + slo_config: dict) -> dict: """Create Service object in Cloud Service Monitoring API. Args: @@ -210,7 +220,7 @@ def create_service(self, slo_config): f'Service Monitoring API.') return SSM.to_json(service) - def get_service(self, slo_config): + def get_service(self, slo_config: dict) -> Optional[dict]: """Get Service object from Cloud Service Monitoring API. Args: @@ -248,7 +258,7 @@ def get_service(self, slo_config): LOGGER.debug(f'Found matching service "{service.name}"') return SSM.to_json(service) - def build_service(self, slo_config): + def build_service(self, slo_config: dict) -> dict: """Build service JSON in Cloud Monitoring API from SLO configuration. @@ -262,7 +272,10 @@ def build_service(self, slo_config): display_name = slo_config.get('service_display_name', service_id) return {'display_name': display_name, 'custom': {}} - def build_service_id(self, slo_config, dest_project_id=None, full=False): + def build_service_id(self, + slo_config: dict, + dest_project_id: Optional[str] = None, + full: bool = False): """Build service id from SLO configuration. Args: @@ -328,7 +341,7 @@ def build_service_id(self, slo_config, dest_project_id=None, full=False): return service_id - def create_slo(self, window, slo_config): + def create_slo(self, window: int, slo_config: dict) -> dict: """Create SLO object in Cloud Service Monitoring API. Args: @@ -345,8 +358,10 @@ def create_slo(self, window, slo_config): parent, slo_json, service_level_objective_id=slo_id) return SSM.to_json(slo) + # pylint: disable=R0912,R0915 @staticmethod - def build_slo(window, slo_config): # pylint: disable=R0912,R0915 + def build_slo(window: int, + slo_config: dict) -> dict: """Get SLO JSON representation in Cloud Service Monitoring API from SLO configuration. @@ -453,11 +468,10 @@ def build_slo(window, slo_config): # pylint: disable=R0912,R0915 raise Exception(f'Method "{method}" is not supported.') return slo - def get_slo(self, window, slo_config): + def get_slo(self, window: int, slo_config: dict) -> Optional[dict]: """Get SLO object from Cloud Service Monssitoring API. Args: - service_id (str): Service identifier. window (int): Window in seconds. slo_config (dict): SLO config. @@ -488,7 +502,7 @@ def get_slo(self, window, slo_config): LOGGER.debug(f'SLO config converted: {slo_json}') return None - def update_slo(self, window, slo_config): + def update_slo(self, window: int, slo_config: dict) -> dict: """Update an existing SLO. Args: @@ -504,16 +518,15 @@ def update_slo(self, window, slo_config): slo_json['name'] = slo_id return SSM.to_json(self.client.update_service_level_objective(slo_json)) - def list_slos(self, service_path): + def list_slos(self, service_path: str) -> list: """List all SLOs from Cloud Service Monitoring API. Args: service_path (str): Service path in the form 'projects/{project_id}/services/{service_id}'. - slo_config (dict): SLO configuration. Returns: - dict: API response. + list: API response. """ slos = self.client.list_service_level_objectives(service_path) slos = list(slos) @@ -521,7 +534,7 @@ def list_slos(self, service_path): # LOGGER.debug(slos) return [SSM.to_json(slo) for slo in slos] - def delete_slo(self, window, slo_config): + def delete_slo(self, window: int, slo_config: dict) -> Optional[dict]: """Delete SLO from Cloud Service Monitoring API. Args: @@ -541,7 +554,10 @@ def delete_slo(self, window, slo_config): f'Skipping.') return None - def build_slo_id(self, window, slo_config, full=False): + def build_slo_id(self, + window: int, + slo_config: dict, + full: bool = False) -> str: """Build SLO id from SLO configuration. Args: @@ -566,7 +582,7 @@ def build_slo_id(self, window, slo_config, full=False): return full_slo_id @staticmethod - def compare_slo(slo1, slo2): + def compare_slo(slo1: dict, slo2: dict) -> bool: """Compares 2 SLO configurations to see if they correspond to the same SLO. @@ -596,7 +612,7 @@ def compare_slo(slo1, slo2): @staticmethod def string_diff(string1: Union[str, Sequence[str]], - string2: Union[str, Sequence[str]]): + string2: Union[str, Sequence[str]]) -> list: """Diff 2 strings. Used to print comparison of JSONs for debugging. Args: @@ -620,7 +636,7 @@ def string_diff(string1: Union[str, Sequence[str]], return lines @staticmethod - def convert_slo_to_ssm_format(slo): + def convert_slo_to_ssm_format(slo: dict) -> dict: """Convert SLO JSON to Cloud Service Monitoring API format. Address edge cases, like `duration` object computation. diff --git a/slo_generator/backends/elasticsearch.py b/slo_generator/backends/elasticsearch.py index 36c6142c..a16371e7 100644 --- a/slo_generator/backends/elasticsearch.py +++ b/slo_generator/backends/elasticsearch.py @@ -25,7 +25,7 @@ LOGGER = logging.getLogger(__name__) -DEFAULT_DATE_FIELD = '@timestamp' +DEFAULT_DATE_FIELD: str = '@timestamp' class ElasticsearchBackend: diff --git a/slo_generator/backends/prometheus.py b/slo_generator/backends/prometheus.py index f69a755f..0bdd24e9 100644 --- a/slo_generator/backends/prometheus.py +++ b/slo_generator/backends/prometheus.py @@ -20,6 +20,7 @@ import logging import os import pprint +from typing import Dict, List, Optional, Tuple from prometheus_http_client import Prometheus @@ -97,7 +98,12 @@ def good_bad_ratio(self, timestamp, window, slo_config): return (good_count, bad_count) # pylint: disable=unused-argument - def distribution_cut(self, timestamp, window, slo_config): + def distribution_cut( + self, + timestamp: int, + window: int, + slo_config: dict + ) -> Tuple[float, float]: """Query events for distributions (histograms). Args: @@ -132,7 +138,12 @@ def distribution_cut(self, timestamp, window, slo_config): return (good_count, bad_count) # pylint: disable=unused-argument - def query(self, filter, window, timestamp=None, operators=[], labels={}): + def query(self, + filter: str, + window: int, + timestamp: Optional[int] = None, + operators: list = [], + labels: dict = {}) -> dict: """Query Prometheus server. Args: @@ -153,7 +164,7 @@ def query(self, filter, window, timestamp=None, operators=[], labels={}): return response @staticmethod - def count(response): + def count(response: dict) -> float: """Count events in Prometheus response. Args: response (dict): Prometheus query response. @@ -170,7 +181,12 @@ def count(response): return NO_DATA # no events in timeseries @staticmethod - def _fmt_query(query, window, operators=[], labels={}): + def _fmt_query( + query: str, + window: int, + operators: List[str] = [], + labels: Dict[str, str] = {} + ) -> str: """Format Prometheus query: * If the PromQL expression has a `window` placeholder, replace it by the diff --git a/slo_generator/compute.py b/slo_generator/compute.py index fa31103b..af48a085 100644 --- a/slo_generator/compute.py +++ b/slo_generator/compute.py @@ -20,6 +20,8 @@ import pprint import time +from typing import Optional + from slo_generator import constants from slo_generator import utils from slo_generator.report import SLOReport @@ -28,12 +30,12 @@ LOGGER = logging.getLogger(__name__) -def compute(slo_config, - config, - timestamp=None, +def compute(slo_config: dict, + config: dict, + timestamp: Optional[float] = None, client=None, - do_export=False, - delete=False): + do_export: bool = False, + delete: bool = False): """Run pipeline to compute SLO, Error Budget and Burn Rate, and export the results (if exporters are specified in the SLO config). @@ -93,7 +95,9 @@ def compute(slo_config, return reports -def export(data, exporters, raise_on_error=False): +def export(data: dict, + exporters: list, + raise_on_error: bool = False) -> list: """Export data using selected exporters. Args: diff --git a/slo_generator/constants.py b/slo_generator/constants.py index 46a75b70..fe4a7f45 100644 --- a/slo_generator/constants.py +++ b/slo_generator/constants.py @@ -80,21 +80,21 @@ # pylint: disable=too-few-public-methods class Colors: """Colors for console output.""" - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' + HEADER: str = '\033[95m' + OKBLUE: str = '\033[94m' + OKGREEN: str = '\033[92m' + WARNING: str = '\033[93m' + FAIL: str = '\033[91m' + ENDC: str = '\033[0m' + BOLD: str = '\033[1m' + UNDERLINE: str = '\033[4m' -GREEN = Colors.OKGREEN -RED = Colors.FAIL -ENDC = Colors.ENDC -BOLD = Colors.BOLD -WARNING = Colors.WARNING -FAIL = '❌' -SUCCESS = '✅' -RIGHT_ARROW = '➞' +GREEN: str = Colors.OKGREEN +RED: str = Colors.FAIL +ENDC: str = Colors.ENDC +BOLD: str = Colors.BOLD +WARNING: str = Colors.WARNING +FAIL: str = '❌' +SUCCESS: str = '✅' +RIGHT_ARROW: str = '➞' diff --git a/slo_generator/exporters/bigquery.py b/slo_generator/exporters/bigquery.py index cea6d3c4..283930b1 100644 --- a/slo_generator/exporters/bigquery.py +++ b/slo_generator/exporters/bigquery.py @@ -21,7 +21,7 @@ import pprint import google.api_core -from google.cloud import bigquery +from google.cloud import bigquery # type: ignore[attr-defined] from slo_generator import constants diff --git a/slo_generator/exporters/cloud_monitoring.py b/slo_generator/exporters/cloud_monitoring.py index 17f54fa6..ac3f012e 100644 --- a/slo_generator/exporters/cloud_monitoring.py +++ b/slo_generator/exporters/cloud_monitoring.py @@ -34,7 +34,7 @@ class CloudMonitoringExporter(MetricsExporter): def __init__(self): self.client = monitoring_v3.MetricServiceClient() - def export_metric(self, data): + def export_metric(self, data: dict): """Export metric to Cloud Monitoring. Create metric descriptor if it doesn't exist. @@ -48,7 +48,7 @@ def export_metric(self, data): self.create_metric_descriptor(data) self.create_timeseries(data) - def create_timeseries(self, data): + def create_timeseries(self, data: dict): """Create Cloud Monitoring timeseries. Args: @@ -96,7 +96,7 @@ def create_timeseries(self, data): f"{labels['slo_name']}-{labels['error_budget_policy_step_name']}") # pylint: enable=E1101 - def get_metric_descriptor(self, data): + def get_metric_descriptor(self, data: dict): """Get Cloud Monitoring metric descriptor. Args: @@ -114,7 +114,7 @@ def get_metric_descriptor(self, data): except google.api_core.exceptions.NotFound: return None - def create_metric_descriptor(self, data): + def create_metric_descriptor(self, data: dict): """Create Cloud Monitoring metric descriptor. Args: diff --git a/slo_generator/exporters/cloudevent.py b/slo_generator/exporters/cloudevent.py index 70313459..6b4618bb 100644 --- a/slo_generator/exporters/cloudevent.py +++ b/slo_generator/exporters/cloudevent.py @@ -25,7 +25,6 @@ LOGGER = logging.getLogger(__name__) - # pylint: disable=too-few-public-methods class CloudeventExporter: """Cloudevent exporter class. diff --git a/slo_generator/exporters/prometheus_self.py b/slo_generator/exporters/prometheus_self.py index 74830000..c62a95c1 100644 --- a/slo_generator/exporters/prometheus_self.py +++ b/slo_generator/exporters/prometheus_self.py @@ -27,8 +27,8 @@ class PrometheusSelfExporter(MetricsExporter): """Prometheus exporter class which uses the API mode of itself to export the metrics.""" - REGISTERED_URL = False - REGISTERED_METRICS = {} + REGISTERED_URL: bool = False + REGISTERED_METRICS: dict = {} def __init__(self): if not self.REGISTERED_URL: diff --git a/slo_generator/exporters/pubsub.py b/slo_generator/exporters/pubsub.py index 481241db..120d15c0 100644 --- a/slo_generator/exporters/pubsub.py +++ b/slo_generator/exporters/pubsub.py @@ -18,7 +18,7 @@ import json import logging -from google.cloud import pubsub_v1 +from google.cloud import pubsub_v1 # type: ignore[attr-defined] LOGGER = logging.getLogger(__name__) diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 8fe1a2e1..bc3e73e5 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -37,18 +37,13 @@ SLO_CONFIG_SCHEMA, GREEN, RED, BOLD, WARNING, ENDC, SUCCESS, FAIL, RIGHT_ARROW) -yaml.explicit_start = True -yaml.default_flow_style = None -yaml.preserve_quotes = True - - def do_migrate(source, target, - error_budget_policy_path, - exporters_path, - version, - quiet=False, - verbose=0): + error_budget_policy_path: list, + exporters_path: list, + version: str, + quiet: bool = False, + verbose: int = 0): """Process all SLO configs in folder and generate new SLO configurations. Args: @@ -61,7 +56,7 @@ def do_migrate(source, quiet (bool, optional): If true, do not prompt for user input. verbose (int, optional): Verbose level. """ - curver = 'v1' + curver: str = 'v1' shared_config = CONFIG_SCHEMA cwd = Path.cwd() source = Path(source).resolve() @@ -213,7 +208,7 @@ def do_migrate(source, # 3.3 - Replace `error_budget_policy.yaml` local variable to `config.yaml` -def exporters_v1tov2(exporters_paths, shared_config={}, quiet=False): +def exporters_v1tov2(exporters_paths: list, shared_config: dict = {}, quiet: bool = False) -> list: """Translate exporters to v2 and put into shared config. Args: @@ -248,7 +243,7 @@ def exporters_v1tov2(exporters_paths, shared_config={}, quiet=False): return exp_keys -def ebp_v1tov2(ebp_paths, shared_config={}, quiet=False): +def ebp_v1tov2(ebp_paths: list, shared_config: dict = {}, quiet: bool = False) -> list: """Translate error budget policies to v2 and put into shared config Args: @@ -286,11 +281,11 @@ def ebp_v1tov2(ebp_paths, shared_config={}, quiet=False): return ebp_keys -def slo_config_v1tov2(slo_config, - shared_config={}, - shared_exporters=[], - quiet=False, - verbose=0): +def slo_config_v1tov2(slo_config: dict, + shared_config: dict = {}, + shared_exporters: list = [], + quiet: bool = False, + verbose: int = 0): """Process old SLO config v1 and generate SLO config v2. Args: @@ -375,7 +370,7 @@ def slo_config_v1tov2(slo_config, return dict(slo_config_v2) -def report_v2tov1(report): +def report_v2tov1(report: dict) -> dict: """Convert SLO report from v2 to v1 format, for exporters to be backward-compatible with v1 data format. @@ -385,7 +380,7 @@ def report_v2tov1(report): Returns: dict: Converted SLO report. """ - mapped_report = {} + mapped_report: dict = {} for key, value in report.items(): # If a metadata label is passed, use the metadata label mapping @@ -428,16 +423,16 @@ def report_v2tov1(report): return mapped_report -def get_random_suffix(): +def get_random_suffix() -> str: """Get random suffix for our backends / exporters when configs clash.""" return ''.join(random.choices(string.digits, k=4)) -def add_to_shared_config(new_obj, - shared_config, - section, - key=None, - quiet=False): +def add_to_shared_config(new_obj: dict, + shared_config: dict, + section: str, + key = None, + quiet: bool = False): """Add an object to the shared_config. If the object with the same config already exists in the shared config, @@ -508,7 +503,7 @@ def add_to_shared_config(new_obj, return key -def detect_config_version(config): +def detect_config_version(config: dict) -> str: """Return version of an slo-generator config based on the format. Args: @@ -522,7 +517,7 @@ def detect_config_version(config): 'Config does not correspond to any known SLO config versions.', fg='red') return None - api_version = config.get('apiVersion', '') + api_version: str = config.get('apiVersion', '') kind = config.get('kind', '') if not kind: # old v1 format return 'v1' @@ -554,7 +549,7 @@ class CustomDumper(yaml.RoundTripDumper): # HACK: insert blank lines between top-level objects # inspired by https://stackoverflow.com/a/44284819/3786245 - def write_line_break(self, data=None): + def write_line_break(self, data: str = None): super().write_line_break(data) if len(self.indents) == 1: diff --git a/slo_generator/report.py b/slo_generator/report.py index e9cde1d3..260848f1 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -42,22 +42,13 @@ class SLOReport: """ # pylint: disable=too-many-instance-attributes - # Metadata - metadata: dict = field(default_factory=dict) - # SLO name: str description: str goal: str backend: str - exporters: list = field(default_factory=list) - error_budget_policy: str = 'default' # SLI - sli_measurement: float = 0 - events_count: int = 0 - bad_events_count: int = 0 - good_events_count: int = 0 gap: float # Error budget @@ -70,6 +61,9 @@ class SLOReport: error_budget_remaining_minutes: float error_minutes: float + # Data validation + valid: bool + # Global (from error budget policy) timestamp: int timestamp_human: str @@ -79,8 +73,20 @@ class SLOReport: consequence_message: str + # SLO + exporters: list = field(default_factory=list) + error_budget_policy: str = 'default' + + # SLI + sli_measurement: float = 0 + events_count: int = 0 + bad_events_count: int = 0 + good_events_count: int = 0 + + # Metadata + metadata: dict = field(default_factory=dict) + # Data validation - valid: bool errors: List[str] = field(default_factory=list) def __init__(self, @@ -265,7 +271,7 @@ def get_sli(self, data): good_count, bad_count = NO_DATA, NO_DATA return sli_measurement, good_count, bad_count - def to_json(self): + def to_json(self) -> dict: """Serialize dataclass to JSON.""" if not self.valid: ebp_name = self.error_budget_policy_step_name @@ -278,7 +284,7 @@ def to_json(self): return asdict(self) # pylint: disable=too-many-return-statements - def _validate(self, data): + def _validate(self, data) -> bool: """Validate backend results. Invalid data will result in SLO report not being built. @@ -360,7 +366,7 @@ def _validate(self, data): return True - def _post_validate(self): + def _post_validate(self) -> bool: """Validate report after build.""" # SLI measurement should be 0 <= x <= 1 @@ -390,11 +396,11 @@ def __set_fields(self, lambdas={}, **kwargs): setattr(self, name, value) @property - def info(self): + def info(self) -> str: """Step information.""" return f"{self.name :<32} | {self.error_budget_policy_step_name :<8}" - def __str__(self): + def __str__(self) -> str: report = self.to_json() if not self.valid: errors_str = ' | '.join(self.errors) diff --git a/slo_generator/utils.py b/slo_generator/utils.py index c896d8db..07789958 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -23,6 +23,7 @@ import pprint import re import sys +from typing import Optional import warnings from collections.abc import Mapping from datetime import datetime @@ -34,7 +35,7 @@ from slo_generator.constants import DEBUG try: - from google.cloud import storage # pytype: disable=import-error + from google.cloud import storage # type: ignore[attr-defined] GCS_ENABLED = True except ImportError: GCS_ENABLED = False @@ -42,7 +43,9 @@ LOGGER = logging.getLogger(__name__) -def load_configs(path, ctx=os.environ, kind=None): +def load_configs(path: str, + ctx: os._Environ = os.environ, + kind: Optional[str] = None) -> list: """Load multiple slo-generator configs from a folder path. Args: @@ -60,7 +63,9 @@ def load_configs(path, ctx=os.environ, kind=None): return [cfg for cfg in configs if cfg] -def load_config(path, ctx=os.environ, kind=None): +def load_config(path: str, + ctx: os._Environ = os.environ, + kind: Optional[str] = None) -> Optional[dict]: """Load any slo-generator config, from a local path, a GCS URL, or directly from a string content. @@ -101,7 +106,9 @@ def load_config(path, ctx=os.environ, kind=None): raise -def parse_config(path=None, content=None, ctx=os.environ): +def parse_config(path: Optional[str] = None, + content=None, + ctx: os._Environ = os.environ): """Load a yaml configuration file and resolve environment variables in it. Args: @@ -115,7 +122,7 @@ def parse_config(path=None, content=None, ctx=os.environ): """ pattern = re.compile(r'.*?\${(\w+)}.*?') - def replace_env_vars(content, ctx): + def replace_env_vars(content, ctx) -> str: """Replace env variables in content from context. Args: @@ -179,7 +186,7 @@ def setup_logging(): pass -def get_human_time(timestamp, timezone=None): +def get_human_time(timestamp: int, timezone: Optional[str] = None) -> str: """Get human-readable timestamp from UNIX UTC timestamp. Args: @@ -210,7 +217,7 @@ def get_human_time(timestamp, timezone=None): return date_str -def get_exporters(config, spec): +def get_exporters(config: dict, spec: dict) -> list: """Get SLO exporters configs from spec and global config. Args: @@ -239,7 +246,7 @@ def get_exporters(config, spec): return exporters -def get_backend(config, spec): +def get_backend(config: dict, spec: dict): """Get SLO backend config from spec and global config. Args: @@ -265,7 +272,7 @@ def get_backend(config, spec): return backend_data -def get_error_budget_policy(config, spec): +def get_error_budget_policy(config: dict, spec: dict): """Get error budget policy from spec and global config. Args: @@ -284,7 +291,7 @@ def get_error_budget_policy(config, spec): return all_ebp[spec_ebp] -def get_backend_cls(backend): +def get_backend_cls(backend: str): """Get backend class. Args: @@ -297,7 +304,7 @@ def get_backend_cls(backend): return import_cls(backend, expected_type) -def get_exporter_cls(exporter): +def get_exporter_cls(exporter: str): """Get exporter class. Args: @@ -335,7 +342,7 @@ def import_cls(cls_name, expected_type): prefix=expected_type) -def import_dynamic(package, name, prefix="class"): +def import_dynamic(package: str, name: str, prefix: str = "class"): """Import class or method dynamically from package and name. Args: @@ -361,7 +368,7 @@ def import_dynamic(package, name, prefix="class"): return None -def capitalize(word): +def capitalize(word: str) -> str: """Only capitalize the first letter of a word, even when written in CamlCase. @@ -374,7 +381,7 @@ def capitalize(word): return re.sub('([a-zA-Z])', lambda x: x.groups()[0].upper(), word, 1) -def snake_to_caml(word): +def snake_to_caml(word: str) -> str: """Convert a string written in snake_case to a string in CamlCase. Args: @@ -386,7 +393,7 @@ def snake_to_caml(word): return re.sub('_.', lambda x: x.group()[1].upper(), word) -def caml_to_snake(word): +def caml_to_snake(word: str) -> str: """Convert a string written in CamlCase to a string written in snake_case. Args: @@ -398,7 +405,7 @@ def caml_to_snake(word): return re.sub(r'(? dict: """Convert dictionary with keys written in snake_case to another one with keys written in CamlCase. @@ -411,7 +418,7 @@ def dict_snake_to_caml(data): return apply_func_dict(data, snake_to_caml) -def apply_func_dict(data, func): +def apply_func_dict(data: dict, func) -> dict: """Apply function on a dictionary keys. Args: @@ -425,7 +432,7 @@ def apply_func_dict(data, func): return data -def str2bool(string): +def str2bool(string: str) -> bool: """Convert a string to a boolean. Args: @@ -446,7 +453,7 @@ def str2bool(string): raise argparse.ArgumentTypeError('Boolean value expected.') -def download_gcs_file(url): +def download_gcs_file(url: str) -> dict: """Download config from GCS. Args: @@ -462,7 +469,7 @@ def download_gcs_file(url): return blob.download_as_string(client=None).decode('utf-8') -def decode_gcs_url(url): +def decode_gcs_url(url: str) -> tuple: """Decode GCS URL. Args: @@ -477,7 +484,7 @@ def decode_gcs_url(url): return (bucket_name, file_path) -def get_files(source, extensions=['yaml', 'yml', 'json']): +def get_files(source, extensions=['yaml', 'yml', 'json']) -> list: """Get all files matching extensions. Args: @@ -486,13 +493,16 @@ def get_files(source, extensions=['yaml', 'yml', 'json']): Returns: list: List of all files matching extensions relative to source folder. """ - all_files = [] + all_files: list = [] for ext in extensions: all_files.extend(Path(source).rglob(f'*.{ext}')) return all_files -def get_target_path(source_dir, target_dir, relative_path, mkdir=True): +def get_target_path(source_dir, + target_dir, + relative_path, + mkdir: bool = True): """Get target file path from a source directory, a target directory and a path relative to the source directory. @@ -515,7 +525,7 @@ def get_target_path(source_dir, target_dir, relative_path, mkdir=True): target_path.parent.mkdir(parents=True, exist_ok=True) return target_path -def fmt_traceback(exc): +def fmt_traceback(exc) -> str: """Format exception to be human-friendly. Args: From fc38839359c61e7c38358854203668de8fbd1350 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Sun, 23 Oct 2022 15:50:28 +0200 Subject: [PATCH 089/107] ci: improve development and CI workflows (#267) * configure setuptools using setup.cfg (PEP 517, PEP 518) * fix TOML syntax thah prevents CI from running * make FIXME messages more explicit * black 22.1.0 depends on click>=8.0.0 * add/configure black and isort * configure pylint in pyproject.toml * remove unused pylint config file * add/configure pre-commit * trim trailing whitespaces * fix linting/formatting globally * install pre-commit hooks in dev mode only * add/configure bandit and safety * add black, isort, bandit and safety to lint target and CI * update versions of GitHub Actions * add `make format` target (with isort then black) * update version of release-please GitHub Action * update contributing doc with new dev workflow and tools + fix/reformat Markdown * add sample output of pre-commit hook * detail how pre-commit hooks work * fix mypy warnings/errors * reformat Markdown * fix discrepancy on Python version and license name * add pytest and pytype temp folders to .gitgnore * configure mypy using pyproject.toml --- .github/workflows/build.yml | 5 +- .github/workflows/deploy.yml | 5 +- .github/workflows/release-please.yml | 2 +- .github/workflows/release.yml | 5 +- .github/workflows/test.yml | 8 +- .gitignore | 2 + .pre-commit-config.yaml | 43 ++ .pylintrc | 492 ------------------ CODE_OF_CONDUCT.md | 87 +--- CONTRIBUTING.md | 82 +-- Makefile | 42 +- README.md | 180 +++---- docs/deploy/cloudbuild.md | 4 +- docs/deploy/cloudrun.md | 8 +- docs/deploy/datastudio_slo_report.md | 4 +- docs/providers/cloudevent.md | 10 +- docs/providers/custom.md | 24 +- docs/providers/dynatrace.md | 8 +- docs/providers/prometheus.md | 10 +- docs/shared/api.md | 68 +-- docs/shared/metrics.md | 22 +- docs/shared/migration.md | 50 +- docs/shared/troubleshooting.md | 10 +- mypy.ini | 2 - pyproject.toml | 31 ++ samples/custom/custom_backend.py | 3 + samples/custom/custom_exporter.py | 13 +- setup.cfg | 119 ++++- setup.py | 75 +-- slo_generator/api/main.py | 148 +++--- slo_generator/backends/cloud_monitoring.py | 144 ++--- .../backends/cloud_monitoring_mql.py | 128 ++--- .../backends/cloud_service_monitoring.py | 353 ++++++------- slo_generator/backends/datadog.py | 83 +-- slo_generator/backends/dynatrace.py | 168 +++--- slo_generator/backends/elasticsearch.py | 33 +- slo_generator/backends/prometheus.py | 90 ++-- slo_generator/cli.py | 269 ++++++---- slo_generator/compute.py | 86 ++- slo_generator/constants.py | 71 +-- slo_generator/exporters/base.py | 125 ++--- slo_generator/exporters/bigquery.py | 347 ++++++------ slo_generator/exporters/cloud_monitoring.py | 58 ++- slo_generator/exporters/cloudevent.py | 33 +- slo_generator/exporters/datadog.py | 30 +- slo_generator/exporters/dynatrace.py | 76 +-- slo_generator/exporters/prometheus.py | 52 +- slo_generator/exporters/prometheus_self.py | 22 +- slo_generator/exporters/pubsub.py | 9 +- slo_generator/migrations/migrator.py | 341 ++++++------ slo_generator/report.py | 180 +++---- slo_generator/utils.py | 175 +++---- tests/unit/__init__.py | 3 +- .../backends/test_cloud_monitoring_mql.py | 31 +- tests/unit/fixtures/dt_slo_get.json | 2 +- tests/unit/fixtures/dummy_backend.py | 12 +- tests/unit/fixtures/fail_exporter.py | 3 +- tests/unit/test_cli.py | 30 +- tests/unit/test_compute.py | 132 ++--- tests/unit/test_migrate.py | 18 +- tests/unit/test_report.py | 3 +- tests/unit/test_stubs.py | 169 +++--- tests/unit/test_utils.py | 45 +- 63 files changed, 2367 insertions(+), 2516 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 .pylintrc delete mode 100644 mypy.ini create mode 100644 pyproject.toml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b303f0ae..51aa4668 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,11 +12,12 @@ jobs: runs-on: ubuntu-latest environment: prod steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: '3.9' architecture: 'x64' + - name: Check release version id: check-tag run: | diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 08ee3d01..dc78f170 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,7 +14,8 @@ jobs: environment: prod concurrency: prod steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 + - name: Check release version id: check-tag run: | @@ -45,7 +46,7 @@ jobs: run: echo job does not exist && true - name: Do something if build fail - if: steps.wait-build.outputs.conclusion == 'failure' || steps.wait-build2.outputs.conclusion == 'failure' + if: steps.wait-build.outputs.conclusion == 'failure' || steps.wait-build2.outputs.conclusion == 'failure' run: echo fail && false # fail if build fail - name: Do something if build timeout diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 1479d0b4..1513b44d 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -9,7 +9,7 @@ jobs: release-pr: runs-on: ubuntu-latest steps: - - uses: GoogleCloudPlatform/release-please-action@v2.4.0 + - uses: GoogleCloudPlatform/release-please-action@v3 with: token: ${{ secrets.RELEASE_PR_TOKEN }} release-type: python diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5ac68a3..c876e8a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,8 @@ jobs: release-pypi: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@master + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: '3.9' architecture: 'x64' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01eb1acb..c70ba39c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,12 +17,12 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} + architecture: ${{ matrix.architecture }} - name: Install dependencies run: make install - - name: Run lint test + - name: Lint run: make lint unit: @@ -43,7 +43,7 @@ jobs: - name: Install dependencies run: make install - - name: Run unittests + - name: Run unit tests run: make unit env: MIN_VALID_EVENTS: "10" @@ -57,7 +57,9 @@ jobs: steps: - uses: actions/checkout@v3 - uses: docker-practice/actions-setup-docker@master + - name: Build Docker image run: make docker_build + - name: Run Docker tests run: make docker_test diff --git a/.gitignore b/.gitignore index 292b7198..b7dd28ef 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ venv*/ .venv*/ reports/ .mypy_cache +.pytest_cache/ +.pytype/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..b3e7945e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +# Usage: +# pip install pre-commit +# pre-commit install +# pre-commit run --all-files +# pre-commit autoupdate +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files +- repo: https://github.com/myint/autoflake + rev: v1.7.6 + hooks: + - id: autoflake + args: + - --in-place + - --remove-unused-variables + - --remove-all-unused-imports +# The configuration of flake8, black, isort and pylint is automatically loaded +# from either pyproject.toml or setup.cfg to remain consistent with the Makefile +# targets. +- repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 +- repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black +- repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort +- repo: https://github.com/PyCQA/pylint + rev: v2.15.4 + hooks: + - id: pylint diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index a1b47771..00000000 --- a/.pylintrc +++ /dev/null @@ -1,492 +0,0 @@ -[MASTER] - -# Add files or directories to the deny list. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the deny list. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - redefined-builtin, - arguments-differ, - dangerous-default-value, - logging-fstring-interpolation, - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, while `new` is for `{}` formatting. -logging-format-style=new - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package.. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local,PublisherClient, - google.cloud.monitoring_v3.types - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=7 - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[STRING] - -# This flag controls whether the implicit-str-concat-in-sequence should -# generate a warning on implicit string concatenation in sequences defined over -# several lines. -check-str-concat-over-line-jumps=no - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=10 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement. -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=30 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=30 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 05e4864e..aee456bb 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,92 +2,55 @@ ## Our Pledge -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of -experience, education, socio-economic status, nationality, personal appearance, -race, religion, or sexual identity and orientation. +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards -Examples of behavior that contributes to creating a positive environment -include: +Examples of behavior that contributes to creating a positive environment include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, or to ban temporarily or permanently any -contributor for other behaviors that they deem inappropriate, threatening, -offensive, or harmful. +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. -This Code of Conduct also applies outside the project spaces when the Project -Steward has a reasonable belief that an individual's behavior may have a -negative impact on the project or its community. +This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community. ## Conflict Resolution -We do not believe that all conflict is bad; healthy debate and disagreement -often yield positive results. However, it is never okay to be disrespectful or -to engage in behavior that violates the project’s code of conduct. +We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project’s code of conduct. -If you see someone violating the code of conduct, you are encouraged to address -the behavior directly with those involved. Many issues can be resolved quickly -and easily, and this gives people more control over the outcome of their -dispute. If you are unable to resolve the matter for any reason, or if the -behavior is threatening or harassing, report it. We are dedicated to providing -an environment where participants feel welcome and safe. +If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe. Reports should be directed to Olivier Cervello (ocervello@google.com) or Bruno Reboul (brunoreboul@google.com), the Project Steward(s) for slo-generator. -It is the Project Steward’s duty to receive and address reported violations of -the code of conduct. They will then work with a committee consisting of -representatives from the Open Source Programs Office and the Google Open Source -Strategy team. If for any reason you are uncomfortable reaching out to the -Project Steward, please email opensource@google.com. + +It is the Project Steward’s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com. We will investigate every complaint, but you may not receive a direct response. -We will use our discretion in determining when and how to follow up on reported -incidents, which may range from not taking action to permanent expulsion from -the project and project-sponsored spaces. We will notify the accused of the -report and provide them an opportunity to discuss it before any action is taken. -The identity of the reporter will be omitted from the details of the report -supplied to the accused. In potentially harmful situations, such as ongoing -harassment or threats to anyone's safety, we may take action without notice. + +We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. + +The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice. ## Attribution -This Code of Conduct is adapted from the Contributor Covenant, version 1.4, -available at -https://www.contributor-covenant.org/version/1/4/code-of-conduct.html +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6722352f..ec27d753 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,61 +1,75 @@ # How to Contribute -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. +We'd love to accept your patches and contributions to this project. There are just a few small guidelines you need to follow. ## Contributor License Agreement -Contributions to this project must be accompanied by a Contributor License -Agreement. You (or your employer) retain the copyright to your contribution; -this simply gives us permission to use and redistribute your contributions as -part of the project. Head over to to see -your current agreements on file or to sign a new one. +Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. Head over to to see your current agreements on file or to sign a new one. -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. +You generally only need to submit a CLA once, so if you've already submitted one (even if it was for a different project), you probably don't need to do it again. ## Code reviews -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. +All submissions, including submissions by project members, require review. We use GitHub Pull Requests (PRs) for this purpose. Consult [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more information on using Pull Requests. ## Community Guidelines -This project follows [Google's Open Source Community -Guidelines](https://opensource.google/conduct/). +This project follows [Google's Open Source Community Guidelines](https://opensource.google/conduct/). ## Contributing guidelines ### Development environment -To prepare for development, you need to fork this repository and work on your -own branch so that you can later submit your changes as a GitHub Pull Request. +To prepare for development, you need to fork this repository and work on your own branch so that you can later submit your changes as a GitHub Pull Request. Once you have forked the repo on GitHub, clone it locally and install the `slo-generator` in a Python virtual environment: -``` + +```sh git clone github.com/google/slo-generator cd slo-generator python3 -m venv venv/ source venv/bin/activate ``` -Install `slo-generator` locally in development mode, so that you can start making changes to it: -``` +Then install `slo-generator` locally in development mode, with all the extra packages as well as pre-commit hooks in order to speed up and simplify the development workflow: + +```sh make develop ``` +Finally, feel free to start making changes. + +Note that [pre-commit](https://pre-commit.com/) hooks are installed during `make develop` and automatically executed by `git`. These checks are responsible for making sure your changes are linted and well-formatted, in order to match the rest of the codebase and simplify the code reviews. You will be warned after issuing `git commit` if at least one of the staged files does not comply. You can also run the checks manually on the staged files at any time with: + +```sh +$ pre-commit run +trim trailing whitespace.................................................Passed +fix end of files.........................................................Passed +check yaml...............................................................Passed +check for added large files..............................................Passed +autoflake................................................................Passed +flake8...................................................................Passed +black....................................................................Passed +isort....................................................................Passed +pylint...................................................................Passed +``` + +If any error is reported, your commit gets canceled. At this point, run `make format` to automatically reformat the code with [`isort`](https://github.com/PyCQA/isort) and [`black`](https://github.com/psf/black). Also make sure to fix any errors returned by linters or static analyzers such as [`flake8`](https://flake8.pycqa.org/en/latest/), [`pylint`](https://pylint.pycqa.org/en/latest/), [`mypy`](http://mypy-lang.org/) or [`pytype`](https://github.com/google/pytype). Then commit again, rinse and repeat. + +Ignoring these pre-commit warnings is not recommended. The Continuous Integration (CI) pipelines will run the exact same checks (and more!) when your commits get pushed. The checks will fail there and prevent you from merging your changes to the `master` branch anyway. So fail fast, fail early, fail often and fix as many errors as possible on your local development machine. Code reviews will be more enjoyable for everyone! + ### Testing environment -Unittests are located in the `tests/unit` folder. + +Unit tests are located in the `tests/unit` folder. To run all the tests, run `make` in the base directory. You can also select which test to run, and do other things: -``` + +```sh make unit # run unit tests only -make flake8 pylint # run linting tests only +make lint # lint code only (with flake8, pylint, mypy and pytype) +make format # format code only (with isort and black) make docker_test # build Docker image and run tests within Docker container make docker_build # build Docker image only make info # see current slo-generator version @@ -63,24 +77,17 @@ make info # see current slo-generator version ### Adding support for a new backend or exporter -The `slo-generator` tool is designed to be modular as it moves forward. -Users, customers and Google folks should be able to easily add the metrics -backend or the exporter of their choosing. +The `slo-generator` tool is designed to be modular as it moves forward. Users, customers and Google folks should be able to easily add the metrics backend or the exporter of their choosing. -**New backend** +#### New backend To add a new backend, one must: -* Add a new file `slo-generator/backends/.py` - +* Add a new file named `slo-generator/backends/.py` * Write a new Python class called `Backend` (CamlCase) - * Test it with a sample config - * Add some unit tests - * Make sure all tests pass - * Submit a PR ***Example with a fake Cat backend:*** @@ -138,6 +145,7 @@ To add a new backend, one must: my_sli_value = self.compute_random_stuff() return my_sli_value ``` + * Write a sample SLO configs (`slo_cat_test_slo_ratio.yaml`): ```yaml @@ -155,21 +163,17 @@ To add a new backend, one must: ``` * Run a live test with the SLO generator: + ```sh slo-generator -f slo_cat_test_slo_ratio.yaml -b samples/error_budget_target.yaml ``` * Create a directory `samples/` for your backend samples. - * Add some YAML samples to show how to write SLO configs for your backend. Samples should be named `slo___.yaml`. - * Add a unit test: in the `tests/unit/test_compute.py`, simply add a method called `test_compute_`. Take the other backends an example when writing the test. - * Add documentation for your backend / exporter in a new file named `docs/providers/cat.md`. - * Make sure all tests pass - * Submit a PR The steps above are similar for adding a new exporter, but the exporter code will go to the `exporters/` directory and the unit test will be named `test_export_`. diff --git a/Makefile b/Makefile index c870f876..59e4178c 100644 --- a/Makefile +++ b/Makefile @@ -13,17 +13,15 @@ ######################################################## # variable section -NAME = "slo_generator" +NAME = slo_generator -PIP=pip3 -PYTHON=python3 -TWINE=twine -COVERAGE=coverage +PIP = pip3 +PYTHON = python3 +TWINE = twine +COVERAGE = coverage SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") -VERSION ?= $(shell grep "version = " setup.py | cut -d\ -f3) - -FLAKE8_IGNORE = E302,E203,E261 +VERSION ?= $(shell grep "version = " setup.cfg | cut -d ' ' -f 3) ######################################################## @@ -56,8 +54,8 @@ deploy: clean install_twine build install_twine: $(PIP) install twine -develop: - $(PIP) install -e . +develop: install + pre-commit install install: clean $(PIP) install -e ."[api, datadog, prometheus, elasticsearch, pubsub, cloud_monitoring, bigquery, dev]" @@ -73,14 +71,24 @@ unit: clean coverage: $(COVERAGE) report --rcfile=".coveragerc" -lint: flake8 pylint pytype mypy +format: + isort . + black . + +lint: black isort flake8 pylint pytype mypy bandit safety + +black: + black . --check + +isort: + isort . --check-only flake8: - flake8 --ignore=$(FLAKE8_IGNORE) $(NAME)/ --max-line-length=80 - flake8 --ignore=$(FLAKE8_IGNORE),E402 tests/ --max-line-length=80 + flake8 $(NAME)/ + flake8 tests/ pylint: - find ./$(NAME) ./tests -name \*.py | xargs pylint --rcfile .pylintrc --ignore-patterns=test_.*?py + find ./$(NAME) ./tests -type f -name "*.py" | xargs pylint pytype: pytype @@ -88,6 +96,12 @@ pytype: mypy: mypy --show-error-codes $(NAME) +bandit: + bandit . + +safety: + safety check + integration: int_cm int_csm int_custom int_dd int_dt int_es int_prom int_cm: diff --git a/README.md b/README.md index 27059be7..361eeacd 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ [![PyPI version](https://badge.fury.io/py/slo-generator.svg)](https://badge.fury.io/py/slo-generator) [![Downloads](https://static.pepy.tech/personalized-badge/slo-generator?period=total&units=international_system&left_color=grey&right_color=orange&left_text=pypi%20downloads)](https://pepy.tech/project/slo-generator) -`slo-generator` is a tool to compute and export **Service Level Objectives** ([SLOs](https://landing.google.com/sre/sre-book/chapters/service-level-objectives/)), -**Error Budgets** and **Burn Rates**, using configurations written in YAML (or JSON) format. +`slo-generator` is a tool to compute and export **Service Level Objectives** ([SLOs](https://landing.google.com/sre/sre-book/chapters/service-level-objectives/)), **Error Budgets** and **Burn Rates**, using configurations written in YAML (or JSON) format. ***IMPORTANT NOTE: the following content is the `slo-generator` v2 documentation. The v1 documentation is available [here](https://github.com/google/slo-generator/tree/v1.5.1), and instructions to migrate to v2 are available [here](https://github.com/google/slo-generator/blob/master/docs/shared/migration.md).*** ## Table of contents + - [Description](#description) - [Local usage](#local-usage) - [Requirements](#requirements) @@ -30,24 +30,22 @@ - [Contribute to the SLO Generator](#contribute-to-the-slo-generator) ## Description -The `slo-generator` runs backend queries computing **Service Level Indicators**, -compares them with the **Service Level Objectives** defined and generates a report -by computing important metrics: -* **Service Level Indicator** (SLI) defined as **SLI = Ngood_events / Nvalid_events** -* **Error Budget** (EB) defined as **EB = 1 - SLI** -* **Error Budget Burn Rate** (EBBR) defined as **EBBR = EB / EBtarget** -* **... and more**, see the [example SLO report](./test/unit/../../tests/unit/fixtures/slo_report_v2.json). +The `slo-generator` runs backend queries computing **Service Level Indicators**, compares them with the **Service Level Objectives** defined and generates a report by computing important metrics: + +- **Service Level Indicator** (SLI) defined as **SLI = Ngood_events / Nvalid_events** +- **Error Budget** (EB) defined as **EB = 1 - SLI** +- **Error Budget Burn Rate** (EBBR) defined as **EBBR = EB / EBtarget** +- **... and more**, see the [example SLO report](./test/unit/../../tests/unit/fixtures/slo_report_v2.json). -The **Error Budget Burn Rate** is often used for [**alerting on SLOs**](https://sre.google/workbook/alerting-on-slos/), as it demonstrates in practice to be more **reliable** and **stable** than -alerting directly on metrics or on **SLI > SLO** thresholds. +The **Error Budget Burn Rate** is often used for [**alerting on SLOs**](https://sre.google/workbook/alerting-on-slos/), as it demonstrates in practice to be more **reliable** and **stable** than alerting directly on metrics or on **SLI > SLO** thresholds. ## Local usage ### Requirements -* `python3.9` and above -* `pip3` +- `python3.7` and above +- `pip3` ### Installation @@ -58,43 +56,46 @@ pip3 install slo-generator ``` ***Notes:*** -* To install **[providers](./docs/providers)**, use `pip3 install slo-generator[, , ... , , ... -c --export ``` -where: - * `` is the [SLO configuration](#slo-configuration) file or folder path. - * `` is the [Shared configuration](#shared-configuration) file path. +where: - * `--export` | `-e` enables exporting data using the `exporters` specified in the SLO +- `` is the [SLO configuration](#slo-configuration) file or folder path. +- `` is the [Shared configuration](#shared-configuration) file path. +- `--export` | `-e` enables exporting data using the `exporters` specified in the SLO configuration file. Use `slo-generator compute --help` to list all available arguments. ### API usage -On top of the CLI, the `slo-generator` can also be run as an API using the Cloud -Functions Framework SDK (Flask) using the `api` subcommand: -``` +On top of the CLI, the `slo-generator` can also be run as an API using the Cloud Functions Framework SDK (Flask) using the `api` subcommand: + +```sh slo-generator api --config ``` + where: - * `` is the [Shared configuration](#shared-configuration) file path or GCS URL. -Once the API is up-and-running, you can make HTTP POST requests with your SLO -configurations (YAML or JSON) in the request body: +- `` is the [Shared configuration](#shared-configuration) file path or GCS URL. -``` +Once the API is up-and-running, you can make HTTP POST requests with your SLO configurations (YAML or JSON) in the request body: + +```sh curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml localhost:8080 # yaml SLO config curl -X POST -H "Content-Type: application/json" -d @slo.json localhost:8080 # json SLO config ``` @@ -103,47 +104,42 @@ To read more about the API and advanced usage, see [docs/shared/api.md](./docs/s ## Configuration -The `slo-generator` requires two configuration files to run, an **SLO configuration** -file, describing your SLO, and the **Shared configuration** file (common -configuration for all SLOs). +The `slo-generator` requires two configuration files to run, an **SLO configuration** file, describing your SLO, and the **Shared configuration** file (common configuration for all SLOs). ### SLO configuration -The **SLO configuration** (JSON or YAML) is following the Kubernetes format and -is composed of the following fields: - -* `api`: `sre.google.com/v2` -* `kind`: `ServiceLevelObjective` -* `metadata`: - * `name`: [**required**] *string* - Full SLO name (**MUST** be unique). - * `labels`: [*optional*] *map* - Metadata labels, **for example**: - * `slo_name`: SLO name (e.g `availability`, `latency128ms`, ...). - * `service_name`: Monitored service (to group SLOs by service). - * `feature_name`: Monitored feature (to group SLOs by feature). - -* `spec`: - * `description`: [**required**] *string* - Description of this SLO. - * `goal`: [**required**] *string* - SLO goal (or target) (**MUST** be between 0 and 1). - * `backend`: [**required**] *string* - Backend name (**MUST** exist in SLO Generator Configuration). - * `method`: [**required**] *string* - Backend method to use (**MUST** exist in backend class definition). - * `service_level_indicator`: [**required**] *map* - SLI configuration. The content of this section is +The **SLO configuration** (JSON or YAML) is following the Kubernetes format and is composed of the following fields: + +- `api`: `sre.google.com/v2` +- `kind`: `ServiceLevelObjective` +- `metadata`: + - `name`: [**required**] *string* - Full SLO name (**MUST** be unique). + - `labels`: [*optional*] *map* - Metadata labels, **for example**: + - `slo_name`: SLO name (e.g `availability`, `latency128ms`, ...). + - `service_name`: Monitored service (to group SLOs by service). + - `feature_name`: Monitored feature (to group SLOs by feature). + +- `spec`: + - `description`: [**required**] *string* - Description of this SLO. + - `goal`: [**required**] *string* - SLO goal (or target) (**MUST** be between 0 and 1). + - `backend`: [**required**] *string* - Backend name (**MUST** exist in SLO Generator Configuration). + - `method`: [**required**] *string* - Backend method to use (**MUST** exist in backend class definition). + - `service_level_indicator`: [**required**] *map* - SLI configuration. The content of this section is specific to each provider, see [`docs/providers`](./docs/providers). - * `error_budget_policy`: [*optional*] *string* - Error budget policy name + - `error_budget_policy`: [*optional*] *string* - Error budget policy name (**MUST** exist in SLO Generator Configuration). If not specified, defaults to `default`. - * `exporters`: [*optional*] *string* - List of exporter names (**MUST** exist in SLO Generator Configuration). + - `exporters`: [*optional*] *string* - List of exporter names (**MUST** exist in SLO Generator Configuration). -***Note:*** *you can use environment variables in your SLO configs by using -`${MY_ENV_VAR}` syntax to avoid having sensitive data in version control. -Environment variables will be replaced automatically at run time.* +***Note:*** *you can use environment variables in your SLO configs by using `${MY_ENV_VAR}` syntax to avoid having sensitive data in version control. Environment variables will be replaced automatically at run time.* **→ See [example SLO configuration](samples/cloud_monitoring/slo_gae_app_availability.yaml).** ### Shared configuration -The shared configuration (JSON or YAML) configures the `slo-generator` and acts -as a shared config for all SLO configs. It is composed of the following fields: -* `backends`: [**required**] *map* - Data backends configurations. Each backend - alias is defined as a key `/`, and a configuration map. +The shared configuration (JSON or YAML) configures the `slo-generator` and acts as a shared config for all SLO configs. It is composed of the following fields: + +- `backends`: [**required**] *map* - Data backends configurations. Each backend alias is defined as a key `/`, and a configuration map. + ```yaml backends: cloud_monitoring/dev: @@ -152,18 +148,18 @@ as a shared config for all SLO configs. It is composed of the following fields: app_key: ${APP_SECRET_KEY} api_key: ${API_SECRET_KEY} ``` + See specific providers documentation for detailed configuration: - * [`cloud_monitoring`](docs/providers/cloud_monitoring.md#backend) - * [`cloud_service_monitoring`](docs/providers/cloud_service_monitoring.md#backend) - * [`prometheus`](docs/providers/prometheus.md#backend) - * [`elasticsearch`](docs/providers/elasticsearch.md#backend) - * [`datadog`](docs/providers/datadog.md#backend) - * [`dynatrace`](docs/providers/dynatrace.md#backend) - * [``](docs/providers/custom.md#backend) - -* `exporters`: A map of exporters to export results to. Each exporter is defined - as a key formatted as `/`, and a map value - detailing the exporter configuration. + - [`cloud_monitoring`](docs/providers/cloud_monitoring.md#backend) + - [`cloud_service_monitoring`](docs/providers/cloud_service_monitoring.md#backend) + - [`prometheus`](docs/providers/prometheus.md#backend) + - [`elasticsearch`](docs/providers/elasticsearch.md#backend) + - [`datadog`](docs/providers/datadog.md#backend) + - [`dynatrace`](docs/providers/dynatrace.md#backend) + - [``](docs/providers/custom.md#backend) + +- `exporters`: A map of exporters to export results to. Each exporter is defined as a key formatted as `/`, and a map value detailing the exporter configuration. + ```yaml exporters: bigquery/dev: @@ -173,24 +169,25 @@ as a shared config for all SLO configs. It is composed of the following fields: prometheus: url: ${PROMETHEUS_URL} ``` + See specific providers documentation for detailed configuration: - * [`bigquery`](docs/providers/bigquery.md#exporter) to export SLO reports to BigQuery for historical analysis and DataStudio reporting. - * [`cloudevent`](docs/providers/cloudevent.md#exporter) to stream SLO reports to Cloudevent receivers. - * [`pubsub`](docs/providers/pubsub.md#exporter) to stream SLO reports to Pubsub. - * [`cloud_monitoring`](docs/providers/cloud_monitoring.md#exporter) to export metrics to Cloud Monitoring. - * [`prometheus`](docs/providers/prometheus.md#exporter) to export metrics to Prometheus. - * [`datadog`](docs/providers/datadog.md#exporter) to export metrics to Datadog. - * [`dynatrace`](docs/providers/dynatrace.md#exporter) to export metrics to Dynatrace. - * [``](docs/providers/custom.md#exporter) to export SLO data or metrics to a custom destination. - -* `error_budget_policies`: [**required**] A map of various error budget policies. - * ``: Name of the error budget policy. - * `steps`: List of error budget policy steps, each containing the following fields: - * `window`: Rolling time window for this error budget. - * `alerting_burn_rate_threshold`: Target burnrate threshold over which alerting is needed. - * `urgent_notification`: boolean whether violating this error budget should trigger a page. - * `overburned_consequence_message`: message to show when the error budget is above the target. - * `achieved_consequence_message`: message to show when the error budget is within the target. + - [`bigquery`](docs/providers/bigquery.md#exporter) to export SLO reports to BigQuery for historical analysis and DataStudio reporting. + - [`cloudevent`](docs/providers/cloudevent.md#exporter) to stream SLO reports to Cloudevent receivers. + - [`pubsub`](docs/providers/pubsub.md#exporter) to stream SLO reports to Pubsub. + - [`cloud_monitoring`](docs/providers/cloud_monitoring.md#exporter) to export metrics to Cloud Monitoring. + - [`prometheus`](docs/providers/prometheus.md#exporter) to export metrics to Prometheus. + - [`datadog`](docs/providers/datadog.md#exporter) to export metrics to Datadog. + - [`dynatrace`](docs/providers/dynatrace.md#exporter) to export metrics to Dynatrace. + - [``](docs/providers/custom.md#exporter) to export SLO data or metrics to a custom destination. + +- `error_budget_policies`: [**required**] A map of various error budget policies. + - ``: Name of the error budget policy. + - `steps`: List of error budget policy steps, each containing the following fields: + - `window`: Rolling time window for this error budget. + - `alerting_burn_rate_threshold`: Target burnrate threshold over which alerting is needed. + - `urgent_notification`: boolean whether violating this error budget should trigger a page. + - `overburned_consequence_message`: message to show when the error budget is above the target. + - `achieved_consequence_message`: message to show when the error budget is within the target. ```yaml error_budget_policies: @@ -217,8 +214,13 @@ as a shared config for all SLO configs. It is composed of the following fields: To go further with the SLO Generator, you can read: ### [Build an SLO achievements report with BigQuery and DataStudio](docs/deploy/datastudio_slo_report.md) + ### [Deploy the SLO Generator in Cloud Run](docs/deploy/cloudrun.md) + ### [Deploy the SLO Generator in Kubernetes (Alpha)](docs/deploy/kubernetes.md) + ### [Deploy the SLO Generator in a CloudBuild pipeline](docs/deploy/cloudbuild.md) + ### [DEPRECATED: Deploy the SLO Generator on Google Cloud Functions (Terraform)](docs/deploy/cloudfunctions.md) + ### [Contribute to the SLO Generator](CONTRIBUTING.md) diff --git a/docs/deploy/cloudbuild.md b/docs/deploy/cloudbuild.md index 53cce786..25988f57 100644 --- a/docs/deploy/cloudbuild.md +++ b/docs/deploy/cloudbuild.md @@ -6,7 +6,7 @@ To do so, you need to build an image for the `slo-generator` and push it to `Goo ## [Optional] Build and push the image to GCR -If you are not allowed to use the public container image, you can build and push +If you are not allowed to use the public container image, you can build and push the image to your project using CloudBuild: ```sh @@ -19,7 +19,7 @@ make cloudbuild ## Run `slo-generator` as a build step -Once the image is built, you can call the SLO generator using the following +Once the image is built, you can call the SLO generator using the following snippet in your `cloudbuild.yaml`: ```yaml diff --git a/docs/deploy/cloudrun.md b/docs/deploy/cloudrun.md index 9c57f632..e70c3d6b 100644 --- a/docs/deploy/cloudrun.md +++ b/docs/deploy/cloudrun.md @@ -1,6 +1,6 @@ # Deploy SLO Generator as a Cloud Run service -`slo-generator` can also be deployed as a Cloud Run service by following the +`slo-generator` can also be deployed as a Cloud Run service by following the instructions below. ## Terraform setup @@ -68,7 +68,7 @@ gcloud scheduler jobs create http slo --schedule=”* * * * */1” \ ### [Optional] Set up the export service -If you decide to split some of the exporters to another dedicated service, you +If you decide to split some of the exporters to another dedicated service, you can deploy an export-only API to Cloud Run: Upload the slo-generator export config to the GCS bucket: @@ -92,6 +92,6 @@ gcloud run deploy slo-generator-export \ --allow-unauthenticated ``` -To send your SLO reports from the standard service to the export-only service, -set up a [cloudevent exporter](../providers/cloudevent.md) in the standard +To send your SLO reports from the standard service to the export-only service, +set up a [cloudevent exporter](../providers/cloudevent.md) in the standard service `config.yaml`. diff --git a/docs/deploy/datastudio_slo_report.md b/docs/deploy/datastudio_slo_report.md index f7c9f92d..8d95a181 100644 --- a/docs/deploy/datastudio_slo_report.md +++ b/docs/deploy/datastudio_slo_report.md @@ -83,7 +83,7 @@ BQ Data Viewer roles. Make a copy of the following data source (copy & paste the below link in a browser) -and clicking on the "Make a Copy" button just left of the "Create Report" button. +and clicking on the "Make a Copy" button just left of the "Create Report" button. ![copy this report](../images/copy_button.png) ##### Step 2 @@ -92,7 +92,7 @@ From the BQ connector settings page use "My projects" search and select your SLO ##### Step 3 -Hit the RECONNECT button top right.The message `Configuration has changed. Do you want to apply the changes?` appears indicating no fields changed. Hit APPLY. +Hit the RECONNECT button top right.The message `Configuration has changed. Do you want to apply the changes?` appears indicating no fields changed. Hit APPLY. ![copy this report](../images/config_has_changed.png) ##### Step 4 diff --git a/docs/providers/cloudevent.md b/docs/providers/cloudevent.md index 40d51b1b..42d30141 100644 --- a/docs/providers/cloudevent.md +++ b/docs/providers/cloudevent.md @@ -2,10 +2,10 @@ ## Exporter -The Cloudevent exporter will make a POST request to a CloudEvent receiver -service. +The Cloudevent exporter will make a POST request to a CloudEvent receiver +service. -This allows to send SLO Reports to another service that can process them, or +This allows to send SLO Reports to another service that can process them, or export them to other destinations, such as an export-only slo-generator service. **Config example:** @@ -24,6 +24,6 @@ Optional fields: * `auth` section allows to specify authentication tokens if needed. Tokens are added as a header * `token` is used to pass an authentication token in the request headers. - * `google_service_account_auth: true` is used to enable Google service account - authentication. Use this if the target service is hosted on GCP (Cloud Run, + * `google_service_account_auth: true` is used to enable Google service account + authentication. Use this if the target service is hosted on GCP (Cloud Run, Cloud Functions, Google Kubernetes Engine ...). diff --git a/docs/providers/custom.md b/docs/providers/custom.md index bb180a58..3fcc6ec7 100644 --- a/docs/providers/custom.md +++ b/docs/providers/custom.md @@ -9,10 +9,10 @@ This enables you to: ## Backend -To create a custom backend, simply create a new file and add the backend code -within it. +To create a custom backend, simply create a new file and add the backend code +within it. -For this example, we will assume the backend code below was added to +For this example, we will assume the backend code below was added to `custom/custom_backend.py`. A sample custom backend will have the following look: @@ -28,7 +28,7 @@ class CustomBackend: def good_bad_ratio(self, timestamp, window, slo_config): # compute your good bad ratio in this method. - # you can do anything here (query your internal API, correlate with + # you can do anything here (query your internal API, correlate with # other data, etc...) # return a tuple (number_good_events, number_bad_events) return (100000, 100) @@ -40,7 +40,7 @@ class CustomBackend: ``` -In order to call the `good_bad_ratio` method in the custom backend above, the +In order to call the `good_bad_ratio` method in the custom backend above, the `backends` block would look like this: ```yaml @@ -61,15 +61,15 @@ service_level_indicator: {} ## Exporter -To create a custom exporter, simply create a new file and add the exporter code -within it. +To create a custom exporter, simply create a new file and add the exporter code +within it. -For the examples below, we will assume the exporter code below was added to +For the examples below, we will assume the exporter code below was added to `custom/custom_exporter.py`. ### Standard -A standard exporter: +A standard exporter: * must implement the `export` method. A sample exporter looks like: @@ -87,7 +87,7 @@ class CustomExporter: Returns: object: Custom exporter response. """ - # export your `data` (SLO report) using `config` to setup export + # export your `data` (SLO report) using `config` to setup export # parameters that need to be configurable. return { 'status': 'ok', @@ -116,7 +116,7 @@ exporters: [custom.custom_exporter.CustomExporter] A metrics exporter: * must inherit from `slo_generator.exporters.base.MetricsExporter`. -* must implement the `export_metric` method which exports **one** metric. +* must implement the `export_metric` method which exports **one** metric. The `export_metric` function takes a metric dict as input, such as: ```py @@ -172,5 +172,5 @@ exporters: The `MetricsExporter` base class has the following behavior: * The `metrics` block in the SLO config is passed to the base class `MetricsExporter` * The base class `MetricsExporter` runs the `export` method which iterates through each metric and add information to it, such as the current value and timestamp. -* The base class `MetricsExporter` calls the derived class `export_metric` for each metric and pass it the metric data to export. +* The base class `MetricsExporter` calls the derived class `export_metric` for each metric and pass it the metric data to export. * The derived class for each metric to export. See [metrics](../shared/metrics.md) for more details on the `metrics` block. diff --git a/docs/providers/dynatrace.md b/docs/providers/dynatrace.md index 90f1a3db..e4e11f3f 100644 --- a/docs/providers/dynatrace.md +++ b/docs/providers/dynatrace.md @@ -35,7 +35,7 @@ purposes as well (see examples). backend: dynatrace method: good_bad_ratio service_level_indicator: - query_good: + query_good: metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) entity_selector: type(HOST) query_valid: @@ -47,8 +47,8 @@ service_level_indicator: ### Threshold -The `threshold` method is used to split a series of values into two buckets -using a threshold as delimiter: one bucket which will represent the good events, +The `threshold` method is used to split a series of values into two buckets +using a threshold as delimiter: one bucket which will represent the good events, the other will represent the bad events. This method can be used for latency SLOs, by defining a latency threshold. @@ -59,7 +59,7 @@ This method can be used for latency SLOs, by defining a latency threshold. backend: dynatrace method: threshold service_level_indicator: - query_valid: + query_valid: metric_selector: ext:app.request_latency:filter(and(eq(app,test_app),eq(env,prod),eq(status_code_class,2xx))) entity_selector: type(HOST) threshold: 40000 # us diff --git a/docs/providers/prometheus.md b/docs/providers/prometheus.md index c0da1ab9..411bbcc5 100644 --- a/docs/providers/prometheus.md +++ b/docs/providers/prometheus.md @@ -133,8 +133,8 @@ set for your metric. Learn more in the [Prometheus docs](https://prometheus.io/d ## Exporter -The `prometheus` exporter allows to export SLO metrics to the -[Prometheus Pushgateway](https://prometheus.io/docs/practices/pushing/) which +The `prometheus` exporter allows to export SLO metrics to the +[Prometheus Pushgateway](https://prometheus.io/docs/practices/pushing/) which needs to be running. ```yaml @@ -149,14 +149,14 @@ Optional fields: * `password`: Password for Basic Auth. * `job`: Name of `Pushgateway` job. Defaults to `slo-generator`. -***Note:*** `prometheus` needs to be setup to **scrape metrics from `Pushgateway`** +***Note:*** `prometheus` needs to be setup to **scrape metrics from `Pushgateway`** (see [documentation](https://github.com/prometheus/pushgateway) for more details). **→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_availability_query_sli.yaml)** ## Self Exporter (API mode) -When running slo-generator as an API, you can enable `prometheus_self` exporter, which will +When running slo-generator as an API, you can enable `prometheus_self` exporter, which will expose all metrics on a standard `/metrics` endpoint, instead of pushing them to a gateway. ```yaml @@ -164,7 +164,7 @@ exporters: prometheus_self: { } ``` -***Note:*** The metrics endpoint will be available after a first successful SLO request. +***Note:*** The metrics endpoint will be available after a first successful SLO request. Before that, it's going to act as if it was endpoint of the generator API. ### Examples diff --git a/docs/shared/api.md b/docs/shared/api.md index a9223d64..a3c6de3a 100644 --- a/docs/shared/api.md +++ b/docs/shared/api.md @@ -2,15 +2,15 @@ ## Description -The **SLO Generator API** is based on the [Functions Framework](https://cloud.google.com/functions/docs/functions-framework) -allowing deployments to hosted services easier, such as -[Kubernetes](./../../../deploy/kubernetes.md), [CloudRun](./../deploy/cloudrun.md) +The **SLO Generator API** is based on the [Functions Framework](https://cloud.google.com/functions/docs/functions-framework) +allowing deployments to hosted services easier, such as +[Kubernetes](./../../../deploy/kubernetes.md), [CloudRun](./../deploy/cloudrun.md) or [Cloud Functions](./../deploy/cloudfunctions.md). ## Standard mode API -In the standard mode, the `slo-generator` API takes SLO configs as inputs and -export SLO reports using the `default_exporters` in the shared config, and the +In the standard mode, the `slo-generator` API takes SLO configs as inputs and +export SLO reports using the `default_exporters` in the shared config, and the `exporters` section in the SLO config: ``` @@ -20,8 +20,8 @@ where: * `CONFIG_PATH` is the [Shared configuration](../../README.md#shared-configuration) file path or a Google Cloud Storage URL. -The standard-mode API can receive HTTP POST requests containing an -**SLO config**, an **SLO config path**, or a **SLO config GCS URL** in the +The standard-mode API can receive HTTP POST requests containing an +**SLO config**, an **SLO config path**, or a **SLO config GCS URL** in the request body: ``` curl -X POST --data-binary /path/to/slo_config.yaml # SLO config (YAML) @@ -41,72 +41,72 @@ Batch mode allows to send multiple file URLs that will be split and: **or** -* Send each URL to PubSub if the section `pubsub_batch_handler` exists in the +* Send each URL to PubSub if the section `pubsub_batch_handler` exists in the `slo-generator` config. This section is populated the same as a [pubsub exporter](../providers/pubsub.md). -This setup is useful to smooth big loads (many SLOs), setting up a Pubsub push +This setup is useful to smooth big loads (many SLOs), setting up a Pubsub push subscription back to the service. ### Export-only mode API -Some use cases require to have a distinct service for the export part. -For instance, some SRE teams might want to let application teams compute their -SLOs, but require them to export the SLO reports to a shared destination like +Some use cases require to have a distinct service for the export part. +For instance, some SRE teams might want to let application teams compute their +SLOs, but require them to export the SLO reports to a shared destination like BigQuery: ![arch](../images/export_service_split.png) -It is possible to run the `slo-generator` API in `export` mode only, by setting +It is possible to run the `slo-generator` API in `export` mode only, by setting the `--target run_export`: ``` slo-generator api --config /path/to/config.yaml --target run_export ``` -In this mode, the API accepts an -[SLO report](../../tests/unit/fixtures/slo_report_v2.json) in the POST request -or in a base64-encoded string under the `message.data` enveloppe., and exports that +In this mode, the API accepts an +[SLO report](../../tests/unit/fixtures/slo_report_v2.json) in the POST request +or in a base64-encoded string under the `message.data` enveloppe., and exports that data to the required exporters. -If `--signature-type cloudevent` is passed, the POST request data needs to be -wrapped into a [CloudEvent message](https://github.com/cloudevents/spec/blob/v1.0.1/spec.md#example) +If `--signature-type cloudevent` is passed, the POST request data needs to be +wrapped into a [CloudEvent message](https://github.com/cloudevents/spec/blob/v1.0.1/spec.md#example) under the `data` key. See the next section to see in which setups this is used. -The exporters which are used for the export are configured using the +The exporters which are used for the export are configured using the `default_exporters` property in the `slo-generator` configuration. ### Sending data from a standard API to an export-only API -There are two ways you can forward SLO reports from the standard `slo-generator` +There are two ways you can forward SLO reports from the standard `slo-generator` API to an export-only API: -* If the export-only API is configured with `--signature-type http` (default), +* If the export-only API is configured with `--signature-type http` (default), then you can: - * Set up a `Pubsub` exporter in the standard API shared config to send the - SLO reports to a Pubsub topic, and set up a Pubsub push subscription + * Set up a `Pubsub` exporter in the standard API shared config to send the + SLO reports to a Pubsub topic, and set up a Pubsub push subscription configured with the export-only API service URL. See official [documentation](https://cloud.google.com/run/docs/triggering/pubsub-push). - * [***not implemented yet***] Set up an `HTTP` exporter in the standard API shared + * [***not implemented yet***] Set up an `HTTP` exporter in the standard API shared config to send the reports. -* If the export-only API is configured with `--signature-type cloudevent`, then +* If the export-only API is configured with `--signature-type cloudevent`, then you can: - * Set up a `Cloudevent` exporter in the standard API shared config to send the + * Set up a `Cloudevent` exporter in the standard API shared config to send the events directly to the export-only API. See cloudevent exporter [documentation](../providers/cloudevent.md#exporter). - * Set up a `Pubsub` exporter in the standard API shared config to send the - events to a Pubsub topic, and set up an [**EventArc**](https://cloud.google.com/eventarc/) + * Set up a `Pubsub` exporter in the standard API shared config to send the + events to a Pubsub topic, and set up an [**EventArc**](https://cloud.google.com/eventarc/) to convert the Pubsub messages to CloudEvent format. See official [documentation](https://cloud.google.com/eventarc/docs/creating-triggers). **Notes:** -* Using a queue like Pubsub as an intermediary between the standard service and -the export-only service is a good way to spread the load over time if you are +* Using a queue like Pubsub as an intermediary between the standard service and +the export-only service is a good way to spread the load over time if you are dealing with lots of SLO computations. -* You can develop your own Cloudevent receivers to use with the `Cloudevent` -exporter if you want to do additional processing / analysis of SLO reports. -An example code for a Cloudevent receiver is given [here](https://cloud.google.com/eventarc/docs/run/event-receivers), -but you can also check out the [Functions Framework](https://cloud.google.com/functions/docs/functions-framework) +* You can develop your own Cloudevent receivers to use with the `Cloudevent` +exporter if you want to do additional processing / analysis of SLO reports. +An example code for a Cloudevent receiver is given [here](https://cloud.google.com/eventarc/docs/run/event-receivers), +but you can also check out the [Functions Framework](https://cloud.google.com/functions/docs/functions-framework) that does the marshalling / unmarshalling of Cloudevents out-of-the-box. diff --git a/docs/shared/metrics.md b/docs/shared/metrics.md index 1c0fefe6..ea3eeca2 100644 --- a/docs/shared/metrics.md +++ b/docs/shared/metrics.md @@ -1,8 +1,8 @@ # Metrics block ## Configuration -The `metrics` block can be added to the configuration of any exporter derived -from [`MetricsExporter`](../../slo_generator/exporters/base.py#L41). It is used +The `metrics` block can be added to the configuration of any exporter derived +from [`MetricsExporter`](../../slo_generator/exporters/base.py#L41). It is used to specify which metrics should be exported to the destination. **Metrics** exported by default: @@ -10,7 +10,7 @@ to specify which metrics should be exported to the destination. - `alerting_burn_rate_threshold`: used for defining the alerting threshold. - `sli_measurement`: used for visualizing the SLI over time. - `slo_target`: used for drawing target over SLI measurement. -- `events_count`: used to split good / bad events in dashboards. Has two +- `events_count`: used to split good / bad events in dashboards. Has two additional labels: `good_events_count` and `bad_events_count`. **Metric labels** exported by default: @@ -31,7 +31,7 @@ additional labels: `good_events_count` and `bad_events_count`. automatically. ### Override config (simple) -If you want to discard some default metrics, but keep the overall defaults, you +If you want to discard some default metrics, but keep the overall defaults, you can use the simple override of the `metrics` block: ```yaml metrics: @@ -55,7 +55,7 @@ metrics: additional_labels: - good_events_count - bad_events_count - + - name: sli_measurement description: SLI measurement. alias: sli @@ -64,22 +64,22 @@ metrics: ``` where: -* `name`: name of the [SLO Report](../../tests/unit/fixtures/slo_report_v2.json) +* `name`: name of the [SLO Report](../../tests/unit/fixtures/slo_report_v2.json) field to export as a metric. The field MUST exist in the SLO report. * `description`: description of the metric (if the metrics exporter supports it) -* `alias` (optional): rename the metric before writing to the monitoring +* `alias` (optional): rename the metric before writing to the monitoring backend. -* `additional_labels` (optional) allow you to specify other labels to the -timeseries written. Each label name must correspond to a field of the +* `additional_labels` (optional) allow you to specify other labels to the +timeseries written. Each label name must correspond to a field of the [SLO Report](../../tests/unit/fixtures/slo_report_v2.json). ## Metric exporters -Some metrics exporters have a specific `prefix` that is pre-prepended to the +Some metrics exporters have a specific `prefix` that is pre-prepended to the metric name: * `cloud_monitoring` exporter prefix: `custom.googleapis.com/` * `datadog` prefix: `custom:` -Some metrics exporters have a limit of `labels` that can be written to their +Some metrics exporters have a limit of `labels` that can be written to their metrics timeseries: * `cloud_monitoring` labels limit: `10`. diff --git a/docs/shared/migration.md b/docs/shared/migration.md index f09122f2..735501b3 100644 --- a/docs/shared/migration.md +++ b/docs/shared/migration.md @@ -2,10 +2,10 @@ ## v1 to v2 -Version `v2` of the slo-generator introduces some changes to the structure of +Version `v2` of the slo-generator introduces some changes to the structure of the SLO configurations. -To migrate your SLO configurations from v1 to v2, please execute the following +To migrate your SLO configurations from v1 to v2, please execute the following instructions: **Upgrade `slo-generator`:** @@ -18,12 +18,12 @@ pip3 install slo-generator -U # upgrades slo-generator version to the latest ver slo-generator migrate -s -t -b -e ``` where: -* `` is the source folder containg SLO configurations in v1 format. -This folder can have nested subfolders containing SLOs. The subfolder structure +* `` is the source folder containg SLO configurations in v1 format. +This folder can have nested subfolders containing SLOs. The subfolder structure will be reproduced on the target folder. * `` is the target folder to drop the SLO configurations in v2 -format. If the target folder is identical to the source folder, the existing SLO +format. If the target folder is identical to the source folder, the existing SLO configurations will be updated in-place. * `` is the path to your error budget policy configuration. You can add more by specifying another `-b ` @@ -46,83 +46,83 @@ $ slo-generator migrate -s slos/ -t slos/ -b slos/error_budget_policy.yaml -b sl Migrating slo-generator configs to v2 ... Config does not correspond to any known SLO config versions. -------------------------------------------------- -slos/exporters.yaml [v1] +slos/exporters.yaml [v1] Invalid configuration: missing required key(s) ['service_name', 'feature_name', 'slo_name', 'backend']. Config does not correspond to any known SLO config versions. -------------------------------------------------- -slos/platform-slos/slo_pubsub_coverage.yaml [v1] +slos/platform-slos/slo_pubsub_coverage.yaml [v1] ➞ slos/platform-slos/slo_pubsub_coverage.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/custom-example/slo_test_custom.yaml [v1] +slos/custom-example/slo_test_custom.yaml [v1] ➞ slos/custom-example/slo_test_custom.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/flask-app-prometheus/slo_flask_latency_query_sli.yaml [v1] +slos/flask-app-prometheus/slo_flask_latency_query_sli.yaml [v1] ➞ slos/flask-app-prometheus/slo_flask_latency_query_sli.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/flask-app-prometheus/slo_flask_availability_ratio.yaml [v1] +slos/flask-app-prometheus/slo_flask_availability_ratio.yaml [v1] ➞ slos/flask-app-prometheus/slo_flask_availability_ratio.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/flask-app-prometheus/slo_flask_availability_query_sli.yaml [v1] +slos/flask-app-prometheus/slo_flask_availability_query_sli.yaml [v1] ➞ slos/flask-app-prometheus/slo_flask_availability_query_sli.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/flask-app-prometheus/slo_flask_latency_distribution_cut.yaml [v1] +slos/flask-app-prometheus/slo_flask_latency_distribution_cut.yaml [v1] ➞ slos/flask-app-prometheus/slo_flask_latency_distribution_cut.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/flask-app-datadog/slo_dd_app_availability_query_sli.yaml [v1] +slos/flask-app-datadog/slo_dd_app_availability_query_sli.yaml [v1] ➞ slos/flask-app-datadog/slo_dd_app_availability_query_sli.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/flask-app-datadog/slo_dd_app_availability_query_slo.yaml [v1] +slos/flask-app-datadog/slo_dd_app_availability_query_slo.yaml [v1] ➞ slos/flask-app-datadog/slo_dd_app_availability_query_slo.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/flask-app-datadog/slo_dd_app_availability_ratio.yaml [v1] +slos/flask-app-datadog/slo_dd_app_availability_ratio.yaml [v1] ➞ slos/flask-app-datadog/slo_dd_app_availability_ratio.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/slo-generator/slo_bq_latency.yaml [v1] +slos/slo-generator/slo_bq_latency.yaml [v1] ➞ slos/slo-generator/slo_bq_latency.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/slo-generator/slo_pubsub_coverage.yaml [v1] +slos/slo-generator/slo_pubsub_coverage.yaml [v1] ➞ slos/slo-generator/slo_pubsub_coverage.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/slo-generator/slo_gcf_throughput.yaml [v1] +slos/slo-generator/slo_gcf_throughput.yaml [v1] ➞ slos/slo-generator/slo_gcf_throughput.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/slo-generator/slo_gcf_latency.yaml [v1] +slos/slo-generator/slo_gcf_latency.yaml [v1] ➞ slos/slo-generator/slo_gcf_latency.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/slo-generator/slo_gcf_latency pipeline.yaml [v1] +slos/slo-generator/slo_gcf_latency pipeline.yaml [v1] ➞ slos/slo-generator/slo_gcf_latency pipeline.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/slo-generator/slo_gcf_errors.yaml [v1] +slos/slo-generator/slo_gcf_errors.yaml [v1] ➞ slos/slo-generator/slo_gcf_errors.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/online-boutique/slo_ob_adservice_availability.yaml [v1] +slos/online-boutique/slo_ob_adservice_availability.yaml [v1] ➞ slos/online-boutique/slo_ob_adservice_availability.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/online-boutique/slo_ob_adservice_latency.yaml [v1] +slos/online-boutique/slo_ob_adservice_latency.yaml [v1] ➞ slos/online-boutique/slo_ob_adservice_latency.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/online-boutique/slo_ob_all_latency_distribution_cut.yaml [v1] +slos/online-boutique/slo_ob_all_latency_distribution_cut.yaml [v1] ➞ slos/online-boutique/slo_ob_all_latency_distribution_cut.yaml [v2] (replaced) ✅ Success ! -------------------------------------------------- -slos/online-boutique/slo_ob_all_availability_basic.yaml [v1] +slos/online-boutique/slo_ob_all_availability_basic.yaml [v1] ➞ slos/online-boutique/slo_ob_all_availability_basic.yaml [v2] (replaced) ✅ Success ! ================================================== diff --git a/docs/shared/troubleshooting.md b/docs/shared/troubleshooting.md index 29d6eef5..01a3807f 100644 --- a/docs/shared/troubleshooting.md +++ b/docs/shared/troubleshooting.md @@ -12,14 +12,14 @@ The new labels would cause the metric custom.googleapis.com/slo_target to have o ### Solutions **Solution 1:** -Delete the metric descriptor, and re-run the SLO Generator. -You can do so using `gmon` (`pip install gmon`) and run: +Delete the metric descriptor, and re-run the SLO Generator. +You can do so using `gmon` (`pip install gmon`) and run: `gmon metrics delete custom.googleapis.com/slo_target -p `. -**Warning:** this will destroy all historical metric data (6 weeks). +**Warning:** this will destroy all historical metric data (6 weeks). If you are using the metric in Cloud Monitoring dashboards, be wary. **Solution 2:** Limit the number of user labels sent with the metric. -The default metrics are exported with 7 labels maximum, which means you have up -to 3 additional user labels (`metadata` labels in SLO config, or +The default metrics are exported with 7 labels maximum, which means you have up +to 3 additional user labels (`metadata` labels in SLO config, or `additional_labels` in the `metrics` config). diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 976ba029..00000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..85d17ba5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] # PEP 508 specifications. +build-backend = "setuptools.build_meta" + +[tool.black] +# Using Black with other tools +# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#using-black-with-other-tools +line-length = 88 # default: 88 + +[tool.isort] +# Make it compatible with black +profile = "black" +extend_skip = [ + ".pytype", +] + +[tool.pylint] +ignore-patterns = [ + "test_.*?py", +] + +[tool.pylint.messages_control] +max-line-length = 88 +disable = [ + "logging-fstring-interpolation", + "import-error", +] + +[tool.mypy] +# https://mypy.readthedocs.io/en/stable/config_file.html#using-a-pyproject-toml-file +ignore_missing_imports = true diff --git a/samples/custom/custom_backend.py b/samples/custom/custom_backend.py index 0bcf76d1..aec3b236 100644 --- a/samples/custom/custom_backend.py +++ b/samples/custom/custom_backend.py @@ -19,12 +19,14 @@ LOGGER = logging.getLogger(__name__) + class CustomBackend: """Custom backend that always return an SLI of 0.999.""" def __init__(self, client=None, **kwargs): pass + # pylint: disable=unused-argument def good_bad_ratio(self, timestamp, window, slo_config): """Good bad ratio method. @@ -38,5 +40,6 @@ def good_bad_ratio(self, timestamp, window, slo_config): """ return 100000, 100 + # pylint: disable=unused-argument,missing-function-docstring def query_sli(self, timestamp, window, slo_config): return 0.999 diff --git a/samples/custom/custom_exporter.py b/samples/custom/custom_exporter.py index 0bd838f5..07cb01d7 100644 --- a/samples/custom/custom_exporter.py +++ b/samples/custom/custom_exporter.py @@ -16,10 +16,12 @@ Dummy sample of a custom exporter. """ import logging + from slo_generator.exporters.base import MetricsExporter LOGGER = logging.getLogger(__name__) + class CustomMetricExporter(MetricsExporter): """Custom exporter for metrics.""" @@ -34,13 +36,16 @@ def export_metric(self, data): """ LOGGER.info(f"Metric data: {data}") return { - 'status': 'ok', - 'code': 200 + "status": "ok", + "code": 200, } + +# pylint: disable=too-few-public-methods class CustomSLOExporter: """Custom exporter for SLO data.""" + # pylint: disable=unused-argument def export(self, data, **config): """Export data to custom destination. @@ -52,6 +57,6 @@ def export(self, data, **config): """ LOGGER.info(f"SLO data: {data}") return { - 'status': 'ok', - 'code': 200 + "status": "ok", + "code": 200, } diff --git a/setup.cfg b/setup.cfg index 0d1debba..4c4c4cf4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,9 +14,122 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Generated by synthtool. DO NOT EDIT! -[bdist_wheel] -universal = 1 +[metadata] +name = slo-generator +version = 2.2.0 +author = Google Inc. +author_email = olivier.cervello@gmail.com +maintainer = Laurent VAYLET +maintainer_email = laurent.vaylet@gmail.com +url = https://github.com/google/slo-generator +description = SLO Generator +long_description = file: README.md +long_description_content_type = text/markdown +keywords = + slo + sli + generator + gcp +license = Apache License 2.0 +license_files = LICENSE +classifiers = + # FIXME: 5 - Production/Stable + Development Status :: 3 - Alpha + Environment :: Console + Intended Audience :: Developers + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Topic :: Software Development :: Build Tools + Topic :: System :: Monitoring + +[options] +packages = find: +python_requires = >=3.7, <4 +install_requires = + pyyaml + ruamel.yaml + python-dateutil + click + +[options.packages.find] +exclude = + contrib + docs + tests + +[options.extras_require] +api = + Flask + gunicorn + cloudevents + functions-framework + requests +prometheus = + prometheus-client + prometheus-http-client +datadog = + datadog + retrying==1.3.3 +dynatrace = + requests +bigquery = + google-api-python-client <2 + google-cloud-bigquery <3 +cloud_monitoring = + google-api-python-client <2 + google-cloud-monitoring <3 +cloud_service_monitoring = + google-api-python-client <2 + google-cloud-monitoring <3 +cloud_storage = + google-api-python-client <2 + google-cloud-storage +pubsub = + google-api-python-client <2 + google-cloud-pubsub <2 +elasticsearch = + elasticsearch +cloudevent = + cloudevents +dev = + pip >=22.3 # avoid known vulnerabilities in pip 20.3.4 (reported by `safety check`) + wheel + flake8 + black + isort + mock + pytest + pytest-cov + pylint + pytype + mypy + types-PyYAML + types-python-dateutil + types-setuptools + types-requests + types-protobuf + pre-commit + bandit + safety + +[options.entry_points] +console_scripts = + slo-generator = slo_generator.cli:main [pytype] inputs = slo_generator + +[flake8] +# Why those options? +# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#id1 +max-line-length = 88 +extend-ignore = E203 + +# Generated by synthtool. DO NOT EDIT! +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py index e54c497c..e042dc15 100644 --- a/setup.py +++ b/setup.py @@ -17,79 +17,12 @@ https://packaging.python.org/en/latest/distributing.html https://github.com/pypa/sampleproject """ -# pylint: disable=invalid-name -from io import open -from os import path import sys -from setuptools import find_packages, setup +from setuptools import setup -here = path.abspath(path.dirname(__file__)) +sys.dont_write_bytecode = True # avoid generating .pyc files -# Package metadata. -name = "slo-generator" -description = "SLO Generator" -version = "2.2.0" -# Should be one of: -# 'Development Status :: 3 - Alpha' -# 'Development Status :: 4 - Beta' -# 'Development Status :: 5 - Production/Stable' -release_status = "Development Status :: 3 - Alpha" -dependencies = ['pyyaml', 'ruamel.yaml', 'python-dateutil', 'click < 8.0'] -extras = { - 'api': [ - 'Flask', 'gunicorn', 'cloudevents', 'functions-framework', 'requests' - ], - 'prometheus': ['prometheus-client', 'prometheus-http-client'], - 'datadog': ['datadog', 'retrying==1.3.3'], - 'dynatrace': ['requests'], - 'bigquery': ['google-api-python-client <2', 'google-cloud-bigquery <3'], - 'cloud_monitoring': [ - 'google-api-python-client <2', 'google-cloud-monitoring <3' - ], - 'cloud_service_monitoring': [ - 'google-api-python-client <2', 'google-cloud-monitoring <3' - ], - 'cloud_storage': ['google-api-python-client <2', 'google-cloud-storage'], - 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], - 'elasticsearch': ['elasticsearch'], - 'cloudevent': ['cloudevents'], - 'dev': [ - 'wheel', 'flake8', 'mock', 'pytest', 'pytest-cov', 'pylint', 'pytype', 'mypy', - 'types-PyYAML', 'types-python-dateutil', 'types-setuptools', 'types-requests', 'types-protobuf' - ] -} - -# Get the long description from the README file -with open(path.join(here, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - -sys.dont_write_bytecode = True -setup(name=name, - version=version, - description=description, - long_description=long_description, - long_description_content_type='text/markdown', - author='Google Inc.', - author_email='ocervello@google.com', - license='Apache 2.0', - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - classifiers=[ - release_status, - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Build Tools', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - keywords='slo sli generator gcp', - install_requires=dependencies, - extras_require=extras, - entry_points={ - 'console_scripts': ['slo-generator=slo_generator.cli:main'], - }, - python_requires='>=3.6') +if __name__ == "__main__": + setup() diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py index 228f0c04..5c3b79e9 100644 --- a/slo_generator/api/main.py +++ b/slo_generator/api/main.py @@ -17,26 +17,26 @@ See https://github.com/GoogleCloudPlatform/functions-framework-python for details on the Functions Framework. """ -# pylint: disable=fixme import base64 -import os import json import logging +import os import pprint -import requests +import requests from flask import jsonify, make_response from slo_generator.compute import compute, export -from slo_generator.utils import setup_logging, load_config, get_exporters +from slo_generator.utils import get_exporters, load_config, setup_logging -CONFIG_PATH = os.environ['CONFIG_PATH'] +CONFIG_PATH = os.environ["CONFIG_PATH"] LOGGER = logging.getLogger(__name__) -TIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' -API_SIGNATURE_TYPE = os.environ['GOOGLE_FUNCTION_SIGNATURE_TYPE'] +TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +API_SIGNATURE_TYPE = os.environ["GOOGLE_FUNCTION_SIGNATURE_TYPE"] setup_logging() + def run_compute(request): """Run slo-generator compute function. Can be configured to export data as well, using the `exporters` key of the SLO config. @@ -48,16 +48,17 @@ def run_compute(request): list: List of SLO reports. """ # Get slo-generator config - LOGGER.info(f'Loading slo-generator config from {CONFIG_PATH}') + LOGGER.info(f"Loading slo-generator config from {CONFIG_PATH}") config = load_config(CONFIG_PATH) # Process request data = process_req(request) - batch_mode = request.args.get('batch', False) + batch_mode = request.args.get("batch", False) if batch_mode: - if not API_SIGNATURE_TYPE == 'http': + if not API_SIGNATURE_TYPE == "http": raise ValueError( - 'Batch mode works only when --signature-type is set to "http".') + 'Batch mode works only when --signature-type is set to "http".' + ) process_batch_req(request, data, config) return jsonify([]) @@ -65,13 +66,15 @@ def run_compute(request): slo_config = load_config(data) # Compute SLO report - LOGGER.debug(f'Config: {pprint.pformat(config)}') - LOGGER.debug(f'SLO Config: {pprint.pformat(slo_config)}') - reports = compute(slo_config, - config, - client=None, - do_export=True) - if API_SIGNATURE_TYPE == 'http': + LOGGER.debug(f"Config: {pprint.pformat(config)}") + LOGGER.debug(f"SLO Config: {pprint.pformat(slo_config)}") + reports = compute( + slo_config, + config, + client=None, + do_export=True, + ) + if API_SIGNATURE_TYPE == "http": reports = jsonify(reports) return reports @@ -90,46 +93,55 @@ def run_export(request): data = process_req(request) slo_report = load_config(data) if not slo_report: - return make_response({ - "error": "SLO report is empty." - }) + return make_response( + { + "error": "SLO report is empty.", + } + ) # Get SLO config - LOGGER.info(f'Loading slo-generator config from {CONFIG_PATH}') + LOGGER.info(f"Loading slo-generator config from {CONFIG_PATH}") config = load_config(CONFIG_PATH) # Construct exporters block spec = {} # pytype: disable=attribute-error + # pylint: disable=fixme # FIXME `load_config()` returns `Optional[dict]` so `config` can be `None` - default_exporters = config.get('default_exporters', []) + default_exporters = config.get("default_exporters", []) # pytype: enable=attribute-error - cli_exporters = os.environ.get('EXPORTERS', None) + cli_exporters = os.environ.get("EXPORTERS", None) if cli_exporters: - cli_exporters = cli_exporters.split(',') + cli_exporters = cli_exporters.split(",") if not default_exporters and not cli_exporters: error = ( - 'No default exporters set for `default_exporters` in shared config ' - f'at {CONFIG_PATH}; and --exporters was not passed to the CLI.' + "No default exporters set for `default_exporters` in shared config " + f"at {CONFIG_PATH}; and --exporters was not passed to the CLI." + ) + return make_response( + { + "error": error, + }, + 500, ) - return make_response({ - 'error': error - }, 500) if cli_exporters: - spec = {'exporters': cli_exporters} + spec = {"exporters": cli_exporters} else: - spec = {'exporters': default_exporters} + spec = {"exporters": default_exporters} exporters = get_exporters(config, spec) # Export data errors = export(slo_report, exporters) - if API_SIGNATURE_TYPE == 'http': - return jsonify({ - "errors": errors - }) + if API_SIGNATURE_TYPE == "http": + return jsonify( + { + "errors": errors, + } + ) return errors + def process_req(request): """Process incoming request. @@ -139,25 +151,26 @@ def process_req(request): Returns: str: Message content. """ - if API_SIGNATURE_TYPE == 'cloudevent': + if API_SIGNATURE_TYPE == "cloudevent": LOGGER.info(f'Loading config from Cloud Event "{request["id"]}"') - if 'message' in request.data: # PubSub enveloppe - LOGGER.info('Unwrapping Pubsub enveloppe') - content = base64.b64decode(request.data['message']['data']) - data = str(content.decode('utf-8')).strip() + if "message" in request.data: # PubSub enveloppe + LOGGER.info("Unwrapping Pubsub enveloppe") + content = base64.b64decode(request.data["message"]["data"]) + data = str(content.decode("utf-8")).strip() else: data = str(request.data) - elif API_SIGNATURE_TYPE == 'http': - data = str(request.get_data().decode('utf-8')) - LOGGER.info('Loading config from HTTP request') + elif API_SIGNATURE_TYPE == "http": + data = str(request.get_data().decode("utf-8")) + LOGGER.info("Loading config from HTTP request") json_data = convert_json(data) - if json_data and 'message' in json_data: # PubSub enveloppe - LOGGER.info('Unwrapping Pubsub enveloppe') - content = base64.b64decode(json_data['message']['data']) - data = str(content.decode('utf-8')).strip() + if json_data and "message" in json_data: # PubSub enveloppe + LOGGER.info("Unwrapping Pubsub enveloppe") + content = base64.b64decode(json_data["message"]["data"]) + data = str(content.decode("utf-8")).strip() LOGGER.debug(data) return data + def convert_json(data): """Convert string to JSON if possible or return None otherwise. @@ -172,6 +185,7 @@ def convert_json(data): except ValueError: return None + def process_batch_req(request, data, config): """Process batch request. Split list of ;-delimited URLs and make one request per URL. @@ -185,29 +199,33 @@ def process_batch_req(request, data, config): list: List of API responses. """ LOGGER.info( - 'Batch request detected. Splitting body and sending individual ' - 'requests separately.') - urls = data.split(';') + "Batch request detected. Splitting body and sending individual " + "requests separately." + ) + urls = data.split(";") service_url = request.base_url - headers = {'User-Agent': 'slo-generator'} - if 'Authorization' in request.headers: - headers['Authorization'] = request.headers['Authorization'] - service_url = service_url.replace('http:', 'https:') # force HTTPS auth + headers = {"User-Agent": "slo-generator"} + if "Authorization" in request.headers: + headers["Authorization"] = request.headers["Authorization"] + service_url = service_url.replace("http:", "https:") # force HTTPS auth for url in urls: - if 'pubsub_batch_handler' in config: - LOGGER.info(f'Sending {url} to pubsub batch handler.') - from google.cloud import pubsub_v1 # pylint: disable=C0415 + if "pubsub_batch_handler" in config: + LOGGER.info(f"Sending {url} to pubsub batch handler.") + from google.cloud import pubsub_v1 # pylint: disable=C0415 + # pytype: disable=attribute-error - # FIXME `load_config()` returns `Optional[dict]` + # pylint: disable=fixme + # FIXME `load_config()` returns `Optional[dict]` so `config` can be `None` # so `config` can be `None` - exporter_conf = config.get('pubsub_batch_handler') + exporter_conf = config.get("pubsub_batch_handler") # pytype: enable=attribute-error client = pubsub_v1.PublisherClient() - project_id = exporter_conf['project_id'] - topic_name = exporter_conf['topic_name'] + project_id = exporter_conf["project_id"] + topic_name = exporter_conf["topic_name"] + # pylint: disable=no-member topic_path = client.topic_path(project_id, topic_name) - data = url.encode('utf-8') + data = url.encode("utf-8") client.publish(topic_path, data=data).result() - else: # http - LOGGER.info(f'Sending {url} to HTTP batch handler.') + else: # http + LOGGER.info(f"Sending {url} to HTTP batch handler.") requests.post(service_url, headers=headers, data=url, timeout=10) diff --git a/slo_generator/backends/cloud_monitoring.py b/slo_generator/backends/cloud_monitoring.py index 27500578..eb9e8036 100644 --- a/slo_generator/backends/cloud_monitoring.py +++ b/slo_generator/backends/cloud_monitoring.py @@ -56,41 +56,47 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: A tuple (good_event_count, bad_event_count) """ - measurement = slo_config['spec']['service_level_indicator'] - filter_good = measurement['filter_good'] - filter_bad = measurement.get('filter_bad') - filter_valid = measurement.get('filter_valid') + measurement = slo_config["spec"]["service_level_indicator"] + filter_good = measurement["filter_good"] + filter_bad = measurement.get("filter_bad") + filter_valid = measurement.get("filter_valid") # Query 'good events' timeseries - good_ts = self.query(timestamp=timestamp, - window=window, - filter=filter_good) + good_ts = self.query( + timestamp=timestamp, + window=window, + filter=filter_good, + ) good_ts = list(good_ts) good_event_count = CM.count(good_ts) # Query 'bad events' timeseries if filter_bad: - bad_ts = self.query(timestamp=timestamp, - window=window, - filter=filter_bad) + bad_ts = self.query( + timestamp=timestamp, + window=window, + filter=filter_bad, + ) bad_ts = list(bad_ts) bad_event_count = CM.count(bad_ts) elif filter_valid: - valid_ts = self.query(timestamp=timestamp, - window=window, - filter=filter_valid) + valid_ts = self.query( + timestamp=timestamp, + window=window, + filter=filter_valid, + ) valid_ts = list(valid_ts) bad_event_count = CM.count(valid_ts) - good_event_count else: - raise Exception( - "One of `filter_bad` or `filter_valid` is required.") + raise Exception("One of `filter_bad` or `filter_valid` is required.") - LOGGER.debug(f'Good events: {good_event_count} | ' - f'Bad events: {bad_event_count}') + LOGGER.debug( + f"Good events: {good_event_count} | " f"Bad events: {bad_event_count}" + ) return good_event_count, bad_event_count - # pylint: disable=duplicate-code + # pylint: disable=duplicate-code,too-many-locals def distribution_cut(self, timestamp, window, slo_config): """Query one timeseries of type 'exponential'. @@ -102,15 +108,17 @@ def distribution_cut(self, timestamp, window, slo_config): Returns: tuple: A tuple (good_event_count, bad_event_count). """ - measurement = slo_config['spec']['service_level_indicator'] - filter_valid = measurement['filter_valid'] - threshold_bucket = int(measurement['threshold_bucket']) - good_below_threshold = measurement.get('good_below_threshold', True) + measurement = slo_config["spec"]["service_level_indicator"] + filter_valid = measurement["filter_valid"] + threshold_bucket = int(measurement["threshold_bucket"]) + good_below_threshold = measurement.get("good_below_threshold", True) # Query 'valid' events - series = self.query(timestamp=timestamp, - window=window, - filter=filter_valid) + series = self.query( + timestamp=timestamp, + window=window, + filter=filter_valid, + ) series = list(series) if not series: @@ -132,7 +140,7 @@ def distribution_cut(self, timestamp, window, slo_config): distribution[i] = { # 'upper_bound': upper_bound, # 'bucket_count': bucket_count, - 'count_sum': count_sum + "count_sum": count_sum } LOGGER.debug(pprint.pformat(distribution)) @@ -141,7 +149,7 @@ def distribution_cut(self, timestamp, window, slo_config): lower_events_count = valid_events_count upper_events_count = 0 else: - lower_events_count = distribution[threshold_bucket]['count_sum'] + lower_events_count = distribution[threshold_bucket]["count_sum"] upper_events_count = valid_events_count - lower_events_count if good_below_threshold: @@ -158,17 +166,22 @@ def exponential_distribution_cut(self, *args, **kwargs): compatibility. """ warnings.warn( - 'exponential_distribution_cut will be deprecated in version 2.0, ' - 'please use distribution_cut instead', FutureWarning) + "exponential_distribution_cut will be deprecated in version 2.0, " + "please use distribution_cut instead", + FutureWarning, + ) return self.distribution_cut(*args, **kwargs) - def query(self, - timestamp, - window, - filter, - aligner='ALIGN_SUM', - reducer='REDUCE_SUM', - group_by=None): + # pylint: disable=redefined-builtin,too-many-arguments + def query( + self, + timestamp, + window, + filter, + aligner="ALIGN_SUM", + reducer="REDUCE_SUM", + group_by=None, + ): """Query timeseries from Cloud Monitoring. Args: @@ -185,10 +198,9 @@ def query(self, if group_by is None: group_by = [] measurement_window = CM.get_window(timestamp, window) - aggregation = CM.get_aggregation(window, - aligner=aligner, - reducer=reducer, - group_by=group_by) + aggregation = CM.get_aggregation( + window, aligner=aligner, reducer=reducer, group_by=group_by + ) request = monitoring_v3.ListTimeSeriesRequest() request.name = self.parent request.filter = filter @@ -228,27 +240,31 @@ def get_window(timestamp, window): :obj:`monitoring_v3.types.TimeInterval`: Measurement window object. """ end_time_seconds = int(timestamp) - end_time_nanos = int((timestamp - end_time_seconds) * 10 ** 9) + end_time_nanos = int((timestamp - end_time_seconds) * 10**9) start_time_seconds = int(timestamp - window) start_time_nanos = end_time_nanos - measurement_window = monitoring_v3.TimeInterval({ - "end_time": { - "seconds": end_time_seconds, - "nanos": end_time_nanos - }, - "start_time": { - "seconds": start_time_seconds, - "nanos": start_time_nanos + measurement_window = monitoring_v3.TimeInterval( + { + "end_time": { + "seconds": end_time_seconds, + "nanos": end_time_nanos, + }, + "start_time": { + "seconds": start_time_seconds, + "nanos": start_time_nanos, + }, } - }) + ) LOGGER.debug(pprint.pformat(measurement_window)) return measurement_window @staticmethod - def get_aggregation(window, - aligner='ALIGN_SUM', - reducer='REDUCE_SUM', - group_by=None): + def get_aggregation( + window, + aligner="ALIGN_SUM", + reducer="REDUCE_SUM", + group_by=None, + ): """Helper for aggregation object. Default aggregation is `ALIGN_SUM`. @@ -265,14 +281,18 @@ def get_aggregation(window, """ if group_by is None: group_by = [] - aggregation = monitoring_v3.Aggregation({ - "alignment_period": {"seconds": window}, - "per_series_aligner": - getattr(monitoring_v3.Aggregation.Aligner, aligner), - "cross_series_reducer": - getattr(monitoring_v3.Aggregation.Reducer, reducer), - "group_by_fields": group_by, - }) + aggregation = monitoring_v3.Aggregation( + { + "alignment_period": {"seconds": window}, + "per_series_aligner": getattr( + monitoring_v3.Aggregation.Aligner, aligner + ), + "cross_series_reducer": getattr( + monitoring_v3.Aggregation.Reducer, reducer + ), + "group_by_fields": group_by, + } + ) LOGGER.debug(pprint.pformat(aggregation)) return aggregation diff --git a/slo_generator/backends/cloud_monitoring_mql.py b/slo_generator/backends/cloud_monitoring_mql.py index f5bb7938..56ea49ba 100644 --- a/slo_generator/backends/cloud_monitoring_mql.py +++ b/slo_generator/backends/cloud_monitoring_mql.py @@ -25,8 +25,9 @@ from google.api.distribution_pb2 import Distribution from google.cloud.monitoring_v3.services.query_service import QueryServiceClient -from google.cloud.monitoring_v3.services.query_service.pagers import \ - QueryTimeSeriesPager +from google.cloud.monitoring_v3.services.query_service.pagers import ( + QueryTimeSeriesPager, +) from google.cloud.monitoring_v3.types import metric_service from google.cloud.monitoring_v3.types.metric import TimeSeriesData @@ -49,16 +50,16 @@ def __init__(self, project_id: str, client: QueryServiceClient = None): self.client = client if client is None: self.client = QueryServiceClient() - self.parent = ( - self.client.common_project_path( # type: ignore[union-attr] - project_id - ) + self.parent = self.client.common_project_path( # type: ignore[union-attr] + project_id ) - def good_bad_ratio(self, - timestamp: int, # pylint: disable=unused-argument - window: int, - slo_config: dict) -> Tuple[int, int]: + def good_bad_ratio( + self, + timestamp: int, # pylint: disable=unused-argument + window: int, + slo_config: dict, + ) -> Tuple[int, int]: """Query two timeseries, one containing 'good' events, one containing 'bad' events. @@ -70,39 +71,41 @@ def good_bad_ratio(self, Returns: tuple: A tuple (good_event_count, bad_event_count) """ - measurement: dict = slo_config['spec']['service_level_indicator'] - filter_good: str = measurement['filter_good'] - filter_bad: typing.Optional[str] = measurement.get('filter_bad') - filter_valid: typing.Optional[str] = measurement.get('filter_valid') + measurement: dict = slo_config["spec"]["service_level_indicator"] + filter_good: str = measurement["filter_good"] + filter_bad: typing.Optional[str] = measurement.get("filter_bad") + filter_valid: typing.Optional[str] = measurement.get("filter_valid") # Query 'good events' timeseries - good_ts: List[TimeSeriesData] = self.query( - query=filter_good, window=window) + good_ts: List[TimeSeriesData] = self.query(query=filter_good, window=window) good_event_count: int = CM.count(good_ts) # Query 'bad events' timeseries bad_event_count: int if filter_bad: - bad_ts: List[TimeSeriesData] = self.query( - query=filter_bad, window=window) + bad_ts: List[TimeSeriesData] = self.query(query=filter_bad, window=window) bad_event_count = CM.count(bad_ts) elif filter_valid: valid_ts: List[TimeSeriesData] = self.query( - query=filter_valid, window=window) + query=filter_valid, window=window + ) bad_event_count = CM.count(valid_ts) - good_event_count else: - raise Exception( - "One of `filter_bad` or `filter_valid` is required.") + raise Exception("One of `filter_bad` or `filter_valid` is required.") - LOGGER.debug(f'Good events: {good_event_count} | ' - f'Bad events: {bad_event_count}') + LOGGER.debug( + f"Good events: {good_event_count} | " f"Bad events: {bad_event_count}" + ) return good_event_count, bad_event_count - def distribution_cut(self, - timestamp: int, # pylint: disable=unused-argument - window: int, - slo_config: dict) -> Tuple[int, int]: + # pylint: disable=too-many-locals,disable=unused-argument + def distribution_cut( + self, + timestamp: int, + window: int, + slo_config: dict, + ) -> Tuple[int, int]: """Query one timeseries of type 'exponential'. Args: @@ -113,12 +116,12 @@ def distribution_cut(self, Returns: tuple: A tuple (good_event_count, bad_event_count). """ - measurement: dict = slo_config['spec']['service_level_indicator'] - filter_valid: str = measurement['filter_valid'] - threshold_bucket: int = int(measurement['threshold_bucket']) + measurement: dict = slo_config["spec"]["service_level_indicator"] + filter_valid: str = measurement["filter_valid"] + threshold_bucket: int = int(measurement["threshold_bucket"]) good_below_threshold: typing.Optional[bool] = measurement.get( - 'good_below_threshold', - True) + "good_below_threshold", True + ) # Query 'valid' events series = self.query(query=filter_valid, window=window) @@ -126,8 +129,9 @@ def distribution_cut(self, if not series: return NO_DATA, NO_DATA # no timeseries - distribution_value: Distribution = series[0].point_data[0].values[ - 0].distribution_value + distribution_value: Distribution = ( + series[0].point_data[0].values[0].distribution_value + ) bucket_counts: list = distribution_value.bucket_counts valid_events_count: int = distribution_value.count @@ -136,9 +140,7 @@ def distribution_cut(self, distribution = OrderedDict() for i, bucket_count in enumerate(bucket_counts): count_sum += bucket_count - distribution[i] = { - 'count_sum': count_sum - } + distribution[i] = {"count_sum": count_sum} LOGGER.debug(pprint.pformat(distribution)) lower_events_count: int @@ -148,8 +150,7 @@ def distribution_cut(self, lower_events_count = valid_events_count upper_events_count = 0 else: - lower_events_count = distribution[threshold_bucket][ - 'count_sum'] + lower_events_count = distribution[threshold_bucket]["count_sum"] upper_events_count = valid_events_count - lower_events_count good_event_count: int @@ -168,13 +169,18 @@ def exponential_distribution_cut(self, *args, **kwargs) -> Tuple[int, int]: compatibility. """ warnings.warn( - 'exponential_distribution_cut will be deprecated in version 2.0, ' - 'please use distribution_cut instead', FutureWarning) + "exponential_distribution_cut will be deprecated in version 2.0, " + "please use distribution_cut instead", + FutureWarning, + ) return self.distribution_cut(*args, **kwargs) - def query_sli(self, - timestamp: int, # pylint: disable=unused-argument - window: int, slo_config: dict) -> float: + def query_sli( + self, + timestamp: int, # pylint: disable=unused-argument + window: int, + slo_config: dict, + ) -> float: """Query SLI value from a given MQL query. Args: @@ -185,8 +191,8 @@ def query_sli(self, Returns: float: SLI value. """ - measurement: dict = slo_config['spec']['service_level_indicator'] - query: str = measurement['query'] + measurement: dict = slo_config["spec"]["service_level_indicator"] + query: str = measurement["query"] series: List[TimeSeriesData] = self.query(query=query, window=window) sli_value: float = series[0].point_data[0].values[0].double_value LOGGER.debug(f"SLI value: {sli_value}") @@ -205,14 +211,14 @@ def query(self, query: str, window: int) -> List[TimeSeriesData]: # Enrich query to aggregate and reduce the time series over the # desired window. formatted_query: str = self._fmt_query(query, window) - request = metric_service.QueryTimeSeriesRequest({ - 'name': self.parent, - 'query': formatted_query - }) - + request = metric_service.QueryTimeSeriesRequest( + {"name": self.parent, "query": formatted_query} + ) + # fmt: off timeseries_pager: QueryTimeSeriesPager = ( self.client.query_time_series(request) # type: ignore[union-attr] ) + # fmt: on timeseries: list = list(timeseries_pager) # convert pager to flat list LOGGER.debug(pprint.pformat(timeseries)) return timeseries @@ -255,19 +261,21 @@ def _fmt_query(query: str, window: int) -> str: str: Formatted query. """ formatted_query: str = query.strip() - if 'group_by' in formatted_query: - formatted_query = re.sub(r'\|\s+group_by\s+\[.*\]\s*', - '| group_by [] ', formatted_query) + if "group_by" in formatted_query: + formatted_query = re.sub( + r"\|\s+group_by\s+\[.*\]\s*", "| group_by [] ", formatted_query + ) else: - formatted_query += '| group_by [] ' - for mql_time_interval_keyword in ['within', 'every']: + formatted_query += "| group_by [] " + for mql_time_interval_keyword in ["within", "every"]: if mql_time_interval_keyword in formatted_query: formatted_query = re.sub( - fr'\|\s+{mql_time_interval_keyword}\s+\w+\s*', - f'| {mql_time_interval_keyword} {window}s ', - formatted_query) + rf"\|\s+{mql_time_interval_keyword}\s+\w+\s*", + f"| {mql_time_interval_keyword} {window}s ", + formatted_query, + ) else: - formatted_query += f'| {mql_time_interval_keyword} {window}s ' + formatted_query += f"| {mql_time_interval_keyword} {window}s " return formatted_query.strip() diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 4668b43f..98ccf462 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -20,29 +20,32 @@ import logging import os import warnings -from typing import Optional, Union, Sequence +from typing import Optional, Sequence, Union import google.api_core.exceptions from google.cloud.monitoring_v3 import ServiceMonitoringServiceClient + # pytype: disable=pyi-error from google.protobuf.json_format import MessageToJson -# pytype: enable=pyi-error from slo_generator.backends.cloud_monitoring import CloudMonitoringBackend from slo_generator.constants import NO_DATA from slo_generator.utils import dict_snake_to_caml +# pytype: enable=pyi-error LOGGER = logging.getLogger(__name__) -SID_GAE: str = 'gae:{project_id}_{module_id}' -SID_CLOUD_ENDPOINT: str = 'ist:{project_id}-{service}' +SID_GAE: str = "gae:{project_id}_{module_id}" +SID_CLOUD_ENDPOINT: str = "ist:{project_id}-{service}" SID_CLUSTER_ISTIO: str = ( - 'ist:{project_id}-{suffix}-{location}-{cluster_name}-{service_namespace}-' - '{service_name}') -SID_MESH_ISTIO: str = ('ist:{mesh_uid}-{service_namespace}-{service_name}') + "ist:{project_id}-{suffix}-{location}-{cluster_name}-{service_namespace}-" + "{service_name}" +) +SID_MESH_ISTIO: str = "ist:{mesh_uid}-{service_namespace}-{service_name}" +# pylint: disable=too-many-public-methods class CloudServiceMonitoringBackend: """Cloud Service Monitoring backend class. @@ -59,13 +62,10 @@ def __init__(self, project_id: str, client=None): if client is None: self.client = ServiceMonitoringServiceClient() self.parent = self.client.project_path(project_id) - self.workspace_path = f'workspaces/{project_id}' - self.project_path = f'projects/{project_id}' + self.workspace_path = f"workspaces/{project_id}" + self.project_path = f"projects/{project_id}" - def good_bad_ratio(self, - timestamp: int, - window: int, - slo_config: dict) -> tuple: + def good_bad_ratio(self, timestamp: int, window: int, slo_config: dict) -> tuple: """Good bad ratio method. Args: @@ -78,10 +78,7 @@ def good_bad_ratio(self, """ return self.retrieve_slo(timestamp, window, slo_config) - def distribution_cut(self, - timestamp: int, - window: int, - slo_config: dict) -> tuple: + def distribution_cut(self, timestamp: int, window: int, slo_config: dict) -> tuple: """Distribution cut method. Args: @@ -122,10 +119,7 @@ def window(self, timestamp: int, window: int, slo_config: dict) -> tuple: return self.retrieve_slo(timestamp, window, slo_config) # pylint: disable=unused-argument - def delete(self, - timestamp: int, - window: int, - slo_config: dict) -> Optional[dict]: + def delete(self, timestamp: int, window: int, slo_config: dict) -> Optional[dict]: """Delete method. Args: @@ -164,7 +158,8 @@ def retrieve_slo(self, timestamp: int, window: int, slo_config: dict): # Now that we have our SLO, retrieve the TimeSeries from Cloud # Monitoring API for that particular SLO id. metric_filter = self.build_slo_id(window, slo_config, full=True) - filter = f"select_slo_counts(\"{metric_filter}\")" + # pylint: disable=redefined-builtin + filter = f'select_slo_counts("{metric_filter}")' # Query SLO timeseries cloud_monitoring = CloudMonitoringBackend(self.project_id) @@ -172,9 +167,10 @@ def retrieve_slo(self, timestamp: int, window: int, slo_config: dict): timestamp, window, filter, - aligner='ALIGN_SUM', - reducer='REDUCE_SUM', - group_by=['metric.labels.event_type']) + aligner="ALIGN_SUM", + reducer="REDUCE_SUM", + group_by=["metric.labels.event_type"], + ) timeseries = list(timeseries) good_event_count, bad_event_count = SSM.count(timeseries) return (good_event_count, bad_event_count) @@ -192,16 +188,15 @@ def count(timeseries: list): """ good_event_count, bad_event_count = NO_DATA, NO_DATA for timeserie in timeseries: - event_type = timeserie.metric.labels['event_type'] + event_type = timeserie.metric.labels["event_type"] value = timeserie.points[0].value.double_value - if event_type == 'bad': + if event_type == "bad": bad_event_count = value - elif event_type == 'good': + elif event_type == "good": good_event_count = value return good_event_count, bad_event_count - def create_service(self, - slo_config: dict) -> dict: + def create_service(self, slo_config: dict) -> dict: """Create Service object in Cloud Service Monitoring API. Args: @@ -213,11 +208,15 @@ def create_service(self, LOGGER.debug("Creating service ...") service_json = self.build_service(slo_config) service_id = self.build_service_id(slo_config) - service = self.client.create_service(self.project_path, - service_json, - service_id=service_id) - LOGGER.info(f'Service "{service_id}" created successfully in Cloud ' - f'Service Monitoring API.') + service = self.client.create_service( + self.project_path, + service_json, + service_id=service_id, + ) + LOGGER.info( + f'Service "{service_id}" created successfully in Cloud ' + f"Service Monitoring API." + ) return SSM.to_json(service) def get_service(self, slo_config: dict) -> Optional[dict]: @@ -234,21 +233,21 @@ def get_service(self, slo_config: dict) -> Optional[dict]: service_id = self.build_service_id(slo_config) services = list(self.client.list_services(self.workspace_path)) matches = [ - service for service in services - if service.name.split("/")[-1] == service_id + service for service in services if service.name.split("/")[-1] == service_id ] # If no match is found for our service name in the API, raise an # exception if the service should have been auto-added (method 'basic'), # else output a warning message. if not matches: - msg = (f'Service "{service_id}" does not exist in ' - f'workspace "{self.project_id}"') - method = slo_config['spec']['method'] - if method == 'basic': + msg = ( + f'Service "{service_id}" does not exist in ' + f'workspace "{self.project_id}"' + ) + method = slo_config["spec"]["method"] + if method == "basic": sids = [service.name.split("/")[-1] for service in services] - LOGGER.debug( - f'List of services in workspace {self.project_id}: {sids}') + LOGGER.debug(f"List of services in workspace {self.project_id}: {sids}") raise Exception(msg) LOGGER.error(msg) return None @@ -269,13 +268,15 @@ def build_service(self, slo_config: dict) -> dict: dict: Service JSON in Cloud Monitoring API. """ service_id = self.build_service_id(slo_config) - display_name = slo_config.get('service_display_name', service_id) - return {'display_name': display_name, 'custom': {}} - - def build_service_id(self, - slo_config: dict, - dest_project_id: Optional[str] = None, - full: bool = False): + display_name = slo_config.get("service_display_name", service_id) + return {"display_name": display_name, "custom": {}} + + def build_service_id( + self, + slo_config: dict, + dest_project_id: Optional[str] = None, + full: bool = False, + ): """Build service id from SLO configuration. Args: @@ -289,55 +290,55 @@ def build_service_id(self, str: Service id. """ project_id = self.project_id - measurement = slo_config['spec']['service_level_indicator'] - app_engine = measurement.get('app_engine') - cluster_istio = measurement.get('cluster_istio') - mesh_istio = measurement.get('mesh_istio') - cloud_endpoints = measurement.get('cloud_endpoints') + measurement = slo_config["spec"]["service_level_indicator"] + app_engine = measurement.get("app_engine") + cluster_istio = measurement.get("cluster_istio") + mesh_istio = measurement.get("mesh_istio") + cloud_endpoints = measurement.get("cloud_endpoints") # Use auto-generated ids for 'custom' SLOs, use system-generated ids # for all other types of SLOs. if app_engine: service_id = SID_GAE.format_map(app_engine) - dest_project_id = app_engine['project_id'] + dest_project_id = app_engine["project_id"] elif cluster_istio: warnings.warn( - 'ClusterIstio is deprecated in the Service Monitoring API.' - 'It will be removed in version 3.0, please use MeshIstio ' - 'instead', FutureWarning) - if 'zone' in cluster_istio: - cluster_istio['suffix'] = 'zone' - cluster_istio['location'] = cluster_istio['zone'] - elif 'location' in cluster_istio: - cluster_istio['suffix'] = 'location' + "ClusterIstio is deprecated in the Service Monitoring API." + "It will be removed in version 3.0, please use MeshIstio " + "instead", + FutureWarning, + ) + if "zone" in cluster_istio: + cluster_istio["suffix"] = "zone" + cluster_istio["location"] = cluster_istio["zone"] + elif "location" in cluster_istio: + cluster_istio["suffix"] = "location" service_id = SID_CLUSTER_ISTIO.format_map(cluster_istio) - dest_project_id = cluster_istio['project_id'] + dest_project_id = cluster_istio["project_id"] elif mesh_istio: service_id = SID_MESH_ISTIO.format_map(mesh_istio) elif cloud_endpoints: service_id = SID_CLOUD_ENDPOINT.format_map(cloud_endpoints) - dest_project_id = cluster_istio['project_id'] + dest_project_id = cluster_istio["project_id"] else: # user-defined service id - service_name = slo_config['metadata']['labels'].get( - 'service_name', '') - feature_name = slo_config['metadata']['labels'].get( - 'feature_name', '') - service_id = slo_config['spec']['service_level_indicator'].get( - 'service_id') + service_name = slo_config["metadata"]["labels"].get("service_name", "") + feature_name = slo_config["metadata"]["labels"].get("feature_name", "") + service_id = slo_config["spec"]["service_level_indicator"].get("service_id") if not service_id: if not service_name or not feature_name: raise Exception( - 'Service id not set in SLO configuration. Please set ' - 'either `spec.service_level_indicator.service_id` or ' - 'both `metadata.labels.service_name` and ' - '`metadata.labels.feature_name` in your SLO ' - 'configuration.') - service_id = f'{service_name}-{feature_name}' + "Service id not set in SLO configuration. Please set " + "either `spec.service_level_indicator.service_id` or " + "both `metadata.labels.service_name` and " + "`metadata.labels.feature_name` in your SLO " + "configuration." + ) + service_id = f"{service_name}-{feature_name}" if full: if dest_project_id: - return f'projects/{dest_project_id}/services/{service_id}' - return f'projects/{project_id}/services/{service_id}' + return f"projects/{dest_project_id}/services/{service_id}" + return f"projects/{project_id}/services/{service_id}" return service_id @@ -355,13 +356,14 @@ def create_slo(self, window: int, slo_config: dict) -> dict: slo_id = self.build_slo_id(window, slo_config) parent = self.build_service_id(slo_config, full=True) slo = self.client.create_service_level_objective( - parent, slo_json, service_level_objective_id=slo_id) + parent, slo_json, service_level_objective_id=slo_id + ) return SSM.to_json(slo) # pylint: disable=R0912,R0915 @staticmethod - def build_slo(window: int, - slo_config: dict) -> dict: + # pylint: disable=R0912,R0915,too-many-locals + def build_slo(window: int, slo_config: dict) -> dict: """Get SLO JSON representation in Cloud Service Monitoring API from SLO configuration. @@ -372,87 +374,86 @@ def build_slo(window: int, Returns: dict: SLO JSON configuration. """ - measurement = slo_config['spec'].get('service_level_indicator', {}) - method = slo_config['spec']['method'] - description = slo_config['spec']['description'] - goal = slo_config['spec']['goal'] + measurement = slo_config["spec"].get("service_level_indicator", {}) + method = slo_config["spec"]["method"] + description = slo_config["spec"]["description"] + goal = slo_config["spec"]["goal"] minutes, _ = divmod(window, 60) hours, _ = divmod(minutes, 60) - display_name = f'{description} ({hours}h)' + display_name = f"{description} ({hours}h)" slo = { - 'display_name': display_name, - 'goal': goal, - 'rolling_period': { - 'seconds': window - } + "display_name": display_name, + "goal": goal, + "rolling_period": {"seconds": window}, } - filter_valid = measurement.get('filter_valid', "") - if method == 'basic': - methods = measurement.get('method', []) - locations = measurement.get('location', []) - versions = measurement.get('version', []) - threshold = measurement.get('latency', {}).get('threshold') - slo['service_level_indicator'] = {'basic_sli': {}} - basic_sli = slo['service_level_indicator']['basic_sli'] + filter_valid = measurement.get("filter_valid", "") + if method == "basic": + methods = measurement.get("method", []) + locations = measurement.get("location", []) + versions = measurement.get("version", []) + threshold = measurement.get("latency", {}).get("threshold") + slo["service_level_indicator"] = {"basic_sli": {}} + basic_sli = slo["service_level_indicator"]["basic_sli"] if methods: - basic_sli['method'] = methods + basic_sli["method"] = methods if locations: - basic_sli['location'] = locations + basic_sli["location"] = locations if versions: - basic_sli['version'] = versions + basic_sli["version"] = versions if threshold: - basic_sli['latency'] = { - 'threshold': { - 'seconds': 0, - 'nanos': int(threshold) * 10**6 + basic_sli["latency"] = { + "threshold": { + "seconds": 0, + "nanos": int(threshold) * 10**6, } } else: - basic_sli['availability'] = {} - - elif method == 'good_bad_ratio': - filter_good = measurement.get('filter_good', "") - filter_bad = measurement.get('filter_bad', "") - slo['service_level_indicator'] = { - 'request_based': { - 'good_total_ratio': {} + basic_sli["availability"] = {} + + elif method == "good_bad_ratio": + filter_good = measurement.get("filter_good", "") + filter_bad = measurement.get("filter_bad", "") + slo["service_level_indicator"] = { + "request_based": { + "good_total_ratio": {}, } } - sli = slo['service_level_indicator'] - ratio = sli['request_based']['good_total_ratio'] + sli = slo["service_level_indicator"] + ratio = sli["request_based"]["good_total_ratio"] if filter_good: - ratio['good_service_filter'] = filter_good + ratio["good_service_filter"] = filter_good if filter_bad: - ratio['bad_service_filter'] = filter_bad + ratio["bad_service_filter"] = filter_bad if filter_valid: - ratio['total_service_filter'] = filter_valid - - elif method == 'distribution_cut': - range_min = measurement.get('range_min', 0) - range_max = measurement['range_max'] - slo['service_level_indicator'] = { - 'request_based': { - 'distribution_cut': { - 'distribution_filter': filter_valid, - 'range': { - 'max': float(range_max) - } + ratio["total_service_filter"] = filter_valid + + elif method == "distribution_cut": + range_min = measurement.get("range_min", 0) + range_max = measurement["range_max"] + slo["service_level_indicator"] = { + "request_based": { + "distribution_cut": { + "distribution_filter": filter_valid, + "range": { + "max": float(range_max), + }, } } } - sli = slo['service_level_indicator']['request_based'] + sli = slo["service_level_indicator"]["request_based"] if range_min != 0: - sli['distribution_cut']['range']['min'] = float(range_min) + sli["distribution_cut"]["range"]["min"] = float(range_min) - elif method == 'windows': - filter = measurement.get('filter') + elif method == "windows": + # pylint: disable=redefined-builtin + filter = measurement.get("filter") # threshold = conf.get('threshold') # mean_in_range = conf.get('filter') # sum_in_range = conf.get('filter') - slo['service_level_indicator'] = { - 'windows_based': { - 'window_period': window, - 'good_bad_metric_filter': filter, + slo["service_level_indicator"] = { + "windows_based": { + "window_period": window, + "good_bad_metric_filter": filter, # 'good_total_ratio_threshold': { # object (PerformanceThreshold) # }, @@ -488,18 +489,18 @@ def get_slo(self, window: int, slo_config: dict) -> Optional[dict]: # Loop through API response to find an existing SLO that corresponds to # our configuration. for slo in slos: - slo_remote_id = slo['name'].split("/")[-1] + slo_remote_id = slo["name"].split("/")[-1] equal = slo_remote_id == slo_local_id if equal: LOGGER.debug(f'Found existing SLO "{slo_remote_id}".') - LOGGER.debug(f'SLO object: {slo}') + LOGGER.debug(f"SLO object: {slo}") strict_equal = SSM.compare_slo(slo_json, slo) if strict_equal: return slo return self.update_slo(window, slo_config) - LOGGER.warning('No SLO found matching configuration.') - LOGGER.debug(f'SLOs from Cloud Service Monitoring API: {slos}') - LOGGER.debug(f'SLO config converted: {slo_json}') + LOGGER.warning("No SLO found matching configuration.") + LOGGER.debug(f"SLOs from Cloud Service Monitoring API: {slos}") + LOGGER.debug(f"SLO config converted: {slo_json}") return None def update_slo(self, window: int, slo_config: dict) -> dict: @@ -515,7 +516,7 @@ def update_slo(self, window: int, slo_config: dict) -> dict: slo_json = SSM.build_slo(window, slo_config) slo_id = self.build_slo_id(window, slo_config, full=True) LOGGER.warning(f"Updating SLO {slo_id} ...") - slo_json['name'] = slo_id + slo_json["name"] = slo_id return SSM.to_json(self.client.update_service_level_objective(slo_json)) def list_slos(self, service_path: str) -> list: @@ -551,13 +552,11 @@ def delete_slo(self, window: int, slo_config: dict) -> Optional[dict]: except google.api_core.exceptions.NotFound: LOGGER.warning( f'SLO "{slo_path}" does not exist in Service Monitoring API. ' - f'Skipping.') + f"Skipping." + ) return None - def build_slo_id(self, - window: int, - slo_config: dict, - full: bool = False) -> str: + def build_slo_id(self, window: int, slo_config: dict, full: bool = False) -> str: """Build SLO id from SLO configuration. Args: @@ -567,18 +566,19 @@ def build_slo_id(self, Returns: str: SLO id. """ - sli = slo_config['spec']['service_level_indicator'] - slo_name = slo_config['metadata']['labels'].get('slo_name') - slo_id = sli.get('slo_id', slo_name) + sli = slo_config["spec"]["service_level_indicator"] + slo_name = slo_config["metadata"]["labels"].get("slo_name") + slo_id = sli.get("slo_id", slo_name) if not slo_id: raise Exception( - 'SLO id not set in SLO configuration. Please set either ' - '`spec.service_level_indicator.slo_id` or ' - '`metadata.labels.slo_name` in your SLO configuration.') - full_slo_id = f'{slo_id}-{window}' + "SLO id not set in SLO configuration. Please set either " + "`spec.service_level_indicator.slo_id` or " + "`metadata.labels.slo_name` in your SLO configuration." + ) + full_slo_id = f"{slo_id}-{window}" if full: service_path = self.build_service_id(slo_config, full=True) - return f'{service_path}/serviceLevelObjectives/{full_slo_id}' + return f"{service_path}/serviceLevelObjectives/{full_slo_id}" return full_slo_id @staticmethod @@ -601,7 +601,7 @@ def compare_slo(slo1: dict, slo2: dict) -> bool: slo2_copy = {k: v for k, v in slo2.items() if k not in exclude_keys} local_json = json.dumps(slo1_copy, sort_keys=True) remote_json = json.dumps(slo2_copy, sort_keys=True) - if os.environ.get('DEBUG') == '2': + if os.environ.get("DEBUG") == "2": LOGGER.info("----------") LOGGER.info(local_json) LOGGER.info("----------") @@ -611,8 +611,9 @@ def compare_slo(slo1: dict, slo2: dict) -> bool: return local_json == remote_json @staticmethod - def string_diff(string1: Union[str, Sequence[str]], - string2: Union[str, Sequence[str]]) -> list: + def string_diff( + string1: Union[str, Sequence[str]], string2: Union[str, Sequence[str]] + ) -> list: """Diff 2 strings. Used to print comparison of JSONs for debugging. Args: @@ -624,13 +625,13 @@ def string_diff(string1: Union[str, Sequence[str]], """ lines = [] for idx, string in enumerate(difflib.ndiff(string1, string2)): - if string[0] == ' ': + if string[0] == " ": continue last = string[-1] - if string[0] == '-': + if string[0] == "-": info = f'Delete "{last}" from position {idx}' lines.append(info) - elif string[0] == '+': + elif string[0] == "+": info = f'Add "{last}" to position {idx}' lines.append(info) return lines @@ -652,16 +653,16 @@ def convert_slo_to_ssm_format(slo: dict) -> dict: # The `rollingPeriod` field is in Duration format, convert it. try: - period = data['rollingPeriod'] - data['rollingPeriod'] = SSM.convert_duration_to_string(period) + period = data["rollingPeriod"] + data["rollingPeriod"] = SSM.convert_duration_to_string(period) except KeyError: pass # The `latency` field is in Duration format, convert it. try: - latency = data['serviceLevelIndicator']['basicSli']['latency'] - threshold = latency['threshold'] - latency['threshold'] = SSM.convert_duration_to_string(threshold) + latency = data["serviceLevelIndicator"]["basicSli"]["latency"] + threshold = latency["threshold"] + latency["threshold"] = SSM.convert_duration_to_string(threshold) except KeyError: pass @@ -678,15 +679,15 @@ def convert_duration_to_string(duration): str: Duration string. """ duration_seconds = 0.000 - if 'seconds' in duration: - duration_seconds += duration['seconds'] - if 'nanos' in duration: - duration_seconds += duration['nanos'] * 10**(-9) + if "seconds" in duration: + duration_seconds += duration["seconds"] + if "nanos" in duration: + duration_seconds += duration["nanos"] * 10 ** (-9) if duration_seconds.is_integer(): duration_str = int(duration_seconds) else: - duration_str = f'{duration_seconds:0.3f}' - return str(duration_str) + 's' + duration_str = f"{duration_seconds:0.3f}" + return str(duration_str) + "s" @staticmethod def to_json(response): diff --git a/slo_generator/backends/datadog.py b/slo_generator/backends/datadog.py index d5fd6f31..6b74874f 100644 --- a/slo_generator/backends/datadog.py +++ b/slo_generator/backends/datadog.py @@ -18,10 +18,11 @@ import logging import pprint + import datadog LOGGER = logging.getLogger(__name__) -logging.getLogger('datadog.api').setLevel(logging.ERROR) +logging.getLogger("datadog.api").setLevel(logging.ERROR) class DatadogBackend: @@ -37,11 +38,12 @@ class DatadogBackend: def __init__(self, client=None, api_key=None, app_key=None, **kwargs): self.client = client if not self.client: - options = {'api_key': api_key, 'app_key': app_key} + options = {"api_key": api_key, "app_key": app_key} options.update(kwargs) datadog.initialize(**options) self.client = datadog.api + # pylint: disable=too-many-locals def good_bad_ratio(self, timestamp, window, slo_config): """Query SLI value from good and valid queries. @@ -53,23 +55,35 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: Good event count, Bad event count. """ - measurement = slo_config['spec']['service_level_indicator'] - operator = measurement.get('operator', 'sum') - operator_suffix = measurement.get('operator_suffix', 'as_count()') + measurement = slo_config["spec"]["service_level_indicator"] + operator = measurement.get("operator", "sum") + operator_suffix = measurement.get("operator_suffix", "as_count()") start = timestamp - window end = timestamp - query_good = measurement['query_good'] - query_valid = measurement['query_valid'] - query_good = self._fmt_query(query_good, window, operator, - operator_suffix) - query_valid = self._fmt_query(query_valid, window, operator, - operator_suffix) - good_event_query = self.client.Metric.query(start=start, - end=end, - query=query_good) - valid_event_query = self.client.Metric.query(start=start, - end=end, - query=query_valid) + query_good = measurement["query_good"] + query_valid = measurement["query_valid"] + query_good = self._fmt_query( + query_good, + window, + operator, + operator_suffix, + ) + query_valid = self._fmt_query( + query_valid, + window, + operator, + operator_suffix, + ) + good_event_query = self.client.Metric.query( + start=start, + end=end, + query=query_good, + ) + valid_event_query = self.client.Metric.query( + start=start, + end=end, + query=query_valid, + ) LOGGER.debug(f"Result good: {pprint.pformat(good_event_query)}") LOGGER.debug(f"Result valid: {pprint.pformat(valid_event_query)}") good_event_count = DatadogBackend.count(good_event_query) @@ -88,10 +102,10 @@ def query_sli(self, timestamp, window, slo_config): Returns: float: SLI value. """ - measurement = slo_config['spec']['service_level_indicator'] + measurement = slo_config["spec"]["service_level_indicator"] start = timestamp - window end = timestamp - query = measurement['query'] + query = measurement["query"] query = self._fmt_query(query, window) response = self.client.Metric.query(start=start, end=end, query=query) LOGGER.debug(f"Result valid: {pprint.pformat(response)}") @@ -108,22 +122,23 @@ def query_slo(self, timestamp, window, slo_config): Returns: tuple: Good event count, bad event count. """ - slo_id = slo_config['spec']['service_level_indicator']['slo_id'] + slo_id = slo_config["spec"]["service_level_indicator"]["slo_id"] from_ts = timestamp - window slo_data = self.client.ServiceLevelObjective.get(id=slo_id) LOGGER.debug(f"SLO data: {slo_id} | Result: {pprint.pformat(slo_data)}") - data = self.client.ServiceLevelObjective.history(id=slo_id, - from_ts=from_ts, - to_ts=timestamp) + data = self.client.ServiceLevelObjective.history( + id=slo_id, + from_ts=from_ts, + to_ts=timestamp, + ) try: - LOGGER.debug( - f"Timeseries data: {slo_id} | Result: {pprint.pformat(data)}") - good_event_count = data['data']['series']['numerator']['sum'] - valid_event_count = data['data']['series']['denominator']['sum'] + LOGGER.debug(f"Timeseries data: {slo_id} | Result: {pprint.pformat(data)}") + good_event_count = data["data"]["series"]["numerator"]["sum"] + valid_event_count = data["data"]["series"]["denominator"]["sum"] bad_event_count = valid_event_count - good_event_count return (good_event_count, bad_event_count) except (KeyError) as exception: # monitor-based SLI - sli_value = data['data']['overall']['sli_value'] / 100 + sli_value = data["data"]["overall"]["sli_value"] / 100 LOGGER.debug(exception) return sli_value @@ -149,12 +164,12 @@ def _fmt_query(query, window, operator=None, operator_suffix=None): """ query = query.strip() if operator: - query = f'{operator}:{query}' - if '[window]' in query: - query = query.replace('[window]', f'{window}') + query = f"{operator}:{query}" + if "[window]" in query: + query = query.replace("[window]", f"{window}") if operator_suffix: - query = f'{query}.{operator_suffix}' - LOGGER.debug(f'Query: {query}') + query = f"{query}.{operator_suffix}" + LOGGER.debug(f"Query: {query}") return query @staticmethod @@ -170,7 +185,7 @@ def count(response, average=False): """ try: values = [] - pointlist = response['series'][0]['pointlist'] + pointlist = response["series"][0]["pointlist"] for point in pointlist: value = point[1] if value is None: diff --git a/slo_generator/backends/dynatrace.py b/slo_generator/backends/dynatrace.py index 87172bf7..410ee673 100644 --- a/slo_generator/backends/dynatrace.py +++ b/slo_generator/backends/dynatrace.py @@ -18,9 +18,10 @@ import json import logging import pprint -import requests +import requests from retrying import retry + from slo_generator.constants import NO_DATA LOGGER = logging.getLogger(__name__) @@ -51,13 +52,13 @@ def query_sli(self, timestamp, window, slo_config): Returns: float: SLI value. """ - measurement = slo_config['spec']['service_level_indicator'] + measurement = slo_config["spec"]["service_level_indicator"] start = (timestamp - window) * 1000 end = timestamp * 1000 - slo_id = measurement['slo_id'] + slo_id = measurement["slo_id"] data = self.retrieve_slo(start, end, slo_id) LOGGER.debug(f"Result SLO: {pprint.pformat(data)}") - sli_value = round(data['evaluatedPercentage'] / 100, 4) + sli_value = round(data["evaluatedPercentage"] / 100, 4) return sli_value def good_bad_ratio(self, timestamp, window, slo_config): @@ -71,11 +72,11 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: Good event count, Bad event count. """ - measurement = slo_config['spec']['service_level_indicator'] + measurement = slo_config["spec"]["service_level_indicator"] start = (timestamp - window) * 1000 end = timestamp * 1000 - query_good = measurement['query_good'] - query_valid = measurement['query_valid'] + query_good = measurement["query_good"] + query_valid = measurement["query_valid"] # Good query good_event_response = self.query(start=start, end=end, **query_good) @@ -103,23 +104,27 @@ def threshold(self, timestamp, window, slo_config): Returns: tuple: Good event count, Bad event count. """ - measurement = slo_config['spec']['service_level_indicator'] + measurement = slo_config["spec"]["service_level_indicator"] start = (timestamp - window) * 1000 end = timestamp * 1000 - query_valid = measurement['query_valid'] - threshold = measurement['threshold'] - good_below_threshold = measurement.get('good_below_threshold', True) + query_valid = measurement["query_valid"] + threshold = measurement["threshold"] + good_below_threshold = measurement.get("good_below_threshold", True) response = self.query(start=start, end=end, **query_valid) LOGGER.debug(f"Result valid: {pprint.pformat(response)}") - return DynatraceBackend.count_threshold(response, threshold, - good_below_threshold) - - def query(self, - start, - end, - metric_selector=None, - entity_selector=None, - aggregation='SUM'): + return DynatraceBackend.count_threshold( + response, threshold, good_below_threshold + ) + + # pylint: disable=too-many-arguments + def query( + self, + start, + end, + metric_selector=None, + entity_selector=None, + aggregation="SUM", + ): """Query Dynatrace Metrics V2. Args: @@ -133,22 +138,16 @@ def query(self, dict: Dynatrace API response. """ params = { - 'from': start, - 'end': end, - 'metricSelector': metric_selector, - 'entitySelector': entity_selector, - 'aggregation': aggregation, - 'includeData': True + "from": start, + "end": end, + "metricSelector": metric_selector, + "entitySelector": entity_selector, + "aggregation": aggregation, + "includeData": True, } - return self.client.request('get', - 'metrics/query', - version='v2', - **params) - - def retrieve_slo(self, - start, - end, - slo_id): + return self.client.request("get", "metrics/query", version="v2", **params) + + def retrieve_slo(self, start, end, slo_id): """Query Dynatrace SLO V2. Args: @@ -159,16 +158,9 @@ def retrieve_slo(self, Returns: dict: Dynatrace API response. """ - params = { - 'from': start, - 'to': end, - 'timeFrame' : 'GTF' - } - endpoint = 'slo/' + slo_id - return self.client.request('get', - endpoint, - version='v2', - **params) + params = {"from": start, "to": end, "timeFrame": "GTF"} + endpoint = "slo/" + slo_id + return self.client.request("get", endpoint, version="v2", **params) @staticmethod def count(response): @@ -181,11 +173,12 @@ def count(response): int: Event count. """ try: - datapoints = response['result'][0]['data'] + datapoints = response["result"][0]["data"] values = [] for point in datapoints: point_values = [ - point for point in point['values'] + point + for point in point["values"] if point is not None and point > 0 ] values.extend(point_values) @@ -209,16 +202,18 @@ def count_threshold(response, threshold, good_below_threshold=True): tuple: Number of good events, Number of bad events. """ try: - datapoints = response['result'][0]['data'] + datapoints = response["result"][0]["data"] below = [] above = [] for point in datapoints: points_below = [ - point for point in point['values'] + point + for point in point["values"] if point is not None and point < threshold ] points_above = [ - point for point in point['values'] + point + for point in point["values"] if point is not None and point > threshold ] below.extend(points_below) @@ -244,11 +239,11 @@ def retry_http(response): bool: True to retry, False otherwise. """ retry_codes = [429] - returned_code = response.get('error', {}) + returned_code = response.get("error", {}) if isinstance(returned_code, str): code = 200 else: - code = int(returned_code.get('code', 200)) + code = int(returned_code.get("code", 200)) return code in retry_codes @@ -259,26 +254,32 @@ class DynatraceClient: api_url (str): Dynatrace API URL. api_token (str): Dynatrace token. """ + # Keys to extract response data for each endpoint - ENDPOINT_KEYS = {'metrics': 'metrics', 'metrics/query': 'result'} + ENDPOINT_KEYS = {"metrics": "metrics", "metrics/query": "result"} def __init__(self, api_url, api_key): self.client = requests.Session() - self.url = api_url.rstrip('/') + self.url = api_url.rstrip("/") self.token = api_key - @retry(retry_on_result=retry_http, - wait_exponential_multiplier=1000, - wait_exponential_max=10000, - stop_max_delay=10000) - def request(self, - method, - endpoint, - name=None, - version='v1', - post_data=None, - key=None, - **params): + @retry( + retry_on_result=retry_http, + wait_exponential_multiplier=1000, + wait_exponential_max=10000, + stop_max_delay=10000, + ) + # pylint: disable=too-many-arguments,too-many-locals + def request( + self, + method, + endpoint, + name=None, + version="v1", + post_data=None, + key=None, + **params, + ): """Request Dynatrace API. Args: @@ -294,33 +295,34 @@ def request(self, obj: API response. """ req = getattr(self.client, method) - url = f'{self.url}/api/{version}/{endpoint}' - params['Api-Token'] = self.token + url = f"{self.url}/api/{version}/{endpoint}" + params["Api-Token"] = self.token headers = { - 'Accept': 'application/json', - 'Content-Type': 'application/json', - 'User-Agent': 'slo-generator' + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "slo-generator", } if name: - url += f'/{name}' - params_str = '&'.join( - f'{key}={val}' for key, val in params.items() if val is not None) - url += f'?{params_str}' + url += f"/{name}" + params_str = "&".join( + f"{key}={val}" for key, val in params.items() if val is not None + ) + url += f"?{params_str}" LOGGER.debug(f'Running "{method}" request to {url} ...') - if method in ['put', 'post']: + if method in ["put", "post"]: response = req(url, headers=headers, json=post_data) else: response = req(url, headers=headers) - LOGGER.debug(f'Response: {response}') + LOGGER.debug(f"Response: {response}") data = DynatraceClient.to_json(response) - next_page_key = data.get('nextPageKey') + next_page_key = data.get("nextPageKey") if next_page_key: - params = {'nextPageKey': next_page_key, 'Api-Token': self.token} - LOGGER.debug(f'Requesting next page: {next_page_key}') + params = {"nextPageKey": next_page_key, "Api-Token": self.token} + LOGGER.debug(f"Requesting next page: {next_page_key}") data_next = self.request(method, endpoint, name, version, **params) - next_page_key = data_next.get('nextPageKey') + next_page_key = data_next.get("nextPageKey") if not key: - key = DynatraceClient.ENDPOINT_KEYS.get(endpoint, 'result') + key = DynatraceClient.ENDPOINT_KEYS.get(endpoint, "result") data[key].extend(data_next[key]) return data @@ -335,6 +337,6 @@ def to_json(resp): Returns: dict: API JSON response. """ - res = resp.content.decode('utf-8').replace('\n', '') + res = resp.content.decode("utf-8").replace("\n", "") data = json.loads(res) return data diff --git a/slo_generator/backends/elasticsearch.py b/slo_generator/backends/elasticsearch.py index a16371e7..97511913 100644 --- a/slo_generator/backends/elasticsearch.py +++ b/slo_generator/backends/elasticsearch.py @@ -25,7 +25,7 @@ LOGGER = logging.getLogger(__name__) -DEFAULT_DATE_FIELD: str = '@timestamp' +DEFAULT_DATE_FIELD: str = "@timestamp" class ElasticsearchBackend: @@ -47,20 +47,19 @@ def __init__(self, client=None, **es_config): # copies. We require a full nested copy of the configuration to # work on. conf = copy.deepcopy(es_config) - url = conf.pop('url', None) - basic_auth = conf.pop('basic_auth', None) - api_key = conf.pop('api_key', None) + url = conf.pop("url", None) + basic_auth = conf.pop("basic_auth", None) + api_key = conf.pop("api_key", None) if url: - conf['hosts'] = url + conf["hosts"] = url if basic_auth: - conf['basic_auth'] = ( - basic_auth['username'], basic_auth['password']) + conf["basic_auth"] = (basic_auth["username"], basic_auth["password"]) if api_key: - conf['api_key'] = (api_key['id'], api_key['value']) + conf["api_key"] = (api_key["id"], api_key["value"]) # Note: Either `hosts` or `cloud_id` must be specified in v8.x.x self.client = Elasticsearch(**conf) - # pylint: disable=unused-argument + # pylint: disable=unused-argument,too-many-locals def good_bad_ratio(self, timestamp, window, slo_config): """Query two timeseries, one containing 'good' events, one containing 'bad' events. @@ -73,12 +72,12 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: A tuple (good_event_count, bad_event_count) """ - measurement = slo_config['spec']['service_level_indicator'] - index = measurement['index'] - query_good = measurement['query_good'] - query_bad = measurement.get('query_bad') - query_valid = measurement.get('query_valid') - date_field = measurement.get('date_field', DEFAULT_DATE_FIELD) + measurement = slo_config["spec"]["service_level_indicator"] + index = measurement["index"] + query_good = measurement["query_good"] + query_bad = measurement.get("query_bad") + query_valid = measurement.get("query_valid") + date_field = measurement.get("date_field", DEFAULT_DATE_FIELD) # Build ELK request bodies good = ES.build_query(query_good, window, date_field) @@ -124,7 +123,7 @@ def count(response): int: Event count. """ try: - return response['hits']['total']['value'] + return response["hits"]["total"]["value"] except KeyError as exception: LOGGER.warning("Couldn't find any values in timeseries response") LOGGER.debug(exception, exc_info=True) @@ -153,7 +152,7 @@ def build_query(query, window, date_field=DEFAULT_DATE_FIELD): range_query = { f"{date_field}": { "gte": f"now-{window}s/s", - "lt": "now/s" + "lt": "now/s", } } diff --git a/slo_generator/backends/prometheus.py b/slo_generator/backends/prometheus.py index 0bdd24e9..411d820d 100644 --- a/slo_generator/backends/prometheus.py +++ b/slo_generator/backends/prometheus.py @@ -36,9 +36,9 @@ def __init__(self, client=None, url=None, headers=None): self.client = client if not self.client: if url: - os.environ['PROMETHEUS_URL'] = url + os.environ["PROMETHEUS_URL"] = url if headers: - os.environ['PROMETHEUS_HEAD'] = json.dumps(headers) + os.environ["PROMETHEUS_HEAD"] = json.dumps(headers) self.client = Prometheus() def query_sli(self, timestamp, window, slo_config): @@ -52,8 +52,8 @@ def query_sli(self, timestamp, window, slo_config): Returns: float: SLI value. """ - measurement = slo_config['spec']['service_level_indicator'] - expr = measurement['expression'] + measurement = slo_config["spec"]["service_level_indicator"] + expr = measurement["expression"] response = self.query(expr, window, timestamp, operators=[]) sli_value = PrometheusBackend.count(response) LOGGER.debug(f"SLI value: {sli_value}") @@ -73,11 +73,11 @@ def good_bad_ratio(self, timestamp, window, slo_config): Returns: tuple: A tuple of (good_count, bad_count). """ - measurement = slo_config['spec']['service_level_indicator'] - good = measurement['filter_good'] - bad = measurement.get('filter_bad') - valid = measurement.get('filter_valid') - operators = measurement.get('operators', ['increase', 'sum']) + measurement = slo_config["spec"]["service_level_indicator"] + good = measurement["filter_good"] + bad = measurement.get("filter_bad") + valid = measurement.get("filter_valid") + operators = measurement.get("operators", ["increase", "sum"]) # Replace window by its value in the error budget policy step res = self.query(good, window, timestamp, operators) @@ -93,16 +93,13 @@ def good_bad_ratio(self, timestamp, window, slo_config): else: raise Exception("`filter_bad` or `filter_valid` is required.") - LOGGER.debug(f'Good events: {good_count} | ' f'Bad events: {bad_count}') + LOGGER.debug(f"Good events: {good_count} | " f"Bad events: {bad_count}") return (good_count, bad_count) # pylint: disable=unused-argument def distribution_cut( - self, - timestamp: int, - window: int, - slo_config: dict + self, timestamp: int, window: int, slo_config: dict ) -> Tuple[float, float]: """Query events for distributions (histograms). @@ -114,36 +111,43 @@ def distribution_cut( Returns: float: SLI value. """ - measurement = slo_config['spec']['service_level_indicator'] - expr = measurement['expression'] - threshold_bucket = measurement['threshold_bucket'] + measurement = slo_config["spec"]["service_level_indicator"] + expr = measurement["expression"] + threshold_bucket = measurement["threshold_bucket"] labels = {"le": threshold_bucket} - res_good = self.query(expr, - window, - operators=['increase', 'sum'], - labels=labels) + res_good = self.query( + expr, + window, + operators=["increase", "sum"], + labels=labels, + ) good_count = PrometheusBackend.count(res_good) # We use the _count metric to figure out the 'valid count'. # Trying to get the valid count from the _bucket metric query is hard # due to Prometheus 'le' syntax that doesn't have the alternative 'ge' # See https://github.com/prometheus/prometheus/issues/2018. - expr_count = expr.replace('_bucket', '_count') - res_valid = self.query(expr_count, - window, - operators=['increase', 'sum']) + expr_count = expr.replace("_bucket", "_count") + res_valid = self.query( + expr_count, + window, + operators=["increase", "sum"], + ) valid_count = PrometheusBackend.count(res_valid) bad_count = valid_count - good_count - LOGGER.debug(f'Good events: {good_count} | ' f'Bad events: {bad_count}') + LOGGER.debug(f"Good events: {good_count} | " f"Bad events: {bad_count}") return (good_count, bad_count) - # pylint: disable=unused-argument - def query(self, - filter: str, - window: int, - timestamp: Optional[int] = None, - operators: list = [], - labels: dict = {}) -> dict: + # pylint: disable=unused-argument,redefined-builtin,dangerous-default-value + # pylint: disable=too-many-arguments + def query( + self, + filter: str, + window: int, + timestamp: Optional[int] = None, + operators: list = [], + labels: dict = {}, + ) -> dict: """Query Prometheus server. Args: @@ -157,7 +161,7 @@ def query(self, dict: Response. """ filter = PrometheusBackend._fmt_query(filter, window, operators, labels) - LOGGER.debug(f'Query: {filter}') + LOGGER.debug(f"Query: {filter}") response = self.client.query(metric=filter) response = json.loads(response) LOGGER.debug(pprint.pformat(response)) @@ -174,18 +178,16 @@ def count(response: dict) -> float: # Note: this function could be replaced by using the `count_over_time` # function that Prometheus provides. try: - return float(response['data']['result'][0]['value'][1]) + return float(response["data"]["result"][0]["value"][1]) except (IndexError, KeyError) as exception: LOGGER.warning("Couldn't find any values in timeseries response.") LOGGER.debug(exception, exc_info=True) return NO_DATA # no events in timeseries @staticmethod + # pylint: disable=dangerous-default-value def _fmt_query( - query: str, - window: int, - operators: List[str] = [], - labels: Dict[str, str] = {} + query: str, window: int, operators: List[str] = [], labels: Dict[str, str] = {} ) -> str: """Format Prometheus query: @@ -207,12 +209,12 @@ def _fmt_query( str: Formatted query. """ query = query.strip() - if '[window' in query: - query = query.replace('[window', f'[{window}s') + if "[window" in query: + query = query.replace("[window", f"[{window}s") else: - query += f'[{window}s]' + query += f"[{window}s]" for operator in operators: - query = f'{operator}({query})' + query = f"{operator}({query})" for key, value in labels.items(): - query = query.replace('}', f', {key}="{value}"}}') + query = query.replace("}", f', {key}="{value}"}}') return query diff --git a/slo_generator/cli.py b/slo_generator/cli.py index e22c3afa..e39680b0 100644 --- a/slo_generator/cli.py +++ b/slo_generator/cli.py @@ -21,13 +21,14 @@ import sys import time from pathlib import Path -from pkg_resources import get_distribution import click +from pkg_resources import get_distribution + from slo_generator import utils from slo_generator.compute import compute as _compute -from slo_generator.migrations import migrator from slo_generator.constants import LATEST_MAJOR_VERSION +from slo_generator.migrations import migrator sys.path.append(os.getcwd()) # dynamic backend loading @@ -35,45 +36,57 @@ @click.group(invoke_without_command=True) -@click.option('--version', - '-v', - is_flag=True, - help='Show slo-generator version.') +@click.option( + "--version", + "-v", + is_flag=True, + help="Show slo-generator version.", +) @click.pass_context def main(ctx, version): """CLI entrypoint.""" utils.setup_logging() if ctx.invoked_subcommand is None or version: - ver = get_distribution('slo-generator').version - print(f'slo-generator v{ver}') + ver = get_distribution("slo-generator").version + print(f"slo-generator v{ver}") sys.exit(0) @main.command() -@click.option('--slo-config', - '-f', - type=click.Path(), - required=True, - help='SLO config path') -@click.option('--config', - '-c', - type=click.Path(), - default='config.yaml', - show_default=True, - help='slo-generator config path') -@click.option('--export', - '-e', - is_flag=True, - help='Export SLO report to exporters') -@click.option('--delete', - '-d', - is_flag=True, - help='Delete mode (used for backends with SLO APIs)') -@click.option('--timestamp', - '-t', - type=float, - default=time.time(), - help='End timestamp for query.') +@click.option( + "--slo-config", + "-f", + type=click.Path(), + required=True, + help="SLO config path", +) +@click.option( + "--config", + "-c", + type=click.Path(), + default="config.yaml", + show_default=True, + help="slo-generator config path", +) +@click.option( + "--export", + "-e", + is_flag=True, + help="Export SLO report to exporters", +) +@click.option( + "--delete", + "-d", + is_flag=True, + help="Delete mode (used for backends with SLO APIs)", +) +@click.option( + "--timestamp", + "-t", + type=float, + default=time.time(), + help="End timestamp for query.", +) def compute(slo_config, config, export, delete, timestamp): """Compute SLO report.""" start = time.time() @@ -84,32 +97,32 @@ def compute(slo_config, config, export, delete, timestamp): # Load SLO config(s) if Path(slo_config).is_dir(): - slo_configs = utils.load_configs(slo_config, - kind='ServiceLevelObjective') + slo_configs = utils.load_configs(slo_config, kind="ServiceLevelObjective") else: - slo_configs = [ - utils.load_config(slo_config, kind='ServiceLevelObjective') - ] + slo_configs = [utils.load_config(slo_config, kind="ServiceLevelObjective")] if not slo_configs: - LOGGER.error(f'No SLO configs found in {slo_config}.') + LOGGER.error(f"No SLO configs found in {slo_config}.") sys.exit(1) # Load SLO configs and compute SLO reports all_reports = {} for slo_config_dict in slo_configs: - reports = _compute(slo_config_dict, - config_dict, - timestamp=timestamp, - do_export=export, - delete=delete) + reports = _compute( + slo_config_dict, + config_dict, + timestamp=timestamp, + do_export=export, + delete=delete, + ) if reports: - name = slo_config_dict['metadata']['name'] + name = slo_config_dict["metadata"]["name"] all_reports[name] = reports end = time.time() duration = round(end - start, 1) - LOGGER.info(f'Run summary | SLO Configs: {len(slo_configs)} | ' - f'Duration: {duration}s') + LOGGER.info( + f"Run summary | SLO Configs: {len(slo_configs)} | " f"Duration: {duration}s" + ) LOGGER.debug(all_reports) return all_reports @@ -117,82 +130,108 @@ def compute(slo_config, config, export, delete, timestamp): # pylint: disable=import-error,import-outside-toplevel @main.command() @click.pass_context -@click.option('--config', - '-c', - envvar='CONFIG_PATH', - required=True, - help='slo-generator configuration file path.') -@click.option('--exporters', - '-e', - envvar='EXPORTERS', - required=False, - default='', - help='List of exporters to send data to') -@click.option('--signature-type', - envvar='GOOGLE_FUNCTION_SIGNATURE_TYPE', - default='http', - type=click.Choice(['http', 'cloudevent']), - help='Signature type') -@click.option('--target', - envvar='GOOGLE_FUNCTION_SIGNATURE_TYPE', - default='run_compute', - help='Target function name') -@click.option('--port', - '-p', - default=8080, - help='HTTP port') +@click.option( + "--config", + "-c", + envvar="CONFIG_PATH", + required=True, + help="slo-generator configuration file path.", +) +@click.option( + "--exporters", + "-e", + envvar="EXPORTERS", + required=False, + default="", + help="List of exporters to send data to", +) +@click.option( + "--signature-type", + envvar="GOOGLE_FUNCTION_SIGNATURE_TYPE", + default="http", + type=click.Choice(["http", "cloudevent"]), + help="Signature type", +) +@click.option( + "--target", + envvar="GOOGLE_FUNCTION_SIGNATURE_TYPE", + default="run_compute", + help="Target function name", +) +@click.option( + "--port", + "-p", + default=8080, + help="HTTP port", +) +# pylint: disable=too-many-arguments def api(ctx, config, exporters, signature_type, target, port): """Run an API that can receive requests (supports both 'http' and 'cloudevents' signature types).""" from functions_framework._cli import _cli - os.environ['EXPORTERS'] = exporters - os.environ['CONFIG_PATH'] = config - os.environ['GOOGLE_FUNCTION_SIGNATURE_TYPE'] = signature_type - os.environ['GOOGLE_FUNCTION_TARGET'] = target - ctx.invoke(_cli, - target=target, - source=Path(__file__).parent / 'api' / 'main.py', - signature_type=signature_type, - port=port) + + os.environ["EXPORTERS"] = exporters + os.environ["CONFIG_PATH"] = config + os.environ["GOOGLE_FUNCTION_SIGNATURE_TYPE"] = signature_type + os.environ["GOOGLE_FUNCTION_TARGET"] = target + ctx.invoke( + _cli, + target=target, + source=Path(__file__).parent / "api" / "main.py", + signature_type=signature_type, + port=port, + ) @main.command() -@click.option('--source', - '-s', - type=click.Path(exists=True, resolve_path=True, readable=True), - required=True, - default=Path.cwd(), - help='Source SLO configs folder') -@click.option('--target', - '-t', - type=click.Path(resolve_path=True), - default=Path.cwd(), - required=True, - help='Target SLO configs folder') -@click.option('--error-budget-policy-path', - '-b', - type=click.Path(exists=True, resolve_path=True, readable=True), - required=False, - multiple=True, - default=['error_budget_policy.yaml'], - help='Error budget policy path') -@click.option('--exporters-path', - '-e', - type=click.Path(exists=True, resolve_path=True, readable=True), - required=False, - multiple=True, - help='Exporters path') -@click.option('--version', - type=str, - required=False, - default=LATEST_MAJOR_VERSION, - show_default=True, - help='SLO generate major version to migrate towards') -@click.option('--quiet', - '-q', - is_flag=True, - default=False, - help='Do not ask for user input and auto-generate config keys') +@click.option( + "--source", + "-s", + type=click.Path(exists=True, resolve_path=True, readable=True), + required=True, + default=Path.cwd(), + help="Source SLO configs folder", +) +@click.option( + "--target", + "-t", + type=click.Path(resolve_path=True), + default=Path.cwd(), + required=True, + help="Target SLO configs folder", +) +@click.option( + "--error-budget-policy-path", + "-b", + type=click.Path(exists=True, resolve_path=True, readable=True), + required=False, + multiple=True, + default=["error_budget_policy.yaml"], + help="Error budget policy path", +) +@click.option( + "--exporters-path", + "-e", + type=click.Path(exists=True, resolve_path=True, readable=True), + required=False, + multiple=True, + help="Exporters path", +) +@click.option( + "--version", + type=str, + required=False, + default=LATEST_MAJOR_VERSION, + show_default=True, + help="SLO generate major version to migrate towards", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + default=False, + help="Do not ask for user input and auto-generate config keys", +) def migrate(**kwargs): """Migrate SLO configs from v1 to v2.""" migrator.do_migrate(**kwargs) diff --git a/slo_generator/compute.py b/slo_generator/compute.py index af48a085..788557cd 100644 --- a/slo_generator/compute.py +++ b/slo_generator/compute.py @@ -19,23 +19,24 @@ import logging import pprint import time - from typing import Optional -from slo_generator import constants -from slo_generator import utils -from slo_generator.report import SLOReport +from slo_generator import constants, utils from slo_generator.migrations.migrator import report_v2tov1 +from slo_generator.report import SLOReport LOGGER = logging.getLogger(__name__) -def compute(slo_config: dict, - config: dict, - timestamp: Optional[float] = None, - client=None, - do_export: bool = False, - delete: bool = False): +# pylint: disable=too-many-arguments,too-many-locals +def compute( + slo_config: dict, + config: dict, + timestamp: Optional[float] = None, + client=None, + do_export: bool = False, + delete: bool = False, +): """Run pipeline to compute SLO, Error Budget and Burn Rate, and export the results (if exporters are specified in the SLO config). @@ -52,27 +53,27 @@ def compute(slo_config: dict, timestamp = time.time() if slo_config is None: - LOGGER.error('SLO configuration is empty') + LOGGER.error("SLO configuration is empty") return [] # Get exporters, backend and error budget policy - spec = slo_config['spec'] + spec = slo_config["spec"] exporters = utils.get_exporters(config, spec) - default_exporters_spec = { - "exporters": config.get('default_exporters', []) - } + default_exporters_spec = {"exporters": config.get("default_exporters", [])} default_exporters = utils.get_exporters(config, default_exporters_spec) exporters.extend(x for x in default_exporters if x not in exporters) error_budget_policy = utils.get_error_budget_policy(config, spec) backend = utils.get_backend(config, spec) reports = [] - for step in error_budget_policy['steps']: - report = SLOReport(config=slo_config, - backend=backend, - step=step, - timestamp=timestamp, - client=client, - delete=delete) + for step in error_budget_policy["steps"]: + report = SLOReport( + config=slo_config, + backend=backend, + step=step, + timestamp=timestamp, + client=client, + delete=delete, + ) json_report = report.to_json() if not report.valid: @@ -86,18 +87,16 @@ def compute(slo_config: dict, LOGGER.info(report) if exporters is not None and do_export is True: errors = export(json_report, exporters) - json_report['errors'].extend(errors) + json_report["errors"].extend(errors) reports.append(json_report) end = time.time() run_duration = round(end - start, 1) LOGGER.debug(pprint.pformat(reports)) - LOGGER.info(f'Run finished successfully in {run_duration}s.') + LOGGER.info(f"Run finished successfully in {run_duration}s.") return reports -def export(data: dict, - exporters: list, - raise_on_error: bool = False) -> list: +def export(data: dict, exporters: list, raise_on_error: bool = False) -> list: """Export data using selected exporters. Args: @@ -107,47 +106,46 @@ def export(data: dict, Returns: list: List of export errors. """ - LOGGER.debug(f'Exporters: {pprint.pformat(exporters)}') - LOGGER.debug(f'Data: {pprint.pformat(data)}') - name = data['metadata']['name'] - ebp_step = data['error_budget_policy_step_name'] - info = f'{name :<32} | {ebp_step :<8}' + LOGGER.debug(f"Exporters: {pprint.pformat(exporters)}") + LOGGER.debug(f"Data: {pprint.pformat(data)}") + name = data["metadata"]["name"] + ebp_step = data["error_budget_policy_step_name"] + info = f"{name :<32} | {ebp_step :<8}" errors = [] # Passing one exporter as a dict will work for convenience if isinstance(exporters, dict): exporters = [exporters] if not exporters: - error = 'No exporters were found.' - LOGGER.error(f'{info} | {error}') + error = "No exporters were found." + LOGGER.error(f"{info} | {error}") errors.append(error) for exporter in exporters: try: - cls = exporter.get('class') - name = exporter.get('name') + cls = exporter.get("class") + name = exporter.get("name") instance = utils.get_exporter_cls(cls) if not instance: - raise ImportError('Exporter not found in shared config.') - LOGGER.debug(f'Exporter config: {pprint.pformat(exporter)}') + raise ImportError("Exporter not found in shared config.") + LOGGER.debug(f"Exporter config: {pprint.pformat(exporter)}") # Convert data to export from v1 to v2 for backwards-compatible # exporters such as BigQuery. json_data = data if cls not in constants.V2_EXPORTERS: - LOGGER.debug(f'{info} | Converting SLO report to v1.') + LOGGER.debug(f"{info} | Converting SLO report to v1.") json_data = report_v2tov1(data) - LOGGER.debug(f'{info} | SLO report: {json_data}') + LOGGER.debug(f"{info} | SLO report: {json_data}") response = instance().export(json_data, **exporter) - LOGGER.info( - f'{info} | SLO report sent to "{name}" exporter successfully.') - LOGGER.debug(f'{info} | {response}') + LOGGER.info(f'{info} | SLO report sent to "{name}" exporter successfully.') + LOGGER.debug(f"{info} | {response}") except Exception as exc: # pylint: disable=broad-except if raise_on_error: raise exc tbk = utils.fmt_traceback(exc) error = f'{cls}Exporter "{name}" failed. | {tbk}' - LOGGER.error(f'{info} | {error}') + LOGGER.error(f"{info} | {error}") LOGGER.exception(exc) errors.append(error) return errors diff --git a/slo_generator/constants.py b/slo_generator/constants.py index fe4a7f45..fe199a85 100644 --- a/slo_generator/constants.py +++ b/slo_generator/constants.py @@ -16,63 +16,63 @@ Constants and environment variables used in `slo-generator`. """ import os -from typing import Tuple, List, Dict +from typing import Dict, List, Tuple # Compute NO_DATA: int = -1 MIN_VALID_EVENTS: int = int(os.environ.get("MIN_VALID_EVENTS", "1")) # Global -LATEST_MAJOR_VERSION: str = 'v2' +LATEST_MAJOR_VERSION: str = "v2" COLORED_OUTPUT: int = int(os.environ.get("COLORED_OUTPUT", "0")) DRY_RUN: bool = bool(int(os.environ.get("DRY_RUN", "0"))) DEBUG: int = int(os.environ.get("DEBUG", "0")) # Exporters supporting v2 SLO report format -V2_EXPORTERS: Tuple[str, ...] = ('Pubsub', 'Cloudevent') +V2_EXPORTERS: Tuple[str, ...] = ("Pubsub", "Cloudevent") # Config skeletons CONFIG_SCHEMA: dict = { - 'backends': {}, - 'exporters': {}, - 'error_budget_policies': {}, + "backends": {}, + "exporters": {}, + "error_budget_policies": {}, } SLO_CONFIG_SCHEMA: dict = { - 'apiVersion': '', - 'kind': '', - 'metadata': {}, - 'spec': { - 'description': '', - 'backend': '', - 'method': '', - 'exporters': [], - 'service_level_indicator': {} - } + "apiVersion": "", + "kind": "", + "metadata": {}, + "spec": { + "description": "", + "backend": "", + "method": "", + "exporters": [], + "service_level_indicator": {}, + }, } # Providers that have changed with v2 YAML config format. This mapping helps # migrate them to their updated names. PROVIDERS_COMPAT: Dict[str, str] = { - 'Stackdriver': 'CloudMonitoring', - 'StackdriverServiceMonitoring': 'CloudServiceMonitoring' + "Stackdriver": "CloudMonitoring", + "StackdriverServiceMonitoring": "CloudServiceMonitoring", } # Fields that have changed name with v2 YAML config format. This mapping helps # migrate them back to their former name, so that exporters are backward- # compatible with v1. METRIC_LABELS_COMPAT: Dict[str, str] = { - 'goal': 'slo_target', - 'description': 'slo_description', - 'error_budget_burn_rate_threshold': 'alerting_burn_rate_threshold' + "goal": "slo_target", + "description": "slo_description", + "error_budget_burn_rate_threshold": "alerting_burn_rate_threshold", } # Fields that used to be specified in top-level of YAML config are now specified # in metadata fields. This mapping helps migrate them back to the top level when # exporting reports, so that exporters are backward-compatible with v1. METRIC_METADATA_LABELS_TOP_COMPAT: List[str] = [ - 'service_name', - 'feature_name', - 'slo_name' + "service_name", + "feature_name", + "slo_name", ] @@ -80,14 +80,15 @@ # pylint: disable=too-few-public-methods class Colors: """Colors for console output.""" - HEADER: str = '\033[95m' - OKBLUE: str = '\033[94m' - OKGREEN: str = '\033[92m' - WARNING: str = '\033[93m' - FAIL: str = '\033[91m' - ENDC: str = '\033[0m' - BOLD: str = '\033[1m' - UNDERLINE: str = '\033[4m' + + HEADER: str = "\033[95m" + OKBLUE: str = "\033[94m" + OKGREEN: str = "\033[92m" + WARNING: str = "\033[93m" + FAIL: str = "\033[91m" + ENDC: str = "\033[0m" + BOLD: str = "\033[1m" + UNDERLINE: str = "\033[4m" GREEN: str = Colors.OKGREEN @@ -95,6 +96,6 @@ class Colors: ENDC: str = Colors.ENDC BOLD: str = Colors.BOLD WARNING: str = Colors.WARNING -FAIL: str = '❌' -SUCCESS: str = '✅' -RIGHT_ARROW: str = '➞' +FAIL: str = "❌" +SUCCESS: str = "✅" +RIGHT_ARROW: str = "➞" diff --git a/slo_generator/exporters/base.py b/slo_generator/exporters/base.py index bc2de873..ec0237d9 100644 --- a/slo_generator/exporters/base.py +++ b/slo_generator/exporters/base.py @@ -23,39 +23,39 @@ # Default metric labels exported by all metrics exporters DEFAULT_METRIC_LABELS = [ - 'error_budget_policy_step_name', 'service_name', 'feature_name', 'slo_name', - 'metadata' + "error_budget_policy_step_name", + "service_name", + "feature_name", + "slo_name", + "metadata", ] # Default metrics that are exported by metrics exporters. DEFAULT_METRICS = [ { - 'name': 'error_budget_burn_rate', - 'description': 'Speed at which the error budget is consumed.', - 'labels': DEFAULT_METRIC_LABELS + "name": "error_budget_burn_rate", + "description": "Speed at which the error budget is consumed.", + "labels": DEFAULT_METRIC_LABELS, }, { - 'name': 'alerting_burn_rate_threshold', - 'description': 'Error Budget burn rate threshold.', - 'labels': DEFAULT_METRIC_LABELS + "name": "alerting_burn_rate_threshold", + "description": "Error Budget burn rate threshold.", + "labels": DEFAULT_METRIC_LABELS, }, { - 'name': - 'events_count', - 'description': - 'Number of events', - 'labels': - DEFAULT_METRIC_LABELS + ['good_events_count', 'bad_events_count'] + "name": "events_count", + "description": "Number of events", + "labels": DEFAULT_METRIC_LABELS + ["good_events_count", "bad_events_count"], }, { - 'name': 'sli_measurement', - 'description': 'Service Level Indicator.', - 'labels': DEFAULT_METRIC_LABELS + "name": "sli_measurement", + "description": "Service Level Indicator.", + "labels": DEFAULT_METRIC_LABELS, }, { - 'name': 'slo_target', - 'description': 'Service Level Objective target.', - 'labels': DEFAULT_METRIC_LABELS + "name": "slo_target", + "description": "Service Level Objective target.", + "labels": DEFAULT_METRIC_LABELS, }, ] @@ -63,6 +63,7 @@ class MetricsExporter: # pytype: disable=ignored-metaclass """Abstract class to export metrics to different backends. Common format for YAML configuration to configure which metrics should be exported.""" + __metaclass__ = ABCMeta # pytype: disable=ignored-metaclass def export(self, data, **config): @@ -76,22 +77,22 @@ class `export_metric` method. Returns: list: List of exporter responses. """ - metrics = config.get('metrics', DEFAULT_METRICS) - required_fields = getattr(self, 'REQUIRED_FIELDS', []) - optional_fields = getattr(self, 'OPTIONAL_FIELDS', []) - LOGGER.debug( - f'Exporting {len(metrics)} metrics with {self.__class__.__name__}') + metrics = config.get("metrics", DEFAULT_METRICS) + required_fields = getattr(self, "REQUIRED_FIELDS", []) + optional_fields = getattr(self, "OPTIONAL_FIELDS", []) + LOGGER.debug(f"Exporting {len(metrics)} metrics with {self.__class__.__name__}") for metric_cfg in metrics: if isinstance(metric_cfg, str): # short form metric_cfg = { - 'name': metric_cfg, - 'alias': metric_cfg, - 'description': "", - 'labels': DEFAULT_METRIC_LABELS + "name": metric_cfg, + "alias": metric_cfg, + "description": "", + "labels": DEFAULT_METRIC_LABELS, } - if metric_cfg['name'] == 'error_budget_burn_rate': + if metric_cfg["name"] == "error_budget_burn_rate": metric_cfg = MetricsExporter.use_deprecated_fields( - config=config, metric=metric_cfg) + config=config, metric=metric_cfg + ) metric = metric_cfg.copy() fields = { key: value @@ -113,29 +114,29 @@ def build_metric(self, data, metric): Returns: dict: Metric configuration. """ - name = metric['name'] - prefix = getattr(self, 'METRIC_PREFIX', None) + name = metric["name"] + prefix = getattr(self, "METRIC_PREFIX", None) # Set value + timestamp - metric['value'] = data[name] - metric['timestamp'] = data['timestamp'] + metric["value"] = data[name] + metric["timestamp"] = data["timestamp"] # Set metric data labels - labels = metric.get('labels', DEFAULT_METRIC_LABELS).copy() - additional_labels = metric.get('additional_labels', []) + labels = metric.get("labels", DEFAULT_METRIC_LABELS).copy() + additional_labels = metric.get("additional_labels", []) labels.extend(additional_labels) labels = MetricsExporter.build_data_labels(data, labels) - metric['labels'] = labels + metric["labels"] = labels # Use metric alias (mapping) - if 'alias' in metric: - metric['name'] = metric['alias'] + if "alias" in metric: + metric["name"] = metric["alias"] - if prefix and not metric['name'].startswith(prefix): - metric['name'] = prefix + metric['name'] + if prefix and not metric["name"].startswith(prefix): + metric["name"] = prefix + metric["name"] # Set description - metric['description'] = metric.get('description', "") + metric["description"] = metric.get("description", "") return metric @staticmethod @@ -151,18 +152,18 @@ def build_data_labels(data, labels): """ data_labels = {} nested_labels = [ - label for label in labels - if label in data and isinstance(data[label], dict) + label for label in labels if label in data and isinstance(data[label], dict) ] flat_labels = [ - label for label in labels + label + for label in labels if label in data and not isinstance(data[label], dict) ] for label in nested_labels: data_labels.update({k: str(v) for k, v in data[label].items()}) for label in flat_labels: data_labels[label] = str(data[label]) - LOGGER.debug(f'Data labels: {data_labels}') + LOGGER.debug(f"Data labels: {data_labels}") return data_labels @staticmethod @@ -179,24 +180,30 @@ def use_deprecated_fields(config, metric): Returns: list: List of metrics to export. """ - old_metric_type = config.get('metric_type') - old_metric_labels = config.get('metric_labels') - old_metric_description = config.get('metric_description') + old_metric_type = config.get("metric_type") + old_metric_labels = config.get("metric_labels") + old_metric_description = config.get("metric_description") if old_metric_type: - metric['alias'] = old_metric_type + metric["alias"] = old_metric_type warnings.warn( - '`metric_type` will be deprecated in favor of `metrics` ' - 'in version 2.0.0, ', FutureWarning) + "`metric_type` will be deprecated in favor of `metrics` " + "in version 2.0.0, ", + FutureWarning, + ) if old_metric_labels: - metric['labels'] = old_metric_labels + metric["labels"] = old_metric_labels warnings.warn( - '`metric_labels` will be deprecated in favor of `metrics` ' - 'in version 2.0.0, ', FutureWarning) + "`metric_labels` will be deprecated in favor of `metrics` " + "in version 2.0.0, ", + FutureWarning, + ) if old_metric_description: warnings.warn( - '`metric_description` will be deprecated in favor of `metrics` ' - 'in version 2.0.0, ', FutureWarning) - metric['description'] = old_metric_description + "`metric_description` will be deprecated in favor of `metrics` " + "in version 2.0.0, ", + FutureWarning, + ) + metric["description"] = old_metric_description return metric @abstractmethod diff --git a/slo_generator/exporters/bigquery.py b/slo_generator/exporters/bigquery.py index 283930b1..1ea573f0 100644 --- a/slo_generator/exporters/bigquery.py +++ b/slo_generator/exporters/bigquery.py @@ -21,7 +21,7 @@ import pprint import google.api_core -from google.cloud import bigquery # type: ignore[attr-defined] +from google.cloud import bigquery # type: ignore[attr-defined] from slo_generator import constants @@ -32,7 +32,7 @@ class BigqueryExporter: """BigQuery exporter class.""" def __init__(self): - self.client = bigquery.Client(project='unset') + self.client = bigquery.Client(project="unset") def export(self, data, **config): """Export results to BigQuery. @@ -53,41 +53,47 @@ def export(self, data, **config): Raises: BigQueryError (object): BigQuery exception object. """ - project_id = config['project_id'] - dataset_id = config['dataset_id'] - table_id = config['table_id'] + project_id = config["project_id"] + dataset_id = config["dataset_id"] + table_id = config["table_id"] self.client.project = project_id table_ref = self.client.dataset(dataset_id).table(table_id) - schema_fields = [element['name'] for element in TABLE_SCHEMA] - keep_fields = config.get('keep_fields', []) + schema_fields = [element["name"] for element in TABLE_SCHEMA] + keep_fields = config.get("keep_fields", []) try: table = self.client.get_table(table_ref) table = self.update_schema(table_ref, keep=keep_fields) except google.api_core.exceptions.NotFound: - table = self.create_table(project_id, - dataset_id, - table_id, - schema=TABLE_SCHEMA) + table = self.create_table( + project_id, + dataset_id, + table_id, + schema=TABLE_SCHEMA, + ) # Format user metadata if needed json_data = {k: v for k, v in data.items() if k in schema_fields} - metadata = json_data.get('metadata', {}) + metadata = json_data.get("metadata", {}) if isinstance(metadata, dict): - metadata_fields = [{ - 'key': key, - 'value': value - } for key, value in metadata.items()] - json_data['metadata'] = metadata_fields + metadata_fields = [ + { + "key": key, + "value": value, + } + for key, value in metadata.items() + ] + json_data["metadata"] = metadata_fields # Write results to BQ table if constants.DRY_RUN: - LOGGER.info(f'[DRY RUN] Writing data to BigQuery: \n{json_data}') + LOGGER.info(f"[DRY RUN] Writing data to BigQuery: \n{json_data}") return [] - LOGGER.debug(f'Writing data to BigQuery:\n{json_data}') + LOGGER.debug(f"Writing data to BigQuery:\n{json_data}") results = self.client.insert_rows_json( table, json_rows=[json_data], - retry=google.api_core.retry.Retry(deadline=30)) + retry=google.api_core.retry.Retry(deadline=30), + ) if results: raise BigQueryError(results) return results @@ -106,17 +112,21 @@ def build_schema(schema): final_schema = [] for row in schema: subschema = [] - if 'fields' in row: + if "fields" in row: subschema = [ - bigquery.SchemaField(subrow['name'], - subrow['type'], - mode=subrow['mode']) - for subrow in row['fields'] + bigquery.SchemaField( + subrow["name"], + subrow["type"], + mode=subrow["mode"], + ) + for subrow in row["fields"] ] - field = bigquery.SchemaField(row['name'], - row['type'], - mode=row['mode'], - fields=subschema) + field = bigquery.SchemaField( + row["name"], + row["type"], + mode=row["mode"], + fields=subschema, + ) final_schema.append(field) return final_schema @@ -135,14 +145,16 @@ def create_table(self, project_id, dataset_id, table_id, schema=None): if schema is not None: schema = TABLE_SCHEMA pyschema = BigqueryExporter.build_schema(schema) - table_name = f'{project_id}.{dataset_id}.{table_id}' - LOGGER.info(f'Creating table {table_name}') - LOGGER.debug(f'Table schema: {pyschema}') + table_name = f"{project_id}.{dataset_id}.{table_id}" + LOGGER.info(f"Creating table {table_name}") + LOGGER.debug(f"Table schema: {pyschema}") table = bigquery.Table(table_name, schema=pyschema) table.time_partitioning = bigquery.TimePartitioning( - type_=bigquery.TimePartitioningType.DAY,) + type_=bigquery.TimePartitioningType.DAY, + ) return self.client.create_table(table) + # pylint: disable=dangerous-default-value def update_schema(self, table_ref, keep=[]): """Updates a BigQuery table schema if needed. @@ -154,39 +166,38 @@ def update_schema(self, table_ref, keep=[]): obj: BigQuery table object. """ table = self.client.get_table(table=table_ref) - iostream = io.StringIO('') + iostream = io.StringIO("") self.client.schema_to_json(table.schema, iostream) existing_schema = json.loads(iostream.getvalue()) - existing_fields = [field['name'] for field in existing_schema] - LOGGER.debug(f'Existing fields: {existing_fields}') + existing_fields = [field["name"] for field in existing_schema] + LOGGER.debug(f"Existing fields: {existing_fields}") # Fields in TABLE_SCHEMA to add / remove updated_fields = [ - field['name'] - for field in TABLE_SCHEMA - if field not in existing_schema + field["name"] for field in TABLE_SCHEMA if field not in existing_schema ] extra_remote_fields = [ - field for field in existing_schema - if field not in TABLE_SCHEMA and field['name'] in keep + field + for field in existing_schema + if field not in TABLE_SCHEMA and field["name"] in keep ] # If extra remote fields are detected in existing schema, update our # TABLE_SCHEMA with those if extra_remote_fields: - LOGGER.info(f'Extra remote BigQuery fields: {extra_remote_fields}') + LOGGER.info(f"Extra remote BigQuery fields: {extra_remote_fields}") TABLE_SCHEMA.extend(extra_remote_fields) # If new fields are detected in TABLE_SCHEMA, update BigQuery schema if updated_fields: - LOGGER.info(f'Updated BigQuery fields: {updated_fields}') + LOGGER.info(f"Updated BigQuery fields: {updated_fields}") table.schema = BigqueryExporter.build_schema(TABLE_SCHEMA) if constants.DRY_RUN: - LOGGER.info('[DRY RUN] Updating BigQuery schema.') + LOGGER.info("[DRY RUN] Updating BigQuery schema.") else: - LOGGER.info('Updating BigQuery schema.') - LOGGER.debug(f'New schema: {pprint.pformat(table.schema)}') - self.client.update_table(table, ['schema']) + LOGGER.info("Updating BigQuery schema.") + LOGGER.debug(f"New schema: {pprint.pformat(table.schema)}") + self.client.update_table(table, ["schema"]) return table @@ -205,113 +216,141 @@ def __init__(self, errors): def _format(errors): err = [] for error in errors: - err.extend(error['errors']) + err.extend(error["errors"]) return json.dumps(err) -TABLE_SCHEMA = [{ - 'name': 'service_name', - 'type': 'STRING', - 'mode': 'REQUIRED' -}, { - 'name': 'feature_name', - 'type': 'STRING', - 'mode': 'REQUIRED' -}, { - 'name': 'slo_name', - 'type': 'STRING', - 'mode': 'REQUIRED' -}, { - 'name': 'slo_target', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'slo_description', - 'type': 'STRING', - 'mode': 'REQUIRED' -}, { - 'name': 'error_budget_policy_step_name', - 'type': 'STRING', - 'mode': 'NULLABLE' -}, { - 'name': 'error_budget_remaining_minutes', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'consequence_message', - 'type': 'STRING', - 'mode': 'NULLABLE' -}, { - 'name': 'error_budget_minutes', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'error_minutes', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'error_budget_target', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'timestamp_human', - 'type': 'TIMESTAMP', - 'mode': 'REQUIRED' -}, { - 'name': 'timestamp', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'cadence', - 'type': 'STRING', - 'mode': 'NULLABLE' -}, { - 'name': 'window', - 'type': 'INTEGER', - 'mode': 'REQUIRED' -}, { - 'name': 'bad_events_count', - 'type': 'INTEGER', - 'mode': 'NULLABLE' -}, { - 'name': 'good_events_count', - 'type': 'INTEGER', - 'mode': 'NULLABLE' -}, { - 'name': 'sli_measurement', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'gap', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'error_budget_measurement', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'error_budget_burn_rate', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'alerting_burn_rate_threshold', - 'type': 'FLOAT', - 'mode': 'NULLABLE' -}, { - 'name': 'alert', - 'type': 'BOOLEAN', - 'mode': 'NULLABLE' -}, { - 'name': 'metadata', - 'type': 'RECORD', - 'mode': 'REPEATED', - 'fields': [{ - 'name': 'key', - 'type': 'STRING', - 'mode': 'NULLABLE' - }, { - 'name': 'value', - 'type': 'STRING', - 'mode': 'NULLABLE' - }] -}] +TABLE_SCHEMA = [ + { + "name": "service_name", + "type": "STRING", + "mode": "REQUIRED", + }, + { + "name": "feature_name", + "type": "STRING", + "mode": "REQUIRED", + }, + { + "name": "slo_name", + "type": "STRING", + "mode": "REQUIRED", + }, + { + "name": "slo_target", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "slo_description", + "type": "STRING", + "mode": "REQUIRED", + }, + { + "name": "error_budget_policy_step_name", + "type": "STRING", + "mode": "NULLABLE", + }, + { + "name": "error_budget_remaining_minutes", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "consequence_message", + "type": "STRING", + "mode": "NULLABLE", + }, + { + "name": "error_budget_minutes", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "error_minutes", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "error_budget_target", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "timestamp_human", + "type": "TIMESTAMP", + "mode": "REQUIRED", + }, + { + "name": "timestamp", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "cadence", + "type": "STRING", + "mode": "NULLABLE", + }, + { + "name": "window", + "type": "INTEGER", + "mode": "REQUIRED", + }, + { + "name": "bad_events_count", + "type": "INTEGER", + "mode": "NULLABLE", + }, + { + "name": "good_events_count", + "type": "INTEGER", + "mode": "NULLABLE", + }, + { + "name": "sli_measurement", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "gap", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "error_budget_measurement", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "error_budget_burn_rate", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "alerting_burn_rate_threshold", + "type": "FLOAT", + "mode": "NULLABLE", + }, + { + "name": "alert", + "type": "BOOLEAN", + "mode": "NULLABLE", + }, + { + "name": "metadata", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "key", + "type": "STRING", + "mode": "NULLABLE", + }, + { + "name": "value", + "type": "STRING", + "mode": "NULLABLE", + }, + ], + }, +] diff --git a/slo_generator/exporters/cloud_monitoring.py b/slo_generator/exporters/cloud_monitoring.py index ac3f012e..f45a803e 100644 --- a/slo_generator/exporters/cloud_monitoring.py +++ b/slo_generator/exporters/cloud_monitoring.py @@ -28,8 +28,9 @@ class CloudMonitoringExporter(MetricsExporter): """Cloud Monitoring exporter class.""" + METRIC_PREFIX = "custom.googleapis.com/" - REQUIRED_FIELDS = ['project_id'] + REQUIRED_FIELDS = ["project_id"] def __init__(self): self.client = monitoring_v3.MetricServiceClient() @@ -58,34 +59,38 @@ def create_timeseries(self, data: dict): object: Metric descriptor. """ series = monitoring_v3.TimeSeries() - series.metric.type = data['name'] - series.resource.type = 'global' - labels = data['labels'] + series.metric.type = data["name"] + series.resource.type = "global" + labels = data["labels"] for key, value in labels.items(): series.metric.labels[key] = value # pylint: disable=E1101 # Define end point timestamp. - timestamp = data['timestamp'] + timestamp = data["timestamp"] seconds = int(timestamp) - nanos = int((timestamp - seconds) * 10 ** 9) - interval = monitoring_v3.TimeInterval({ - "end_time": { - "seconds": seconds, - "nanos": nanos + nanos = int((timestamp - seconds) * 10**9) + interval = monitoring_v3.TimeInterval( + { + "end_time": { + "seconds": seconds, + "nanos": nanos, + } } - }) + ) # Create a new data point and set the metric value. - point = monitoring_v3.Point({ - "interval": interval, - "value": { - "double_value": data['value'] + point = monitoring_v3.Point( + { + "interval": interval, + "value": { + "double_value": data["value"], + }, } - }) + ) series.points = [point] # Record the timeseries to Cloud Monitoring. - project = self.client.common_project_path(data['project_id']) + project = self.client.common_project_path(data["project_id"]) self.client.create_time_series(name=project, time_series=[series]) # pylint: disable=E1101 labels = series.metric.labels @@ -93,7 +98,8 @@ def create_timeseries(self, data: dict): f"timestamp: {timestamp}" f"value: {point.value.double_value}" f"{labels['service_name']}-{labels['feature_name']}-" - f"{labels['slo_name']}-{labels['error_budget_policy_step_name']}") + f"{labels['slo_name']}-{labels['error_budget_policy_step_name']}" + ) # pylint: enable=E1101 def get_metric_descriptor(self, data: dict): @@ -105,10 +111,11 @@ def get_metric_descriptor(self, data: dict): Returns: object: Metric descriptor (or None if not found). """ - project_id = data['project_id'] - metric_id = data['name'] + project_id = data["project_id"] + metric_id = data["name"] request = monitoring_v3.GetMetricDescriptorRequest( - name=f"projects/{project_id}/metricDescriptors/{metric_id}") + name=f"projects/{project_id}/metricDescriptors/{metric_id}" + ) try: return self.client.get_metric_descriptor(request) except google.api_core.exceptions.NotFound: @@ -123,14 +130,15 @@ def create_metric_descriptor(self, data: dict): Returns: object: Metric descriptor. """ - project = self.client.common_project_path(data['project_id']) + project = self.client.common_project_path(data["project_id"]) descriptor = ga_metric.MetricDescriptor() - descriptor.type = data['name'] + descriptor.type = data["name"] # pylint: disable=E1101 descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE # pylint: enable=E1101 - descriptor.description = data['description'] + descriptor.description = data["description"] descriptor = self.client.create_metric_descriptor( - name=project, metric_descriptor=descriptor) + name=project, metric_descriptor=descriptor + ) return descriptor diff --git a/slo_generator/exporters/cloudevent.py b/slo_generator/exporters/cloudevent.py index 6b4618bb..6974b553 100644 --- a/slo_generator/exporters/cloudevent.py +++ b/slo_generator/exporters/cloudevent.py @@ -16,15 +16,15 @@ CloudEvents exporter class. """ import logging -import requests import google.auth.transport.requests - +import requests from cloudevents.http import CloudEvent, to_structured from google.oauth2.id_token import fetch_id_token LOGGER = logging.getLogger(__name__) + # pylint: disable=too-few-public-methods class CloudeventExporter: """Cloudevent exporter class. @@ -33,8 +33,9 @@ class CloudeventExporter: client (obj, optional): Existing Datadog client to pass. service_url (str): Cloudevent receiver service URL. """ - REQUIRED_FIELDS = ['service_url'] - OPTIONAL_FIELDS = ['auth'] + + REQUIRED_FIELDS = ["service_url"] + OPTIONAL_FIELDS = ["auth"] def export(self, data, **config): """Export data as CloudEvent to an HTTP service receiving cloud events. @@ -45,22 +46,26 @@ def export(self, data, **config): """ attributes = { "source": "https://github.com/cloudevents/spec/pull", - "type": "com.google.slo_generator.slo_report" + "type": "com.google.slo_generator.slo_report", } event = CloudEvent(attributes, data) headers, data = to_structured(event) - service_url = config['service_url'] - if 'auth' in config: - auth = config['auth'] + service_url = config["service_url"] + if "auth" in config: + auth = config["auth"] id_token = None - if 'token' in auth: - id_token = auth['token'] - elif auth.get('google_service_account_auth', False): # Google oauth + if "token" in auth: + id_token = auth["token"] + elif auth.get("google_service_account_auth", False): # Google oauth auth = google.auth.transport.requests.Request() id_token = fetch_id_token(auth, service_url) if id_token: - headers["Authorization"] = f'Bearer {id_token}' - resp = requests.post(service_url, headers=headers, data=data, - timeout=10) + headers["Authorization"] = f"Bearer {id_token}" + resp = requests.post( + service_url, + headers=headers, + data=data, + timeout=10, + ) resp.raise_for_status() return resp diff --git a/slo_generator/exporters/datadog.py b/slo_generator/exporters/datadog.py index 71a505ec..cef641ec 100644 --- a/slo_generator/exporters/datadog.py +++ b/slo_generator/exporters/datadog.py @@ -22,10 +22,11 @@ from .base import MetricsExporter LOGGER = logging.getLogger(__name__) -logging.getLogger('datadog.api').setLevel(logging.ERROR) +logging.getLogger("datadog.api").setLevel(logging.ERROR) DEFAULT_API_HOST = "https://api.datadoghq.com" + # pylint: disable=too-few-public-methods class DatadogExporter(MetricsExporter): """Datadog exporter class. @@ -36,8 +37,9 @@ class DatadogExporter(MetricsExporter): app_key (str): Datadog APP key. kwargs (dict): Extra arguments to pass to initialize function. """ - REQUIRED_FIELDS = ['api_key', 'app_key'] - OPTIONAL_FIELDS = ['api_host'] + + REQUIRED_FIELDS = ["api_key", "app_key"] + OPTIONAL_FIELDS = ["api_host"] def export_metric(self, data): """Export a metric to Datadog. @@ -49,16 +51,18 @@ def export_metric(self, data): DatadogError (object): Datadog exception object. """ options = { - 'api_key': data['api_key'], - 'app_key': data['app_key'], - 'api_host': data.get('api_host', DEFAULT_API_HOST) + "api_key": data["api_key"], + "app_key": data["app_key"], + "api_host": data.get("api_host", DEFAULT_API_HOST), } datadog.initialize(**options) client = datadog.api - timestamp = data['timestamp'] - tags = data['labels'] - name = data['name'] - value = data['value'] - return client.Metric.send(metric=name, - points=[(timestamp, value)], - tags=tags) + timestamp = data["timestamp"] + tags = data["labels"] + name = data["name"] + value = data["value"] + return client.Metric.send( + metric=name, + points=[(timestamp, value)], + tags=tags, + ) diff --git a/slo_generator/exporters/dynatrace.py b/slo_generator/exporters/dynatrace.py index 646a397d..3b4db0ae 100644 --- a/slo_generator/exporters/dynatrace.py +++ b/slo_generator/exporters/dynatrace.py @@ -25,6 +25,7 @@ LOGGER = logging.getLogger(__name__) DEFAULT_DEVICE_ID = "slo_report" + class DynatraceExporter(MetricsExporter): """Backend for querying metrics from Dynatrace. @@ -33,8 +34,9 @@ class DynatraceExporter(MetricsExporter): api_url (str): Dynatrace API URL. api_token (str): Dynatrace token. """ - METRIC_PREFIX = 'custom:' - REQUIRED_FIELDS = ['api_url', 'api_token'] + + METRIC_PREFIX = "custom:" + REQUIRED_FIELDS = ["api_url", "api_token"] def __init__(self): self.client = None @@ -48,10 +50,10 @@ def export_metric(self, data): Returns: object: Dynatrace API response. """ - api_url, api_token = data['api_url'], data['api_token'] + api_url, api_token = data["api_url"], data["api_token"] self.client = DynatraceClient(api_url, api_token) metric = self.get_custom_metric(data) - code = int(metric.get('error', {}).get('code', '200')) + code = int(metric.get("error", {}).get("code", "200")) if code == 404: LOGGER.warning("Custom metric doesn't exist. Creating it.") metric = self.create_custom_metric(data) @@ -67,28 +69,30 @@ def create_timeseries(self, data): Returns: object: Dynatrace API response. """ - name = data['name'] - labels = data['labels'] - value = data['value'] - tags = data.get('tags', []) - device_id = data.get('device_id', DEFAULT_DEVICE_ID) + name = data["name"] + labels = data["labels"] + value = data["value"] + tags = data.get("tags", []) + device_id = data.get("device_id", DEFAULT_DEVICE_ID) timestamp_ms = time.time() * 1000 timeseries = { - "type": - DEFAULT_DEVICE_ID, - "tags": - tags, + "type": DEFAULT_DEVICE_ID, + "tags": tags, "properties": {}, - "series": [{ - "timeseriesId": name, - "dimensions": labels, - "dataPoints": [[timestamp_ms, value]] - }] + "series": [ + { + "timeseriesId": name, + "dimensions": labels, + "dataPoints": [[timestamp_ms, value]], + } + ], } - return self.client.request('post', - 'entity/infrastructure/custom', - name=device_id, - post_data=timeseries) + return self.client.request( + "post", + "entity/infrastructure/custom", + name=device_id, + post_data=timeseries, + ) def create_custom_metric(self, data): """Create a metric descriptor in Dynatrace API. @@ -99,19 +103,21 @@ def create_custom_metric(self, data): Returns: obj: Dynatrace API response. """ - name = data['name'] - device_ids = [data.get('device_id', DEFAULT_DEVICE_ID)] - labelkeys = list(data['labels'].keys()) + name = data["name"] + device_ids = [data.get("device_id", DEFAULT_DEVICE_ID)] + labelkeys = list(data["labels"].keys()) metric = { "displayName": name, "unit": "Count", "dimensions": labelkeys, - "types": device_ids + "types": device_ids, } - return self.client.request('put', - 'timeseries', - name=name, - post_data=metric) + return self.client.request( + "put", + "timeseries", + name=name, + post_data=metric, + ) def get_custom_metric(self, data): """Get a custom metric descriptor from Dynatrace API. @@ -122,7 +128,9 @@ def get_custom_metric(self, data): Returns: obj: Dynatrace API response. """ - name = data['name'] - return self.client.request('get', - 'timeseries', - name=name) + name = data["name"] + return self.client.request( + "get", + "timeseries", + name=name, + ) diff --git a/slo_generator/exporters/prometheus.py b/slo_generator/exporters/prometheus.py index 1d25531b..800a1952 100644 --- a/slo_generator/exporters/prometheus.py +++ b/slo_generator/exporters/prometheus.py @@ -25,10 +25,12 @@ LOGGER = logging.getLogger(__name__) DEFAULT_PUSHGATEWAY_JOB = "slo-generator" + class PrometheusExporter(MetricsExporter): """Prometheus exporter class.""" - REQUIRED_FIELDS = ['url'] - OPTIONAL_FIELDS = ['job', 'username', 'password'] + + REQUIRED_FIELDS = ["url"] + OPTIONAL_FIELDS = ["job", "username", "password"] def __init__(self): self.username = None @@ -54,34 +56,39 @@ def create_timeseries(self, data): Returns: object: Metric descriptor. """ - name = data['name'] - description = data['description'] - prometheus_push_url = data['url'] - prometheus_push_job_name = data.get('job', DEFAULT_PUSHGATEWAY_JOB) - value = data['value'] + name = data["name"] + description = data["description"] + prometheus_push_url = data["url"] + prometheus_push_job_name = data.get("job", DEFAULT_PUSHGATEWAY_JOB) + value = data["value"] # Write timeseries w/ metric labels. - labels = data['labels'] + labels = data["labels"] registry = CollectorRegistry() - gauge = Gauge(name, - description, - registry=registry, - labelnames=labels.keys()) + gauge = Gauge( + name, + description, + registry=registry, + labelnames=labels.keys(), + ) gauge.labels(*labels.values()).set(value) # Handle headers handler = default_handler - if 'username' in data and 'password' in data: - self.username = data['username'] - self.password = data['password'] + if "username" in data and "password" in data: + self.username = data["username"] + self.password = data["password"] handler = PrometheusExporter.auth_handler - return pushadd_to_gateway(prometheus_push_url, - job=prometheus_push_job_name, - grouping_key=labels, - registry=registry, - handler=handler) + return pushadd_to_gateway( + prometheus_push_url, + job=prometheus_push_job_name, + grouping_key=labels, + registry=registry, + handler=handler, + ) + # pylint: disable=too-many-arguments def auth_handler(self, url, method, timeout, headers, data): """Handles authentication for pushing to Prometheus gateway. @@ -97,5 +104,6 @@ def auth_handler(self, url, method, timeout, headers, data): """ username = self.username password = self.password - return basic_auth_handler(url, method, timeout, headers, data, username, - password) + return basic_auth_handler( + url, method, timeout, headers, data, username, password + ) diff --git a/slo_generator/exporters/prometheus_self.py b/slo_generator/exporters/prometheus_self.py index c62a95c1..9bee834e 100644 --- a/slo_generator/exporters/prometheus_self.py +++ b/slo_generator/exporters/prometheus_self.py @@ -24,15 +24,17 @@ LOGGER = logging.getLogger(__name__) + class PrometheusSelfExporter(MetricsExporter): """Prometheus exporter class which uses the API mode of itself to export the metrics.""" + REGISTERED_URL: bool = False REGISTERED_METRICS: dict = {} def __init__(self): if not self.REGISTERED_URL: - current_app.add_url_rule('/metrics', view_func=self.serve_metrics) + current_app.add_url_rule("/metrics", view_func=self.serve_metrics) PrometheusSelfExporter.REGISTERED_URL = True @staticmethod @@ -43,7 +45,7 @@ def serve_metrics(): object: Flask HTTP Response """ resp = make_response(generate_latest(), 200) - resp.mimetype = 'text/plain' + resp.mimetype = "text/plain" return resp def export_metric(self, data): @@ -52,16 +54,18 @@ def export_metric(self, data): Args: data (dict): Metric data. """ - name = data['name'] - description = data['description'] - value = data['value'] + name = data["name"] + description = data["description"] + value = data["value"] # Write timeseries w/ metric labels. - labels = data['labels'] + labels = data["labels"] gauge = self.REGISTERED_METRICS.get(name) if gauge is None: - gauge = Gauge(name, - description, - labelnames=labels.keys()) + gauge = Gauge( + name, + description, + labelnames=labels.keys(), + ) PrometheusSelfExporter.REGISTERED_METRICS[name] = gauge gauge.labels(*labels.values()).set(value) diff --git a/slo_generator/exporters/pubsub.py b/slo_generator/exporters/pubsub.py index 120d15c0..7733a924 100644 --- a/slo_generator/exporters/pubsub.py +++ b/slo_generator/exporters/pubsub.py @@ -18,7 +18,7 @@ import json import logging -from google.cloud import pubsub_v1 # type: ignore[attr-defined] +from google.cloud import pubsub_v1 # type: ignore[attr-defined] LOGGER = logging.getLogger(__name__) @@ -41,8 +41,9 @@ def export(self, data, **config): Returns: str: Pub/Sub topic id. """ - project_id = config['project_id'] - topic_name = config['topic_name'] + project_id = config["project_id"] + topic_name = config["topic_name"] + # pylint: disable=no-member topic_path = self.publisher.topic_path(project_id, topic_name) - data = json.dumps(data, indent=4).encode('utf-8') + data = json.dumps(data, indent=4).encode("utf-8") return self.publisher.publish(topic_path, data=data).result() diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index bc3e73e5..5b678c0a 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -31,19 +31,37 @@ from ruamel import yaml from slo_generator import utils -from slo_generator.constants import (METRIC_LABELS_COMPAT, - METRIC_METADATA_LABELS_TOP_COMPAT, - PROVIDERS_COMPAT, CONFIG_SCHEMA, - SLO_CONFIG_SCHEMA, GREEN, RED, BOLD, - WARNING, ENDC, SUCCESS, FAIL, RIGHT_ARROW) - -def do_migrate(source, - target, - error_budget_policy_path: list, - exporters_path: list, - version: str, - quiet: bool = False, - verbose: int = 0): +from slo_generator.constants import ( + BOLD, + CONFIG_SCHEMA, + ENDC, + FAIL, + GREEN, + METRIC_LABELS_COMPAT, + METRIC_METADATA_LABELS_TOP_COMPAT, + PROVIDERS_COMPAT, + RED, + RIGHT_ARROW, + SLO_CONFIG_SCHEMA, + SUCCESS, + WARNING, +) + +yaml.explicit_start = True # type: ignore[attr-defined] +yaml.default_flow_style = None # type: ignore[attr-defined] +yaml.preserve_quotes = True # type: ignore[attr-defined] + + +# pylint: disable=too-many-arguments +def do_migrate( + source, + target, + error_budget_policy_path: list, + exporters_path: list, + version: str, + quiet: bool = False, + verbose: int = 0, +): """Process all SLO configs in folder and generate new SLO configurations. Args: @@ -56,7 +74,7 @@ def do_migrate(source, quiet (bool, optional): If true, do not prompt for user input. verbose (int, optional): Verbose level. """ - curver: str = 'v1' + curver: str = "v1" shared_config = CONFIG_SCHEMA cwd = Path.cwd() source = Path(source).resolve() @@ -80,8 +98,9 @@ def do_migrate(source, # Translate exporters to v2 and put into shared config if exporters_paths: - exporters_func = getattr(sys.modules[__name__], - f"exporters_{curver}to{version}") + exporters_func = getattr( + sys.modules[__name__], f"exporters_{curver}to{version}" + ) exp_keys = exporters_func( exporters_paths, shared_config=shared_config, @@ -89,26 +108,26 @@ def do_migrate(source, ) # Process SLO configs - click.secho('=' * 50) - click.secho(f"Migrating slo-generator configs to {version} ...", - fg='cyan', - bold=True) + click.secho("=" * 50) + click.secho( + f"Migrating slo-generator configs to {version} ...", fg="cyan", bold=True + ) paths = utils.get_files(source) if not paths: - click.secho(f"{FAIL} No SLO configs found in {source}", - fg='red', - bold=True) + click.secho(f"{FAIL} No SLO configs found in {source}", fg="red", bold=True) sys.exit(1) - curver = '' + curver = "" for source_path in paths: if source_path in ebp_paths + exporters_paths: continue source_path_str = source_path.relative_to(cwd) - target_path = utils.get_target_path(source, - target, - source_path, - mkdir=True) + target_path = utils.get_target_path( + source, + target, + source_path, + mkdir=True, + ) target_path_str = target_path.resolve().relative_to(cwd) slo_config_str = source_path.open().read() slo_config, ind, blc = yaml.util.load_yaml_guess_indent(slo_config_str) @@ -123,17 +142,17 @@ def do_migrate(source, # If config version is same as target version, continue if curver == version: click.secho( - f'{FAIL} {source_path_str} is already in {version} format', - fg='red', - bold=True) + f"{FAIL} {source_path_str} is already in {version} format", + fg="red", + bold=True, + ) continue # Create target dirs if needed target_path.parent.mkdir(parents=True, exist_ok=True) # Run vx to vy migrator method - slo_func = getattr(sys.modules[__name__], - f"slo_config_{curver}to{version}") + slo_func = getattr(sys.modules[__name__], f"slo_config_{curver}to{version}") slo_config_v2 = slo_func( slo_config, shared_config=shared_config, @@ -144,10 +163,9 @@ def do_migrate(source, continue # Write resulting config to target path - extra = '(replaced)' if target_path_str == source_path_str else '' - click.secho( - f"{RIGHT_ARROW} {GREEN}{target_path_str}{ENDC} [{version}] {extra}") - with target_path.open('w', encoding='utf8') as conf: + extra = "(replaced)" if target_path_str == source_path_str else "" + click.secho(f"{RIGHT_ARROW} {GREEN}{target_path_str}{ENDC} [{version}] {extra}") + with target_path.open("w", encoding="utf8") as conf: yaml.round_trip_dump( slo_config_v2, conf, @@ -155,17 +173,18 @@ def do_migrate(source, block_seq_indent=blc, default_flow_style=None, ) - click.secho(f'{SUCCESS} Success !', fg='green', bold=True) + click.secho(f"{SUCCESS} Success !", fg="green", bold=True) # Write shared config to file - click.secho('=' * 50) - shared_config_path = target / 'config.yaml' + click.secho("=" * 50) + shared_config_path = target / "config.yaml" shared_config_path_str = shared_config_path.relative_to(cwd) - with shared_config_path.open('w', encoding='utf8') as conf: + with shared_config_path.open("w", encoding="utf8") as conf: click.secho( - f'Writing slo-generator config to {shared_config_path_str} ...', - fg='cyan', - bold=True) + f"Writing slo-generator config to {shared_config_path_str} ...", + fg="cyan", + bold=True, + ) yaml.round_trip_dump( shared_config, conf, @@ -174,7 +193,7 @@ def do_migrate(source, block_seq_indent=0, explicit_start=True, ) - click.secho(f'{SUCCESS} Success !', fg='green', bold=True) + click.secho(f"{SUCCESS} Success !", fg="green", bold=True) # Remove error budget policy file # click.secho('=' * 50) @@ -186,29 +205,36 @@ def do_migrate(source, # Print next steps relative_ebp_path = ebp_paths[0].relative_to(cwd) - click.secho('=' * 50) + click.secho("=" * 50) click.secho( - f'\n{SUCCESS} Migration of `slo-generator` configs to v2 completed successfully ! Configs path: {target_str}/.\n', - fg='green', - bold=True) - click.secho('=' * 50) + f"\n{SUCCESS} Migration of `slo-generator` configs to v2 completed successfully ! Configs path: {target_str}/.\n", + fg="green", + bold=True, + ) + click.secho("=" * 50) click.secho( - f'{BOLD}PLEASE FOLLOW THE MANUAL STEPS BELOW TO FINISH YOUR MIGRATION:', - fg='red', - bold=True) - click.secho(f""" + f"{BOLD}PLEASE FOLLOW THE MANUAL STEPS BELOW TO FINISH YOUR MIGRATION:", + fg="red", + bold=True, + ) + click.secho( + f""" 1 - Commit the updated SLO configs and your shared SLO config to version control. 2 - [local/k8s/cloudbuild] Update your slo-generator command: {RED} [-] slo-generator -f {source_str} -b {relative_ebp_path}{ENDC} {GREEN} [+] slo-generator -f {target_str} -c {target_str}/config.yaml{ENDC} - """) + """ + ) # 3 - [terraform] Upgrade your `terraform-google-slo` modules: # 3.1 - Upgrade the module `version` to 2.0.0. # 3.2 - Replace `error_budget_policy` field in your `slo` and `slo-pipeline` modules by `shared_config` # 3.3 - Replace `error_budget_policy.yaml` local variable to `config.yaml` -def exporters_v1tov2(exporters_paths: list, shared_config: dict = {}, quiet: bool = False) -> list: +# pylint: disable=dangerous-default-value +def exporters_v1tov2( + exporters_paths: list, shared_config: dict = {}, quiet: bool = False +) -> list: """Translate exporters to v2 and put into shared config. Args: @@ -221,7 +247,7 @@ def exporters_v1tov2(exporters_paths: list, shared_config: dict = {}, quiet: boo """ exp_keys = [] for exp_path in exporters_paths: - with open(exp_path, encoding='utf-8') as conf: + with open(exp_path, encoding="utf-8") as conf: content = yaml.load(conf, Loader=yaml.SafeLoader) exporters = content @@ -235,14 +261,14 @@ def exporters_v1tov2(exporters_paths: list, shared_config: dict = {}, quiet: boo # exporter. Refer to the alias in the SLO config file. for exporter in exporters: exporter = OrderedDict(exporter) - exp_key = add_to_shared_config(exporter, - shared_config, - 'exporters', - quiet=quiet) + exp_key = add_to_shared_config( + exporter, shared_config, "exporters", quiet=quiet + ) exp_keys.append(exp_key) return exp_keys +# pylint: disable=dangerous-default-value def ebp_v1tov2(ebp_paths: list, shared_config: dict = {}, quiet: bool = False) -> list: """Translate error budget policies to v2 and put into shared config @@ -256,36 +282,40 @@ def ebp_v1tov2(ebp_paths: list, shared_config: dict = {}, quiet: bool = False) - """ ebp_keys = [] for ebp_path in ebp_paths: - with open(ebp_path, encoding='utf-8') as conf: + with open(ebp_path, encoding="utf-8") as conf: error_budget_policy = yaml.load(conf, Loader=yaml.SafeLoader) for step in error_budget_policy: - step['name'] = step.pop('error_budget_policy_step_name') - step['burn_rate_threshold'] = step.pop( - 'alerting_burn_rate_threshold') - step['alert'] = step.pop('urgent_notification') - step['message_alert'] = step.pop('overburned_consequence_message') - step['message_ok'] = step.pop('achieved_consequence_message') - step['window'] = step.pop('measurement_window_seconds') - - ebp = {'steps': error_budget_policy} - if ebp_path.name == 'error_budget_policy.yaml': - ebp_key = 'default' + step["name"] = step.pop("error_budget_policy_step_name") + step["burn_rate_threshold"] = step.pop("alerting_burn_rate_threshold") + step["alert"] = step.pop("urgent_notification") + step["message_alert"] = step.pop("overburned_consequence_message") + step["message_ok"] = step.pop("achieved_consequence_message") + step["window"] = step.pop("measurement_window_seconds") + + ebp = {"steps": error_budget_policy} + if ebp_path.name == "error_budget_policy.yaml": + ebp_key = "default" else: - ebp_key = ebp_path.stem.replace('error_budget_policy_', '') - ebp_key = add_to_shared_config(ebp, - shared_config, - 'error_budget_policies', - ebp_key, - quiet=quiet) + ebp_key = ebp_path.stem.replace("error_budget_policy_", "") + ebp_key = add_to_shared_config( + ebp, + shared_config, + "error_budget_policies", + ebp_key, + quiet=quiet, + ) ebp_keys.append(ebp_key) return ebp_keys -def slo_config_v1tov2(slo_config: dict, - shared_config: dict = {}, - shared_exporters: list = [], - quiet: bool = False, - verbose: int = 0): +# pylint: disable=dangerous-default-value +def slo_config_v1tov2( + slo_config: dict, + shared_config: dict = {}, + shared_exporters: list = [], + quiet: bool = False, + verbose: int = 0, +): """Process old SLO config v1 and generate SLO config v2. Args: @@ -300,70 +330,73 @@ def slo_config_v1tov2(slo_config: dict, """ # SLO config v2 skeleton slo_config_v2 = OrderedDict(copy.deepcopy(SLO_CONFIG_SCHEMA)) - slo_config_v2['apiVersion'] = 'sre.google.com/v2' - slo_config_v2['kind'] = 'ServiceLevelObjective' + slo_config_v2["apiVersion"] = "sre.google.com/v2" + slo_config_v2["kind"] = "ServiceLevelObjective" missing_keys = [ - key for key in ['service_name', 'feature_name', 'slo_name', 'backend'] + key + for key in ["service_name", "feature_name", "slo_name", "backend"] if key not in slo_config ] if missing_keys: click.secho( - f'Invalid SLO configuration: missing key(s) {missing_keys}.', - fg='red') + f"Invalid SLO configuration: missing key(s) {missing_keys}.", fg="red" + ) return None # Get fields from old config - slo_metadata_name_fmt = '{service_name}-{feature_name}-{slo_name}' + slo_metadata_name_fmt = "{service_name}-{feature_name}-{slo_name}" slo_metadata_name = slo_metadata_name_fmt.format(**slo_config) - slo_description = slo_config.pop('slo_description') - slo_target = slo_config.pop('slo_target') - service_level_indicator = slo_config['backend'].pop('measurement', {}) - backend = slo_config['backend'] - method = backend.pop('method') - exporters = slo_config.get('exporters', []) + slo_description = slo_config.pop("slo_description") + slo_target = slo_config.pop("slo_target") + service_level_indicator = slo_config["backend"].pop("measurement", {}) + backend = slo_config["backend"] + method = backend.pop("method") + exporters = slo_config.get("exporters", []) if isinstance(exporters, dict): # single exporter, deprecated exporters = [exporters] # Fill spec - slo_config_v2['metadata']['name'] = slo_metadata_name - slo_config_v2['metadata']['labels'] = { - 'service_name': slo_config['service_name'], - 'feature_name': slo_config['feature_name'], - 'slo_name': slo_config['slo_name'], + slo_config_v2["metadata"]["name"] = slo_metadata_name + slo_config_v2["metadata"]["labels"] = { + "service_name": slo_config["service_name"], + "feature_name": slo_config["feature_name"], + "slo_name": slo_config["slo_name"], } other_labels = { - k: v for k, v in slo_config.items() if k not in - ['service_name', 'feature_name', 'slo_name', 'backend', 'exporters'] + k: v + for k, v in slo_config.items() + if k not in ["service_name", "feature_name", "slo_name", "backend", "exporters"] } - slo_config_v2['metadata']['labels'].update(other_labels) - slo_config_v2['spec']['description'] = slo_description - slo_config_v2['spec']['goal'] = slo_target + slo_config_v2["metadata"]["labels"].update(other_labels) + slo_config_v2["spec"]["description"] = slo_description + slo_config_v2["spec"]["goal"] = slo_target # Process backend backend = OrderedDict(backend) - backend_key = add_to_shared_config(backend, - shared_config, - 'backends', - quiet=quiet) - slo_config_v2['spec']['backend'] = backend_key - slo_config_v2['spec']['method'] = method + backend_key = add_to_shared_config( + backend, + shared_config, + "backends", + quiet=quiet, + ) + slo_config_v2["spec"]["backend"] = backend_key + slo_config_v2["spec"]["method"] = method # If exporter not in general config, add it and add an alias for the # exporter. Refer to the alias in the SLO config file. for exporter in exporters: exporter = OrderedDict(exporter) - exp_key = add_to_shared_config(exporter, - shared_config, - 'exporters', - quiet=quiet) - slo_config_v2['spec']['exporters'].append(exp_key) + exp_key = add_to_shared_config( + exporter, shared_config, "exporters", quiet=quiet + ) + slo_config_v2["spec"]["exporters"].append(exp_key) # Add shared exporters to slo config for exp_key in shared_exporters: - slo_config_v2['spec']['exporters'].append(exp_key) + slo_config_v2["spec"]["exporters"].append(exp_key) # Fill spec - slo_config_v2['spec']['service_level_indicator'] = service_level_indicator + slo_config_v2["spec"]["service_level_indicator"] = service_level_indicator if verbose > 0: pprint.pprint(dict(slo_config_v2)) @@ -384,13 +417,13 @@ def report_v2tov1(report: dict) -> dict: for key, value in report.items(): # If a metadata label is passed, use the metadata label mapping - if key == 'metadata': - mapped_report['metadata'] = {} + if key == "metadata": + mapped_report["metadata"] = {} for subkey, subvalue in value.items(): # v2 `metadata.labels` attributes map to `metadata` attributes # in v1 - if subkey == 'labels': + if subkey == "labels": labels = subvalue for labelkey, labelval in labels.items(): @@ -402,17 +435,17 @@ def report_v2tov1(report: dict) -> dict: # Other labels that are mapped to 'metadata' in the v1 # report else: - mapped_report['metadata'][labelkey] = labelval + mapped_report["metadata"][labelkey] = labelval # ignore the name attribute which is just a concatenation of # service_name, feature_name and slo_name - elif subkey == 'name': + elif subkey == "name": continue # other metadata labels are still mapped to the v1 `metadata` # attributes else: - mapped_report['metadata'][subkey] = subvalue + mapped_report["metadata"][subkey] = subvalue # If a key in the default label mapping is passed, use the default # label mapping @@ -425,14 +458,12 @@ def report_v2tov1(report: dict) -> dict: def get_random_suffix() -> str: """Get random suffix for our backends / exporters when configs clash.""" - return ''.join(random.choices(string.digits, k=4)) + return "".join(random.choices(string.digits, k=4)) # nosec B311 -def add_to_shared_config(new_obj: dict, - shared_config: dict, - section: str, - key = None, - quiet: bool = False): +def add_to_shared_config( + new_obj: dict, shared_config: dict, section: str, key=None, quiet: bool = False +): """Add an object to the shared_config. If the object with the same config already exists in the shared config, @@ -453,16 +484,16 @@ def add_to_shared_config(new_obj: dict, str: Object key in the shared config. """ shared_obj = shared_config[section] - key = key or new_obj.pop('class', None) + key = key or new_obj.pop("class", None) if not key: raise ValueError("Object key is undefined.") - if '.' not in key: + if "." not in key: key = utils.caml_to_snake(PROVIDERS_COMPAT.get(key, key)) existing_obj = { k: v for k, v in shared_obj.items() - if k.startswith(key.split('/')[0]) and str(v) == str(dict(new_obj)) + if k.startswith(key.split("/")[0]) and str(v) == str(dict(new_obj)) } if existing_obj: key = next(iter(existing_obj)) @@ -470,32 +501,34 @@ def add_to_shared_config(new_obj: dict, else: if key in shared_obj.keys(): # key conflicts if quiet: - key += '/' + get_random_suffix() + key += "/" + get_random_suffix() else: - name = section.rstrip('s') + name = section.rstrip("s") cfg = pprint.pformat({key: dict(new_obj)}) valid = False while not valid: click.secho( - f'\nNew {name} found with the following config:\n{cfg}', - fg='cyan', - blink=True) + f"\nNew {name} found with the following config:\n{cfg}", + fg="cyan", + blink=True, + ) user_input = click.prompt( - f'\n{RED}{BOLD}Please give this {name} a name{ENDC}', - type=str) + f"\n{RED}{BOLD}Please give this {name} a name{ENDC}", type=str + ) former_key = key - key += '/' + user_input.lower() + key += "/" + user_input.lower() if key in shared_obj.keys(): click.secho( f'{name.capitalize()} "{key}" already exists in shared config', - fg='red', - bold=True) + fg="red", + bold=True, + ) key = former_key else: valid = True - click.secho(f'Backend {key} was added to shared config.', - fg='green', - bold=True) + click.secho( + f"Backend {key} was added to shared config.", fg="green", bold=True + ) # click.secho(f"Adding new {section} {key}") shared_obj[key] = dict(new_obj) @@ -514,14 +547,14 @@ def detect_config_version(config: dict) -> str: """ if not isinstance(config, dict): click.secho( - 'Config does not correspond to any known SLO config versions.', - fg='red') + "Config does not correspond to any known SLO config versions.", fg="red" + ) return None - api_version: str = config.get('apiVersion', '') - kind = config.get('kind', '') + api_version: str = config.get("apiVersion", "") + kind = config.get("kind", "") if not kind: # old v1 format - return 'v1' - return api_version.split('/')[-1] + return "v1" + return api_version.split("/")[-1] def peek(iterable): @@ -540,6 +573,7 @@ def peek(iterable): return first, itertools.chain([first], iterable) +# pylint: disable=too-few-public-methods class CustomDumper(yaml.RoundTripDumper): """Dedicated YAML dumper to insert lines between top-level objects. @@ -549,6 +583,7 @@ class CustomDumper(yaml.RoundTripDumper): # HACK: insert blank lines between top-level objects # inspired by https://stackoverflow.com/a/44284819/3786245 + # pylint: disable=missing-function-docstring def write_line_break(self, data: str = None): super().write_line_break(data) diff --git a/slo_generator/report.py b/slo_generator/report.py index 260848f1..84a6d3df 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -17,12 +17,11 @@ """ import logging -from dataclasses import asdict, dataclass, fields, field +from dataclasses import asdict, dataclass, field, fields from typing import List from slo_generator import utils -from slo_generator.constants import (COLORED_OUTPUT, MIN_VALID_EVENTS, NO_DATA, - Colors) +from slo_generator.constants import COLORED_OUTPUT, MIN_VALID_EVENTS, NO_DATA, Colors LOGGER = logging.getLogger(__name__) @@ -40,6 +39,7 @@ class SLOReport: client (obj): Existing backend client. delete (bool): Backend delete action. """ + # pylint: disable=too-many-instance-attributes # SLO @@ -75,7 +75,7 @@ class SLOReport: # SLO exporters: list = field(default_factory=list) - error_budget_policy: str = 'default' + error_budget_policy: str = "default" # SLI sli_measurement: float = 0 @@ -89,31 +89,27 @@ class SLOReport: # Data validation errors: List[str] = field(default_factory=list) - def __init__(self, - config, - backend, - step, - timestamp, - client=None, - delete=False): + # pylint: disable=too-many-arguments + def __init__(self, config, backend, step, timestamp, client=None, delete=False): # Init dataclass fields from SLO config and Error Budget Policy - spec = config['spec'] + spec = config["spec"] self.exporters = [] - self.__set_fields(**spec, - **step, - lambdas={ - 'goal': float, - 'step': int, - 'error_budget_burn_rate_threshold': float - }) + self.__set_fields( + **spec, + **step, + lambdas={ + "goal": float, + "step": int, + "error_budget_burn_rate_threshold": float, + }, + ) # Set other fields - self.metadata = config['metadata'] + self.metadata = config["metadata"] self.timestamp = int(timestamp) - self.name = self.metadata['name'] - self.error_budget_policy_step_name = step['name'] - self.error_budget_burn_rate_threshold = float( - step['burn_rate_threshold']) + self.name = self.metadata["name"] + self.error_budget_policy_step_name = step["name"] + self.error_budget_burn_rate_threshold = float(step["burn_rate_threshold"]) self.timestamp_human = utils.get_human_time(timestamp) self.valid = True self.errors = [] @@ -163,27 +159,30 @@ def build(self, step, data): # Manage alerting message. if alert: - consequence_message = step['message_alert'] + consequence_message = step["message_alert"] elif eb_burn_rate <= 1: - consequence_message = step['message_ok'] + consequence_message = step["message_ok"] else: - consequence_message = \ - 'Missed for this measurement window, but not enough to alert' + consequence_message = ( + "Missed for this measurement window, but not enough to alert" + ) # Set fields in dataclass. - self.__set_fields(sli_measurement=sli, - good_events_count=int(good_count), - bad_events_count=int(bad_count), - events_count=int(good_count + bad_count), - gap=gap, - error_budget_target=eb_target, - error_budget_measurement=eb_value, - error_budget_burn_rate=eb_burn_rate, - error_budget_remaining_minutes=eb_remaining_minutes, - error_budget_minutes=eb_target_minutes, - error_minutes=eb_minutes, - alert=alert, - consequence_message=consequence_message) + self.__set_fields( + sli_measurement=sli, + good_events_count=int(good_count), + bad_events_count=int(bad_count), + events_count=int(good_count + bad_count), + gap=gap, + error_budget_target=eb_target, + error_budget_measurement=eb_value, + error_budget_burn_rate=eb_burn_rate, + error_budget_remaining_minutes=eb_remaining_minutes, + error_budget_minutes=eb_target_minutes, + error_minutes=eb_minutes, + alert=alert, + consequence_message=consequence_message, + ) def run_backend(self, config, backend, client=None, delete=False): """Get appropriate backend method from SLO configuration and run it on @@ -200,27 +199,27 @@ def run_backend(self, config, backend, client=None, delete=False): obj: Backend data. """ # Grab backend class and method dynamically. - cls_name = backend.get('class') - method = config['spec']['method'] - excluded_keys = ['class', 'service_level_indicator', 'name'] - backend_cfg = { - k: v for k, v in backend.items() if k not in excluded_keys - } + cls_name = backend.get("class") + method = config["spec"]["method"] + excluded_keys = ["class", "service_level_indicator", "name"] + backend_cfg = {k: v for k, v in backend.items() if k not in excluded_keys} cls = utils.get_backend_cls(cls_name) if not cls: - LOGGER.warning(f'{self.info} | Backend {cls_name} not loaded.') + LOGGER.warning(f"{self.info} | Backend {cls_name} not loaded.") self.valid = False return None instance = cls(client=client, **backend_cfg) method = getattr(instance, method) - LOGGER.debug(f'{self.info} | ' - f'Using backend {cls_name}.{method.__name__} (from ' - f'SLO config file).') + LOGGER.debug( + f"{self.info} | " + f"Using backend {cls_name}.{method.__name__} (from " + f"SLO config file)." + ) # Delete mode activation. - if delete and hasattr(instance, 'delete'): + if delete and hasattr(instance, "delete"): method = instance.delete - LOGGER.info(f'{self.info} | Delete mode enabled.') + LOGGER.info(f"{self.info} | Delete mode enabled.") # Set offset from class attribute if it exists in the class, otherwise # keep the value defined in config. @@ -233,7 +232,7 @@ def run_backend(self, config, backend, client=None, delete=False): # Run backend method and return results. try: data = method(self.timestamp, self.window, config) - LOGGER.debug(f'{self.info} | Backend response: {data}') + LOGGER.debug(f"{self.info} | Backend response: {data}") except Exception as exc: # pylint:disable=broad-except self.errors.append(utils.fmt_traceback(exc)) return None @@ -264,7 +263,7 @@ def get_sli(self, data): good_count = 0 if bad_count == NO_DATA: bad_count = 0 - LOGGER.debug(f'{self.info} | Good: {good_count} | Bad: {bad_count}') + LOGGER.debug(f"{self.info} | Good: {good_count} | Bad: {bad_count}") sli_measurement = round(good_count / (good_count + bad_count), 6) else: # sli value sli_measurement = round(data, 6) @@ -276,10 +275,10 @@ def to_json(self) -> dict: if not self.valid: ebp_name = self.error_budget_policy_step_name return { - 'metadata': self.metadata, - 'errors': self.errors, - 'error_budget_policy_step_name': ebp_name, - 'valid': self.valid + "metadata": self.metadata, + "errors": self.errors, + "error_budget_policy_step_name": ebp_name, + "valid": self.valid, } return asdict(self) @@ -301,9 +300,10 @@ def _validate(self, data) -> bool: # Backend result is the wrong type if not isinstance(data, (tuple, float, int)): error = ( - f'Backend method returned an object of type ' - f'{type(data).__name__}. It should instead return a tuple ' - '(good_count, bad_count) or a numeric SLI value (float / int).') + f"Backend method returned an object of type " + f"{type(data).__name__}. It should instead return a tuple " + "(good_count, bad_count) or a numeric SLI value (float / int)." + ) self.errors.append(error) return False @@ -313,8 +313,9 @@ def _validate(self, data) -> bool: # Tuple length should be 2 if len(data) != 2: error = ( - f'Backend method returned a tuple with {len(data)} items.' - f'Expected 2 items.') + f"Backend method returned a tuple with {len(data)} items." + f"Expected 2 items." + ) self.errors.append(error) return False good, bad = data @@ -322,24 +323,27 @@ def _validate(self, data) -> bool: # Tuple should contain only elements of type int or float if not all(isinstance(n, (float, int)) for n in data): error = ( - 'Backend method returned a tuple with some elements having' - ' a type different than float or int') + "Backend method returned a tuple with some elements having" + " a type different than float or int" + ) self.errors.append(error) return False # Tuple should not contain any element with value None. if good is None or bad is None: error = ( - f'Backend method returned a valid tuple {data} but one of ' - 'the values is None.') + f"Backend method returned a valid tuple {data} but one of " + "the values is None." + ) self.errors.append(error) return False # Tuple should not have NO_DATA everywhere if (good, bad) == (NO_DATA, NO_DATA): error = ( - f'Backend method returned a valid tuple {data} but the ' - 'good and bad count is NO_DATA (-1).') + f"Backend method returned a valid tuple {data} but the " + "good and bad count is NO_DATA (-1)." + ) self.errors.append(error) return False @@ -347,20 +351,21 @@ def _validate(self, data) -> bool: # minimum valid events threshold if (good + bad) < MIN_VALID_EVENTS: error = ( - f'Not enough valid events ({good + bad}) found. Minimum ' - f'valid events: {MIN_VALID_EVENTS}.') + f"Not enough valid events ({good + bad}) found. Minimum " + f"valid events: {MIN_VALID_EVENTS}." + ) self.errors.append(error) return False # Check backend float / int value if isinstance(data, (float, int)) and data == NO_DATA: - error = 'Backend returned NO_DATA (-1).' + error = "Backend returned NO_DATA (-1)." self.errors.append(error) return False # Check backend None if data is None: - error = 'Backend returned None.' + error = "Backend returned None." self.errors.append(error) return False @@ -371,13 +376,13 @@ def _post_validate(self) -> bool: # SLI measurement should be 0 <= x <= 1 if not 0 <= self.sli_measurement <= 1: - error = ( - f'SLI is not between 0 and 1 (value = {self.sli_measurement})') + error = f"SLI is not between 0 and 1 (value = {self.sli_measurement})" self.errors.append(error) return False return True + # pylint: disable=dangerous-default-value def __set_fields(self, lambdas={}, **kwargs): """Set all fields in dataclasses from configs passed and apply function on values whose key match one in the dictionaries. @@ -403,22 +408,23 @@ def info(self) -> str: def __str__(self) -> str: report = self.to_json() if not self.valid: - errors_str = ' | '.join(self.errors) - return f'{self.info} | {errors_str}' + errors_str = " | ".join(self.errors) + return f"{self.info} | {errors_str}" goal_per = self.goal * 100 sli_per = round(self.sli_measurement * 100, 6) gap = round(self.gap * 100, 2) gap_str = str(gap) if gap >= 0: - gap_str = f'+{gap}' - - sli_str = (f'SLI: {sli_per:<7} % | SLO: {goal_per} % | ' - f'Gap: {gap_str:<6}%') - result_str = ('BR: {error_budget_burn_rate:<2} / ' - '{error_budget_burn_rate_threshold} | ' - 'Alert: {alert:<1} | Good: {good_events_count:<8} | ' - 'Bad: {bad_events_count:<8}').format_map(report) - full_str = f'{self.info} | {sli_str} | {result_str}' + gap_str = f"+{gap}" + + sli_str = f"SLI: {sli_per:<7} % | SLO: {goal_per} % | " f"Gap: {gap_str:<6}%" + result_str = ( + "BR: {error_budget_burn_rate:<2} / " + "{error_budget_burn_rate_threshold} | " + "Alert: {alert:<1} | Good: {good_events_count:<8} | " + "Bad: {bad_events_count:<8}" + ).format_map(report) + full_str = f"{self.info} | {sli_str} | {result_str}" if COLORED_OUTPUT == 1: if self.alert: full_str = Colors.FAIL + full_str + Colors.ENDC diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 07789958..302524e3 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -23,11 +23,11 @@ import pprint import re import sys -from typing import Optional import warnings from collections.abc import Mapping from datetime import datetime from pathlib import Path +from typing import Optional import yaml from dateutil import tz @@ -35,7 +35,9 @@ from slo_generator.constants import DEBUG try: - from google.cloud import storage # type: ignore[attr-defined] + # pytype: disable=import-error + from google.cloud import storage # type: ignore[attr-defined] + GCS_ENABLED = True except ImportError: GCS_ENABLED = False @@ -43,9 +45,10 @@ LOGGER = logging.getLogger(__name__) -def load_configs(path: str, - ctx: os._Environ = os.environ, - kind: Optional[str] = None) -> list: +# pylint: disable=dangerous-default-value +def load_configs( + path: str, ctx: os._Environ = os.environ, kind: Optional[str] = None +) -> list: """Load multiple slo-generator configs from a folder path. Args: @@ -58,14 +61,15 @@ def load_configs(path: str, """ configs = [ load_config(str(p), ctx=ctx, kind=kind) - for p in sorted(Path(path).glob('*.yaml')) + for p in sorted(Path(path).glob("*.yaml")) ] return [cfg for cfg in configs if cfg] -def load_config(path: str, - ctx: os._Environ = os.environ, - kind: Optional[str] = None) -> Optional[dict]: +# pylint: disable=dangerous-default-value +def load_config( + path: str, ctx: os._Environ = os.environ, kind: Optional[str] = None +) -> Optional[dict]: """Load any slo-generator config, from a local path, a GCS URL, or directly from a string content. @@ -79,24 +83,24 @@ def load_config(path: str, """ abspath = Path(path) try: - if path.startswith('gs://'): + if path.startswith("gs://"): if not GCS_ENABLED: warnings.warn( - 'To load a file from GCS, you need `google-cloud-storage` ' - 'installed. Please install it using pip by running ' - '`pip install google-cloud-storage`', ImportWarning) + "To load a file from GCS, you need `google-cloud-storage` " + "installed. Please install it using pip by running " + "`pip install google-cloud-storage`", + ImportWarning, + ) sys.exit(1) config = parse_config(content=download_gcs_file(str(path)), ctx=ctx) elif abspath.is_file(): config = parse_config(path=str(abspath.resolve()), ctx=ctx) else: - LOGGER.debug(f'Path {path} not found. Trying to load from string') + LOGGER.debug(f"Path {path} not found. Trying to load from string") config = parse_config(content=str(path), ctx=ctx) # Filter on 'kind' - if kind and ( - not isinstance(config, dict) or kind != config.get('kind', '') - ): + if kind and (not isinstance(config, dict) or kind != config.get("kind", "")): config = None return config @@ -106,9 +110,10 @@ def load_config(path: str, raise -def parse_config(path: Optional[str] = None, - content=None, - ctx: os._Environ = os.environ): +# pylint: disable=dangerous-default-value +def parse_config( + path: Optional[str] = None, content=None, ctx: os._Environ = os.environ +): """Load a yaml configuration file and resolve environment variables in it. Args: @@ -120,7 +125,7 @@ def parse_config(path: Optional[str] = None, Returns: dict: Parsed YAML dictionary. """ - pattern = re.compile(r'.*?\${(\w+)}.*?') + pattern = re.compile(r".*?\${(\w+)}.*?") def replace_env_vars(content, ctx) -> str: """Replace env variables in content from context. @@ -137,24 +142,26 @@ def replace_env_vars(content, ctx) -> str: full_value = content for var in match: try: - full_value = full_value.replace(f'${{{var}}}', ctx[var]) + full_value = full_value.replace(f"${{{var}}}", ctx[var]) except KeyError as exception: - LOGGER.error(f'Environment variable "{var}" should be set.', - exc_info=True) + LOGGER.error( + f'Environment variable "{var}" should be set.', exc_info=True + ) raise exception content = full_value return content if path: - with Path(path).open(encoding='utf8') as config: + with Path(path).open(encoding="utf8") as config: content = config.read() if ctx: content = replace_env_vars(content, ctx) data = yaml.safe_load(content) if isinstance(data, str): error = ( - 'Error serializing config into dict. This might be due to a syntax ' - 'error in the YAML / JSON config file.') + "Error serializing config into dict. This might be due to a syntax " + "error in the YAML / JSON config file." + ) LOGGER.error(error) LOGGER.debug(pprint.pformat(data)) @@ -164,24 +171,23 @@ def replace_env_vars(content, ctx) -> str: def setup_logging(): """Setup logging for the CLI.""" if DEBUG == 1: - print(f'DEBUG mode is enabled. DEBUG={DEBUG}') + print(f"DEBUG mode is enabled. DEBUG={DEBUG}") level = logging.DEBUG - format_str = '%(name)s - %(levelname)s - %(message)s' + format_str = "%(name)s - %(levelname)s - %(message)s" else: level = logging.INFO - format_str = '%(levelname)s - %(message)s' - logging.basicConfig(stream=sys.stdout, - level=level, - format=format_str, - datefmt='%m/%d/%Y %I:%M:%S') - logging.getLogger('googleapiclient').setLevel(logging.ERROR) + format_str = "%(levelname)s - %(message)s" + logging.basicConfig( + stream=sys.stdout, level=level, format=format_str, datefmt="%m/%d/%Y %I:%M:%S" + ) + logging.getLogger("googleapiclient").setLevel(logging.ERROR) # Ignore Cloud SDK warning when using a user instead of service account try: # pylint: disable=import-outside-toplevel from google.auth._default import _CLOUD_SDK_CREDENTIALS_WARNING - warnings.filterwarnings("ignore", - message=_CLOUD_SDK_CREDENTIALS_WARNING) + + warnings.filterwarnings("ignore", message=_CLOUD_SDK_CREDENTIALS_WARNING) except ImportError: pass @@ -209,11 +215,11 @@ def get_human_time(timestamp: int, timezone: Optional[str] = None) -> str: to_zone = tz.tzlocal() dt_utc = datetime.utcfromtimestamp(timestamp) dt_tz = dt_utc.replace(tzinfo=to_zone) - timeformat = '%Y-%m-%dT%H:%M:%S.%f%z' + timeformat = "%Y-%m-%dT%H:%M:%S.%f%z" date_str = datetime.strftime(dt_tz, timeformat) core_str = date_str[:-2] tz_str = date_str[-2:] - date_str = f'{core_str}:{tz_str}' + date_str = f"{core_str}:{tz_str}" return date_str @@ -227,21 +233,19 @@ def get_exporters(config: dict, spec: dict) -> list: Returns: list: List of dict containing exporters configurations. """ - all_exporters = config.get('exporters', {}) - spec_exporters = spec.get('exporters', []) + all_exporters = config.get("exporters", {}) + spec_exporters = spec.get("exporters", []) exporters = [] for exporter in spec_exporters: if exporter not in all_exporters.keys(): - LOGGER.error( - f'Exporter "{exporter}" not found in config.') + LOGGER.error(f'Exporter "{exporter}" not found in config.') continue exporter_data = all_exporters[exporter] - exporter_data['name'] = exporter - if '.' in exporter: # support custom exporter - exporter_data['class'] = exporter - else: # core exporter - exporter_data['class'] = capitalize( - snake_to_caml(exporter.split('/')[0])) + exporter_data["name"] = exporter + if "." in exporter: # support custom exporter + exporter_data["class"] = exporter + else: # core exporter + exporter_data["class"] = capitalize(snake_to_caml(exporter.split("/")[0])) exporters.append(exporter_data) return exporters @@ -256,19 +260,18 @@ def get_backend(config: dict, spec: dict): Returns: list: List of dict containing exporters configurations. """ - all_backends = config.get('backends', {}) - spec_backend = spec['backend'] + all_backends = config.get("backends", {}) + spec_backend = spec["backend"] backend_data = {} if spec_backend not in all_backends.keys(): LOGGER.error(f'Backend "{spec_backend}" not found in config. Exiting.') sys.exit(0) backend_data = all_backends[spec_backend] - backend_data['name'] = spec_backend - if '.' in spec_backend: # custom backend - backend_data['class'] = spec_backend - else: # built-in backend - backend_data['class'] = capitalize(snake_to_caml( - spec_backend.split('/')[0])) + backend_data["name"] = spec_backend + if "." in spec_backend: # custom backend + backend_data["class"] = spec_backend + else: # built-in backend + backend_data["class"] = capitalize(snake_to_caml(spec_backend.split("/")[0])) return backend_data @@ -282,11 +285,10 @@ def get_error_budget_policy(config: dict, spec: dict): Returns: list: List of dict containing exporters configurations. """ - all_ebp = config.get('error_budget_policies', {}) - spec_ebp = spec.get('error_budget_policy', 'default') + all_ebp = config.get("error_budget_policies", {}) + spec_ebp = spec.get("error_budget_policy", "default") if spec_ebp not in all_ebp.keys(): - LOGGER.error( - f'Error budget policy "{spec_ebp}" not found in config. Exiting.') + LOGGER.error(f'Error budget policy "{spec_ebp}" not found in config. Exiting.') sys.exit(0) return all_ebp[spec_ebp] @@ -335,11 +337,11 @@ def import_cls(cls_name, expected_type): # slo-generator core class modules_name = f"{expected_type.lower()}s" - full_cls_name = f'{cls_name}{expected_type}' - filename = re.sub(r'(? str: Returns: str: Input string with first letter capitalized. """ - return re.sub('([a-zA-Z])', lambda x: x.groups()[0].upper(), word, 1) + return re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), word, 1) def snake_to_caml(word: str) -> str: @@ -390,7 +393,7 @@ def snake_to_caml(word: str) -> str: Returns: str: Output string. """ - return re.sub('_.', lambda x: x.group()[1].upper(), word) + return re.sub("_.", lambda x: x.group()[1].upper(), word) def caml_to_snake(word: str) -> str: @@ -402,7 +405,7 @@ def caml_to_snake(word: str) -> str: Returns: str: Output string. """ - return re.sub(r'(? dict: @@ -446,11 +449,11 @@ def str2bool(string: str) -> bool: """ if isinstance(string, bool): return string - if string.lower() in ('yes', 'true', 't', 'y', '1'): + if string.lower() in ("yes", "true", "t", "y", "1"): return True - if string.lower() in ('no', 'false', 'f', 'n', '0'): + if string.lower() in ("no", "false", "f", "n", "0"): return False - raise argparse.ArgumentTypeError('Boolean value expected.') + raise argparse.ArgumentTypeError("Boolean value expected.") def download_gcs_file(url: str) -> dict: @@ -466,7 +469,7 @@ def download_gcs_file(url: str) -> dict: bucket, filepath = decode_gcs_url(url) bucket = client.get_bucket(bucket) blob = bucket.blob(filepath) - return blob.download_as_string(client=None).decode('utf-8') + return blob.download_as_string(client=None).decode("utf-8") def decode_gcs_url(url: str) -> tuple: @@ -478,13 +481,14 @@ def decode_gcs_url(url: str) -> tuple: Returns: tuple: (bucket_name, file_path) """ - split_url = url.split('/') + split_url = url.split("/") bucket_name = split_url[2] - file_path = '/'.join(split_url[3:]) + file_path = "/".join(split_url[3:]) return (bucket_name, file_path) -def get_files(source, extensions=['yaml', 'yml', 'json']) -> list: +# pylint: disable=dangerous-default-value +def get_files(source, extensions=["yaml", "yml", "json"]) -> list: """Get all files matching extensions. Args: @@ -495,14 +499,11 @@ def get_files(source, extensions=['yaml', 'yml', 'json']) -> list: """ all_files: list = [] for ext in extensions: - all_files.extend(Path(source).rglob(f'*.{ext}')) + all_files.extend(Path(source).rglob(f"*.{ext}")) return all_files -def get_target_path(source_dir, - target_dir, - relative_path, - mkdir: bool = True): +def get_target_path(source_dir, target_dir, relative_path, mkdir: bool = True): """Get target file path from a source directory, a target directory and a path relative to the source directory. @@ -519,12 +520,12 @@ def get_target_path(source_dir, target_dir = target_dir.resolve() relative_path = relative_path.relative_to(source_dir) common_path = os.path.commonpath([source_dir, target_dir]) - target_path = common_path / target_dir.relative_to( - common_path) / relative_path + target_path = common_path / target_dir.relative_to(common_path) / relative_path if mkdir: target_path.parent.mkdir(parents=True, exist_ok=True) return target_path + def fmt_traceback(exc) -> str: """Format exception to be human-friendly. diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index bb72f5be..0417b91d 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -16,4 +16,5 @@ Init test environment variables. """ import os -os.environ['MIN_VALID_EVENTS'] = '10' + +os.environ["MIN_VALID_EVENTS"] = "10" diff --git a/tests/unit/backends/test_cloud_monitoring_mql.py b/tests/unit/backends/test_cloud_monitoring_mql.py index b215ef17..9e5e9d2e 100644 --- a/tests/unit/backends/test_cloud_monitoring_mql.py +++ b/tests/unit/backends/test_cloud_monitoring_mql.py @@ -16,53 +16,46 @@ import unittest -from slo_generator.backends.cloud_monitoring_mql import \ - CloudMonitoringMqlBackend +from slo_generator.backends.cloud_monitoring_mql import CloudMonitoringMqlBackend class TestCloudMonitoringMqlBackend(unittest.TestCase): - def test_fmt_query(self): - # pylint: disable=trailing-whitespace queries = [ - ''' fetch gae_app + """ fetch gae_app | metric 'appengine.googleapis.com/http/server/response_count' | filter resource.project_id == '${GAE_PROJECT_ID}' | filter metric.response_code == 429 || metric.response_code == 200 - | group_by [metric.response_code] | within 1h ''', - - ''' fetch gae_app + | group_by [metric.response_code] | within 1h """, + """ fetch gae_app | metric 'appengine.googleapis.com/http/server/response_count' | filter resource.project_id == '${GAE_PROJECT_ID}' | filter metric.response_code == 429 || metric.response_code == 200 | group_by [metric.response_code, response_code_class] - | within 1h - | every 1h ''', - - ''' fetch gae_app + | within 1h + | every 1h """, + """ fetch gae_app | metric 'appengine.googleapis.com/http/server/response_count' | filter resource.project_id == '${GAE_PROJECT_ID}' | filter metric.response_code == 429 || metric.response_code == 200 - | group_by [metric.response_code,response_code_class] + | group_by [metric.response_code,response_code_class] | within 1h - | every 1h ''', + | every 1h """, ] - # pylint: enable=trailing-whitespace - formatted_query = '''fetch gae_app + formatted_query = """fetch gae_app | metric 'appengine.googleapis.com/http/server/response_count' | filter resource.project_id == '${GAE_PROJECT_ID}' | filter metric.response_code == 429 || metric.response_code == 200 - | group_by [] | within 3600s | every 3600s''' + | group_by [] | within 3600s | every 3600s""" for query in queries: - assert CloudMonitoringMqlBackend._fmt_query(query, - 3600) == formatted_query + assert CloudMonitoringMqlBackend._fmt_query(query, 3600) == formatted_query diff --git a/tests/unit/fixtures/dt_slo_get.json b/tests/unit/fixtures/dt_slo_get.json index d888c53a..a0956d9b 100644 --- a/tests/unit/fixtures/dt_slo_get.json +++ b/tests/unit/fixtures/dt_slo_get.json @@ -18,4 +18,4 @@ "evaluationType": "AGGREGATE", "timeframe": "-30m", "filter": "type(\"sERVICE\"),tag(\"XXXXXX\")" -} \ No newline at end of file +} diff --git a/tests/unit/fixtures/dummy_backend.py b/tests/unit/fixtures/dummy_backend.py index 0022e541..be12591b 100644 --- a/tests/unit/fixtures/dummy_backend.py +++ b/tests/unit/fixtures/dummy_backend.py @@ -2,18 +2,20 @@ Dummy backend implementation for testing. """ -# pylint:disable=missing-class-docstring,missing-function-docstring,unused-argument +# pylint: disable=missing-class-docstring class DummyBackend: - + # pylint: disable=unused-argument def __init__(self, client=None, **config): - self.good_events = config.get('good_events', None) - self.bad_events = config.get('bad_events', None) - self.sli_value = config.get('sli', None) + self.good_events = config.get("good_events", None) + self.bad_events = config.get("bad_events", None) + self.sli_value = config.get("sli", None) + # pylint: disable=missing-function-docstring,unused-argument def good_bad_ratio(self, timestamp, window, slo_config): return (self.good_events, self.bad_events) + # pylint: disable=missing-function-docstring,unused-argument def sli(self, timestamp, window, slo_config): return self.sli_value diff --git a/tests/unit/fixtures/fail_exporter.py b/tests/unit/fixtures/fail_exporter.py index 430c3da9..d3797a85 100644 --- a/tests/unit/fixtures/fail_exporter.py +++ b/tests/unit/fixtures/fail_exporter.py @@ -2,12 +2,11 @@ Dummy exporter implementation for testing. """ -# pylint: disable=missing-class-docstring from slo_generator.exporters.base import MetricsExporter +# pylint: disable=missing-class-docstring class FailExporter(MetricsExporter): - def export_metric(self, data): raise ValueError("Oops !") diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 036731f0..30948f5e 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -15,9 +15,9 @@ import os import unittest +from click.testing import CliRunner from mock import patch -from click.testing import CliRunner from slo_generator.cli import main from slo_generator.utils import load_config @@ -28,40 +28,30 @@ class TestCLI(unittest.TestCase): - def setUp(self): for key, value in CTX.items(): os.environ[key] = value - slo_config = f'{root}/samples/cloud_monitoring/slo_gae_app_availability.yaml' # noqa: E501 - config = f'{root}/samples/config.yaml' + slo_config = f"{root}/samples/cloud_monitoring/slo_gae_app_availability.yaml" # noqa: E501 + config = f"{root}/samples/config.yaml" self.slo_config = slo_config - self.slo_metadata_name = load_config(slo_config, - ctx=CTX)['metadata']['name'] + self.slo_metadata_name = load_config(slo_config, ctx=CTX)["metadata"]["name"] self.config = config self.cli = CliRunner() - @patch('google.api_core.grpc_helpers.create_channel', - return_value=mock_sd(8)) + @patch("google.api_core.grpc_helpers.create_channel", return_value=mock_sd(8)) def test_cli_compute(self, mock): - args = ['compute', '-f', self.slo_config, '-c', self.config] + args = ["compute", "-f", self.slo_config, "-c", self.config] result = self.cli.invoke(main, args) self.assertEqual(result.exit_code, 0) - @patch('google.api_core.grpc_helpers.create_channel', - return_value=mock_sd(40)) + @patch("google.api_core.grpc_helpers.create_channel", return_value=mock_sd(40)) def test_cli_compute_folder(self, mock): - args = [ - 'compute', '-f', f'{root}/samples/cloud_monitoring', '-c', - self.config - ] + args = ["compute", "-f", f"{root}/samples/cloud_monitoring", "-c", self.config] result = self.cli.invoke(main, args) self.assertEqual(result.exit_code, 0) def test_cli_compute_no_config(self): - args = [ - 'compute', '-f', f'{root}/samples', '-c', - f'{root}/samples/config.yaml' - ] + args = ["compute", "-f", f"{root}/samples", "-c", f"{root}/samples/config.yaml"] result = self.cli.invoke(main, args) self.assertEqual(result.exit_code, 1) @@ -74,5 +64,5 @@ def test_cli_migrate(self): pass -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index 1a4abaf1..d17762ae 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -20,30 +20,43 @@ from google.auth._default import _CLOUD_SDK_CREDENTIALS_WARNING from mock import MagicMock, patch from prometheus_http_client import Prometheus + from slo_generator.backends.dynatrace import DynatraceClient from slo_generator.compute import compute, export -from slo_generator.exporters.bigquery import BigQueryError from slo_generator.exporters.base import MetricsExporter -from .test_stubs import (CTX, load_fixture, load_sample, load_slo_samples, - mock_dd_metric_query, mock_dd_metric_send, - mock_dd_slo_get, mock_dd_slo_history, mock_dt, - mock_dt_errors, mock_es, mock_prom, mock_sd, - mock_ssm_client) +from slo_generator.exporters.bigquery import BigQueryError + +from .test_stubs import ( + CTX, + load_fixture, + load_sample, + load_slo_samples, + mock_dd_metric_query, + mock_dd_metric_send, + mock_dd_slo_get, + mock_dd_slo_history, + mock_dt, + mock_dt_errors, + mock_es, + mock_prom, + mock_sd, + mock_ssm_client, +) warnings.filterwarnings("ignore", message=_CLOUD_SDK_CREDENTIALS_WARNING) -CONFIG = load_sample('config.yaml', CTX) -STEPS = len(CONFIG['error_budget_policies']['default']['steps']) -SLO_CONFIGS_SD = load_slo_samples('cloud_monitoring', CTX) -SLO_CONFIGS_SDSM = load_slo_samples('cloud_service_monitoring', CTX) -SLO_CONFIGS_PROM = load_slo_samples('prometheus', CTX) -SLO_CONFIGS_ES = load_slo_samples('elasticsearch', CTX) -SLO_CONFIGS_DD = load_slo_samples('datadog', CTX) -SLO_CONFIGS_DT = load_slo_samples('dynatrace', CTX) -SLO_REPORT = load_fixture('slo_report_v2.json') -SLO_REPORT_V1 = load_fixture('slo_report_v1.json') -EXPORTERS = load_fixture('exporters.yaml', CTX) -BQ_ERROR = load_fixture('bq_error.json') +CONFIG = load_sample("config.yaml", CTX) +STEPS = len(CONFIG["error_budget_policies"]["default"]["steps"]) +SLO_CONFIGS_SD = load_slo_samples("cloud_monitoring", CTX) +SLO_CONFIGS_SDSM = load_slo_samples("cloud_service_monitoring", CTX) +SLO_CONFIGS_PROM = load_slo_samples("prometheus", CTX) +SLO_CONFIGS_ES = load_slo_samples("elasticsearch", CTX) +SLO_CONFIGS_DD = load_slo_samples("datadog", CTX) +SLO_CONFIGS_DT = load_slo_samples("dynatrace", CTX) +SLO_REPORT = load_fixture("slo_report_v2.json") +SLO_REPORT_V1 = load_fixture("slo_report_v1.json") +EXPORTERS = load_fixture("exporters.yaml", CTX) +BQ_ERROR = load_fixture("bq_error.json") # Pub/Sub methods to patch PUBSUB_MOCKS = [ @@ -53,38 +66,41 @@ ] # Service Monitoring method to patch -# pylint: ignore=E501 SSM_MOCKS = [ - "slo_generator.backends.cloud_service_monitoring.ServiceMonitoringServiceClient", # noqa: E501 - "slo_generator.backends.cloud_service_monitoring.SSM.to_json" + "slo_generator.backends.cloud_service_monitoring.ServiceMonitoringServiceClient", + "slo_generator.backends.cloud_service_monitoring.SSM.to_json", ] class TestCompute(unittest.TestCase): maxDiff = None - @patch('google.api_core.grpc_helpers.create_channel', - return_value=mock_sd(2 * STEPS * len(SLO_CONFIGS_SD))) + @patch( + "google.api_core.grpc_helpers.create_channel", + return_value=mock_sd(2 * STEPS * len(SLO_CONFIGS_SD)), + ) def test_compute_stackdriver(self, mock): for config in SLO_CONFIGS_SD: with self.subTest(config=config): compute(config, CONFIG) @patch(SSM_MOCKS[0], return_value=mock_ssm_client()) - @patch(SSM_MOCKS[1], - return_value=MagicMock(side_effect=mock_ssm_client.to_json)) - @patch('google.api_core.grpc_helpers.create_channel', - return_value=mock_sd(2 * STEPS * len(SLO_CONFIGS_SDSM))) + @patch(SSM_MOCKS[1], return_value=MagicMock(side_effect=mock_ssm_client.to_json)) + @patch( + "google.api_core.grpc_helpers.create_channel", + return_value=mock_sd(2 * STEPS * len(SLO_CONFIGS_SDSM)), + ) def test_compute_ssm(self, *mocks): for config in SLO_CONFIGS_SDSM: with self.subTest(config=config): compute(config, CONFIG) @patch(SSM_MOCKS[0], return_value=mock_ssm_client()) - @patch(SSM_MOCKS[1], - return_value=MagicMock(side_effect=mock_ssm_client.to_json)) - @patch('google.api_core.grpc_helpers.create_channel', - return_value=mock_sd(2 * STEPS * len(SLO_CONFIGS_SDSM))) + @patch(SSM_MOCKS[1], return_value=MagicMock(side_effect=mock_ssm_client.to_json)) + @patch( + "google.api_core.grpc_helpers.create_channel", + return_value=mock_sd(2 * STEPS * len(SLO_CONFIGS_SDSM)), + ) @patch(PUBSUB_MOCKS[0]) @patch(PUBSUB_MOCKS[1]) def test_compute_ssm_delete_export(self, *mocks): @@ -92,27 +108,27 @@ def test_compute_ssm_delete_export(self, *mocks): with self.subTest(config=config): compute(config, CONFIG, delete=True, do_export=True) - @patch.object(Prometheus, 'query', mock_prom) + @patch.object(Prometheus, "query", mock_prom) def test_compute_prometheus(self): for config in SLO_CONFIGS_PROM: with self.subTest(config=config): compute(config, CONFIG) - @patch.object(Elasticsearch, 'search', mock_es) + @patch.object(Elasticsearch, "search", mock_es) def test_compute_elasticsearch(self): for config in SLO_CONFIGS_ES: with self.subTest(config=config): compute(config, CONFIG) - @patch.object(Metric, 'query', mock_dd_metric_query) - @patch.object(ServiceLevelObjective, 'history', mock_dd_slo_history) - @patch.object(ServiceLevelObjective, 'get', mock_dd_slo_get) + @patch.object(Metric, "query", mock_dd_metric_query) + @patch.object(ServiceLevelObjective, "history", mock_dd_slo_history) + @patch.object(ServiceLevelObjective, "get", mock_dd_slo_get) def test_compute_datadog(self): for config in SLO_CONFIGS_DD: with self.subTest(config=config): compute(config, CONFIG) - @patch.object(DynatraceClient, 'request', side_effect=mock_dt) + @patch.object(DynatraceClient, "request", side_effect=mock_dt) def test_compute_dynatrace(self, mock): for config in SLO_CONFIGS_DT: with self.subTest(config=config): @@ -124,8 +140,7 @@ def test_compute_dynatrace(self, mock): def test_export_pubsub(self, *mocks): export(SLO_REPORT, EXPORTERS[0]) - @patch("google.api_core.grpc_helpers.create_channel", - return_value=mock_sd(STEPS)) + @patch("google.api_core.grpc_helpers.create_channel", return_value=mock_sd(STEPS)) def test_export_stackdriver(self, mock): export(SLO_REPORT, EXPORTERS[1]) @@ -139,8 +154,7 @@ def test_export_bigquery(self, *mocks): @patch("google.cloud.bigquery.Client.get_table") @patch("google.cloud.bigquery.Client.create_table") @patch("google.cloud.bigquery.Client.update_table") - @patch("google.cloud.bigquery.Client.insert_rows_json", - return_value=BQ_ERROR) + @patch("google.cloud.bigquery.Client.insert_rows_json", return_value=BQ_ERROR) def test_export_bigquery_error(self, *mocks): with self.assertRaises(BigQueryError): export(SLO_REPORT, EXPORTERS[2], raise_on_error=True) @@ -152,63 +166,59 @@ def test_export_prometheus(self, mock): def test_export_prometheus_self(self): export(SLO_REPORT, EXPORTERS[7]) - @patch.object(Metric, 'send', mock_dd_metric_send) + @patch.object(Metric, "send", mock_dd_metric_send) def test_export_datadog(self): export(SLO_REPORT, EXPORTERS[4]) - @patch.object(DynatraceClient, 'request', side_effect=mock_dt) + @patch.object(DynatraceClient, "request", side_effect=mock_dt) def test_export_dynatrace(self, mock): export(SLO_REPORT, EXPORTERS[5]) - @patch.object(DynatraceClient, 'request', side_effect=mock_dt_errors) + @patch.object(DynatraceClient, "request", side_effect=mock_dt_errors) def test_export_dynatrace_error(self, mock): responses = export(SLO_REPORT, EXPORTERS[5]) - codes = [r[0]['response']['error']['code'] for r in responses] + codes = [r[0]["response"]["error"]["code"] for r in responses] self.assertTrue(all(code == 429 for code in codes)) def test_metrics_exporter_build_data_labels(self): exporter = MetricsExporter() data = SLO_REPORT_V1 - labels = ['service_name', 'slo_name', 'metadata'] + labels = ["service_name", "slo_name", "metadata"] result = exporter.build_data_labels(data, labels) expected = { - 'service_name': SLO_REPORT_V1['service_name'], - 'slo_name': SLO_REPORT_V1['slo_name'], - 'env': SLO_REPORT_V1['metadata']['env'], - 'team': SLO_REPORT_V1['metadata']['team'] + "service_name": SLO_REPORT_V1["service_name"], + "slo_name": SLO_REPORT_V1["slo_name"], + "env": SLO_REPORT_V1["metadata"]["env"], + "team": SLO_REPORT_V1["metadata"]["team"], } self.assertEqual(result, expected) - @patch("google.api_core.grpc_helpers.create_channel", - return_value=mock_sd(STEPS)) + @patch("google.api_core.grpc_helpers.create_channel", return_value=mock_sd(STEPS)) @patch("google.cloud.bigquery.Client.get_table") @patch("google.cloud.bigquery.Client.create_table") @patch("google.cloud.bigquery.Client.update_table") - @patch("google.cloud.bigquery.Client.insert_rows_json", - return_value=BQ_ERROR) + @patch("google.cloud.bigquery.Client.insert_rows_json", return_value=BQ_ERROR) def test_export_multiple_error(self, *mocks): exporters = [EXPORTERS[1], EXPORTERS[2]] errors = export(SLO_REPORT, exporters) self.assertEqual(len(errors), 1) - self.assertIn('BigQueryError', errors[0]) + self.assertIn("BigQueryError", errors[0]) - @patch("google.api_core.grpc_helpers.create_channel", - return_value=mock_sd(STEPS)) + @patch("google.api_core.grpc_helpers.create_channel", return_value=mock_sd(STEPS)) @patch("google.cloud.bigquery.Client.get_table") @patch("google.cloud.bigquery.Client.create_table") @patch("google.cloud.bigquery.Client.update_table") - @patch("google.cloud.bigquery.Client.insert_rows_json", - return_value=BQ_ERROR) + @patch("google.cloud.bigquery.Client.insert_rows_json", return_value=BQ_ERROR) def test_export_multiple_error_raise(self, *mocks): exporters = [EXPORTERS[1], EXPORTERS[2]] with self.assertRaises(BigQueryError): export(SLO_REPORT, exporters, raise_on_error=True) def test_export_wrong_class(self): - exporters = [{'class': 'Unknown'}] + exporters = [{"class": "Unknown"}] with self.assertRaises(ImportError): export(SLO_REPORT, exporters, raise_on_error=True) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_migrate.py b/tests/unit/test_migrate.py index c3ac5498..91e86c5b 100644 --- a/tests/unit/test_migrate.py +++ b/tests/unit/test_migrate.py @@ -15,22 +15,22 @@ import unittest from slo_generator.migrations.migrator import slo_config_v1tov2 + from .test_stubs import load_fixture class TestMigrator(unittest.TestCase): - def setUp(self): - self.slo_config_v1 = load_fixture('slo_config_v1.yaml') - self.slo_config_v2 = load_fixture('slo_config_v2.yaml') + self.slo_config_v1 = load_fixture("slo_config_v1.yaml") + self.slo_config_v2 = load_fixture("slo_config_v2.yaml") self.shared_config = { - 'backends': {}, - 'exporters': {}, - 'error_budget_policies': {} + "backends": {}, + "exporters": {}, + "error_budget_policies": {}, } def test_migrate_v1_to_v2(self): - slo_config_migrated = slo_config_v1tov2(self.slo_config_v1, - self.shared_config, - quiet=True) + slo_config_migrated = slo_config_v1tov2( + self.slo_config_v1, self.shared_config, quiet=True + ) self.assertDictEqual(slo_config_migrated, self.slo_config_v2) diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index 347ae8b9..06ca1e88 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -20,7 +20,6 @@ class TestReport(unittest.TestCase): - def test_report_enough_events(self): report_cfg = mock_slo_report("enough_events") report = SLOReport(**report_cfg) @@ -44,7 +43,7 @@ def test_report_valid_sli_value(self): report_cfg = mock_slo_report("valid_sli_value") report = SLOReport(**report_cfg) self.assertTrue(report.valid) - self.assertEqual(report.sli_measurement, report_cfg['backend']['sli']) + self.assertEqual(report.sli_measurement, report_cfg["backend"]["sli"]) self.assertEqual(report.alert, False) def test_report_no_events(self): diff --git a/tests/unit/test_stubs.py b/tests/unit/test_stubs.py index cad83efc..3038c17e 100644 --- a/tests/unit/test_stubs.py +++ b/tests/unit/test_stubs.py @@ -22,40 +22,40 @@ from types import ModuleType from google.cloud import monitoring_v3 -from slo_generator.utils import load_configs, load_config + +from slo_generator.utils import load_config, load_configs TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -SAMPLE_DIR = os.path.join(os.path.dirname(os.path.dirname(TEST_DIR)), - "samples/") +SAMPLE_DIR = os.path.join(os.path.dirname(os.path.dirname(TEST_DIR)), "samples/") CTX = { - 'PROJECT_ID': 'fake', - 'PUBSUB_PROJECT_ID': 'fake', - 'PUBSUB_TOPIC_NAME': 'fake', - 'GAE_PROJECT_ID': 'fake', - 'GAE_MODULE_ID': 'fake', - 'GKE_MESH_UID': 'fake', - 'GKE_PROJECT_ID': 'fake', - 'GKE_CLUSTER_NAME': 'fake', - 'GKE_LOCATION': 'fake', - 'GKE_SERVICE_NAMESPACE': 'fake', - 'GKE_SERVICE_NAME': 'fake', - 'LB_PROJECT_ID': 'fake', - 'PROMETHEUS_URL': 'http://localhost:9090', - 'PROMETHEUS_PUSHGATEWAY_URL': 'http://localhost:9091', - 'ELASTICSEARCH_URL': 'http://localhost:9200', - 'STACKDRIVER_HOST_PROJECT_ID': 'fake', - 'STACKDRIVER_LOG_METRIC_NAME': 'fake', - 'BIGQUERY_PROJECT_ID': 'fake', - 'BIGQUERY_TABLE_ID': 'fake', - 'BIGQUERY_DATASET_ID': 'fake', - 'BIGQUERY_TABLE_NAME': 'fake', - 'DATADOG_API_KEY': 'fake', - 'DATADOG_APP_KEY': 'fake', - 'DATADOG_SLO_ID': 'fake', - 'DYNATRACE_API_URL': 'fake', - 'DYNATRACE_API_TOKEN': 'fake', - 'DYNATRACE_SLO_ID': 'fake' + "PROJECT_ID": "fake", + "PUBSUB_PROJECT_ID": "fake", + "PUBSUB_TOPIC_NAME": "fake", + "GAE_PROJECT_ID": "fake", + "GAE_MODULE_ID": "fake", + "GKE_MESH_UID": "fake", + "GKE_PROJECT_ID": "fake", + "GKE_CLUSTER_NAME": "fake", + "GKE_LOCATION": "fake", + "GKE_SERVICE_NAMESPACE": "fake", + "GKE_SERVICE_NAME": "fake", + "LB_PROJECT_ID": "fake", + "PROMETHEUS_URL": "http://localhost:9090", + "PROMETHEUS_PUSHGATEWAY_URL": "http://localhost:9091", + "ELASTICSEARCH_URL": "http://localhost:9200", + "STACKDRIVER_HOST_PROJECT_ID": "fake", + "STACKDRIVER_LOG_METRIC_NAME": "fake", + "BIGQUERY_PROJECT_ID": "fake", + "BIGQUERY_TABLE_ID": "fake", + "BIGQUERY_DATASET_ID": "fake", + "BIGQUERY_TABLE_NAME": "fake", + "DATADOG_API_KEY": "fake", + "DATADOG_APP_KEY": "fake", + "DATADOG_SLO_ID": "fake", + "DYNATRACE_API_URL": "fake", + "DYNATRACE_API_TOKEN": "fake", + "DYNATRACE_SLO_ID": "fake", } @@ -68,7 +68,7 @@ def add_dynamic(name, code, type): type (str): 'backends' or 'exporters'. """ mod = ModuleType(name) - module_name = f'slo_generator.{type}.{name}' + module_name = f"slo_generator.{type}.{name}" sys.modules[module_name] = mod exec(code, mod.__dict__) @@ -82,14 +82,13 @@ def mock_slo_report(key): Returns: dict: Dict configuration for SLOReport class. """ - slo_config = load_fixture('dummy_slo_config.json') - ebp_step = load_fixture( - 'dummy_config.json')['error_budget_policies']['default'][0] - dummy_tests = load_fixture('dummy_tests.json') + slo_config = load_fixture("dummy_slo_config.json") + ebp_step = load_fixture("dummy_config.json")["error_budget_policies"]["default"][0] + dummy_tests = load_fixture("dummy_tests.json") backend = dummy_tests[key] - slo_config['spec']['method'] = backend['method'] - backend['name'] = 'dummy' - backend['class'] = 'Dummy' + slo_config["spec"]["method"] = backend["method"] + backend["name"] = "dummy" + backend["class"] = "Dummy" timestamp = time.time() return { "config": slo_config, @@ -97,7 +96,7 @@ def mock_slo_report(key): "step": ebp_step, "timestamp": timestamp, "client": None, - "delete": False + "delete": False, } @@ -133,10 +132,7 @@ def __init__(self, responses=[]): self.requests = [] # pylint: disable=C0116,W0613 - def unary_unary(self, - method, - request_serializer=None, - response_deserializer=None): + def unary_unary(self, method, request_serializer=None, response_deserializer=None): return MultiCallableStub(method, self) @@ -164,12 +160,13 @@ def mock_sd(nresp=1): Returns: ChannelStub: Mocked gRPC channel stub. """ - timeseries = load_fixture('time_series_proto.json') + timeseries = load_fixture("time_series_proto.json") response = {"next_page_token": "", "time_series": [timeseries]} return mock_grpc_stub( response=response, proto_method=monitoring_v3.types.ListTimeSeriesResponse, - nresp=nresp) + nresp=nresp, + ) # pylint: disable=W0613,R1721 @@ -183,11 +180,13 @@ def mock_prom(self, metric): dict: Fake response. """ data = { - 'data': { - 'result': [{ - 'values': [x for x in range(5)], - 'value': [0, 1] - }] + "data": { + "result": [ + { + "values": [x for x in range(5)], + "value": [0, 1], + } + ] } } return json.dumps(data) @@ -204,64 +203,65 @@ def mock_es(self, index, body): Returns: dict: Fake response. """ - return {'hits': {'total': {'value': 120}}} + return {"hits": {"total": {"value": 120}}} def mock_dd_metric_query(*args, **kwargs): """Mock Datadog response for datadog.api.Metric.query.""" - return load_fixture('dd_timeseries.json') + return load_fixture("dd_timeseries.json") def mock_dd_slo_history(*args, **kwargs): """Mock Datadog response for datadog.api.ServiceLevelObjective.history.""" - return load_fixture('dd_slo_history.json') + return load_fixture("dd_slo_history.json") def mock_dd_slo_get(*args, **kwargs): """Mock Datadog response for datadog.api.ServiceLevelObjective.get.""" - return load_fixture('dd_slo.json') + return load_fixture("dd_slo.json") def mock_dd_metric_send(*args, **kwargs): """Mock Datadog response for datadog.api.Metric.send.""" - return load_fixture('dd_success.json') + return load_fixture("dd_success.json") def mock_dt(*args, **kwargs): """Mock Dynatrace response.""" - if args[0] == 'get' and args[1] == 'timeseries': - return load_fixture('dt_metric_get.json') + if args[0] == "get" and args[1] == "timeseries": + return load_fixture("dt_metric_get.json") - elif args[0] == 'get' and args[1] == 'metrics/query': - return load_fixture('dt_timeseries_get.json') + elif args[0] == "get" and args[1] == "metrics/query": + return load_fixture("dt_timeseries_get.json") - elif args[0] == 'get' and args[1].startswith('slo/'): - return load_fixture('dt_slo_get.json') + elif args[0] == "get" and args[1].startswith("slo/"): + return load_fixture("dt_slo_get.json") - elif args[0] == 'post' and args[1] == 'entity/infrastructure/custom': - return load_fixture('dt_metric_send.json') + elif args[0] == "post" and args[1] == "entity/infrastructure/custom": + return load_fixture("dt_metric_send.json") - elif args[0] == 'put' and args[1] == 'timeseries': + elif args[0] == "put" and args[1] == "timeseries": return {} def mock_dt_errors(*args, **kwargs): """Mock Dynatrace response with errors.""" - if args[0] == 'get' and args[1] == 'timeseries': - return load_fixture('dt_metric_get.json') + if args[0] == "get" and args[1] == "timeseries": + return load_fixture("dt_metric_get.json") - elif args[0] == 'get' and args[1] == 'metrics/query': - return load_fixture('dt_timeseries_get.json') + elif args[0] == "get" and args[1] == "metrics/query": + return load_fixture("dt_timeseries_get.json") - elif args[0] == 'post' and args[1] == 'entity/infrastructure/custom': - return load_fixture('dt_error_rate.json') + elif args[0] == "post" and args[1] == "entity/infrastructure/custom": + return load_fixture("dt_error_rate.json") - elif args[0] == 'put' and args[1] == 'timeseries': - return load_fixture('dt_error_rate.json') + elif args[0] == "put" and args[1] == "timeseries": + return load_fixture("dt_error_rate.json") class dotdict(dict): """dot.notation access to dictionary attributes""" + __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ @@ -287,17 +287,17 @@ class mock_ssm_client: """Fake Service Monitoring API client.""" def __init__(self): - self.services = [dotize(s) for s in load_fixture('ssm_services.json')] + self.services = [dotize(s) for s in load_fixture("ssm_services.json")] self.service_level_objectives = [ - dotize(slo) for slo in load_fixture('ssm_slos.json') + dotize(slo) for slo in load_fixture("ssm_slos.json") ] def project_path(self, project_id): - return f'projects/{project_id}' + return f"projects/{project_id}" def service_path(self, project_id, service_id): project_path = self.project_path(project_id) - return f'{project_path}/services/{service_id}' + return f"{project_path}/services/{service_id}" def create_service(self, parent, service, service_id=None): return self.services[0] @@ -308,10 +308,9 @@ def list_services(self, parent): def delete_service(self, name): return None - def create_service_level_objective(self, - parent, - service_level_objective, - service_level_objective_id=None): + def create_service_level_objective( + self, parent, service_level_objective, service_level_objective_id=None + ): return self.service_level_objectives[0] def update_service_level_objective(self, service_level_objective): @@ -380,11 +379,11 @@ def load_slo_samples(folder_path, ctx=os.environ): Returns: list: List of loaded SLO configs. """ - return load_configs(f'{SAMPLE_DIR}/{folder_path}', ctx) + return load_configs(f"{SAMPLE_DIR}/{folder_path}", ctx) # Add custom backends / exporters for testing purposes -DUMMY_BACKEND_CODE = open(get_fixture_path('dummy_backend.py')).read() -FAIL_EXPORTER_CODE = open(get_fixture_path('fail_exporter.py')).read() -add_dynamic('dummy', DUMMY_BACKEND_CODE, 'backends') -add_dynamic('fail', FAIL_EXPORTER_CODE, 'exporters') +DUMMY_BACKEND_CODE = open(get_fixture_path("dummy_backend.py")).read() +FAIL_EXPORTER_CODE = open(get_fixture_path("fail_exporter.py")).read() +add_dynamic("dummy", DUMMY_BACKEND_CODE, "backends") +add_dynamic("fail", FAIL_EXPORTER_CODE, "exporters") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 012a4ab6..9152944d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -14,16 +14,19 @@ import unittest -from slo_generator.utils import (get_backend_cls, get_exporter_cls, - get_human_time, import_dynamic) +from slo_generator.utils import ( + get_backend_cls, + get_exporter_cls, + get_human_time, + import_dynamic, +) class TestUtils(unittest.TestCase): - def test_get_human_time(self): # Timezones - tz_1 = 'Europe/Paris' - tz_2 = 'America/Chicago' + tz_1 = "Europe/Paris" + tz_2 = "America/Chicago" # Timestamp 1 timestamp = 1565092435 @@ -46,8 +49,7 @@ def test_get_backend_cls(self): res1 = get_backend_cls("CloudMonitoring") res2 = get_backend_cls("Prometheus") self.assertEqual(res1.__name__, "CloudMonitoringBackend") - self.assertEqual(res1.__module__, - "slo_generator.backends.cloud_monitoring") + self.assertEqual(res1.__module__, "slo_generator.backends.cloud_monitoring") self.assertEqual(res2.__name__, "PrometheusBackend") self.assertEqual(res2.__module__, "slo_generator.backends.prometheus") with self.assertWarns(ImportWarning): @@ -65,8 +67,7 @@ def test_get_exporter_cls(self): res2 = get_exporter_cls("Pubsub") res3 = get_exporter_cls("Bigquery") self.assertEqual(res1.__name__, "CloudMonitoringExporter") - self.assertEqual(res1.__module__, - "slo_generator.exporters.cloud_monitoring") + self.assertEqual(res1.__module__, "slo_generator.exporters.cloud_monitoring") self.assertEqual(res2.__name__, "PubsubExporter") self.assertEqual(res2.__module__, "slo_generator.exporters.pubsub") self.assertEqual(res3.__name__, "BigqueryExporter") @@ -82,19 +83,25 @@ def test_get_exporter_dynamic_cls(self): get_exporter_cls("foo.bar.DoesNotExist") def test_import_dynamic(self): - res1 = import_dynamic("slo_generator.backends.cloud_monitoring", - "CloudMonitoringBackend", - prefix="backend") - res2 = import_dynamic("slo_generator.exporters.cloud_monitoring", - "CloudMonitoringExporter", - prefix="exporter") + res1 = import_dynamic( + "slo_generator.backends.cloud_monitoring", + "CloudMonitoringBackend", + prefix="backend", + ) + res2 = import_dynamic( + "slo_generator.exporters.cloud_monitoring", + "CloudMonitoringExporter", + prefix="exporter", + ) self.assertEqual(res1.__name__, "CloudMonitoringBackend") self.assertEqual(res2.__name__, "CloudMonitoringExporter") with self.assertWarns(ImportWarning): - import_dynamic("slo_generator.backends.unknown", - "CloudMonitoringUnknown", - prefix="unknown") + import_dynamic( + "slo_generator.backends.unknown", + "CloudMonitoringUnknown", + prefix="unknown", + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 8cb0f1b3c898d57673811cc638bc167add190b96 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Mon, 24 Oct 2022 14:45:14 +0200 Subject: [PATCH 090/107] fix: update "Development Status" classifier --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4c4c4cf4..7bae7dbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,8 +33,7 @@ keywords = license = Apache License 2.0 license_files = LICENSE classifiers = - # FIXME: 5 - Production/Stable - Development Status :: 3 - Alpha + Development Status :: 5 - Production/Stable Environment :: Console Intended Audience :: Developers Intended Audience :: System Administrators From fdad50c35b08b20d8af69f2935462c30b3d209df Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Mon, 24 Oct 2022 15:48:31 +0200 Subject: [PATCH 091/107] ci: increase timeout on `build` stage to account for tests added in PR #267 (#273) --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index dc78f170..6101d981 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,7 +39,7 @@ jobs: checkName: cloudbuild ref: ${{ github.event.pull_request.head.sha || github.sha }} intervalSeconds: 10 - timeoutSeconds: 600 # 10m + timeoutSeconds: 900 # 15m - name: Do something if build isn't launch if: steps.wait-build.outputs.conclusion == 'does not exist' || steps.wait-build2.outputs.conclusion == 'does not exist' From 3003356f334727556936e4829ea15965abff7d64 Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Mon, 24 Oct 2022 18:11:49 +0200 Subject: [PATCH 092/107] chore(master): release 2.3.0 (#270) Co-authored-by: Laurent Vaylet --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ setup.cfg | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 168d62f6..d8011d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [2.3.0](https://github.com/google/slo-generator/compare/v2.2.0...v2.3.0) (2022-10-24) + + +### Features + +* add Cloud Monitoring MQL backend ([#245](https://github.com/google/slo-generator/issues/245)) ([159f4d5](https://github.com/google/slo-generator/commit/159f4d5f93fd389f991fd1df3981ab28a2a80572)) +* add Prometheus Self exporter for API mode ([#209](https://github.com/google/slo-generator/issues/209)) ([53c0fdf](https://github.com/google/slo-generator/commit/53c0fdfb1030b84ca7ec11e2c54ab2d22cb046f4)) +* add pytype linting ([#249](https://github.com/google/slo-generator/issues/249)) ([b622d09](https://github.com/google/slo-generator/commit/b622d098a1f489988c8f4b2e92f52e87cad703bc)) + + +### Bug Fixes + +* add timeFrame to retrieve_slo dynatrace ([#212](https://github.com/google/slo-generator/issues/212)) ([2db0140](https://github.com/google/slo-generator/commit/2db01404e1b9e9d216a26f987fbc5531092312ea)) +* make unit tests pass again with elasticsearch 8.x client ([#223](https://github.com/google/slo-generator/issues/223)) ([39dd26c](https://github.com/google/slo-generator/commit/39dd26cb7197fdae0c4cb6f4fbc6808053615a37)) +* prevent gcloud crash with python 3.10 during release workflow ([39a257e](https://github.com/google/slo-generator/commit/39a257e7244c53990063fb63f0edf88cfbb30681)) +* remove useless and unknown Pylint options ([#247](https://github.com/google/slo-generator/issues/247)) ([5053251](https://github.com/google/slo-generator/commit/50532511b4de13becabd5b78d92eb32d59fefde7)) +* support custom exporters ([#235](https://github.com/google/slo-generator/issues/235)) ([b72b8f4](https://github.com/google/slo-generator/commit/b72b8f46d33b42ceb805c45eccdd7275c5495dd9)) +* update "Development Status" classifier ([c82eea3](https://github.com/google/slo-generator/commit/c82eea3a843dadf8720e8c828b2eaed0064eee4e)) + + +### Documentation + +* add missing 'method' field in readme ([#213](https://github.com/google/slo-generator/issues/213)) ([5d2a9a0](https://github.com/google/slo-generator/commit/5d2a9a00ba3cb45b2fe1d5144e4aee735abaf655)) +* add Python 3.9 classifier ([#226](https://github.com/google/slo-generator/issues/226)) ([83c36b9](https://github.com/google/slo-generator/commit/83c36b93693d4d6cb231ba38c5ea3ea0c8e01c2a)) +* document how to write and configure filters in Cloud Monitoring provider ([#266](https://github.com/google/slo-generator/issues/266)) ([29ab2e1](https://github.com/google/slo-generator/commit/29ab2e1dc043bbaa5203c2b5219e474d292fe7f9)) + ## [2.2.0](https://www.github.com/google/slo-generator/compare/v2.1.0...v2.2.0) (2022-02-02) diff --git a/setup.cfg b/setup.cfg index 7bae7dbf..bd234794 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ [metadata] name = slo-generator -version = 2.2.0 +version = 2.3.0 author = Google Inc. author_email = olivier.cervello@gmail.com maintainer = Laurent VAYLET From dfb70a4c04a6f701fb77217d0805a74b5943909a Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Tue, 25 Oct 2022 11:01:48 +0200 Subject: [PATCH 093/107] docs: refine development workflow instructions (#275) * add details on development workflow * fix markdown style violations --- CONTRIBUTING.md | 110 +++++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec27d753..c475ada8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,43 +20,65 @@ This project follows [Google's Open Source Community Guidelines](https://opensou ### Development environment -To prepare for development, you need to fork this repository and work on your own branch so that you can later submit your changes as a GitHub Pull Request. +1. To prepare for development, you need to fork this repository and work on your own branch so that you can later submit your changes as a GitHub Pull Request. -Once you have forked the repo on GitHub, clone it locally and install the `slo-generator` in a Python virtual environment: +1. Once you have forked the repo on GitHub, clone it locally and create a Python virtual environment to work in: -```sh -git clone github.com/google/slo-generator -cd slo-generator -python3 -m venv venv/ -source venv/bin/activate -``` + ```sh + git clone github.com/google/slo-generator + cd slo-generator + python3 -m venv venv/ + source venv/bin/activate + ``` -Then install `slo-generator` locally in development mode, with all the extra packages as well as pre-commit hooks in order to speed up and simplify the development workflow: +1. Install `slo-generator` locally in development mode, with all the extra packages as well as pre-commit hooks in order to speed up and simplify the development workflow: -```sh -make develop -``` + ```sh + make develop + ``` -Finally, feel free to start making changes. +1. Start making changes. -Note that [pre-commit](https://pre-commit.com/) hooks are installed during `make develop` and automatically executed by `git`. These checks are responsible for making sure your changes are linted and well-formatted, in order to match the rest of the codebase and simplify the code reviews. You will be warned after issuing `git commit` if at least one of the staged files does not comply. You can also run the checks manually on the staged files at any time with: +1. Commit your changes locally, in small chunks with meaningful commit messages to simplify the code reviewing process later on. -```sh -$ pre-commit run -trim trailing whitespace.................................................Passed -fix end of files.........................................................Passed -check yaml...............................................................Passed -check for added large files..............................................Passed -autoflake................................................................Passed -flake8...................................................................Passed -black....................................................................Passed -isort....................................................................Passed -pylint...................................................................Passed -``` + Note that [pre-commit](https://pre-commit.com/) hooks are installed during `make develop` and automatically executed by `git` on every `git commit` operation. These checks are responsible for making sure your changes are linted and well-formatted, in order to match the rest of the codebase and simplify the code reviews. You will be warned after issuing `git commit` if at least one of the staged files does not comply. You can also run the checks manually on the staged files at any time with: + + ```sh + $ pre-commit run + trim trailing whitespace.................................................Passed + fix end of files.........................................................Passed + check yaml...............................................................Passed + check for added large files..............................................Passed + autoflake................................................................Passed + flake8...................................................................Passed + black....................................................................Passed + isort....................................................................Passed + pylint...................................................................Passed + ``` -If any error is reported, your commit gets canceled. At this point, run `make format` to automatically reformat the code with [`isort`](https://github.com/PyCQA/isort) and [`black`](https://github.com/psf/black). Also make sure to fix any errors returned by linters or static analyzers such as [`flake8`](https://flake8.pycqa.org/en/latest/), [`pylint`](https://pylint.pycqa.org/en/latest/), [`mypy`](http://mypy-lang.org/) or [`pytype`](https://github.com/google/pytype). Then commit again, rinse and repeat. + If any error is reported, your commit gets canceled and you can start working on fixing the issues. If a formatting error gets reported, run `make format` to automatically organize the imports and reformat the code with [`isort`](https://github.com/PyCQA/isort) and [`black`](https://github.com/psf/black). Also make sure to fix any errors returned by linters or static analyzers such as [`flake8`](https://flake8.pycqa.org/en/latest/), [`pylint`](https://pylint.pycqa.org/en/latest/), [`mypy`](http://mypy-lang.org/) or [`pytype`](https://github.com/google/pytype). Then try `git commit`-ting your changes again. + + Ignoring these pre-commit warnings is not recommended. The Continuous Integration (CI) pipelines will run the exact same checks (and more!) when your commits get pushed. The checks will fail there and prevent you from merging your changes to the `master` branch anyway. So fail fast, fail early, fail often and fix as many errors as possible on your local development machine. Code reviews will be more enjoyable for everyone! + +1. Push your changes when all the pre-commit checks complete successfully. + +1. Create a Pull Request (PR) and ask for a review. + +***Notes:*** + +- IDEs such as Visual Studio Code and PyCharm can help with linting and reformatting. For example, Visual Studio Code can be configured to run `isort` and `black` automatically on every file save. Just add these lines to your `settings.json` file, as documented in [this article](https://cereblanco.medium.com/setup-black-and-isort-in-vscode-514804590bf9): + + ```json + "editor.formatOnSave": true, + "python.formatting.provider": "black", + "[python]": { + "editor.codeActionsOnSave": { + "source.organizeImports": true // isort + } + }, + ``` -Ignoring these pre-commit warnings is not recommended. The Continuous Integration (CI) pipelines will run the exact same checks (and more!) when your commits get pushed. The checks will fail there and prevent you from merging your changes to the `master` branch anyway. So fail fast, fail early, fail often and fix as many errors as possible on your local development machine. Code reviews will be more enjoyable for everyone! +- [`pyenv`](https://github.com/pyenv/pyenv) can be used to easily switch between multiple versions of Python. For example, you might want to create multiple virtual environments like `venv3.9.14` and `venv3.10.7` to confirm a syntax or feature is supported by all Python versions, while keeping the environments isolated from each other. ### Testing environment @@ -83,22 +105,22 @@ The `slo-generator` tool is designed to be modular as it moves forward. Users, c To add a new backend, one must: -* Add a new file named `slo-generator/backends/.py` -* Write a new Python class called `Backend` (CamlCase) -* Test it with a sample config -* Add some unit tests -* Make sure all tests pass -* Submit a PR +- Add a new file named `slo-generator/backends/.py` +- Write a new Python class called `Backend` (CamlCase) +- Test it with a sample config +- Add some unit tests +- Make sure all tests pass +- Submit a PR ***Example with a fake Cat backend:*** -* Add a new backend file: +- Add a new backend file: ```sh touch slo-generator/backends/cat.py ``` -* Fill the content of `cat.py`: +- Fill the content of `cat.py`: ```python from provider import CatClient @@ -146,7 +168,7 @@ To add a new backend, one must: return my_sli_value ``` -* Write a sample SLO configs (`slo_cat_test_slo_ratio.yaml`): +- Write a sample SLO configs (`slo_cat_test_slo_ratio.yaml`): ```yaml service_name: cat @@ -162,18 +184,18 @@ To add a new backend, one must: query_valid: avg:system.disk.used{*}.rollup(avg, {window}) ``` -* Run a live test with the SLO generator: +- Run a live test with the SLO generator: ```sh slo-generator -f slo_cat_test_slo_ratio.yaml -b samples/error_budget_target.yaml ``` -* Create a directory `samples/` for your backend samples. -* Add some YAML samples to show how to write SLO configs for your backend. Samples should be named `slo___.yaml`. -* Add a unit test: in the `tests/unit/test_compute.py`, simply add a method called `test_compute_`. Take the other backends an example when +- Create a directory `samples/` for your backend samples. +- Add some YAML samples to show how to write SLO configs for your backend. Samples should be named `slo___.yaml`. +- Add a unit test: in the `tests/unit/test_compute.py`, simply add a method called `test_compute_`. Take the other backends an example when writing the test. -* Add documentation for your backend / exporter in a new file named `docs/providers/cat.md`. -* Make sure all tests pass -* Submit a PR +- Add documentation for your backend / exporter in a new file named `docs/providers/cat.md`. +- Make sure all tests pass +- Submit a PR The steps above are similar for adding a new exporter, but the exporter code will go to the `exporters/` directory and the unit test will be named `test_export_`. From 79e2cb83a78ad654ec01650db94e2065d7ef78f4 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Tue, 25 Oct 2022 16:15:33 +0200 Subject: [PATCH 094/107] docs: document Cloud Monitoring MQL backend (#277) * add documentation for Cloud Monitoring MQL backend * fix markdown style violations --- docs/providers/bigquery.md | 4 +- docs/providers/cloud_monitoring.md | 273 +++++++++++++-------- docs/providers/cloud_service_monitoring.md | 136 ++++------ docs/providers/cloudevent.md | 14 +- docs/providers/custom.md | 27 +- docs/providers/datadog.md | 74 ++---- docs/providers/dynatrace.md | 34 +-- docs/providers/elasticsearch.md | 42 ++-- docs/providers/prometheus.md | 83 +++---- docs/providers/pubsub.md | 3 +- 10 files changed, 326 insertions(+), 364 deletions(-) diff --git a/docs/providers/bigquery.md b/docs/providers/bigquery.md index fe963753..ed59a513 100644 --- a/docs/providers/bigquery.md +++ b/docs/providers/bigquery.md @@ -2,9 +2,7 @@ ## Exporter -The BigQuery exporter will export SLO reports to a BigQuery table. This allows -teams to conduct historical analysis of SLO reports, and figure out what to do -to improve on the long-run (months, years). +The BigQuery exporter will export SLO reports to a BigQuery table. This allows teams to conduct historical analysis of SLO reports, and figure out what to do to improve on the long-run (months, years). **Config example:** diff --git a/docs/providers/cloud_monitoring.md b/docs/providers/cloud_monitoring.md index d01b7aef..019ab55c 100644 --- a/docs/providers/cloud_monitoring.md +++ b/docs/providers/cloud_monitoring.md @@ -1,129 +1,214 @@ # Cloud Monitoring -## Backend +## Backends -Using the `cloud_monitoring` backend class, you can query any metrics available -in `Cloud Monitoring` to create an SLO. +Using the `cloud_monitoring` and `cloud_monitoring_mql` backend classes, you can query any metrics available in `Cloud Monitoring` to create an SLO. -```yaml -backends: - cloud_monitoring: - project_id: "${WORKSPACE_PROJECT_ID}" -``` +- To query data with Monitoring Query Filters (MQF), use the `cloud_monitoring` backend: + + ```yaml + backends: + cloud_monitoring: + project_id: "${WORKSPACE_PROJECT_ID}" + ``` + + In this case, the syntax of the filters for SLI definition follows [Cloud Monitoring v3 APIs' definitions](https://cloud.google.com/monitoring/api/v3/filters). + +- To query data with Monitoring Query Language (MQL), use the `cloud_monitoring_mql` backend: + + ```yaml + backends: + cloud_monitoring_mql: + project_id: "${WORKSPACE_PROJECT_ID}" + ``` -The following methods are available to compute SLOs with the `cloud_monitoring` -backend: + In this case, SLI definitions use the MQL language. Refer to the [MQL Reference](https://cloud.google.com/monitoring/mql/reference) and [MQL Examples](https://cloud.google.com/monitoring/mql/examples) pages for more details. -* `good_bad_ratio` for metrics of type `DELTA`, `GAUGE`, or `CUMULATIVE`. -* `distribution_cut` for metrics of type `DELTA` and unit `DISTRIBUTION`. +The following methods are available to compute SLOs with the `cloud_monitoring` and `cloud_monitoring_mql` backends: -The syntax of the filters for SLI definition follows [Cloud Monitoring v3 APIs' definitions](https://cloud.google.com/monitoring/api/v3/filters). +| `method` | `cloud_monitoring` | `cloud_monitoring_mql` | For metrics of type: | +| --- | --- | --- | --- | +| `good_bad_ratio` | ✅ | ✅ |`DELTA`, `GAUGE`, or `CUMULATIVE` | +| `distribution_cut` | ✅ | ✅ | `DELTA` and unit `DISTRIBUTION` | +| `query_sli` | ❌ | ✅ | any | ### Good / bad ratio The `good_bad_ratio` method is used to compute the ratio between two metrics: -* **Good events**, i.e events we consider as 'good' from the user perspective. -* **Bad or valid events**, i.e events we consider either as 'bad' from the user -perspective, or all events we consider as 'valid' for the computation of the -SLO. +- **Good events**, i.e events we consider as 'good' from the user perspective. +- **Bad or valid events**, i.e events we consider either as 'bad' from the user perspective, or all events we consider as 'valid' for the computation of the SLO. -This method is often used for availability SLOs, but can be used for other -purposes as well (see examples). +This method is often used for availability SLOs, but can be used for other purposes as well (see examples). **SLO config blob:** -```yaml -backend: cloud_monitoring -method: good_bad_ratio -service_level_indicator: - filter_good: > - project="${GAE_PROJECT_ID}" - metric.type="appengine.googleapis.com/http/server/response_count" - metric.labels.response_code >= 200 - metric.labels.response_code < 500 - filter_valid: > - project="${GAE_PROJECT_ID}" - metric.type="appengine.googleapis.com/http/server/response_count" -``` - -You can also use the `filter_bad` field which identifies bad events instead of -the `filter_valid` field which identifies all valid events. - -**→ [Full SLO config](../../samples/cloud_monitoring/slo_gae_app_availability.yaml)** +- For queries defined using Monitoring Query Filters (MQF): + + ```yaml + backend: cloud_monitoring + method: good_bad_ratio + service_level_indicator: + filter_good: > + project="${GAE_PROJECT_ID}" + metric.type="appengine.googleapis.com/http/server/response_count" + metric.labels.response_code >= 200 + metric.labels.response_code < 500 + filter_valid: > + project="${GAE_PROJECT_ID}" + metric.type="appengine.googleapis.com/http/server/response_count" + ``` + + You can also use the `filter_bad` field which identifies bad events instead of the `filter_valid` field which identifies all valid events. + + **→ [Full SLO config](../../samples/cloud_monitoring/slo_gae_app_availability.yaml)** + +- For queries defined using Monitoring Query Language (MQL) : + + ```yaml + backend: cloud_monitoring_mql + method: good_bad_ratio + service_level_indicator: + filter_good: > + fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | filter + metric.response_code == 429 + || metric.response_code == 200 + filter_valid: > + fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + ``` + + You can also use the `filter_bad` field which identifies bad events instead of the `filter_valid` field which identifies all valid events. + + **→ [Full SLO config](../../samples/cloud_monitoring_mql/slo_gae_app_availability.yaml)** ### Distribution cut -The `distribution_cut` method is used for Cloud Monitoring distribution-type -metrics, which are usually used for latency metrics. +The `distribution_cut` method is used for Cloud Monitoring distribution-type metrics, which are usually used for latency metrics. -A distribution metric records the **statistical distribution of the extracted -values** in **histogram buckets**. The extracted values are not recorded -individually, but their distribution across the configured buckets are recorded, -along with the `count`, `mean`, and `sum` of squared deviation of the values. +A distribution metric records the **statistical distribution of the extracted values** in **histogram buckets**. The extracted values are not recorded individually, but their distribution across the configured buckets are recorded, along with the `count`, `mean`, and `sum` of squared deviation of the values. -In Cloud Monitoring, there are three different ways to specify bucket -boundaries: +In Cloud Monitoring, there are three different ways to specify bucket boundaries: -* **Linear:** Every bucket has the same width. -* **Exponential:** Bucket widths increases for higher values, using an -exponential growth factor. -* **Explicit:** Bucket boundaries are set for each bucket using a bounds array. +- **Linear:** Every bucket has the same width. +- **Exponential:** Bucket widths increases for higher values, using an exponential growth factor. +- **Explicit:** Bucket boundaries are set for each bucket using a bounds array. **SLO config blob:** +- For queries defined using Monitoring Query Filters (MQF): + + ```yaml + backend: cloud_monitoring + method: exponential_distribution_cut + service_level_indicator: + filter_valid: > + project=${GAE_PROJECT_ID} AND + metric.type=appengine.googleapis.com/http/server/response_latencies AND + metric.labels.response_code >= 200 AND + metric.labels.response_code < 500 + good_below_threshold: true + threshold_bucket: 19 + ``` + + **→ [Full SLO config](../../samples/cloud_monitoring/slo_gae_app_latency.yaml)** + +- For queries defined using Monitoring Query Language (MQL) : + + ```yaml + backend: cloud_monitoring_mql + method: distribution_cut + exporters: + - cloud_monitoring + service_level_indicator: + filter_valid: > + fetch https_lb_rule + | metric 'loadbalancing.googleapis.com/https/total_latencies' + | filter resource.project_id == '${LB_PROJECT_ID}' + | filter metric.label.response_code_class = "200" + || metric.response_code_class = "300" + || metric.response_code_class = "400" + good_below_threshold: true + threshold_bucket: 19 + ``` + + **→ [Full SLO config](../../samples/cloud_monitoring_mql/slo_gae_app_latency.yaml)** + +The `threshold_bucket` number to reach our 724ms target latency will depend on how the buckets boundaries are set. Learn how to [inspect your distribution metrics](https://cloud.google.com/logging/docs/logs-based-metrics/distribution-metrics#inspecting_distribution_metrics) to figure out the bucketization. + +### Query SLI + +As MQL is a much richer language than MQF, the `cloud_monitoring_mql` backend has an extra `query_sli` method that can be used to retrieve the value of a given SLI with a single API call. + +For example, the `ratio` keyword lets us compute a ratio of good and valid events directly. Here is an example for availability: + ```yaml -backend: cloud_monitoring -method: exponential_distribution_cut -service_level_indicator: - filter_valid: > - project=${GAE_PROJECT_ID} AND - metric.type=appengine.googleapis.com/http/server/response_latencies AND - metric.labels.response_code >= 200 AND - metric.labels.response_code < 500 - good_below_threshold: true - threshold_bucket: 19 + backend: cloud_monitoring_mql + method: query_sli + exporters: + - cloud_monitoring + service_level_indicator: + query: > + fetch gae_app + | metric 'appengine.googleapis.com/http/server/response_count' + | filter resource.project_id == '${GAE_PROJECT_ID}' + | { filter + metric.response_code == 429 + || metric.response_code == 200 + || metric.response_code == 201 + || metric.response_code == 202 + || metric.response_code == 203 + || metric.response_code == 204 + || metric.response_code == 205 + || metric.response_code == 206 + || metric.response_code == 207 + || metric.response_code == 208 + || metric.response_code == 226 + || metric.response_code == 304 + ; ident } + | sum + | ratio ``` -**→ [Full SLO config](../../samples/cloud_monitoring/slo_gae_app_latency.yaml)** +Refer to the [MQL Examples](https://cloud.google.com/monitoring/mql/examples) page for more details and more interesting keywords/functions. -The `threshold_bucket` number to reach our 724ms target latency will depend on -how the buckets boundaries are set. Learn how to [inspect your distribution metrics](https://cloud.google.com/logging/docs/logs-based-metrics/distribution-metrics#inspecting_distribution_metrics) to figure out the bucketization. +**→ [Full SLO config](../../samples/cloud_monitoring_mql/slo_gae_app_availability_ratio.yaml)** + +Generally speaking, any query that returns a single value after being aggregated (within a given time series) and reduced (across all time series) with a sum can be retrieved directly with `query_sli`. ## Exporter The `cloud_monitoring` exporter allows to export SLO metrics to Cloud Monitoring API. ```yaml -backends: +exporter: cloud_monitoring: project_id: "${WORKSPACE_PROJECT_ID}" ``` Optional fields: -* `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). +- `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). **→ [Full SLO config](../../samples/cloud_monitoring/slo_lb_request_availability.yaml)** ## Alerting -Alerting is essential in any SRE approach. Having all the right metrics without -being able to alert on them is simply useless. +Alerting is essential in any SRE approach. Having all the right metrics without being able to alert on them is simply useless. -**Too many alerts** can be daunting, and page your SRE engineers for no valid -reasons. +**Too many alerts** can be daunting, and page your SRE engineers for no valid reasons. -**Too little alerts** can mean that your applications are not monitored at all -(no application have 100% reliability). +**Too little alerts** can mean that your applications are not monitored at all (no application have 100% reliability). -**Alerting on high error budget burn rates** for some hand-picked SLOs can help -reduce the noise and page only when it's needed. +**Alerting on high error budget burn rates** for some hand-picked SLOs can help reduce the noise and page only when it's needed. **Example:** -We will define a `Cloud Monitoring` alert that we will **filter out on the -corresponding error budget step**. +We will define a `Cloud Monitoring` alert that we will **filter out on the corresponding error budget step**. Consider the following error budget policy step config: @@ -136,37 +221,23 @@ Consider the following error budget policy step config: message_ok: Last hour on track ``` -Using Cloud Monitoring UI, let's set up an alert when our error budget burn rate -is burning **9X faster** than it should in the last hour: - -* Open `Cloud Monitoring` and click on `Alerting > Create Policy` - -* Fill the alert name and click on `Add Condition`. - -* Search for `custom/error_budget_burn_rate` and click on the metric. - -* Filter on `error_budget_policy_step_name` label with value `1 hour`. - -* Set the `Condition` field to `is above`. - -* Set the `Threshold` field to `9`. - -* Set the `For` field to `most_recent_value`. - -* Click `Add` - -* Fill the notification options for your alert. +Using Cloud Monitoring UI, let's set up an alert when our error budget burn rate is burning **9X faster** than it should in the last hour: -* Click `Save`. +1. Open `Cloud Monitoring` and click `Alerting > Create Policy`. +1. Fill the alert name and click `Add Condition`. +1. Search for `custom/error_budget_burn_rate` and click the metric. +1. Filter on `error_budget_policy_step_name` label with value `1 hour`. +1. Set the `Condition` field to `is above`. +1. Set the `Threshold` field to `9`. +1. Set the `For` field to `most_recent_value`. +1. Click `Add` +1. Fill the notification options for your alert. +1. Click `Save`. Repeat the above steps for every item in your error budget policy. -Alerts can be filtered out more (e.g: `service_name`, `feature_name`), but you -can keep global ones filtered only on `error_budget_policy_step_name` if you -want your SREs to have visibility on all the incidents. Labels will be used to -differentiate the alert messages. +Alerts can be filtered out more (e.g: `service_name`, `feature_name`), but you can keep global ones filtered only on `error_budget_policy_step_name` if you want your SREs to have visibility on all the incidents. Labels will be used to differentiate the alert messages. ## Examples -Complete SLO samples using Cloud Monitoring are available in -[samples/cloud_monitoring](../../samples/cloud_monitoring). Check them out ! +Complete SLO samples using Cloud Monitoring are available in [samples/cloud_monitoring](../../samples/cloud_monitoring). Check them out! diff --git a/docs/providers/cloud_service_monitoring.md b/docs/providers/cloud_service_monitoring.md index bea7d835..58b77055 100644 --- a/docs/providers/cloud_service_monitoring.md +++ b/docs/providers/cloud_service_monitoring.md @@ -2,8 +2,7 @@ ## Backend -Using the `cloud_service_monitoring` backend, you can use the -`Cloud Service Monitoring API` to manage your SLOs. +Using the `cloud_service_monitoring` backend, you can use the `Cloud Service Monitoring API` to manage your SLOs. ```yaml backends: @@ -11,30 +10,23 @@ backends: project_id: "${WORKSPACE_PROJECT_ID}" ``` -SLOs are created from standard metrics available in Cloud Monitoring and -the data is stored in `Cloud Service Monitoring API` (see - [docs](https://cloud.google.com/monitoring/service-monitoring/using-api)). +SLOs are created from standard metrics available in Cloud Monitoring and the data is stored in `Cloud Service Monitoring API` (see [docs](https://cloud.google.com/monitoring/service-monitoring/using-api)). -The following methods are available to compute SLOs with the `cloud_service_monitoring` -backend: +The following methods are available to compute SLOs with the `cloud_service_monitoring` backend: -* `basic` to create standard SLOs for Google App Engine, Google Kubernetes -Engine, and Cloud Endpoints. +* `basic` to create standard SLOs for Google App Engine, Google Kubernetes Engine, and Cloud Endpoints. * `good_bad_ratio` for metrics of type `DELTA` or `CUMULATIVE`. * `distribution_cut` for metrics of type `DELTA` and unit `DISTRIBUTION`. - ### Basic -The `basic` method is used to let the `Cloud Service Monitoring API` -automatically generate standardized SLOs for the following GCP services: +The `basic` method is used to let the `Cloud Service Monitoring API` automatically generate standardized SLOs for the following GCP services: + * **Google App Engine** * **Google Kubernetes Engine** (with Istio) * **Google Cloud Endpoints** -The SLO configuration uses Cloud Monitoring -[GCP metrics](https://cloud.google.com/monitoring/api/metrics_gcp) and only -requires minimal configuration compared to custom SLOs. +The SLO configuration uses Cloud Monitoring [GCP metrics](https://cloud.google.com/monitoring/api/metrics_gcp) and only requires minimal configuration compared to custom SLOs. **Example config (App Engine availability):** @@ -47,8 +39,8 @@ service_level_indicator: module_id: ${GAE_MODULE_ID} availability: {} ``` -For details on filling the `app_engine` fields, see [AppEngine](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#appengine) -spec. + +For details on filling the `app_engine` fields, see [AppEngine](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#appengine) spec. **→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml)** @@ -63,10 +55,11 @@ service_level_indicator: latency: threshold: 724 # ms ``` -For details on filling the `cloud_endpoints` fields, see [CloudEndpoint](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#cloudendpoints) -spec. + +For details on filling the `cloud_endpoints` fields, see [CloudEndpoint](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#cloudendpoints) spec. **Example config (Istio service latency):** + ```yaml backend: cloud_service_monitoring method: basic @@ -78,12 +71,13 @@ service_level_indicator: latency: threshold: 500 # ms ``` -For details on filling the `mesh_istio` fields, see [MeshIstio](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#meshistio) -spec. + +For details on filling the `mesh_istio` fields, see [MeshIstio](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#meshistio) spec. **→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gke_app_latency_basic.yaml)** **Example config (Istio service latency) [DEPRECATED]:** + ```yaml backend: cloud_service_monitoring method: basic @@ -97,25 +91,22 @@ service_level_indicator: latency: threshold: 500 # ms ``` -For details on filling the `cluster_istio` fields, see [ClusterIstio](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#clusteristio) -spec. -**→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml)** +For details on filling the `cluster_istio` fields, see [ClusterIstio](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/services#clusteristio) spec. +**→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml)** ### Good / bad ratio The `good_bad_ratio` method is used to compute the ratio between two metrics: -- **Good events**, i.e events we consider as 'good' from the user perspective. -- **Bad or valid events**, i.e events we consider either as 'bad' from the user -perspective, or all events we consider as 'valid' for the computation of the -SLO. +* **Good events**, i.e events we consider as 'good' from the user perspective. +* **Bad or valid events**, i.e events we consider either as 'bad' from the user perspective, or all events we consider as 'valid' for the computation of the SLO. -This method is often used for availability SLOs, but can be used for other -purposes as well (see examples). +This method is often used for availability SLOs, but can be used for other purposes as well (see examples). **Example config:** + ```yaml backend: cloud_service_monitoring method: good_bad_ratio @@ -131,20 +122,15 @@ service_level_indicator: metric.type="appengine.googleapis.com/http/server/response_count" ``` -You can also use the `filter_bad` field which identifies bad events instead of -the `filter_valid` field which identifies all valid events. +You can also use the `filter_bad` field which identifies bad events instead of the `filter_valid` field which identifies all valid events. **→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gae_app_availability.yaml)** ## Distribution cut -The `distribution_cut` method is used for Cloud distribution-type metrics, -which are usually used for latency metrics. +The `distribution_cut` method is used for Cloud distribution-type metrics, which are usually used for latency metrics. -A distribution metric records the **statistical distribution of the extracted -values** in **histogram buckets**. The extracted values are not recorded -individually, but their distribution across the configured buckets are recorded, -along with the `count`, `mean`, and `sum` of squared deviation of the values. +A distribution metric records the **statistical distribution of the extracted values** in **histogram buckets**. The extracted values are not recorded individually, but their distribution across the configured buckets are recorded, along with the `count`, `mean`, and `sum` of squared deviation of the values. **Example config:** @@ -161,104 +147,88 @@ service_level_indicator: range_max: 724 # ms ``` -The `range_min` and `range_max` are used to specify the latency range that we -consider 'good'. +The `range_min` and `range_max` are used to specify the latency range that we consider 'good'. **→ [Full SLO config](../../samples/cloud_service_monitoring/slo_gae_app_latency.yaml)** - ## Service Monitoring API considerations ### Tracking objects -Since `Cloud Service Monitoring API` persists `Service` and -`ServiceLevelObjective` objects, we need ways to keep our local SLO YAML -configuration synced with the remote objects. + +Since `Cloud Service Monitoring API` persists `Service` and `ServiceLevelObjective` objects, we need ways to keep our local SLO YAML configuration synced with the remote objects. **Auto-imported** -Some services are auto-imported by the `Service Monitoring API`: they correspond -to SLO configurations using the [`basic`](#basic) method. +Some services are auto-imported by the `Service Monitoring API`: they correspond to SLO configurations using the [`basic`](#basic) method. -The following conventions are used by the `Service Monitoring API` to give a -unique id to an auto-imported `Service`: +The following conventions are used by the `Service Monitoring API` to give a unique id to an auto-imported `Service`: - * **App Engine:** +* **App Engine:** ``` gae:{project_id}_{module_id} ``` - → *Make sure that the `app_engine` block in your config has the - correct fields corresponding to your App Engine service.* - * **Cloud Endpoints:** + → *Make sure that the `app_engine` block in your config has the correct fields corresponding to your App Engine service.* + +* **Cloud Endpoints:** ``` ist:{project_id}-{service} ``` - → *Make sure that the `cloud_endpoints` block in your config has - the correct fields corresponding to your Cloud Endpoint service.* - * **Mesh Istio [NOT YET RELEASED]:** + → *Make sure that the `cloud_endpoints` block in your config has the correct fields corresponding to your Cloud Endpoint service.* + +* **Mesh Istio [NOT YET RELEASED]:** ``` ist:{project_id}-{mesh_uid}-{service_namespace}-{service_name} ``` - → *Make sure that the `mesh_istio` block in your config has the - correct fields corresponding to your Istio service.* - * **Cluster Istio [DEPRECATED SOON]:** + → *Make sure that the `mesh_istio` block in your config has the correct fields corresponding to your Istio service.* + +* **Cluster Istio [DEPRECATED SOON]:** ``` ist:{project_id}-{suffix}-{location}-{cluster_name}-{service_namespace}-{service_name} ``` - → *Make sure that the `cluster_istio` block in your config has - the correct fields corresponding to your Istio service.* -You cannot import an existing `ServiceLevelObjective` object, since they use a -random id. + → *Make sure that the `cluster_istio` block in your config has the correct fields corresponding to your Istio service.* + +You cannot import an existing `ServiceLevelObjective` object, since they use a random id. **Custom** -Custom services are the ones you create yourself using the -`Cloud Service Monitoring API` and the `slo-generator`. +Custom services are the ones you create yourself using the `Cloud Service Monitoring API` and the `slo-generator`. -The following conventions are used by the `slo-generator` to give a unique id -to a custom `Service` and `Service Level Objective` objects: +The following conventions are used by the `slo-generator` to give a unique id to a custom `Service` and `Service Level Objective` objects: * `service_id = ${metadata.service_name}-${metadata.feature_name}` * `slo_id = ${metadata.service_name}-${metadata.feature_name}-${metadata.slo_name}-${window}` -To keep track of those, **do not update any of the following fields** in your -configs: +To keep track of those, **do not update any of the following fields** in your configs: - * `metadata.service_name`, `metadata.feature_name` and `metadata.slo_name` in the SLO config. +* `metadata.service_name`, `metadata.feature_name` and `metadata.slo_name` in the SLO config. - * `window` in the Error Budget Policy. +* `window` in the Error Budget Policy. -If you need to make updates to any of those fields, first run the -`slo-generator` with the `-d` (delete) option (see - [#deleting-objects](#deleting-objects)), then re-run normally. +If you need to make updates to any of those fields, first run the `slo-generator` with the `-d` (delete) option (see [#deleting-objects](#deleting-objects)), then re-run normally. -To import an existing custom `Service` objects, find out your service id from -the API and fill the `service_id` in the `service_level_indicator` configuration. +To import an existing custom `Service` objects, find out your service id from the API and fill the `service_id` in the `service_level_indicator` configuration. ### Deleting objects -To delete an SLO object in `Cloud Monitoring API` using the -`cloud_service_monitoring` class, run the `slo-generator` with the -`-d` (or `--delete`) flag: +To delete an SLO object in `Cloud Monitoring API` using the `cloud_service_monitoring` class, run the `slo-generator` with the `-d` (or `--delete`) flag: -``` +```sh slo-generator -f -b --delete ``` ## Alerting -See the Cloud Service Monitoring [docs](https://cloud.google.com/monitoring/service-monitoring/alerting-on-budget-burn-rate) -for instructions on alerting. +See the Cloud Service Monitoring [docs](https://cloud.google.com/monitoring/service-monitoring/alerting-on-budget-burn-rate) for instructions on alerting. ### Examples -Complete SLO samples using `Cloud Service Monitoring` are available in [ samples/cloud_service_monitoring](../../samples/cloud_service_monitoring). -Check them out ! +Complete SLO samples using `Cloud Service Monitoring` are available in [samples/cloud_service_monitoring](../../samples/cloud_service_monitoring). Check them out! diff --git a/docs/providers/cloudevent.md b/docs/providers/cloudevent.md index 42d30141..1dc2c5ee 100644 --- a/docs/providers/cloudevent.md +++ b/docs/providers/cloudevent.md @@ -2,11 +2,9 @@ ## Exporter -The Cloudevent exporter will make a POST request to a CloudEvent receiver -service. +The Cloudevent exporter will make a POST request to a CloudEvent receiver service. -This allows to send SLO Reports to another service that can process them, or -export them to other destinations, such as an export-only slo-generator service. +This allows to send SLO Reports to another service that can process them, or export them to other destinations, such as an export-only slo-generator service. **Config example:** @@ -21,9 +19,7 @@ exporters: ``` Optional fields: -* `auth` section allows to specify authentication tokens if needed. -Tokens are added as a header + +* `auth` section allows to specify authentication tokens if needed. Tokens are added as a header. * `token` is used to pass an authentication token in the request headers. - * `google_service_account_auth: true` is used to enable Google service account - authentication. Use this if the target service is hosted on GCP (Cloud Run, - Cloud Functions, Google Kubernetes Engine ...). + * `google_service_account_auth: true` is used to enable Google service account authentication. Use this if the target service is hosted on GCP (Cloud Run, Cloud Functions, Google Kubernetes Engine...). diff --git a/docs/providers/custom.md b/docs/providers/custom.md index 3fcc6ec7..df5111ea 100644 --- a/docs/providers/custom.md +++ b/docs/providers/custom.md @@ -3,17 +3,16 @@ `slo-generator` allows you to load custom backends / exporters dynamically. This enables you to: + * Support other backends or exporters that are not part of `slo-generator` core. * Query or export from / to internal custom APIs. * Create SLOs based on more complicated logic (e.g: fetch a Datastore record or run a BQ query). ## Backend -To create a custom backend, simply create a new file and add the backend code -within it. +To create a custom backend, simply create a new file and add the backend code within it. -For this example, we will assume the backend code below was added to -`custom/custom_backend.py`. +For this example, we will assume the backend code below was added to `custom/custom_backend.py`. A sample custom backend will have the following look: @@ -39,9 +38,7 @@ class CustomBackend: return 0.999 ``` - -In order to call the `good_bad_ratio` method in the custom backend above, the -`backends` block would look like this: +In order to call the `good_bad_ratio` method in the custom backend above, the `backends` block would look like this: ```yaml backends: @@ -51,6 +48,7 @@ backends: ``` The `spec` section in the SLO config would look like: + ```yaml backend: custom.custom_backend.CustomBackend method: good_bad_ratio # name of the method to run @@ -61,18 +59,18 @@ service_level_indicator: {} ## Exporter -To create a custom exporter, simply create a new file and add the exporter code -within it. +To create a custom exporter, simply create a new file and add the exporter code within it. -For the examples below, we will assume the exporter code below was added to -`custom/custom_exporter.py`. +For the examples below, we will assume the exporter code below was added to `custom/custom_exporter.py`. ### Standard A standard exporter: + * must implement the `export` method. A sample exporter looks like: + ```py class CustomExporter: """Custom exporter.""" @@ -107,6 +105,7 @@ exporters: ``` The `spec` section in the SLO config would look like: + ```yaml exporters: [custom.custom_exporter.CustomExporter] ``` @@ -116,8 +115,7 @@ exporters: [custom.custom_exporter.CustomExporter] A metrics exporter: * must inherit from `slo_generator.exporters.base.MetricsExporter`. -* must implement the `export_metric` method which exports **one** metric. -The `export_metric` function takes a metric dict as input, such as: +* must implement the `export_metric` method which exports **one** metric. The `export_metric` function takes a metric dict as input, such as: ```py { @@ -133,8 +131,6 @@ The `export_metric` function takes a metric dict as input, such as: } ``` - - A sample metrics exporter will look like: ```py @@ -170,6 +166,7 @@ exporters: **Note:** The `MetricsExporter` base class has the following behavior: + * The `metrics` block in the SLO config is passed to the base class `MetricsExporter` * The base class `MetricsExporter` runs the `export` method which iterates through each metric and add information to it, such as the current value and timestamp. * The base class `MetricsExporter` calls the derived class `export_metric` for each metric and pass it the metric data to export. diff --git a/docs/providers/datadog.md b/docs/providers/datadog.md index f6e13f61..f3b82237 100644 --- a/docs/providers/datadog.md +++ b/docs/providers/datadog.md @@ -2,8 +2,7 @@ ## Backend -Using the `datadog` backend class, you can query any metrics available in -Datadog to create an SLO. +Using the `datadog` backend class, you can query any metrics available in Datadog to create an SLO. ```yaml backends: @@ -12,29 +11,22 @@ backends: app_key: ${DATADOG_APP_KEY} ``` -The following methods are available to compute SLOs with the `datadog` -backend: +The following methods are available to compute SLOs with the `datadog` backend: * `good_bad_ratio` for computing good / bad metrics ratios. * `query_sli` for computing SLIs directly with Datadog. * `query_slo` for getting SLO value from Datadog SLO endpoint. -Optional arguments to configure Datadog are documented in the Datadog -`initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). -You can pass them in the `backend` section, such as specifying -`api_host: api.datadoghq.eu` in order to use the EU site. +Optional arguments to configure Datadog are documented in the Datadog `initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). You can pass them in the `backend` section, such as specifying `api_host: api.datadoghq.eu` in order to use the EU site. ### Good / bad ratio The `good_bad_ratio` method is used to compute the ratio between two metrics: -- **Good events**, i.e events we consider as 'good' from the user perspective. -- **Bad or valid events**, i.e events we consider either as 'bad' from the user -perspective, or all events we consider as 'valid' for the computation of the -SLO. +* **Good events**, i.e events we consider as 'good' from the user perspective. +* **Bad or valid events**, i.e events we consider either as 'bad' from the user perspective, or all events we consider as 'valid' for the computation of the SLO. -This method is often used for availability SLOs, but can be used for other -purposes as well (see examples). +This method is often used for availability SLOs, but can be used for other purposes as well (see examples). **Config example:** @@ -45,15 +37,14 @@ service_level_indicator: filter_good: app.requests.count{http.path:/, http.status_code_class:2xx} filter_valid: app.requests.count{http.path:/} ``` + **→ [Full SLO config](../../samples/datadog/slo_dd_app_availability_ratio.yaml)** ### Query SLI -The `query_sli` method is used to directly query the needed SLI with Datadog: -Datadog's query language is powerful enough that it can do ratios natively. +The `query_sli` method is used to directly query the needed SLI with Datadog: Datadog's query language is powerful enough that it can do ratios natively. -This method makes it more flexible to input any `datadog` SLI computation and -eventually reduces the number of queries made to Datadog. +This method makes it more flexible to input any `datadog` SLI computation and eventually reduces the number of queries made to Datadog. ```yaml backend: datadog @@ -66,14 +57,11 @@ service_level_indicator: ### Query SLO -The `query_slo` method is used to directly query the needed SLO with Datadog: -indeed, Datadog has SLO objects that you can directly refer to in your config by inputing their `slo_id`. +The `query_slo` method is used to directly query the needed SLO with Datadog: indeed, Datadog has SLO objects that you can directly refer to in your config by inputing their `slo_id`. -This method makes it more flexible to input any `datadog` SLI computation and -eventually reduces the number of queries made to Datadog. +This method makes it more flexible to input any `datadog` SLI computation and eventually reduces the number of queries made to Datadog. -To query the value from Datadog SLO, simply add a `slo_id` field in the -`measurement` section: +To query the value from Datadog SLO, simply add a `slo_id` field in the `measurement` section: ```yaml backend: datadog @@ -86,8 +74,7 @@ service_level_indicator: ### Examples -Complete SLO samples using `datadog` are available in -[samples/datadog](../../samples/datadog). Check them out! +Complete SLO samples using `datadog` are available in [samples/datadog](../../samples/datadog). Check them out! ## Exporter @@ -99,37 +86,24 @@ exporters: api_key: ${DATADOG_API_KEY} app_key: ${DATADOG_APP_KEY} ``` -Optional arguments to configure Datadog are documented in the Datadog -`initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). -You can pass them in the `backend` section, such as specifying -`api_host: api.datadoghq.eu` in order to use the EU site. + +Optional arguments to configure Datadog are documented in the Datadog `initialize` method [here](https://github.com/DataDog/datadogpy/blob/058114cc3d65483466684c96a5c23e36c3aa052e/datadog/__init__.py#L33). You can pass them in the `backend` section, such as specifying `api_host: api.datadoghq.eu` in order to use the EU site. Optional fields: - * `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). -**→ [Full SLO config](../../samples/datadog/slo_dd_app_availability_ratio.yaml)** +* `metrics`: [*optional*] `list` - List of metrics to export ([see docs](../shared/metrics.md)). +**→ [Full SLO config](../../samples/datadog/slo_dd_app_availability_ratio.yaml)** ## Datadog API considerations The `distribution_cut` method is not currently implemented for Datadog. -The reason for this is that Datadog distributions (or histograms) do not conform -to what histograms should be (see [old issue](https://github.com/DataDog/dd-agent/issues/349)), -i.e a set of configurable bins, each providing the number of events falling into -each bin. +The reason for this is that Datadog distributions (or histograms) do not conform to what histograms should be (see [old issue](https://github.com/DataDog/dd-agent/issues/349)), i.e a set of configurable bins, each providing the number of events falling into each bin. -Standard histograms representations (see [wikipedia](https://en.wikipedia.org/wiki/Histogram)) -already implement this, but the approach Datadog took is to pre-compute -(client-side) or post-compute (server-side) percentiles, resulting in a -different metric for each percentile representing the percentile value instead -of the number of events in the percentile. +Standard histograms representations (see [wikipedia](https://en.wikipedia.org/wiki/Histogram)) already implement this, but the approach Datadog took is to pre-compute (client-side) or post-compute (server-side) percentiles, resulting in a different metric for each percentile representing the percentile value instead of the number of events in the percentile. -This implementation has a couple of advantages, like making it easy to query and -graph the value of the 99th, 95p, or 50p percentiles; but it makes it -effectively very hard to compute a standard SLI for it, since it's not possible -to see how many requests fall in each bin; hence there is no way to know how -many good and bad events there are. +This implementation has a couple of advantages, like making it easy to query and graph the value of the 99th, 95p, or 50p percentiles; but it makes it effectively very hard to compute a standard SLI for it, since it's not possible to see how many requests fall in each bin; hence there is no way to know how many good and bad events there are. Three options can be considered to implement this: @@ -138,12 +112,8 @@ in `datadog-agent`. **OR** -* Implement support for standard histograms where bucketization is configurable -and where it's possible to query the number of events falling into each bucket. +* Implement support for standard histograms where bucketization is configurable and where it's possible to query the number of events falling into each bucket. **OR** -* Design an implementation that tries to reconstitute the original distribution -by assimilating it to a Gaussian distribution and estimating its parameters. -This is a complex and time-consuming approach that will give approximate results -and is not a straightforward problem (see [StackExchange thread](https://stats.stackexchange.com/questions/6022/estimating-a-distribution-based-on-three-percentiles)) +* Design an implementation that tries to reconstitute the original distribution by assimilating it to a Gaussian distribution and estimating its parameters. This is a complex and time-consuming approach that will give approximate results and is not a straightforward problem (see [StackExchange thread](https://stats.stackexchange.com/questions/6022/estimating-a-distribution-based-on-three-percentiles)) diff --git a/docs/providers/dynatrace.md b/docs/providers/dynatrace.md index e4e11f3f..304d0f01 100644 --- a/docs/providers/dynatrace.md +++ b/docs/providers/dynatrace.md @@ -2,8 +2,7 @@ ## Backend -Using the `dynatrace` backend class, you can query any metrics available in -Dynatrace to create an SLO. +Using the `dynatrace` backend class, you can query any metrics available in Dynatrace to create an SLO. ```yaml backends: @@ -12,8 +11,7 @@ backends: api_url: ${DYNATRACE_API_URL} ``` -The following methods are available to compute SLOs with the `dynatrace` -backend: +The following methods are available to compute SLOs with the `dynatrace` backend: * `good_bad_ratio` for computing good / bad metrics ratios. @@ -21,13 +19,10 @@ backend: The `good_bad_ratio` method is used to compute the ratio between two metrics: -- **Good events**, i.e events we consider as 'good' from the user perspective. -- **Bad or valid events**, i.e events we consider either as 'bad' from the user -perspective, or all events we consider as 'valid' for the computation of the -SLO. +* **Good events**, i.e events we consider as 'good' from the user perspective. +* **Bad or valid events**, i.e events we consider either as 'bad' from the user perspective, or all events we consider as 'valid' for the computation of the SLO. -This method is often used for availability SLOs, but can be used for other -purposes as well (see examples). +This method is often used for availability SLOs, but can be used for other purposes as well (see examples). **Config example:** @@ -42,14 +37,12 @@ service_level_indicator: metric_selector: ext:app.request_count:filter(and(eq(app,test_app),eq(env,prod))) entity_selector: type(HOST) ``` -**→ [Full SLO config](../../samples/dynatrace/slo_dt_app_availability_ratio.yaml)** +**→ [Full SLO config](../../samples/dynatrace/slo_dt_app_availability_ratio.yaml)** ### Threshold -The `threshold` method is used to split a series of values into two buckets -using a threshold as delimiter: one bucket which will represent the good events, -the other will represent the bad events. +The `threshold` method is used to split a series of values into two buckets using a threshold as delimiter: one bucket which will represent the good events, the other will represent the bad events. This method can be used for latency SLOs, by defining a latency threshold. @@ -64,16 +57,16 @@ service_level_indicator: entity_selector: type(HOST) threshold: 40000 # us ``` + **→ [Full SLO config](../../samples/dynatrace/slo_dt_app_latency_threshold.yaml)** Optional fields: - * `good_below_threshold`: Boolean, specify if good events are above or below threshold. Default: `true`. +* `good_below_threshold`: Boolean, specify if good events are above or below threshold. Default: `true`. ### Examples -Complete SLO samples using `dynatrace` are available in -[samples/dynatrace](../../samples/dynatrace). Check them out! +Complete SLO samples using `dynatrace` are available in [samples/dynatrace](../../samples/dynatrace). Check them out! ## Exporter @@ -87,12 +80,11 @@ exporters: ``` Optional fields: - * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`custom:error_budget_burn_rate`, `custom:sli_service_level_indicator`]. -**→ [Full SLO config](../../samples/dynatrace/slo_dt_app_availability_ratio.yaml)** +* `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`custom:error_budget_burn_rate`, `custom:sli_service_level_indicator`]. +**→ [Full SLO config](../../samples/dynatrace/slo_dt_app_availability_ratio.yaml)** ## Dynatrace API considerations -The `distribution_cut` method is not currently implemented for Dynatrace, since -there are no metric type corresponding to a distribution in the API. +The `distribution_cut` method is not currently implemented for Dynatrace, since there are no metric type corresponding to a distribution in the API. diff --git a/docs/providers/elasticsearch.md b/docs/providers/elasticsearch.md index 1f4e08b8..400f385f 100644 --- a/docs/providers/elasticsearch.md +++ b/docs/providers/elasticsearch.md @@ -2,8 +2,7 @@ ## Backend -Using the `elasticsearch` backend class, you can query any metrics available in -Elasticsearch to create an SLO. +Using the `elasticsearch` backend class, you can query any metrics available in Elasticsearch to create an SLO. ```yaml backends: @@ -11,8 +10,7 @@ backends: url: ${ELASTICSEARCH_URL} ``` -Note that `url` can be either a single string (when connecting to a single node) -or a list of strings (when connecting to multiple nodes): +Note that `url` can be either a single string (when connecting to a single node) or a list of strings (when connecting to multiple nodes): ```yaml backends: @@ -28,8 +26,7 @@ backends: - https://localhost:9201 ``` -The following methods are available to compute SLOs with the `elasticsearch` -backend: +The following methods are available to compute SLOs with the `elasticsearch` backend: * `good_bad_ratio` for computing good / bad metrics ratios. @@ -37,15 +34,13 @@ backend: The `good_bad_ratio` method is used to compute the ratio between two metrics: -- **Good events**, i.e events we consider as 'good' from the user perspective. -- **Bad or valid events**, i.e events we consider either as 'bad' from the user -perspective, or all events we consider as 'valid' for the computation of the -SLO. +* **Good events**, i.e events we consider as 'good' from the user perspective. +* **Bad or valid events**, i.e events we consider either as 'bad' from the user perspective, or all events we consider as 'valid' for the computation of the SLO. -This method is often used for availability SLOs, but can be used for other -purposes as well (see examples). +This method is often used for availability SLOs, but can be used for other purposes as well (see examples). **Config example:** + ```yaml backend: class: Elasticsearch @@ -60,25 +55,21 @@ backend: term: name: JAgOZE8 ``` + Optional fields: - * `date_field`: Alternative field to filter time on. Has to be an ELK `date` - field. Defaults to `@timestamp` which is the Logstash-generated one. + +* `date_field`: Alternative field to filter time on. Has to be an ELK `date` field. Defaults to `@timestamp` which is the Logstash-generated one. **→ [Full SLO config](../../samples/elasticsearch/slo_elk_test_ratio.yaml)** -You can also use the `filter_bad` field which identifies bad events instead of -the `filter_valid` field which identifies all valid events. +You can also use the `filter_bad` field which identifies bad events instead of the `filter_valid` field which identifies all valid events. + +The Lucene query entered in either the `query_good`, `query_bad` or `query_valid` fields will be combined (using the `bool` operator) into a larger query that filters results on the `window` specified in your Error Budget Policy steps. -The Lucene query entered in either the `query_good`, `query_bad` or -`query_valid` fields will be combined (using the `bool` operator) into a larger -query that filters results on the `window` specified in your Error Budget Policy -steps. +You can specify a different field to filter error budget policy windows on, using the `date_field` field. -You can specify a different field to filter error budget policy windows on, -using the `date_field` field. +The full `ElasticSearch` query body for the `query_bad` above will therefore look like: -The full `ElasticSearch` query body for the `query_bad` above will therefore -look like: ```json { "query": { @@ -104,5 +95,4 @@ look like: ### Examples -Complete SLO samples using the `elasticsearch` backend are available in -[samples/elasticsearch](../../samples/elasticsearch). Check them out ! +Complete SLO samples using the `elasticsearch` backend are available in [samples/elasticsearch](../../samples/elasticsearch). Check them out! diff --git a/docs/providers/prometheus.md b/docs/providers/prometheus.md index 411bbcc5..f35dc409 100644 --- a/docs/providers/prometheus.md +++ b/docs/providers/prometheus.md @@ -2,8 +2,7 @@ ## Backend -Using the `prometheus` backend class, you can query any metrics available in -Prometheus to create an SLO. +Using the `prometheus` backend class, you can query any metrics available in Prometheus to create an SLO. ```yaml backends: @@ -15,6 +14,7 @@ backends: ``` Optional fields: + * `headers` allows to specify Basic Authentication credentials if needed. The following methods are available to compute SLOs with the `prometheus` @@ -27,13 +27,10 @@ backend: The `good_bad_ratio` method is used to compute the ratio between two metrics: -- **Good events**, i.e events we consider as 'good' from the user perspective. -- **Bad or valid events**, i.e events we consider either as 'bad' from the user -perspective, or all events we consider as 'valid' for the computation of the -SLO. +* **Good events**, i.e events we consider as 'good' from the user perspective. +* **Bad or valid events**, i.e events we consider either as 'bad' from the user perspective, or all events we consider as 'valid' for the computation of the SLO. -This method is often used for availability SLOs, but can be used for other -purposes as well (see examples). +This method is often used for availability SLOs, but can be used for other purposes as well (see examples). **Config example:** @@ -45,28 +42,20 @@ service_level_indicator: filter_valid: http_requests_total{handler="/metrics"}[window] # operators: ['sum', 'rate'] ``` -* The `window` placeholder is needed in the query and will be replaced by the -corresponding `window` field set in each step of the Error Budget Policy. -* The `operators` section defines which PromQL functions to apply on the -timeseries. The default is to compute `sum(increase([METRIC_NAME][window]))` to -get an accurate count of good and bad events. Be aware that changing will likely -result in good / bad counts that do not accurately reflect actual load. +* The `window` placeholder is needed in the query and will be replaced by the corresponding `window` field set in each step of the Error Budget Policy. -**→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_availability_ratio.yaml)** +* The `operators` section defines which PromQL functions to apply on the timeseries. The default is to compute `sum(increase([METRIC_NAME][window]))` to get an accurate count of good and bad events. Be aware that changing will likely result in good / bad counts that do not accurately reflect actual load. +**→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_availability_ratio.yaml)** ### Query SLI -The `query_sli` method is used to directly query the needed SLI with Prometheus: -indeed, Prometheus' `PromQL` language is powerful enough that it can do ratios -natively. +The `query_sli` method is used to directly query the needed SLI with Prometheus: indeed, Prometheus' `PromQL` language is powerful enough that it can do ratios natively. -This method makes it more flexible to input any `PromQL` SLI computation and -eventually reduces the number of queries made to Prometheus. +This method makes it more flexible to input any `PromQL` SLI computation and eventually reduces the number of queries made to Prometheus. -See Bitnami's [article](https://engineering.bitnami.com/articles/implementing-slos-using-prometheus.html) -on engineering SLOs with Prometheus. +See Bitnami's [article](https://engineering.bitnami.com/articles/implementing-slos-using-prometheus.html) on engineering SLOs with Prometheus. **Config example:** @@ -79,8 +68,8 @@ service_level_indicator: / sum(rate(http_requests_total{handler="/metrics"}[window])) ``` -* The `window` placeholder is needed in the query and will be replaced by the -corresponding `window` field set in each step of the Error Budget Policy. + +* The `window` placeholder is needed in the query and will be replaced by the corresponding `window` field set in each step of the Error Budget Policy. **→ [Full SLO config (availability)](../../samples/prometheus/slo_prom_metrics_availability_query_sli.yaml)** @@ -90,18 +79,12 @@ corresponding `window` field set in each step of the Error Budget Policy. The `distribution_cut` method is used for Prometheus distribution-type metrics (histograms), which are usually used for latency metrics. -A distribution metric records the **statistical distribution of the extracted -values** in **histogram buckets**. The extracted values are not recorded -individually, but their distribution across the configured buckets are recorded. -Prometheus creates 3 separate metrics `_count`, `_bucket`, -and `_sum` metrics. +A distribution metric records the **statistical distribution of the extracted values** in **histogram buckets**. The extracted values are not recorded individually, but their distribution across the configured buckets are recorded. Prometheus creates 3 separate metrics `_count`, `_bucket`, and `_sum` metrics. -When computing SLOs on histograms, we're usually interested in -taking the ratio of the number of events that are located in particular buckets -(considered 'good', e.g: all requests in the `le=0.25` bucket) over the total -count of valid events. +When computing SLOs on histograms, we're usually interested in taking the ratio of the number of events that are located in particular buckets (considered 'good', e.g: all requests in the `le=0.25` bucket) over the total count of valid events. The resulting PromQL expression would be similar to: + ``` increase( _bucket{le="0.25"}[window] @@ -111,13 +94,14 @@ increase( _count[window] ) ``` + which you can very well use directly with the method `query_sli`. The `distribution_cut` method does this calculus under the hood - while -additionally gathering exact good / bad counts - and proposes a simpler way of -expressing it, as shown in the config example below. +additionally gathering exact good / bad counts - and proposes a simpler way of expressing it, as shown in the config example below. **Config example:** + ```yaml backend: prometheus method: distribution_cut @@ -125,17 +109,14 @@ service_level_indicator: expression: http_requests_duration_bucket{path='/', code=~"2.."} threshold_bucket: 0.25 # corresponds to 'le' attribute in Prometheus histograms ``` -**→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_latency_distribution_cut.yaml)** -The `threshold_bucket` allowed will depend on how the buckets boundaries are -set for your metric. Learn more in the [Prometheus docs](https://prometheus.io/docs/concepts/metric_types/#histogram). +**→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_latency_distribution_cut.yaml)** +The `threshold_bucket` allowed will depend on how the buckets boundaries are set for your metric. Learn more in the [Prometheus docs](https://prometheus.io/docs/concepts/metric_types/#histogram). ## Exporter -The `prometheus` exporter allows to export SLO metrics to the -[Prometheus Pushgateway](https://prometheus.io/docs/practices/pushing/) which -needs to be running. +The `prometheus` exporter allows to export SLO metrics to the [Prometheus Pushgateway](https://prometheus.io/docs/practices/pushing/) which needs to be running. ```yaml exporters: @@ -144,30 +125,28 @@ exporters: ``` Optional fields: - * `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`error_budget_burn_rate`, `sli_service_level_indicator`]. - * `username`: Username for Basic Auth. - * `password`: Password for Basic Auth. - * `job`: Name of `Pushgateway` job. Defaults to `slo-generator`. -***Note:*** `prometheus` needs to be setup to **scrape metrics from `Pushgateway`** -(see [documentation](https://github.com/prometheus/pushgateway) for more details). +* `metrics`: List of metrics to export ([see docs](../shared/metrics.md)). Defaults to [`error_budget_burn_rate`, `sli_service_level_indicator`]. +* `username`: Username for Basic Auth. +* `password`: Password for Basic Auth. +* `job`: Name of `Pushgateway` job. Defaults to `slo-generator`. + +***Note:*** `prometheus` needs to be setup to **scrape metrics from `Pushgateway`** (see [documentation](https://github.com/prometheus/pushgateway) for more details). **→ [Full SLO config](../../samples/prometheus/slo_prom_metrics_availability_query_sli.yaml)** ## Self Exporter (API mode) -When running slo-generator as an API, you can enable `prometheus_self` exporter, which will -expose all metrics on a standard `/metrics` endpoint, instead of pushing them to a gateway. +When running slo-generator as an API, you can enable `prometheus_self` exporter, which will expose all metrics on a standard `/metrics` endpoint, instead of pushing them to a gateway. ```yaml exporters: prometheus_self: { } ``` -***Note:*** The metrics endpoint will be available after a first successful SLO request. -Before that, it's going to act as if it was endpoint of the generator API. +***Note:*** The metrics endpoint will be available after a first successful SLO request. Before that, it's going to act as if it was endpoint of the generator API. ### Examples Complete SLO samples using `prometheus` are available in -[samples/prometheus](../../samples/prometheus). Check them out ! +[samples/prometheus](../../samples/prometheus). Check them out! diff --git a/docs/providers/pubsub.md b/docs/providers/pubsub.md index 0f9f2437..a57e0ce1 100644 --- a/docs/providers/pubsub.md +++ b/docs/providers/pubsub.md @@ -11,7 +11,6 @@ exporters: topic_name: "${PUBSUB_TOPIC_NAME}" ``` -This allows teams to consume SLO reports in real-time, and take appropriate -actions when they see a need. +This allows teams to consume SLO reports in real-time, and take appropriate actions when they see a need. **→ [Full SLO config](../../samples/cloud_monitoring/slo_pubsub_subscription_throughput.yaml)** From 3345ca1d7358e319b8d87a533d9c5db88531d872 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Thu, 27 Oct 2022 11:58:48 +0200 Subject: [PATCH 095/107] fix: migrate `cloud_service_monitoring` backend to `google-cloud-monitoring` v2 (with breaking changes) (#280) --- .../backends/cloud_service_monitoring.py | 47 +++++++++++++++---- tests/unit/test_stubs.py | 2 +- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 98ccf462..54f509fc 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -61,7 +61,7 @@ def __init__(self, project_id: str, client=None): self.client = client if client is None: self.client = ServiceMonitoringServiceClient() - self.parent = self.client.project_path(project_id) + self.parent = self.client.common_project_path(project_id) self.workspace_path = f"workspaces/{project_id}" self.project_path = f"projects/{project_id}" @@ -209,9 +209,11 @@ def create_service(self, slo_config: dict) -> dict: service_json = self.build_service(slo_config) service_id = self.build_service_id(slo_config) service = self.client.create_service( - self.project_path, - service_json, - service_id=service_id, + request={ + "parent": self.project_path, + "service": service_json, + "service_id": service_id, + } ) LOGGER.info( f'Service "{service_id}" created successfully in Cloud ' @@ -231,7 +233,13 @@ def get_service(self, slo_config: dict) -> Optional[dict]: # Look for API services in workspace matching our config. service_id = self.build_service_id(slo_config) - services = list(self.client.list_services(self.workspace_path)) + services = list( + self.client.list_services( + request={ + "parent": self.workspace_path, + } + ) + ) matches = [ service for service in services if service.name.split("/")[-1] == service_id ] @@ -356,7 +364,11 @@ def create_slo(self, window: int, slo_config: dict) -> dict: slo_id = self.build_slo_id(window, slo_config) parent = self.build_service_id(slo_config, full=True) slo = self.client.create_service_level_objective( - parent, slo_json, service_level_objective_id=slo_id + request={ + "parent": parent, + "service_level_objective": slo_json, + "service_level_objective_id": slo_id, + } ) return SSM.to_json(slo) @@ -517,7 +529,13 @@ def update_slo(self, window: int, slo_config: dict) -> dict: slo_id = self.build_slo_id(window, slo_config, full=True) LOGGER.warning(f"Updating SLO {slo_id} ...") slo_json["name"] = slo_id - return SSM.to_json(self.client.update_service_level_objective(slo_json)) + return SSM.to_json( + self.client.update_service_level_objective( + request={ + "service_level_objective": slo_json, + } + ) + ) def list_slos(self, service_path: str) -> list: """List all SLOs from Cloud Service Monitoring API. @@ -529,7 +547,11 @@ def list_slos(self, service_path: str) -> list: Returns: list: API response. """ - slos = self.client.list_service_level_objectives(service_path) + slos = self.client.list_service_level_objectives( + request={ + "parent": service_path, + } + ) slos = list(slos) LOGGER.debug(f"{len(slos)} SLOs found in Cloud Service Monitoring API.") # LOGGER.debug(slos) @@ -548,7 +570,11 @@ def delete_slo(self, window: int, slo_config: dict) -> Optional[dict]: slo_path = self.build_slo_id(window, slo_config, full=True) LOGGER.info(f'Deleting SLO "{slo_path}"') try: - return self.client.delete_service_level_objective(slo_path) + return self.client.delete_service_level_objective( + request={ + "name": slo_path, + } + ) except google.api_core.exceptions.NotFound: LOGGER.warning( f'SLO "{slo_path}" does not exist in Service Monitoring API. ' @@ -700,7 +726,8 @@ def to_json(response): Returns: dict: Response object serialized as JSON. """ - return json.loads(MessageToJson(response)) + # pylint: disable=protected-access + return json.loads(MessageToJson(response._pb)) SSM = CloudServiceMonitoringBackend diff --git a/tests/unit/test_stubs.py b/tests/unit/test_stubs.py index 3038c17e..768ef192 100644 --- a/tests/unit/test_stubs.py +++ b/tests/unit/test_stubs.py @@ -292,7 +292,7 @@ def __init__(self): dotize(slo) for slo in load_fixture("ssm_slos.json") ] - def project_path(self, project_id): + def common_project_path(self, project_id): return f"projects/{project_id}" def service_path(self, project_id, service_id): From df335de718a6a8a12bb62f035cd97d19a7638bd4 Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Thu, 27 Oct 2022 14:26:35 +0200 Subject: [PATCH 096/107] chore(master): release 2.3.1 (#276) --- CHANGELOG.md | 13 +++++++++++++ setup.cfg | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8011d23..e5140b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [2.3.1](https://github.com/google/slo-generator/compare/v2.3.0...v2.3.1) (2022-10-27) + + +### Bug Fixes + +* migrate `cloud_service_monitoring` backend to `google-cloud-monitoring` v2 (with breaking changes) ([#280](https://github.com/google/slo-generator/issues/280)) ([affd157](https://github.com/google/slo-generator/commit/affd157fc5b1e253a3e6f02baa22a100b4da244d)) + + +### Documentation + +* document Cloud Monitoring MQL backend ([#277](https://github.com/google/slo-generator/issues/277)) ([8c931cd](https://github.com/google/slo-generator/commit/8c931cd69c8a1be5c59e8b431271da210ea986d7)) +* refine development workflow instructions ([#275](https://github.com/google/slo-generator/issues/275)) ([2a35754](https://github.com/google/slo-generator/commit/2a357546d0e6019110a76c7238349a92677cabd1)) + ## [2.3.0](https://github.com/google/slo-generator/compare/v2.2.0...v2.3.0) (2022-10-24) diff --git a/setup.cfg b/setup.cfg index bd234794..da3ee1ff 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ [metadata] name = slo-generator -version = 2.3.0 +version = 2.3.1 author = Google Inc. author_email = olivier.cervello@gmail.com maintainer = Laurent VAYLET From 60b7dcee5ae21b146a52c5c48baa215776905e6e Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Thu, 27 Oct 2022 23:06:56 +0200 Subject: [PATCH 097/107] docs: update docs and samples with all backends (#283) * add `cloud_monitoring_mql` backend section to sample config * update docs with missing backends --- samples/README.md | 176 +++++++++++++++++++++++++++++--------------- samples/config.yaml | 2 + 2 files changed, 117 insertions(+), 61 deletions(-) diff --git a/samples/README.md b/samples/README.md index 201181fe..b2732c69 100644 --- a/samples/README.md +++ b/samples/README.md @@ -1,98 +1,152 @@ # SLO Library -This folder is an SLO library to facilitate writing new SLOs by starting from -already written SLO configurations. - -All samples are classified into a folder named after their respective backend -or exporter class. - -Each sample contains environmental variables that should be set prior to -running it. - -## Environmental variables - -The following is listing all environmental variables found in the SLO configs, -per backend: - -`cloud_monitoring/`: - - `WORKSPACE_PROJECT_ID`: Cloud Monitoring host project id. - - `LOG_METRIC_NAME`: Cloud Logging log-based metric name. - - `GAE_PROJECT_ID`: Google App Engine application project id. - - `GAE_MODULE_ID`: Google App Engine application module id. - - `PUBSUB_PROJECT_ID`: Pub/Sub project id. - - `PUBSUB_TOPIC_NAME`: Pub/Sub topic name. - -`cloud_service_monitoring/`: - - `WORKSPACE_PROJECT_ID`: Cloud Monitoring host project id. - - `LOG_METRIC_NAME`: Cloud Logging log-based metric name. - - `GAE_PROJECT_ID`: Google App Engine application project id. - - `GAE_MODULE_ID`: Google App Engine application module id. - - `PUBSUB_PROJECT_ID`: Pub/Sub project id. - - `PUBSUB_TOPIC_NAME`: Pub/Sub topic name. - - `GKE_PROJECT_ID`: GKE project id. - - `GKE_LOCATION`: GKE location. - - `GKE_CLUSTER_NAME`: GKE cluster name. - - `GKE_SERVICE_NAMESPACE`: GKE service namespace. - - `GKE_SERVICE_NAME`: GKE service name. - -`elasticsearch/`: - - `ELASTICSEARCH_URL`: ElasticSearch instance URL. - -`prometheus/`: - - `PROMETHEUS_URL`: Prometheus instance URL. - - `PROMETHEUS_PUSHGATEWAY_URL`: Prometheus Pushgateway instance URL. - -You can either set those variables for the backends you want to try, or set all -of those in an `.env` file and then `source` it. Note that the actual GCP resources -you're pointing to need to exist. +This folder is an SLO library to facilitate writing new SLOs by starting from already written SLO configurations. + +All samples are classified within folders named after their respective backend or exporter class. + +Each sample references environment variables that must be set prior to running it. + +## Environment variables + +The following is listing all environment variables found in the SLO configs, per backend: + +You can either set those variables for the backends you want to try, or set all of those in an `.env` file and then `source` it. Note that the actual GCP resources you're pointing to need to exist. + +### `cloud_monitoring` + +| Environment variable | Description | +| --- | --- | +| `WORKSPACE_PROJECT_ID` | Cloud Monitoring host project ID | +| `LOG_METRIC_NAME` | Cloud Logging log-based metric name | +| `GAE_PROJECT_ID` | Google App Engine application project ID | +| `GAE_MODULE_ID` | Google App Engine application module ID | +| `PUBSUB_PROJECT_ID` | Pub/Sub project ID | +| `PUBSUB_TOPIC_NAME` | Pub/Sub topic name | + +### `cloud_monitoring_mql` + +| Environment variable | Description | +| --- | --- | +| `WORKSPACE_PROJECT_ID` | Cloud Monitoring host project ID | +| `LOG_METRIC_NAME` | Cloud Logging log-based metric name | +| `GAE_PROJECT_ID` | Google App Engine application project ID | +| `GAE_MODULE_ID` | Google App Engine application module ID | +| `PUBSUB_PROJECT_ID` | Pub/Sub project ID | +| `PUBSUB_TOPIC_NAME` | Pub/Sub topic name | + +### `cloud_service_monitoring` + +| Environment variable | Description | +| --- | --- | +| `WORKSPACE_PROJECT_ID` | Cloud Monitoring host project ID | +| `LOG_METRIC_NAME` | Cloud Logging log-based metric name | +| `GAE_PROJECT_ID` | Google App Engine application project ID | +| `GAE_MODULE_ID` | Google App Engine application module ID | +| `PUBSUB_PROJECT_ID` | Pub/Sub project ID | +| `PUBSUB_TOPIC_NAME` | Pub/Sub topic name | +| `GKE_PROJECT_ID` | GKE project ID | +| `GKE_LOCATION` | GKE location | +| `GKE_CLUSTER_NAME` | GKE cluster name | +| `GKE_SERVICE_NAMESPACE` | GKE service namespace | +| `GKE_SERVICE_NAME` | GKE service name | + +### `datadog` + +| Environment variable | Description | +| --- | --- | +| `DATADOG_API_KEY` | Datadog API key | +| `DATADOG_APP_KEY` | Datadog APP key | + +### `dynatrace` + +| Environment variable | Description | +| --- | --- | +| `DYNATRACE_API_URL` | Dynatrace API URL | +| `DYNATRACE_API_TOKEN` | Dynatrace API token | + +### `elasticsearch` + +| Environment variable | Description | +| --- | --- | +| `ELASTICSEARCH_URL` | ElasticSearch instance URL | + +### `prometheus` + +| Environment variable | Description | +| --- | --- | +| `PROMETHEUS_URL` | Prometheus instance URL | +| `PROMETHEUS_PUSHGATEWAY_URL` | Prometheus Pushgateway instance URL | ## Running the samples To run one sample: -``` + +```sh slo-generator -f samples/cloud_monitoring/.yaml ``` To run all the samples for a backend: -``` +```sh slo-generator -f samples/ -b samples/ ``` + *where:* -* `` is the backend name (lowercase) -* `` is the path to the error budget policy YAML file. -***Note:*** *if you want to enable the exporters as well, you can add the -`--export` flag.* +- `` is the backend name (lowercase) +- `` is the path to the error budget policy YAML file. +***Note:*** *if you want to enable the exporters as well, you can add the `--export` flag.* ### Examples -##### Cloud Monitoring -``` +#### Cloud Monitoring (MQF) + +```sh slo-generator -f samples/cloud_monitoring -b error_budget_policy.yaml ``` -##### Cloud Service Monitoring +#### Cloud Monitoring (MQL) + +```sh +slo-generator -f samples/cloud_monitoring_mql -b error_budget_policy.yaml ``` + +#### Cloud Service Monitoring + +```sh slo-generator -f samples/cloud_service_monitoring -b error_budget_policy_ssm.yaml ``` -***Note:*** *the Error Budget Policy is different for this backend, because it only -supports steps where the `window` is a multiple of 24 hours.* +***Note:*** *the Error Budget Policy is different for this backend, because it only supports steps where `window` is a multiple of 24 hours.* -##### Prometheus -``` -slo-generator -f samples/prometheus -b error_budget_policy.yaml +#### Datadog + +```sh +slo-generator -f samples/datadog -b error_budget_policy.yaml ``` -##### Elasticsearch +#### Dynatrace + +```sh +slo-generator -f samples/dynatrace -b error_budget_policy.yaml ``` + +#### Elasticsearch + +```sh slo-generator -f samples/elasticsearch -b error_budget_policy.yaml ``` -##### Custom Class +#### Prometheus + +```sh +slo-generator -f samples/prometheus -b error_budget_policy.yaml ``` + +#### Custom Class + +```sh cd samples/ slo-generator -f custom -b error_budget_policy.yaml -e ``` diff --git a/samples/config.yaml b/samples/config.yaml index 9ef03fc8..5978cdad 100644 --- a/samples/config.yaml +++ b/samples/config.yaml @@ -4,6 +4,8 @@ default_exporters: [cloudevent] backends: cloud_monitoring: project_id: ${STACKDRIVER_HOST_PROJECT_ID} + cloud_monitoring_mql: + project_id: ${STACKDRIVER_HOST_PROJECT_ID} cloud_service_monitoring: project_id: ${STACKDRIVER_HOST_PROJECT_ID} samples.custom.custom_backend.CustomBackend: {} From 4f3c81562d175a04a0e410cee32c0e79059b2049 Mon Sep 17 00:00:00 2001 From: Joshua Pollock Date: Sat, 29 Oct 2022 03:21:41 -0400 Subject: [PATCH 098/107] fix: remove calls to list and create metric descriptors in Cloud Monitoring exporter to prevent Quota Exceeded errors (#286) --- slo_generator/exporters/cloud_monitoring.py | 48 +-------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/slo_generator/exporters/cloud_monitoring.py b/slo_generator/exporters/cloud_monitoring.py index f45a803e..71e49e98 100644 --- a/slo_generator/exporters/cloud_monitoring.py +++ b/slo_generator/exporters/cloud_monitoring.py @@ -17,8 +17,6 @@ """ import logging -import google.api_core.exceptions -from google.api import metric_pb2 as ga_metric from google.cloud import monitoring_v3 from .base import MetricsExporter @@ -36,8 +34,7 @@ def __init__(self): self.client = monitoring_v3.MetricServiceClient() def export_metric(self, data: dict): - """Export metric to Cloud Monitoring. Create metric descriptor if - it doesn't exist. + """Export metric to Cloud Monitoring. Args: data (dict): Data to send to Cloud Monitoring. @@ -45,8 +42,6 @@ def export_metric(self, data: dict): Returns: object: Cloud Monitoring API result. """ - if not self.get_metric_descriptor(data): - self.create_metric_descriptor(data) self.create_timeseries(data) def create_timeseries(self, data: dict): @@ -101,44 +96,3 @@ def create_timeseries(self, data: dict): f"{labels['slo_name']}-{labels['error_budget_policy_step_name']}" ) # pylint: enable=E1101 - - def get_metric_descriptor(self, data: dict): - """Get Cloud Monitoring metric descriptor. - - Args: - data (dict): Metric data. - - Returns: - object: Metric descriptor (or None if not found). - """ - project_id = data["project_id"] - metric_id = data["name"] - request = monitoring_v3.GetMetricDescriptorRequest( - name=f"projects/{project_id}/metricDescriptors/{metric_id}" - ) - try: - return self.client.get_metric_descriptor(request) - except google.api_core.exceptions.NotFound: - return None - - def create_metric_descriptor(self, data: dict): - """Create Cloud Monitoring metric descriptor. - - Args: - data (dict): Metric data. - - Returns: - object: Metric descriptor. - """ - project = self.client.common_project_path(data["project_id"]) - descriptor = ga_metric.MetricDescriptor() - descriptor.type = data["name"] - # pylint: disable=E1101 - descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE - descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE - # pylint: enable=E1101 - descriptor.description = data["description"] - descriptor = self.client.create_metric_descriptor( - name=project, metric_descriptor=descriptor - ) - return descriptor From 186b0d7bea48b8728363f459cdc6f509de441445 Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Sat, 29 Oct 2022 09:36:38 +0200 Subject: [PATCH 099/107] chore(master): release 2.3.2 (#284) --- CHANGELOG.md | 12 ++++++++++++ setup.cfg | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5140b06..3a343de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [2.3.2](https://github.com/google/slo-generator/compare/v2.3.1...v2.3.2) (2022-10-29) + + +### Bug Fixes + +* remove calls to list and create metric descriptors in Cloud Monitoring exporter to prevent Quota Exceeded errors ([#286](https://github.com/google/slo-generator/issues/286)) ([0a6a0fb](https://github.com/google/slo-generator/commit/0a6a0fb75d6c83deddbf81288c4d020e22bbd6d5)) + + +### Documentation + +* update docs and samples with all backends ([#283](https://github.com/google/slo-generator/issues/283)) ([61f2f32](https://github.com/google/slo-generator/commit/61f2f3291671fbfc1afc607255c1366f90c55b98)) + ## [2.3.1](https://github.com/google/slo-generator/compare/v2.3.0...v2.3.1) (2022-10-27) diff --git a/setup.cfg b/setup.cfg index da3ee1ff..d1df59a6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ [metadata] name = slo-generator -version = 2.3.1 +version = 2.3.2 author = Google Inc. author_email = olivier.cervello@gmail.com maintainer = Laurent VAYLET From b1146b556c82be04dbc1933dac99bfbfe217a865 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Wed, 2 Nov 2022 21:36:13 +0100 Subject: [PATCH 100/107] fix: compute the time horizon of MQL requests more accurately so they return the same results as MQF requests (#290) --- .../backends/cloud_monitoring_mql.py | 112 ++++++++---------- .../backends/test_cloud_monitoring_mql.py | 60 ++++------ 2 files changed, 73 insertions(+), 99 deletions(-) diff --git a/slo_generator/backends/cloud_monitoring_mql.py b/slo_generator/backends/cloud_monitoring_mql.py index 56ea49ba..1e934f98 100644 --- a/slo_generator/backends/cloud_monitoring_mql.py +++ b/slo_generator/backends/cloud_monitoring_mql.py @@ -17,19 +17,19 @@ """ import logging import pprint -import re import typing import warnings from collections import OrderedDict -from typing import List, Tuple +from datetime import datetime +from typing import List, Optional, Tuple from google.api.distribution_pb2 import Distribution +from google.cloud.monitoring_v3 import QueryTimeSeriesRequest from google.cloud.monitoring_v3.services.query_service import QueryServiceClient from google.cloud.monitoring_v3.services.query_service.pagers import ( QueryTimeSeriesPager, ) -from google.cloud.monitoring_v3.types import metric_service -from google.cloud.monitoring_v3.types.metric import TimeSeriesData +from google.cloud.monitoring_v3.types.metric import TimeSeries from slo_generator.constants import NO_DATA @@ -56,7 +56,7 @@ def __init__(self, project_id: str, client: QueryServiceClient = None): def good_bad_ratio( self, - timestamp: int, # pylint: disable=unused-argument + timestamp: int, window: int, slo_config: dict, ) -> Tuple[int, int]: @@ -73,22 +73,20 @@ def good_bad_ratio( """ measurement: dict = slo_config["spec"]["service_level_indicator"] filter_good: str = measurement["filter_good"] - filter_bad: typing.Optional[str] = measurement.get("filter_bad") - filter_valid: typing.Optional[str] = measurement.get("filter_valid") + filter_bad: Optional[str] = measurement.get("filter_bad") + filter_valid: Optional[str] = measurement.get("filter_valid") # Query 'good events' timeseries - good_ts: List[TimeSeriesData] = self.query(query=filter_good, window=window) + good_ts: List[TimeSeries] = self.query(timestamp, window, filter_good) good_event_count: int = CM.count(good_ts) # Query 'bad events' timeseries bad_event_count: int if filter_bad: - bad_ts: List[TimeSeriesData] = self.query(query=filter_bad, window=window) + bad_ts: List[TimeSeries] = self.query(timestamp, window, filter_bad) bad_event_count = CM.count(bad_ts) elif filter_valid: - valid_ts: List[TimeSeriesData] = self.query( - query=filter_valid, window=window - ) + valid_ts: List[TimeSeries] = self.query(timestamp, window, filter_valid) bad_event_count = CM.count(valid_ts) - good_event_count else: raise Exception("One of `filter_bad` or `filter_valid` is required.") @@ -124,7 +122,7 @@ def distribution_cut( ) # Query 'valid' events - series = self.query(query=filter_valid, window=window) + series = self.query(timestamp, window, filter_valid) if not series: return NO_DATA, NO_DATA # no timeseries @@ -193,38 +191,66 @@ def query_sli( """ measurement: dict = slo_config["spec"]["service_level_indicator"] query: str = measurement["query"] - series: List[TimeSeriesData] = self.query(query=query, window=window) + series: List[TimeSeries] = self.query(timestamp, window, query) sli_value: float = series[0].point_data[0].values[0].double_value LOGGER.debug(f"SLI value: {sli_value}") return sli_value - def query(self, query: str, window: int) -> List[TimeSeriesData]: + def query(self, timestamp: float, window: int, query: str) -> List[TimeSeries]: """Query timeseries from Cloud Monitoring using MQL. Args: - query (str): MQL query. + timestamp (float): Current timestamp. window (int): Window size (in seconds). - + query (str): MQL query. Returns: list: List of timeseries objects. """ - # Enrich query to aggregate and reduce the time series over the - # desired window. - formatted_query: str = self._fmt_query(query, window) - request = metric_service.QueryTimeSeriesRequest( - {"name": self.parent, "query": formatted_query} + # Enrich query to aggregate and reduce time series over target window. + query_with_time_horizon_and_period: str = ( + self.enrich_query_with_time_horizon_and_period(timestamp, window, query) + ) + request = QueryTimeSeriesRequest( + {"name": self.parent, "query": query_with_time_horizon_and_period} ) # fmt: off timeseries_pager: QueryTimeSeriesPager = ( self.client.query_time_series(request) # type: ignore[union-attr] ) # fmt: on - timeseries: list = list(timeseries_pager) # convert pager to flat list + timeseries: List[TimeSeries] = list(timeseries_pager) LOGGER.debug(pprint.pformat(timeseries)) return timeseries @staticmethod - def count(timeseries: List[TimeSeriesData]) -> int: + def enrich_query_with_time_horizon_and_period( + timestamp: float, + window: int, + query: str, + ) -> str: + """Enrich MQL query with time period and horizon. + Args: + timestamp (float): UNIX timestamp. + window (int): Query window (in seconds). + query (str): Base query in YAML config. + Returns: + str: Enriched query. + """ + # Python uses floating point numbers to represent time in seconds since the + # epoch, in UTC, with decimal part representing nanoseconds. + # MQL expects dates formatted like "%Y/%m/%d %H:%M:%S" or "%Y/%m/%d-%H:%M:%S". + # Reference: https://cloud.google.com/monitoring/mql/reference#lexical-elements + end_time_str: str = datetime.fromtimestamp(timestamp).strftime( + "%Y/%m/%d %H:%M:%S" + ) + query_with_time_horizon_and_period: str = ( + query + + f"| group_by [] | within {window}s, d'{end_time_str}' | every {window}s" + ) + return query_with_time_horizon_and_period + + @staticmethod + def count(timeseries: List[TimeSeries]) -> int: """Count events in time series assuming it was aligned with ALIGN_SUM and reduced with REDUCE_SUM (default). @@ -240,43 +266,5 @@ def count(timeseries: List[TimeSeriesData]) -> int: LOGGER.debug(exception, exc_info=True) return NO_DATA # no events in timeseries - @staticmethod - def _fmt_query(query: str, window: int) -> str: - """Format MQL query: - - * If the MQL expression has a `window` placeholder, replace it by the - current window. Otherwise, append it to the expression. - - * If the MQL expression has a `every` placeholder, replace it by the - current window. Otherwise, append it to the expression. - - * If the MQL expression has a `group_by` placeholder, replace it. - Otherwise, append it to the expression. - - Args: - query (str): Original query in YAMLconfig. - window (int): Query window (in seconds). - - Returns: - str: Formatted query. - """ - formatted_query: str = query.strip() - if "group_by" in formatted_query: - formatted_query = re.sub( - r"\|\s+group_by\s+\[.*\]\s*", "| group_by [] ", formatted_query - ) - else: - formatted_query += "| group_by [] " - for mql_time_interval_keyword in ["within", "every"]: - if mql_time_interval_keyword in formatted_query: - formatted_query = re.sub( - rf"\|\s+{mql_time_interval_keyword}\s+\w+\s*", - f"| {mql_time_interval_keyword} {window}s ", - formatted_query, - ) - else: - formatted_query += f"| {mql_time_interval_keyword} {window}s " - return formatted_query.strip() - CM = CloudMonitoringMqlBackend diff --git a/tests/unit/backends/test_cloud_monitoring_mql.py b/tests/unit/backends/test_cloud_monitoring_mql.py index 9e5e9d2e..addb6ef3 100644 --- a/tests/unit/backends/test_cloud_monitoring_mql.py +++ b/tests/unit/backends/test_cloud_monitoring_mql.py @@ -20,42 +20,28 @@ class TestCloudMonitoringMqlBackend(unittest.TestCase): - def test_fmt_query(self): - queries = [ - """ fetch gae_app - | metric 'appengine.googleapis.com/http/server/response_count' - | filter resource.project_id == '${GAE_PROJECT_ID}' - | filter - metric.response_code == 429 - || metric.response_code == 200 - | group_by [metric.response_code] | within 1h """, - """ fetch gae_app - | metric 'appengine.googleapis.com/http/server/response_count' - | filter resource.project_id == '${GAE_PROJECT_ID}' - | filter - metric.response_code == 429 - || metric.response_code == 200 - | group_by [metric.response_code, response_code_class] - | within 1h - | every 1h """, - """ fetch gae_app - | metric 'appengine.googleapis.com/http/server/response_count' - | filter resource.project_id == '${GAE_PROJECT_ID}' - | filter - metric.response_code == 429 - || metric.response_code == 200 - | group_by [metric.response_code,response_code_class] - | within 1h - | every 1h """, - ] + def test_enrich_query_with_time_horizon_and_period(self): + timestamp: float = 1666995015.5144777 # = 2022/10/28 22:10:15.5144777 + window: int = 3600 # in seconds + query: str = """fetch gae_app +| metric 'appengine.googleapis.com/http/server/response_count' +| filter resource.project_id == 'slo-generator-demo' +| filter + metric.response_code == 429 + || metric.response_code == 200 +""" - formatted_query = """fetch gae_app - | metric 'appengine.googleapis.com/http/server/response_count' - | filter resource.project_id == '${GAE_PROJECT_ID}' - | filter - metric.response_code == 429 - || metric.response_code == 200 - | group_by [] | within 3600s | every 3600s""" + enriched_query = """fetch gae_app +| metric 'appengine.googleapis.com/http/server/response_count' +| filter resource.project_id == 'slo-generator-demo' +| filter + metric.response_code == 429 + || metric.response_code == 200 +| group_by [] | within 3600s, d'2022/10/28 22:10:15' | every 3600s""" - for query in queries: - assert CloudMonitoringMqlBackend._fmt_query(query, 3600) == formatted_query + assert ( + CloudMonitoringMqlBackend.enrich_query_with_time_horizon_and_period( + timestamp, window, query + ) + == enriched_query + ) From d355e5ba53d90d0bf3b9274487f940adf95f1c6b Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Wed, 2 Nov 2022 22:14:31 +0100 Subject: [PATCH 101/107] chore(master): release 2.3.3 (#291) --- CHANGELOG.md | 7 +++++++ setup.cfg | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a343de0..a8a61c0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [2.3.3](https://github.com/google/slo-generator/compare/v2.3.2...v2.3.3) (2022-11-02) + + +### Bug Fixes + +* compute the time horizon of MQL requests more accurately so they return the same results as MQF requests ([#290](https://github.com/google/slo-generator/issues/290)) ([41b814b](https://github.com/google/slo-generator/commit/41b814b6119f7a43b229317f8da0f4006c987656)) + ## [2.3.2](https://github.com/google/slo-generator/compare/v2.3.1...v2.3.2) (2022-10-29) diff --git a/setup.cfg b/setup.cfg index d1df59a6..fa247a2c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ [metadata] name = slo-generator -version = 2.3.2 +version = 2.3.3 author = Google Inc. author_email = olivier.cervello@gmail.com maintainer = Laurent VAYLET From 6e947def8171ac87a2c808445dcf258448bae552 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Wed, 16 Nov 2022 16:11:49 +0100 Subject: [PATCH 102/107] chore: add templates for bug reports and feature requests (#299) * add bug report template * add feature request template * prevent users from creating blank issues (i.e. force them to use a template) * fix config file extension --- .github/ISSUE_TEMPLATE/BUG-REPORT.yml | 72 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml | 57 +++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + 3 files changed, 130 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/BUG-REPORT.yml create mode 100644 .github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml new file mode 100644 index 00000000..377249ca --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -0,0 +1,72 @@ +name: "🐛 Bug Report" +description: File a bug report. +title: "🐛 [BUG] - " +labels: [ + "bug", + "triage", +] +assignees: + - lvaylet +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: input + id: slo-generator-version + attributes: + label: SLO Generator Version + description: Which version of SLO Generator are you using? + placeholder: ex. v1.5.0, v2.3.3 + validations: + required: true + - type: input + id: python-version + attributes: + label: Python Version + description: Which version of Python are you using? + placeholder: ex. 2.7, 3.10 + validations: + required: true + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Please provide as many details as possible, including SLO definitions and shared config. Ideally reproduction steps too, so we can troubleshoot the issue on our own machines. + placeholder: Tell us what you see! + validations: + required: true + - type: textarea + id: what-did-you-expect + attributes: + label: What did you expect? + description: What did you expect to happen? + placeholder: Tell us what you expected! + validations: + required: true + - type: textarea + id: screenshot + attributes: + label: "Screenshots" + description: If applicable, add screenshots to help explain your problem. + value: | + ![DESCRIPTION](LINK.png) + render: bash + validations: + required: false + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell + validations: + required: false + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/google/slo-generator/blob/master/CODE_OF_CONDUCT.md). + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml new file mode 100644 index 00000000..399023a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml @@ -0,0 +1,57 @@ +name: "💡 Feature Request" +description: File a new feature request. +title: "💡 [REQUEST] - <title>" +labels: [ + "question" +] +body: + - type: textarea + id: summary + attributes: + label: "Summary" + description: Provide a brief explanation of the feature. + placeholder: Describe in a few lines your feature request. + validations: + required: true + - type: textarea + id: basic_example + attributes: + label: "Basic Example" + description: Indicate here some basic examples of your feature. + placeholder: A few specific words about your feature request. + validations: + required: true + - type: textarea + id: screenshot + attributes: + label: "Screenshots" + description: If applicable, add screenshots to help explain your problem. + value: | + ![DESCRIPTION](LINK.png) + render: bash + validations: + required: false + - type: textarea + id: drawbacks + attributes: + label: "Drawbacks" + description: What are the drawbacks/impacts of your feature request ? + placeholder: Identify the drawbacks and impacts while being neutral on your feature request. + validations: + required: true + - type: textarea + id: unresolved_question + attributes: + label: "Unresolved questions" + description: What questions still remain unresolved ? + placeholder: Identify any unresolved issues. + validations: + required: false + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/google/slo-generator/blob/master/CODE_OF_CONDUCT.md). + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..3ba13e0c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false From ea482674d6492f4572c98722c31fe9d63469359b Mon Sep 17 00:00:00 2001 From: Laurent Vaylet <lvaylet@users.noreply.github.com> Date: Wed, 16 Nov 2022 16:57:04 +0100 Subject: [PATCH 103/107] fix: implicit Optional type hints are now forbidden (cf. PEP 484) (#301) * fix no_implicit_optional errors (cf. PEP 484) * fix package name --- slo_generator/backends/cloud_monitoring_mql.py | 2 +- slo_generator/migrations/migrator.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/slo_generator/backends/cloud_monitoring_mql.py b/slo_generator/backends/cloud_monitoring_mql.py index 1e934f98..419e20fb 100644 --- a/slo_generator/backends/cloud_monitoring_mql.py +++ b/slo_generator/backends/cloud_monitoring_mql.py @@ -46,7 +46,7 @@ class CloudMonitoringMqlBackend: if omitted. """ - def __init__(self, project_id: str, client: QueryServiceClient = None): + def __init__(self, project_id: str, client=None): self.client = client if client is None: self.client = QueryServiceClient() diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 5b678c0a..ab065115 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -26,6 +26,7 @@ import sys from collections import OrderedDict from pathlib import Path +from typing import Optional import click from ruamel import yaml @@ -584,7 +585,7 @@ class CustomDumper(yaml.RoundTripDumper): # HACK: insert blank lines between top-level objects # inspired by https://stackoverflow.com/a/44284819/3786245 # pylint: disable=missing-function-docstring - def write_line_break(self, data: str = None): + def write_line_break(self, data: Optional[str] = None): super().write_line_break(data) if len(self.indents) == 1: From 09e9dcf1934c6ae7672bde5d8110b682f03bd541 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet <lvaylet@users.noreply.github.com> Date: Wed, 23 Nov 2022 17:24:47 +0100 Subject: [PATCH 104/107] doc: update BigQuery and Data Studio report instructions for Looker Studio (#304) --- docs/deploy/datastudio_slo_report.md | 119 --------------------- docs/deploy/lookerstudio_slo_report.md | 129 +++++++++++++++++++++++ docs/images/confirm_copy_data_source.png | Bin 0 -> 15977 bytes docs/images/copy_button.png | Bin 2827 -> 0 bytes docs/images/copy_data_source.png | Bin 0 -> 4618 bytes docs/images/copy_report.png | Bin 0 -> 11780 bytes 6 files changed, 129 insertions(+), 119 deletions(-) delete mode 100644 docs/deploy/datastudio_slo_report.md create mode 100644 docs/deploy/lookerstudio_slo_report.md create mode 100644 docs/images/confirm_copy_data_source.png delete mode 100644 docs/images/copy_button.png create mode 100644 docs/images/copy_data_source.png create mode 100644 docs/images/copy_report.png diff --git a/docs/deploy/datastudio_slo_report.md b/docs/deploy/datastudio_slo_report.md deleted file mode 100644 index 8d95a181..00000000 --- a/docs/deploy/datastudio_slo_report.md +++ /dev/null @@ -1,119 +0,0 @@ -# Build an SLO achievements report using BigQuery and DataStudio - -This template provides a basic dashboard with 3 views: - -* `Morning snapshot`: last error budget and SLI achievement, support decisions agreed in the Error Budget Policy. - -* `Trends`: SLI vs SLO by service/feature over a period of time. - -* `Alerting on burnrate`: visualize when alerting engage and fade off by sliding window sizes - -## Prerequisites - -### Setup the BigQuery exporter - -In order to setup a DataStudio report, make sure `slo-generator` is configured -to export to a BigQuery dataset (see [instructions here](../providers/bigquery.md)). - -### Create a BigQuery view - -Replace the variables `PROJECT_ID`, `DATASET_ID` and `TABLE_ID` in the -content below by the values configured in your BigQuery exporter, and put it -into a file `create_view.sql`: - -```sql -CREATE VIEW <PROJECT_ID>.<DATASET_ID>.last_report AS -SELECT - r2.* -FROM - ( - SELECT - r.service_name, - r.feature_name, - r.slo_name, - r.window, - MAX(r.timestamp_human) AS timestamp_human - FROM - <PROJECT_ID>.<DATASET_ID>.<TABLE_ID> AS r - GROUP BY - r.service_name, - r.feature_name, - r.slo_name, - r.window - ORDER BY - r.window - ) - AS latest_report - INNER JOIN - <PROJECT_ID>.<DATASET_ID>.<TABLE_ID> AS r2 - ON r2.service_name = latest_report.service_name - AND r2.feature_name = latest_report.feature_name - AND r2.slo_name = latest_report.slo_name - AND r2.window = latest_report.window - AND r2.timestamp_human = latest_report.timestamp_human -ORDER BY - r2.service_name, - r2.feature_name, - r2.slo_name, - r2.error_budget_policy_step_name -``` - -Run it with the BigQuery CLI using: - -```sh -bq query `cat create_view.sql` -``` - -Alternatively, you can create the view above with Terraform. - -## Setup DataStudio SLO achievements report - -1. The user that would own this report has sufficient access to the BQ data set where the aforementioned -view and table resides. This would require provisioning BQ access to the report owner, at minimum as BQ User and -BQ Data Viewer roles. -2. You also need the fully qualified name of view or table in this format `<project-id>.<dataset-name>.<view-or-table-name>` -3. Table `report` in SLO Generator Bigquery dataset -4. View `last_report` in SLO Generator Bigquery dataset. - -### 1. Copy the data sources - -#### 1.a. `report` data source - -##### Step 1 - -Make a copy of the following data source (copy & paste the below link in a browser) -<https://datastudio.google.com/datasources/24648dbe-fa29-48bf-9227-7e8673319968> -and clicking on the "Make a Copy" button just left of the "Create Report" button. -![copy this report](../images/copy_button.png) - -##### Step 2 - -From the BQ connector settings page use "My projects" search and select your SLO-generator project ID, then your dataset, usually `slo_reports`, then table "reports". - -##### Step 3 - -Hit the RECONNECT button top right.The message `Configuration has changed. Do you want to apply the changes?` appears indicating no fields changed. Hit APPLY. -![copy this report](../images/config_has_changed.png) - -##### Step 4 - -Change the data source name at the top left corner from "Copy of..." to something appropriate. -You can close this browser window now. - -#### 1.b. `last_report` data source - -Repeat the previous steps for the second data source starting by using this URL: -<https://datastudio.google.com/datasources/e21bfc1b-70c7-46e9-b851-652c6ce0eb15> -and selecting the table "last_report" from your project, your dataset. - -### 2. Copy the report template - -#### Step A - -Open the report template from this URL: <https://datastudio.google.com/reporting/964e185c-6ca0-4ed8-809d-425e22568aa0> check you are in view mode, and click on the "Make a Copy" button between the fullscreen button and the refresh button on top right of the screen. - -#### Step B - -Point the "New Data source" to the two newly created data sources from the above steps and click `Copy Report`. - -![copy this report](../images/copy_this_report.png) diff --git a/docs/deploy/lookerstudio_slo_report.md b/docs/deploy/lookerstudio_slo_report.md new file mode 100644 index 00000000..00cb46f9 --- /dev/null +++ b/docs/deploy/lookerstudio_slo_report.md @@ -0,0 +1,129 @@ +# Build an SLO achievements report using BigQuery and Looker Studio + +This template provides a basic dashboard with 3 views: + +* `Morning snapshot`: last error budget and SLI achievement, to support decisions documented in the Error Budget Policy. + +* `Trends`: SLI vs SLO by service/feature over a given window. + +* `Alerting on burnrate`: visualize when alerting engages and fades off by sliding window sizes + +## Prerequisites + +### Setup the BigQuery exporter + +In order to setup a Looker Studio report, make sure `slo-generator` is configured to export to a BigQuery dataset (see [instructions here](../providers/bigquery.md)). + +### Create a BigQuery view + +Populate the `PROJECT_ID`, `DATASET_ID` and `TABLE_ID` environment variables below so they match the values configured in your BigQuery exporter, then generate a BigQuery SQL query responsible for creating a view and store it in `create_view.sql` with: + +```sh +export PROJECT_ID=<PROJECT_ID> +export DATASET_ID=<DATASET_ID> +export TABLE_ID=<TABLE_ID> + +cat > create_view.sql << EOF +CREATE VIEW `${PROJECT_ID}.${DATASET_ID}.last_report` AS +SELECT + r2.* +FROM + ( + SELECT + r.service_name, + r.feature_name, + r.slo_name, + r.window, + MAX(r.timestamp_human) AS timestamp_human + FROM + `${PROJECT_ID}.${DATASET_ID}.${TABLE_ID}` AS r + GROUP BY + r.service_name, + r.feature_name, + r.slo_name, + r.window + ORDER BY + r.window + ) + AS latest_report + INNER JOIN + `${PROJECT_ID}.${DATASET_ID}.${TABLE_ID}` AS r2 + ON r2.service_name = latest_report.service_name + AND r2.feature_name = latest_report.feature_name + AND r2.slo_name = latest_report.slo_name + AND r2.window = latest_report.window + AND r2.timestamp_human = latest_report.timestamp_human +ORDER BY + r2.service_name, + r2.feature_name, + r2.slo_name, + r2.error_budget_policy_step_name +EOF +``` + +Run the query using the BigQuery CLI with: + +```sh +bq query --use_legacy_sql=false < create_view.sql +``` + +Alternatively, you can create the view above with Terraform. + +## Setup Looker Studio report + +Prerequisites: + +1. The user owning this report must have access to the BigQuery dataset where the aforementioned view and table reside, for example with the **BigQuery User** and **BigQuery Data Viewer** roles at the dataset or project level. +2. You also need the fully qualified names of the `last_report` view and the `report` table, formatted as `<project-id>.<dataset-name>.<view-or-table-name>`. + +### 1. Copy the data sources + +#### 1.a. `report` data source + +##### Step 1 + +Make a copy of the following data source (copy & paste the link in a browser) by clicking the **Make a copy of this datasource** button (just left of the **Create Report** button) and confirming with **Copy Data Source**: + +https://datastudio.google.com/datasources/24648dbe-fa29-48bf-9227-7e8673319968 + +![copy this data source](../images/copy_data_source.png) + +![confirm copy this data source](../images/confirm_copy_data_source.png) + +##### Step 2 + +From the BigQuery connector settings page, select your SLO Generator project ID from the **My projects** category on the left, then select your dataset (usually `slo_reports`) and finally your table (usually `reports`). + +##### Step 3 + +Hit the **Reconnect** button at the top right, then click **Apply** on the next confirmation dialog. + +##### Step 4 + +Change the name of the data source name at the top left, from **Copy of...** to something more relevant and meaningful. + +At this point, feel free to close the current browser tab and move on to the next step. + +#### 1.b. `last_report` data source + +Repeat the previous steps for the second data source available at the URL below. Select the `last_report` view from the same project and dataset. + +https://datastudio.google.com/datasources/e21bfc1b-70c7-46e9-b851-652c6ce0eb15 + +### 2. Copy the report template + +#### Step A + +Open the report template at this URL: + +https://datastudio.google.com/reporting/964e185c-6ca0-4ed8-809d-425e22568aa0 + +Confirm that you are in **View** mode, then click the **Make a Copy** button from the **More options** (three dots) menu. + +![copy report](../images/copy_report.png) + +#### Step B + +Configure the two **New Data source** fields so they point to the two newly-created data sources from the above steps, then `Copy Report`. + +![copy this report](../images/copy_this_report.png) diff --git a/docs/images/confirm_copy_data_source.png b/docs/images/confirm_copy_data_source.png new file mode 100644 index 0000000000000000000000000000000000000000..ce13e86d6e3cca28817dcde12af70888e9e8fe77 GIT binary patch literal 15977 zcmbt*WmJ{n_a#yiiZmjnl9Ea{0@6r#OLv!as)QgQT>=71hjfX8AR*liQqtWp=l=dP zA7;(WS~F`tC>QR%@B73#d+)Q)^M)xYNZrMHfQ5vFbXP|Dg$feVEj@T$hk*|No36e6 z3O{bSs7Q$+mHZ)JM?!jtB=bU4%`<H)-NT1)<os=B^2}if4K7BLj3qL1j6$tiZa7Ae z-c?{w&_m)EY*^XjtPdYEVucgN7-wM)rdr+!G5G9xYn?Xauv*65W96j7oPMe@3pM1x zNBH-42De3*-)j0v<7yG-QUsg2J(7A;)<5LH=Z+nZWd>i`N%;%KhjwD226^AoEpEWu zQ&%UFi4CW(<zf6L_5}^Ob!){dbg00PM`ezwm8C>!GX5mH<ijiYdZiP6g!{QEb;c~# z`g;tAn4pc~@q+M0{wB{WJPg#J)F7To>WQ_~^o!>S5%w+LUn-GLW7#W<6&4i8ZH%b0 zEU4gOpw^J-#<;4{*hHn@dLm;VT2(t*BG-CE6Il{6s&4hD1UYbCp)H<U8*PYNDo#ng zYWRtmXA_=2eXH9DsfVz3N31~uRmgp8)FAv{aS!XmCCl>0F-zYbH{$6Fw7L<BIfRUw zq#Hkcq4f}VnpqDslji}oOl;igBxWgBUtU^zL^j62$P}^b6d#GPP3lf>)N~O$tjj#6 z)PSqzFCqnF8|nha<IxhXAFax!LSZGsEG)fDOjTdlCC+`DUc4)L{;<{{oPcZoYmte8 za%K}t&OK~w7l&fv#*@(k_34l|uM4PQinKDy*~|?Xn3csQzHCHvp3D{zvt;E*G~Wu` zQ2Wq$Hs^?tS#UAJ7tzNA90?^07Z@g9(2my;+g{eD7><Q{x5nGFR^6+?D6eXkl$0E- zkZ>tbFMMO*uiwp%fr{Kp@%Fdr;&h!$+s<U~NTFt+Sa}t}f^m3Br7ZEguZzF<N7}0O z_|Omi^)QNIxEuW&N7Ej2zG{UU@2UJR!wSQ~Qat&w3a~n3P3h~mKNQE?I*}OKTuf$5 zU_Yvc)i04O9UFpQ5+7Jb<nv)x@0h(+k*LQjDusJBTsOh=g8VTwiVtY&HRNi(IMHht zs)dYC;zWHdU@X#w7*YH*wjpGlMI<%gmYbUrx=}1sR8TM#---&?wtt~c7@cCNIVvvT zEPA&gy!x(|I89MO2}8J+voh?M>KCnV{iF3&=^pc+)9s7&CMJ`ZY}-f8auBh5Z(adq z=H;<-{)Y;IjnR?9oEx!+xER}Ldh3SDr6us;cvt@L0(z<P=WwO1S~AYcmo5cM*%S?B zPZd?2wtQ;E93i%aLW+-xs)hr_gJz`i!|q8G$3N7=f}t7wB%?U|oTH(4t&-C1Ml{XP z_+OXt`Eep<RP0pu$ElW*S-0C};5|0%!OXj+l?fY@7<)9bG@sh2(c;Jdxd>%Pyqu`A zIz3YjgWX4cBFB<1*$j`F<zYOj<JOtkydrxIL&~Z5o+~S39h=!bAP!=J3DA;@n-qV= z$qlb?q`<Co#t^X6!TZk}6oS?nsaB1D+SN{!u}0&6lJ^?hWFSVxWXbfTJ_YxUclW3= z+l=IsaeG-JF}M^e&YLgZF>Y}S@8|h>@(PS!OTx14d@7jjD`U~nN^&t9J9vxspSk+s zk&T^YIFb^-p$~Mcd=v<QqFe5XD;U!6g@IkE%d(kpqy&qL>duQ<LjSfDanl3OchU^u zkNxRp(se}<3r`xJB(t)G`VOW(%Tj>2y@vqhD9XWu=pY-O+{DFDNkp{HgmBS}x@U_! zE%z7Cd*N$5szVImN8Acg2gXQ#e!f6k!q-q)gEFxs<I3HVY=)%X6|y-L7|`k44>2); z3i;v*W;8Nl31+4;VzD$d+G4TH>rwszGv2<ImS5t=C$B4lk1A<jw%Rre1*OKsw6+G{ zu5(_-3JeV7v>twEJyH<wDN=G<LYdcf?SsCKk`gWvlLieh?<x0D!RhC`yb6yUgV@+u z3{1={zpL}%BJBfr54nf6%ON2lpN7`^*G*|;sxeAz*(#zk?LsxGv9`CSF&&<!_WJDq zrf1eF3rI^NZ)kMintb+p-?S@{%|F#^m7=b$?r>{r%*?>mm9tOyg881v<?dZOY7UOL zj~@w5wew`-8F_{isL#&ME>Bu+%znqxCdtz^G_Lffa##)0SPy@3QmuJ{JN=&b!f~fe zw4B7yCq}!{JSHwqeCgNw^|4YayQyl4{Gq*zotA?(0>ux>$*&t4*16ORHE8+yQ_$}| zz`cLJv)T9J{xf6Lo#u<}yHyRY>ni-NYtQ>Kgd9{9u}OuzIOlv%DCpwC*48YquXg+o zc7NVm+Susco@sauGn0{(tzVxa!njuU%u1ituElA&R#GX_sS3YK#*Oq8rh0y|Q@TvS z>lkng1>N!b@^$n1KPChJt0Vl4vG-3+J8@wUs9MY&$s88@XRDc&My<&IhQIzKP!x1Z z^|tH_BcRu6@e?w_n}xY4cMH7Ur->FmXTZe7^f~C|X*^p=ttm`J7R8*u#Kvh~j>E zb28(B)YjHE9>(%Kj_&2!WEGRmXwj>!$tr~)GAgRRe<Ou;jtj`5qN3kyCt|O!E(q@5 z{}O)&3#46Pih1kStveL_AM0FJolkdPAtNKN4SyxLIzNKQlBXkxqgRwvS2s2`ww!Bj z8vV6Cl&{_B`3x!e_Psoa9)q3C(o(k4#(k_rR(<2M{bl)7?)M#3{^aQj`SCLH@_+L) z{TBPP#9;>uG>T<!uCMBDt}kxYx^F!&^gSYyil#(k-i|n#^Uv(;lzwxUEU(Jta7|gG zMDO=k)%@LqgB*yRI}bSDRhV^qZ7xkzT2!4-2)KXijtCFW7JrM=_Uo7P<*8Fc<F{{| ze&_4-IOM#~&CTh!tcO22%r$e`Pmh;O7im{Y%gScIOyzdp+L^A4W6`a~B`5D$8_2me zUTsI_<m9AQYLNcxJsF3^AM%QdiX;w;s1+sw$9W_sjiUL@iOSRS^ZohYd;5Rm6jch; zo-}Nw^qP;B8umdzLLg+T<jNlY8;M9tx*y#k{*VEupul>JZn`zdhJpsgb1mzwUIMA0 z=hG^iG06n828z_DuY%};G`Q_1+k()rwZ2*6#WSiS&o8z|^kj-qr7LtG_R4cFTf^^Q zr5C2hZu0BSp_h<U6#1)@?HO_b_Z|gD&0?MQ%d-Pa9Gn%Ao3lQ*t&RECVC`lfzTdxp zZ;qFXUkrA1bR0~(^u?s7+iCg@WJ^R+iDWzyfAb+Fr3=1TC73hN2|vGTmS)h^R$BC} z<U|XLDPtv$OgFyfI9TnoXbnPpAtlwYmAdu)Z?wpDXImIS%(9r`FvR16+Uj_@$sJPm zkhZoLEOs4H6wC&VNzY7v?f$@Kvs~{^WLrI4A38ldo50+Tp_WP#@fXI!!%LZm)JZ%$ zSXFKIc|ABZ^z8l}_o;L)8)DUbg(|s=qcKB7BmxiwyU)5fJ0nuFyu94qERjX`*E>Aw z>t~8`LH?8;n~PkUC3*vkQ`6e-2L=WrA8>`kJ*<r{u1@AEJ@<@hWELTxEJg~{eU2wA zK0Z-><Ffkane{M3el1_+guJIGFZ^z{@FUc9eGn}G_Qaa?{cMvrN&=g&wRM4p$Hr)J zz;fRF>Z*tSg-)q~P;Y^d+do31Fk8=~5zXI!{?O6Uy(!kM!6zY+Dm7?QQjdS4n)e|g zL9)tvM61&LZlBNzITAoCgxnia4s#@8rdh`7pKtHc3kzqsZHy{6xZ629JA3Xlod_Nm zE2Q&J-gSV@Ib0hEmWUvppg_@ec5xx)bEXLk3j?T)gN&^3IdeWg+PMAwo7K=bV3r#N zJrm#n>h0SH!>WqU07__NjK@m!Q^s-bKMjay(%kRmnN#|zLd&XO-xh*N@a<eRPyP`% zcS2d&Q{l6}1fxaTId!t;y(#U00uEpNkpl&`wXk`Fg!H|saB=6|njpnc`tud3dKYcS z%O1hb37-9pBj<O0cY5lnQ)O+Yxzo+o^4WR03+bEnNcf+00hi-VZH*!=Q5eMfXfYW9 zy+WHMXNB8_1`JzDObi(U1f@2sl`&mG%F*%Z`ue(FgZtewjIoxRD+eUX*MIT0@;J<U z?)#lBlSE2VcSKX=15gi_8fL-`i;LkRHg-(mKbT4zgJ@*#PzjSa87KmQ-RE~@FzJX| z?u?-ZXgTycN&Ep(m(1(*xI2Mm$^Yhp%Aw_o8_=Eg*|TSn_n!s9l&#EoY{$Pt491ux z!_rz4I|EkodTf(>A8#%cmNr*-%QCR$%ElY_rf|7$e}D@R2LMn%1=3;BFC9DR;pp21 z$c&4N3m&li3$Jb)U7qd<p0DK~A;#-_aa?ZFfs??ZD>;<+vb5p9*0!~(eJYt8f_4EU zX+R}XHLTjZcNoB_<CPZCar5YCXjVIB+IhM)_B|<FatH6<goX~3=nGWa&vXI60G(x; z&%Am2cCKuv@4VKb$`u6#g_O&hFg)9FcV5(eds;?ABO%?pbuVU;NLg9=g@gox2=(;H zN!Se9jGpd}8T#K!FiFkK+);K!K}JquH^TzlM38EcenZ0Fzoyj=bAOJvrbyXMF&vtX zh>iA|f|lu4@6%s(Qb(6r^ijflZzpW457bHC0z}i#9R#A>MzRobfmA_2juGBmZ7na$ zH<qVQKfsG(+hle#q_oQc<ksHBD5B}XTsfPa%8H7{0+S7nBBiV#RX-pJfX%@oX+<p) zZ#$MPDazL&h@2uLi)b}6g6%_c-5A}fZ%1Jkme>%U{6)9}$N~7`2x+@vPiC~Y*e;(T zIPsfAPtSU+q!*wm`ROZUfbpKBXTh)u&m3k2|BV*QwfJ3qGudF|p7c`aG3T(dvP$_r zbEk+k3SxI6_u}-l|FaCiR^=9#@rGPAEj9I0-jUqHvSk=E=#Q+ijc3X7Y5Z>CI28Q6 z@e|*y2vT_*0Kn-KGi+DI-OR4ETco3?M@!0n9iN|{TU(_Ic+kNLfZPFO|E%rO<aNY2 z`-iX0`^3IC+@b$d;OLk|rf))I<ehrgbwo6a0Y?4q>G=$>U{FIrMOAvn1{f~`1o=d_ znpwBnc40ZukhS6((v-`|XFl*YQUuW~#0BJ_1f&|D%gP;*n@bL$>7hTO_Vy*Jy47OD z`1p%Q!k)n`Eh3UR16IzF)T6gVWMUE6r9aEiR40rYw6Opa3<yA){ATqN0{quxl}(lC zBVA%P19@9pX19OCqOmjunx#?E(f1^CVgS0PLLbu7(mI{(n|<F;(kIl=4B9Fxh6^C3 z0c95=o<OL*kd&M}dm0fjVLOUmyWOxO{Bn44X^AT^1>S!3>Qw-(dkwGcY$Ive9lxgI zNgJJddXgX(1Qf&g0qImL*NRV1Pn7^4|D^HdXeU7=+fLWUEOkT+oV~e6C45H<U9^~o z-QV9IX@7q|yvi_Y_=|Ed49{h;4YT2Jcvy|4akp+YW5~SiYk@3;=2nqm;Jk)QhvX9^ z9j&2-pKl>7!&kgj2}1!B6tFi)pBf<<o$c#L4O{eO6o<Y}?lp(4jr?u;n1SIPpqhZ) zMxvqL!0gUkiy=V)h>(f+N=YUf8XBbj9I1N^PoFqEOYKF#BIs$C`R}((wO4?G#`f)s zkB0IUZFWdlbUruL9NFnEMchRXPIt4iqkr;5rg9Q~Ss8P+?T7(7wDC<6_T`t%0fpyt z`qck&@3s-1Ynj$XEKN*O(w7<O@AxZKAWV_$K4D=wSPFP-4^RH-?Chu0^^}7MxFu&A z=Jt?aKGmUlp|Vfd1kwT(6?GScign3*P>``{X+`l>3VG!jx`FfN)V}Ojsw4J$4<tmL z^m=~TvPxGn>zQyf`JA?1NYwZdoSrU4C-b?`8ND86GHe#qEHxPMDFhuoVr+Gb)v&pT zI$BUnRu&7S;<W$Gm0qK#@&&%{;eeFRyFo-4ar6o1*cIB()FXzln(1%Mo{Vwto&`u4 z_iIxi-UwnAX&ISYM$29CDeC{CAWeGJjkJ-(-ws#~hp;|;7yvauzrwqU%F6Y@Tx=k? z(~FBC$WH`|0bp0LgG?A1wdD2OeHblteBa2aW4Ks1(V_Y54aC4uje{X%IB?Z)iGC`? z%v7jb`>$Wt&kCZx6m)@D(XO^7HF82zE7l<jZv76T8;JYE2V78@Us+5!%(@7hug`71 z<9F!RyIMt&^92o9(l9cHXZl}jM+j;+xEB-0s%MFzF=`Zb?k#n2jXi(y0=2<oXY|jE zrlD`H_CWRhJLr_Yhsp{Wf?*(rp$4D<KE<xA6w+w9xfTGpZMWqnAS7&q2vKx*=LSX; zS68Qdneut24GlS9&*A0s{J4R2*MxQ(8cgl{O2bCU*bn5yWw*Ww*8XJC3m7x-eh;!+ zwNQhFot>T6X%Q7z^dpOI!a$A`2Y{V`h+ca*5n__Sb$^$a%b{k&$H)JooP!RMcDQYH z#2vAPfZI=#<$)l|9T&s|%F!Z>bc&lHg^8b3M<w-|?;jjs-noOA_-clC)X~P+R2Yj^ zHr>jfbh%*kyPdtgW~+bFKY#wb`$u3$soYrNlcOo+C#R)fmvc8)R7jxy2Sf6db0q(x zyO=biK7Xch-y9bO$_D{5@Xsq;Af6@+s+e~$lc7(dDC`^@d|rnP`HGo<s{HPoo%_q( zTN-!C1l*YsG?XisxB`T!BXV)L{!>xp%9Lbf3=&WB{ew`0($%hbrp>9E=oBs+b|?pp zDaHVh{7x5cdma4?&1OKn1@WSsD@!DNvB}ZU9t<eg5lbs)Is650Vq#)*wkb77DjIY< zk?_@_>Z9r)-Cq<A3zNXfV#dagA3b_x&%&-dI5=1fAfA<#g#fpg<tX0R9&^pUpu7-@ z=koGWP+`D#t7=TE+!&2oD(c%o#4ICR0m3{BhT$IUSt9-NB`qIcGNXFIcpWXF!~)Fl z{(DJ4*KJqkMp_}E^g_+j$1?cp1!@C6WhF|4q1Ys>Mw}16s(p>^@cegq>GLO3L{UlU zLpMg;;)l4npY0JOm>3wJfWIhDxZuJzV<mXR#O@uX{PeZH)6Cc>Q52jm5|BXY%nFJj zp`lj6FK6eMmQ?=+Od_f_(5FgkeN7`TUcA7?!&^|F*xhyf@$+Z;pqch4o%=Kf%rxZ9 z8!vT@1hdjH=HsKIT#aHKo<Xkn-@iWxqTN!TU^nZU9^T~){7&G}D7T=nu)FY6aIcNv zaC0IKM$7N9y{Y~!7`zazGb!!bnm9mZyWKpip*(kCB`_>@-+1ddPF33#7E5cWJ@f;j zuNrWBH32s3#zF?MUb@2SQPE=&5s?R6)~*rWVD)aBhuqTyCq)+HQ7sr;N@5tTMBuLB zi(!Dzi2D&2;a3HI3c*E~L8!DLxO#9ChDRauSg24NKSKSVzPdIvG_*C^td2d}R8{oe z=FC=ukfqO-+eTy^oJQ0W<mBY!&I&gny>6Wv{*B3+3b!lWiz-LEv7c4VV{hNQaTdb< zfbg}N#H6J=oAw9<5QbaL!~gjn{~yEqzxwL7XKQa?A3ccq_hdZI*OzCy%}pZLrzklC zp4~Mr^f4bkJooVt+}qpRoUDof9sVIRvvipr$a}5-QNGv3$qu|m3xImw+HP~MB@@DD zB3|hA{^zL!kCh(M>+1`LhHt69vp1IqiV(<1r|bEdr-ws|>9Zv5P)UqK34Zrz2?X~Y zGVVJFV+R=C;Y$-)>v7UFMuYp-Vi<kKC(z`P<b3#Xztv1Brk3X44Op7ae-G4a_BnfV z;hyg#g}n`gX*E_t0SRgbN)Vx*<<t3(671O6*rEYWoQ^jqbgSRT(mZ<>1C?iOr*kT| z-Qt8rrWr^UpUb@txO{B2-PHb+L(8p%)PATu)Be4OI#9pC{nk_8b5{f3BlWzOslU8; znxx+1P}~>sYt(Q*KU`<l`PR973>empKyBY<qSdPiwq~P?<1Hk;TE{?GFWx4$aKHuU zqkki0|D?RU_~AX2`O=o_#jTpT=B#5Rge?KuAVR<IILXtym^T)Z5^`X@KMN((_n361 z-c7u8j6}Qdn3p%30SlJohtX5Q1`!W_3996&7xwn7fD?be=4Tj#ctO&ucMU7iZy2n$ z(*;fEHdM2|y4tbRa^p8N)uc~DOS=a(@xi1`DF8dvF3kG%Y8Q<F+y;JUW-pbRvOu;( zlJVe_Y4vVR)#M}04!g<W4Q~69*Ei=IrKN?l#h)SbY$w01znPjSdxZk5#C%*{E=1Q} zr(ugwj{t;-jg*PC94~uS{%_7^x}4Y^G$49#Ajl;fE3hRNAp%x@gklFQZv#SWQF_kb z(y6iU9sa5c;s?nG=wCjKHwG-O{<2r%89w_mKr@jOwt3zs+u&SSZ-KYgfY>r%7x0gG z;igC^w{c8o8($0VcT#g&_6OimOMXa9y#3tSnH^{{7x+W)c-+KixAit$dB92pVUv6k z4Mx}jEIQwim^DUJ6@?^#kJe_J1b~Cl=K<QkIm~6SwOqzR<uzEK&IF8AShZ~BPf=3| z>gmlL((J7`0h=EbP#Pd)em&JHH8nLR7S>}ft}3mgM!p@5q7Ehj0kSf!U!X$Dj-|=S z$hzS+Xy38f8o!4VF@=vL!p8L%YO(<|wE?Ymc6ANzpC518cyOdZ#p$xrb8FhrI|{)& z22Ea8nTlz=qW|6LdVRX^_6&Bis?n@FVF32}0k>`E_dt}YgTZ259%B;|P$M{Ojl0N5 zu<j84d*4y+;Sv!1gpS0j^G#7l+f3DdJBeXGG#noA*!RHAr4<!x3R_%O6^acY!p5O8 z0hML)BLq_?8NeVI5mH~2sWbe}txELj7Ax4mkf8_H0%B<IH+|;#ayk?>h%EySM$m4( z+lF5Cp5i=|1t1obn!Mb6e0(+sJ0eNNVG6e(eFPEC9;5=ZVRPE}xK{J&A~uqWipqGs z8`DnX0S-dqw6?Z3UY~EQx$+~X2M*RPuz<l2m94}91^4;Pu~e__2mCo%$eSCH&V3>` ze?eZ~nlG-I6;mekyE@VozT6eBww=g6<FEr4>k%t!)NLxC4`7SD;!b_Nl5Al)Rn2l6 zhkPCu<1bh|^c)-nl@mRn4S#ocn}F{tpTZdimSF|t#p9DPCjsB{!&fUk$uJ~t`)O(T zK!n@nJxalM3i&x=p;!iE2tO3a2=UcON}&|cLHGm&szqB~F0;j;t%wZ`4PTzZS37|^ ztNIX=1h)vCIl8AhNShEU9>|@@C<+0D6SOniw1s*Dn&j`wigl){iVCNwv&R-+Y-(yD zrW0g6$QMpn^+nirgb$Z!;1vd&LI6?rStd>k%v6{cg4G*OX1xO>z93Qz{Jdmty9Y+k z9wJt{+~hXw@zOz`$e8`VYOUz7LK|9O0niH-)+3D9r%U(1QL8#G=5&}%gPRe;*b$S? zS4i*d>7n7{BMm^fV9tM~XZ<ehj<;vzigl{=bo%KP(u}5Soe(*U@I(m-O%rc^kva53 z4nfu11_0X&UXRbulJu!|3Z`20;MVo;Q?cpkV2{{%f)R~*?_NHv<>ji#%`2$Qb&}sw zdc6a~hJcdt4^$-xsDujl?|>AQ84Mq!4?ugcL1{i@0&5M2aT^9S1QbEJhDAlKfO!Kx z4!6TB6!9hus(E(+gexo6$YD!y02ydOmd-SI#DRYas|e;H*kNzLMrmMcgW5;V(vksz zg1~4{H4YSBRn0^hly=O`Q3Zf@aph0%Sn^#Tq@DF#x{LTH8F{}IV4IZ3o*V*~a$~|7 zibw#DU$BLIP+>svImwpjC8xYP69Wr@OG0w{J6JIY7Z*$b^oL;P!Ji3jE;PU!k%A3o zYdXXS)d-$qC`iB8`g1c?t5INv5K6oPYz6yx5DIrNnBTS<eS&(wt)qhg53dzs7YtZR zdc_P>tiEC>xUa9TA%gZ{n>l<=9igI$iHQ+B>=&PW<`i0JBk><M9^w6=p@AE62WIg~ zQ!#(&b>lCD{Z^<^Y}x<m7W7sSb~)G))`~P5qb2%skh$yN=YzU)SL87|8<_&D99(X2 z^MxQ4A9Hekd&l8u;oNcVNd5S+w49tvs{hs`7u+38^!`Cms|rx}@$vB~EElSMMg9(5 zPOFIuT4*Hzuv4|Qw?h$f50?1Y(>|iuNQ><U2!BF}EC^?Vtq40ZaG_|Ru>&YD9`96B z3fA6oPcnzWmV;A!Z^EZf4=H_iMQLPU6dyfiU%v%~Oj7tBj8Q9LJ*XP{GBz{DCXtXR zNf2iNgb)rfv>lG?-)H}6poR^%SZoWsgrW+HMGn_BB})HGC&_5aUa&B^codvuq@_O@ zw}(s2eraqyo^<Jek$=@H&z|8U=XHE&MA}2aVSWcJmn=}ZpL27w7{a^V0Rn#a^?Bg4 zbZiteAS@EVThFVbF&Rb0d}viP?sqY@TQ7lEkC=Cziv<hI<HtXq1kpZvBq|_TlEm-E zh;Um0Z2?vHFAw^reD7V$OqU0Nxu>s?Dg4sQtA1t%eAQQAu5(!SKLooP_zPOkcgqA< z&`?n=J@j>p=<DQ@*`;G&0IX=&*i!(@nS*?3|Cd_z9@h59k00UcHMSEK|FK*?6oVzf zn@dYar;;a6`k0zJqN=5xmYF#cnss$OO#1cT;@C8l2>~9S!0T>){`9CXUl{0HY?KMz zW+_5HLs0-1C^P+y4}K2hGSRzt?~H(-!P~_$G@LWGo0EO<BIMX|6H7ZmS$(9i+O|~4 zBU=|deEP&;z#j~3Y!gs%G<0+oc2nG@+UDSVHbR5P{(?mYAKE9olMfvwKuLkkvy^O+ zfhd6*k4AL?1`S+Zy%wLSFaysx6$%`=MAnI;iQ`HZ^GZ^1dCdQ$h5a@)27pG$J=i&$ z?QDO1k0QB8(7}T?(LNNRmdzCq1C?O8^1J<e@%%ZGe!c4g0M-j-LNJmm&3m(f3py*z zY5?&HwfznKued-bJ{A-t4-ob}FY`LGR#RsgW#A8h)B~r(4A6-oTnyR(Ge~dg5pfAl z69S!$_GVrtv$wNS4nBx-gI*rD?f7#@s-{UpDX0RWY>SMHG~UXj*0|ZJne)RAC@APj z<BRL>S4zwoG_M58JKhB-9cX-gU~E_T<zV|AKp-@unDuHu0AOMS0AsU5=V=mLj3PUM zV`>qROhCw63AZ>mOa!$u1dE!FpfD_hqQ2ogqeMuP(`GcXr$-(T{!7&vn6!?2ixPLx zQ6TdHV&MgJ{>r@{92^`BWndwrpdiF{rq4bmEUdb^cA3#FC<IPwTcXCgV6@#I9*KAM z_2omJfE@dzCF~gRmdY;)s(+Wiop?wH+u-l-xO96pSld@nvFUiu0#kl#Y-|*SMQ;Ob z91g1o((L=#SmNfUZKR}0%!OONQwJ97cd&sJK-WR7LI;}$jufJm1ua9g@4)QOq0sJ4 z<v~=XYe!xU<BC*NRP`P^3P2PS(3FEBWpqC~JNsRD`2L#rQ5%@;eV|HTgAa;OegL)x zLW8%V2{NK-XhjUImtKS{x?nHSt%(B7*bmwY^jRUMl=8)fG*HP`!xmw?$!|_p%HDEc zRdP>3BA;Df>{x*z05M?<cLh{>3uI9|oyO<#MZ1y?aC#YZ+bX!Qqc8fo=gvteD7Kuv zCDJ{lKsHH3&~s4%-{=Dujcv7{ML5396i%0kqZ_DoG6ATU0(-aW5;RUSNcs%uB4>F0 zdjk2n_0PNhSPu^${|DUgY{&mbPG0^FXqAO@kJ&5`k9Wv<aiPNjopA`s_PM!CaOhc# z^WaX1Zn|hN`Yl9z8Ok%u*+xF-fB~3fN!yI=>AZ%17bZ4#4ulRgg3H{ubbduq=oXob zgNsHdpY$CRg%wCBA+N(v;BPC;#UUD=APKxqwqHUU<XuF>P=-8&3qp1{H144z_!JcL z;m%wpI8_Kwj~xQUU#+p^@b{sjy1M$nH6dF2ab-bvQZJen7DQL2_QKD5dLo0<Sm5m_ z6Sf!U2-4HT$%0LEPXy}*1qCI6PyhAnSA|h9X<XO(0|BU^3`{pZdBVd(0$C513&!2M zh-J?Q2t@R`4QgT%65eEHQi1Up(4O00?`DHTDfs36pzcBVq@jlv%Web-8=^7GzkKih z)9<X2`3%5q|7Gu$Zs@T|q@Tn-S62tyOz*hWmOCycX2I2t$Zd;A!rGb<$X_Am0SxFP zqk31aAs<M}bOj3!l%C(eTP-=gd@qZ}WWi10Qts*M`eDgQ>`i5>tR4>=0vIf)5Dt0M zVah(S6>K+G#44!E6O+wgpSf#Kw=|a$pg^B9wElsHR6F*A8$XTA;5FiOhFJsW^X_1Q z@wYZ|%7*ru^qs^V5zEkqF4C)u?d<H_z}F2xv=hOFf^!In5{-=8mcczOpZ=zlDXU6l z_<HhjH&`$qvtV&1b7tNvrF{t{%cXhI#84FcuB3n6RRU|^2qV6MKV#C_tti|j{rN(b zad$88Tr3TEqspdypbKk+J+u>^XqKdPANu3pV6-+-rf}Q+JN^p|m?-Gy3PbU~Veb!3 zWGI*4fddM*c~=>*4VnAmZB*3$1`h}NZpq!gA4662wKZo_wS<UcOYY7?dRX8|x-S+G zV-!mGuw?dzXD@@LUh^yfVU+O1CMSn6>!OW4&f_nv?a~OWBGJ`t{U6Q>F~ZH6W`YwX z(}dV3nafVZh$B=_|6kml|110Ce{T#;SBaFHNT{pJn@cdK_(gm$BW+SR{znpV3&)Ui ze|^Q)PasCIM46>N=(!`x>4o%VVKrN+D`vLx^3=_cx<}P~L4j=ZfNMj8lV%PB)>(K^ zX>_ynfRN9CbS9f>Xg`*0q+U+fubkJH?vFF$*+S;r2bv1?vFP@?32EQ(bes?MPKeo} z)r7)_@>44vJb1S4TZ!yLVt=NZTup^t;;)50pHC53IBzIK({!|WRHWOw^;e<x`hP!_ z|I;dX6a4OKxOzojohHM6Frk6cQIh1~p(#tAen-@A_9aiVCdOg>?ns1*vp`*8{&51T zIhyc<v5}JxD`_2Ls~>(7*#&#G&BNB3Gi-l4nT5(?bjHx`;nGhH#lBCTJPtAwULz2z zLBC#ebL0|#HCj61Ly)Sv=u0~B=-q&R5Yp4+1>=I`_}k9OSKi!oY&(I0&G=l8U1>}i zXe{*SUStIJqIlN1;Ht}RD5<{_N$(5E<lys@koRV@3JLWNZmeTZ)t<U5=-R?IbaDH@ z6kEh^Jgy@tx5od$Lera41LKKzj+FIt(S0@M2aB<;#~EDg<sKM)?WhT!{ywh6t(Sf^ zfo#=gF6Sv>dqP2Hue5l**jirSWn7;)v?isrS@hTUT^s0(j=o4-l#fluix_;pu`0jK zl)sw4`V;dmuHj9rq*g7@_qnki=BY%fonOs9!(+z34}T>_>c#JPtU3uxWLaC0s)ugT zN0K*NW~&9X{XFvhdf&WmT#+ti!v9a3_>u3~(#M#CC+0fCy9=%VLi2wWeSed4mL|)o z`baI$g4z(-7rS`XCK=1-riXqEqt|B5O6O=+uoxxfjDYCt>QOE!^Mo1F{3=SFt#|RA zpHsK3HA<L8-`4wa78TJy6MwZdAlU15XD@N>C_tp4?|v9#evUnwkQ7_$xGQHg5>=S+ z%d5H%#&pM@%~CAiV^9=tADEqxQs?9A>qgq(ELU7a;ssdrJ$PN?ff1~GoBiaToIkQJ ztrzEhzO@xa;$TmaaFU!tvuygl7ysA^$|ox>Dt?0llPSa3NSsA~f42NPMaK1D%tTpz z&suY(<Qt(Bef=yw#I;-4eEEsIR}mq_zw`ZrKs}_cw_S&pYl@#r+nNs~-P0J;(5KiY zGe0wjZrxOV9x42kFQZO1dX+4t6D5{OcTHo#^f&dBwi}hXZTDzY@6WrJ{oVKi*MBT- z>V62HUq~PNe2?y0;J@~0Z-dlg@JNu_PWYs#s3@hb{#K-Yzd%bkl}MtSBemD~eGsy- zq7Lh)?Yll(gPAv)NOm@7^O~Q$``G>%cklU7ekoe@2~Yk?cK4E#IRriaaPxAiv(3;s z=_vJ9FkXeTdY&Ek)0=hNYxSwy5}`M#iN8xYBn#q4o%m*YBJI#Mns8;;ABKsOyu6c= zi22+ypiPR5Xm*ph)8*Ni1>RNMnI=U9@0Bxyrktvp=W@p##?bZt4Q8IwiQu1Wqu--l zew%m2$C6I5B>DF#EU%dn#N@il`!*8FXk#_=W6D&#+qKZGII|QdW^=`O{9W_$`IXHV zMnN)y&LKNws~hDCP2Wp*DIAjqv;%1q|CQ?)@%4wvTx{`qw(ok%yXhWijB*s1;LZrY z%oC**YN9Ml>qs~`i;d5(*eVswup#MelM>^7JrWYFu-LoZ8mSxeJBZ?seVm*X@mza< zb)Zk<vsA&ju8O8Q<|$pzNQhpuh00e*ur-YXLP_lncP=fg(2=MPpR_l-NWOFw7&%yi z%aGT9kk^U&ArKw%PzINhA?FJtN442da}?Vg4kk6dAuW^QGJMLXmOCQ3#hiecq0-ot zm#h09Z>DjcWJrcx$H9F;w&L3S#x^i6UL=^g@>9gpJd(lZpQvE|N+XA6RgqwEg|a!j zAucYy)vlqeM2@QUEERi@5Xx2aax4<0u;hd#CE?-gfg;&j`FO_&3`1Gf&OfTJYcz2c zs}7f}--s3dwz@2%kG{u9q?3J>eRY52Q2>XGIlpi0pL6OnK=*rvI4gMmlb!w-sCzFs zM4}9z4IM<d5^wgHxvnDpD(<1{R5YHE%L+l0Zah~d9;&(Tc`4cwD}d{P5%Y+h?~F0I z?l^C-^*|DbSStXKbEr6wDsIf|nTam<vbY&{6zk8Aez|LBufsJ~cO|G5`{aIV(wF;~ z^8UOb>KtoXF57?3n$v)~_ne3dCxd6Ou9>QXHI;Y$SG`-?zwTn4UyG{#W%c(+RC*dO zefhorJXjxU%Vf*AlWU?clCaagEEb*|&-T%LrCLnHwenvxb2!gacWv|v)Rtz0H(Aea zv7()H8&+Bw{Cpcj5#(85_OONgkC{wkU@!?*ac?x`y+_oV3k&N{Xe^#YZawIcOApq% zv7l@r)*6<;TVs0mcZw5-=-b6yhpdxuyZnhFEUf0)`OS~}Q(Q@P<$RIb_%GBxXUrOY zF1tyK@vp%!`nQz3rd87))G`FeM@921u}UJDWjeVRnPl)QXdeBpkZX;{p2epBbN3Um zPpQz|_v%=2+Pm@?@juMWJ60+zH7WT?OUx!B#JV3OxbfS(Slb;DS81i*&ZwlL!KAb? z#_RVYa%HA42se1|Bh#bC>(?${HONjD{r6xgL-5j<#`!%Xed(-s3}&pFr^56=^Zc{R zDP$*EX-zF}-o#((J{Ws!f^kx;DNmV<o)%nw0|iPIHW|aOu~}2gSkJwiV}K?U|I@9A zKWU@!(Y-I$3tW}!mXzWxFLSI(DcIa_vi0rcZn{EiZ>(`HSl{n>klml;{xC*~^Gk|e znYHcclcjq`*OFoG{$V4hvEs@Az+Dwv;!2a0IMNsLk{k^_AsZZ`di52;T7E#6F|*VX zUayI^1e4c#-D>ZcX;q6Oqul&gV&e0&3!W~l-pNQx52w*){tJ*<zkGDrAQ9j`O{QLV zq5sO`hH_TyaS)4Pc$@r%!)uj<9}_g87hUTZyMcjorq2Ao<!jKJKjQ4OI=3>CucH%D zQJhYuJt>SnR?C)91C~mz$=Q{oBKk_*=X#H~ja<4*#ZpE6ai`+rDY_S>=Gqi^t?U<l zq?m{CPm{|lgN9txXuj_H@W&?G;AQtePZqLxPhM~IdE?6OFQKi8+cQ*6Hu3mD{_8}e z!SiZsH7j-(^=isK@q~zG;aPvqko+msit9eXKeW2`of*~nU&_w3C{v@C@rW5XOpKnq zslIEvgctD~`SV{w;i^0`<{fImcg^%akXeo@Y@_x@?t~VS5+}ZR*Ro75WW({{tOb?e zBq(16WBYD@<D2DOJH8CXj`~oh-~MMdtk~`=6?^eYfASf~>n#_TI)XWw+zirA4Eq@u z*w5&?(a5yBoFB1`C(9kN^XrTbQ_I06?s42u6pcJ%O5h!~^RJ_nuG0+@Nmoof_Y+i` zi;lmI6}`79`s(A|7^}-|E(QAJfsIdIAyJhw31k}SSYG=t88O$f;(w&GwCgN0ub!zC z<X(1N)yZ#C{7@}M9{2yl-oMVGScOH?uW3A!nIN@#V?!zph+Q%iSNqXC5192_qT!_F z?WvsL5Z#B&;|HEcQW4QreiBsKg>}Ro!35p0vBmxN;q4u12|2q<vwHla2RS;V6yALg zl-1rbYBHd?J!aYIqt`uV>NBld_wEiQ<R}PV;lAbhDf}ML{wt@(z)Bh7#4hnn8WTob z?R48Z`yKa?`B|j9OzZ1P9dD^lR`53I2O8gGwBYwJPo&GG>IlDxABs^k61Ba_jKRuc zFTA%L=YSkHA%J~Yp6A!|zT50a0eMEmp2eyab*BBZr(YFu{k+2b2K<<35<7af&{Nb; zUUke#j#B)#SbnEc<2<$_ov_Kit0#{ecb<?fU)B>!Sb4lAVkS;pfF-iYy?fb$e0I90 zB0u%<WpFqMjsuf*0~GYsBb5st?D=^#YGJRieBIm5j02ywa{;E=vorxLTzjsI*G^|~ z+^)W;8Ym7G2p@Vs4G<@Z?0K@=vGP~9dLsX;2!N>P*Gk$+3C&OHr08;f5f?raE~A}> zm~@fm@wW?3`|}dm?lNsy>2@dj@Qe+eG~9{Y#BHNTk=^Ug61+UxRmp!SkG08XLeuwo ze~A($TIzP$Nl{E(7#)zke6x(%x<SOn-yIve6pe!{DQXdf@;rH=(A<^sv+mGpv2UB> zjOH_w&#ooLt!8`|zrKZrbUq*9ryu*!-|Wxepm25l^UQU@6>omIG``#*z^w%V&2D1! zbf@_DtJXhyE0?drnl?u|qxXcENz4i_eC|(oWrpt674fAhUQ7zBVZ`&r{hirrbpIT+ zeii)M<0TG?xy5{b;(g*J#hW0}@)~dS$1=O1`z0#_Rj%>bJaEde$r>~}pME(^kN+|3 zm+M*M@h;(b;0<&0xHYCaf1<7VaO@r7O@bd4cne8Ha%w-P3fZ?N`7@2pCj(SMZg=VR zDtR7t++neO@=W#7wwkbAj*qtQm$m%(M1Hha>_0=*I5?`!IZSN0&(K&iOa1%uBwkX@ zZc;|d1m=3P@cc<JV=?R{(0tjJ-j%P5BJx<=)T>u|;mC~ivPSWVpO<i2OKI}B3wNf* z$r-+O*Px~@U1{HAdP6hmspsgN*!@wh??SC6pU*WEa$w&3cvcooJ2@{~pS;>Ew<_OC z@cna@c8@QA<V5ff>2Bq5w_nm{N`1i<OB$y8PX1wv=Wny)7D`bG`g#<9e*V7iTlW#? z<Ljd=BTBo)=-+qbPDnXPCT1cL4#&twn9AvH@QH)RTm%hU%U{Z`2QNRQ<ly2l=eX>- zCdQQ-bLxuKlq7<3e{_w1E&Rt0mdbAecWjCc^U_D?=DA_+SV@AC;=^%8&~Af1AEP?) zKi(UOu~*u>Y^v&7Kid*9JN1#{R^>5QF8SC~CEvd8bdpFS7iXce&ck$+cgQ2C^vg(1 z|C%w7Msxx<`H)=wpm)P9fq(nB53laidp(n^=+PKl|FG{tCM)*|nB<5vx2xoHI|F5O zf%|09q-o4+XU*$W*ptymw$~RmoY$8+Hwt*Ji!pg<WksQ=!88;~i5?;1`MJov@d{U* zUf^y1Ot<OsSIV(!CAHeLm5FQ>wQs+XH9-00DysTJXutZKeV96^SvFlXlU2ZrneaPu z%osbt*Xf%d#qS<BoUfI|I|`gaF&is%#>-xlm-N)lS;&ER&*<V|L#6+?v!HHmoDVjX zi&R!ykO+{&Q$JAo8~r>ZtmMw#34gLtnN%TMi0VyKn~Heje?8#Q=7w4F*NQa@;TD)p zGxr`SI94UGZ)75Vx6wLOjHMA|HCnh;6J&!q-83783CA^Nq9weWNMsX(nDo$tbOQ@X zosmUloR~^QJW|$^wwY@^V#d2F?h`OSY2+Ji*EUptSUWMjztM-i)mTU*<KI9`RC{}G zMXP0yR$8krXey{Bx{V2~<&zK8$?!lRjgRl)kHMzU&Wgy==vgKnwVW>DBA-WTZ+A`z zelRiM!8TAf18=Z}41K9sZI71?YVflzrI9&Zb9??wLy35r{ek-Nt67d$H4S9Qt#(=C zd701&d5U-%N>zF~%?P;^b&Rp%u@^i{G~GS@g}7@}a+;Ivg$A66(Np3j?a|LT@E&uj z<*6}5cr2Pr@#eCzmGi`A1F~pDh^Lq`S|iGt92##`jIp9Ts({bTR}t3_YSeu}R&WW$ z9@TpKKyLf?1H>oYUvWNQd9X2`=$6|cL0XnF&rkgAYD8ugofcX#>Wv17lP@B;5u<fu z;+m7|IAg))tmfu?*PA=XTP>JOaL#6W1gz(3hnrr73q7pT)QKXUi6BjSY)x;LqQz+p zM>src?tgF>&b`6Hz_}hq#6j;U3X@}=ImCM&t%lq;>0HA^=g^u$Qd1u>*t#Jk%-yZb zW|%ZQzfo|!9CzwL%j`Ah2+szoI6C<}G$=E##=avw-1-9XsFnSlsYmqFg4=Qlf`#pj z-R!Z<)ZOg->3{YayF1Z#>~63&q@`6BETq^13N?MTlFSSmD*O2(O4MHvwMS0qF*Isq zRpk|i2W1mulo;dbMf}l`XfoPS&nknZA<hZQNHD@ge=Zh_-KJWBQKCRk-OVqs5N|hF zytt`g8-Z`YMp7DsG@>G-9s_=#ifDq8`d1?^Rw6Z0*H6;;v5Z*^jMB$Jy5!~Rp}%4B z;ol2K^fzDBC^?pwtC>waZmeLoN+_Z&ey&&UM&e0DXQMlqGp}P>-LGuBf%Cx>Q#(eP zmZ)kTClH55LM)b$P(mz0jgU+;RNa(ilBrD%ktWpyB(}^-+Khzy5%IdajM^ciDduw@ z-=paku}ZJmg$_@?D=0|$&c6%`la0h)@L8+J@%x?5f{>sf9H=sDADb3!hX)lG{BbYL zG1TOBK|>bZ+au2~6ObEq_X`}U4;Aay(3~HVsBoen{L5Fp!x|*2t35#1n;TjfR?Hln zZK+YB9#GW~%&d#5mXC5f5S|Cy9IZI|%ZmF89*kpP%~&C~JTAtT8#^xTk2i99G>#Kn z7yh^6P4H_06HSK=?p@0XVp~0B!rsI}9UjY<D6SQeDo@AJ6GS-}zG-6@Yg?%Q=ixnj z!m-I!{!MkpGTUNS)R!UUc&eLR{qeLL`y(ZatmwUZVJv8i|FrP5B{c}uWW;W{e$hEi z(lZ(<2w<Ezb!6EQ#4c3QheT=eEaNOjF;6(*whsuBpK@gDZ9+<6YW*F4^ss)rUl*J2 zLU~4Wlz9$0F^E}L?{<aWeOV2ylJ8wCmU7DK(RnRP++cZrVrcaJoM<O8_^ZCEdV#gE zJJl!g%89`QPY;V~*N$T=8=t{ds1t3$DC#VHR8rKpJvL|ZGA&8%;g4-xeu#n+lD_(l zho43@S+H6|ZA33zwqROAp}Y<2fh#JGX3LXdYzeQw%Zieh#97MXS6DkNJSNfl!A(FJ zekZRiyy^lU(%I<k*n)QK7NcQF&YfxP#sIlz?z174g{|SU|8lC%pV{8-QoB7@Jkp~3 zS9pOTX_UX9m!-4nvvtYR<C@2C=IyvPk^bcIT32*oZi5pG@2H1$w}eI7qN5#?KzJcB zorY2Cqys8p?o(5F7F@?Ecm1#`o0f9bSU_7no>`r&YO9L({$uKC#R4|9U$*{XJEx`P zm;7k6Eq|%1URI)DJ{4(VnM|J;T8N1UV^z{4rzhiGZYb9%Zq~73%0L~hF6qwq<5|?b zpBrHIyqlYDES&3l^sCdzsDzQPI@Z62Ipst=jOx5v1iG6FYS)K~qE+Z;FlnL?%AY%r hM7?~iId$bP_tt3RD{UzzJUfmgBcbr3M9e7YzX88-W_kbs literal 0 HcmV?d00001 diff --git a/docs/images/copy_button.png b/docs/images/copy_button.png deleted file mode 100644 index faecd8b8c9e30f9672cba3cf00a82efd0f9133f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2827 zcmV+m3-t7fP)<h;3K|Lk000e1NJLTq008g+001}$0ssI2OpM|j00009a7bBm000ie z000ie0hKEb8vp<by-7qtRCt{2oqKRoRUXH`H;>!gBu$%?SKIB{Hm(X`VP|aR(V~O4 zJOnB`ELBDnE7N5xD7ys#!N-ELAPUMNC=cDD3y#3h5uIV7l!t;8Y=>c2>Z>NKwzLX0 zY0@M&O>T1ckA%>srFpbT+UxoJ(Mj(8-Ftq2eD2A~`JIzeJkNtb5DKDrCqN+dCq!t1 zpddmM1O*YAASj5?1VKTBCI|{5G(k`hp$US52u%<aL}-Gbki<KIFyMLqdaKE1XE~0G zcP>IEfu@y8xkjB7??fVj8Z2u$9FC5r=ATkDYNbLhmG<-;avXmA5!u->Pi^4Rp#R}_ z+S|gtisLx5#maHq@L@xukyTU~G(!%XolViG)yb+JZ030!JP4f*4?YODJ095y3W-FL zqDht|NX=$*ymJYg0W7r1gezD4@7k@lM5UsKu#+b-ZXAk>@!fZzC^R<4E<-`1R4VOu zd%Uv<n}I5{*^H@E@$9qy1DuN!i^W}HE?vU3X(%W_Z7tH%v1=DDUq)tTj58G~VzJoi zbjCZ2uo;j-x3^>Zbkx@3&>_^;dXMn{McoLvas~6}<L<k0>==@guwerl8c<q_0OJY2 zJ_er9HXCNlKuwMB?Q_p{IeYy&mM+D}k=VBnG>zrUq1R*8Dlkm!GbIc_BA>R?>-E*u z)gcCFXJ;1{76y;A+c9ev&YZb<<irUaKaRpek==%M>#%t<EEb5xC@I0(waCbbD0hNj ziBM>kW#51QeU@cIjMwY+YPC8iC#Sa{hXb=`<J76Z<EvKT+ix@1u0?ArC<;YISic@R zT||i!1WtrPH#awXZ4Ra1)$8>JgTY`h^d@vi2j<Md@#8(5x_A*6FNy)=<>8fAke?q} z&V*kBeXSIRG>`xc27}RPyya=96UD_ia^#i)G~M#X*3_~^;U`G=buj?G6uMlPHxD0w z+*^#(DK_2uc_0vO5bw1d9EZmqLuF;qK+V>F;IB&|mq*aZ{#y2`$zx_c85;52943jO zf?q?3GAmcE3^-CyP~iW-Hy8{%ckT=b*tl_H(A&mUXtx_BCD^||lvv5NpRsKlR;&oC z1vASp+|yR0zgZyI{h9sUlH}28@qUsC(IJc>$BrF4R;*ZYyvOft5#aHlv=n>yhH=S; z4KSI)YUD2;n!Ps9&5<pfBu|u4O(yrkiZ(MFxFl<TE&F<v?VLXB+L*<`udcL=`P=o| zS6`d8)p(}CcLy`er!W8c#l!94-jL82;xvZ#c<|IysHh0<s<t+C`xxqlMx(o`z5~F) zr_@^>Nm}<%;)OLsy-pv!><9=xP|I$vvetYbw$NpJ+ul3Jj?b1Zn85sGanIjvs{6qe z?o9~&MLa%vc^=O^gI&9VN76J#jl!5Q7(4dr&=Kj=3NdtO=<PI`d@JuJYH9C_B!<$Z zNzNNwO~(FNl=q9(bLt!Cym8E?xhepEzE834lXgbhKVx*K8#Jlqhe^zZ=6RHtWBc}= z&yE=3)fZ#OV$2xabr+;k?-{%Php=o0pkjQVKaECrhQ_Cc*K+@<b9yHQ3P8Jq2he2l z)aZTv>jbSdXcKapqA2{Y)$QOQlVG??wEfX8>*7Tep8u%*P#r6kP(^=`y)ZK|K>`4u z{Kv85wEdeFmqI4GCrkSL%*4zTv6<yZy>PAcUb*4A`)otUcrE=wsTzQX$F`HUedpP0 z7WZvK#3gx(1rsAI^_yA#tLC1!M~q672;l2-Y=!1|tXzpLTYPV&Qhx{R`#Wq{KejeH zO_Ht_T{U?Y?rB>#MX6#$yU*CJnmhoejZW~!%zeAdp!`4`d!Wu&{=B+rc+e&onkt*D z*nZNMxBhx@E>oP#WDV)ksebqco1zdsI-WVFcUGLWr>aDY?^m!+e&?rlUAkmbv8vVT zSyO45zuQ#3+_#$kpJ(lvL&PO{a*e|G)U7&X`QQROeRM*GCgF=KuGN(mhl^iyUnCPg zNes2W)_$-qaKtq?N4DjWm~K%iC@2UxqSb2suhnXG_kd7{t<c-I<I*K8Uw*R#rs=3Q zxaE<gCo9_Oo4F_VwD}Ld{;<m5VfUTY4=R73E_wU3ZR-i!n<s4#=Q4l)qe^z8bC-%y zxmYt?C30}cefipnZ#ov=uV85E@DdGxUu>St6!G&j6H6-EzHD^eF+u`hh*ET9X^Jwz z*IDYXx%ZuC7u_c>$&&+kA`fMI%{xxpA`=>bSKQ7{$KGZHE6J(V>JV#o0mN45B}=em zNh}+jsHKmWr@VN$y}sFHVR>DeWZ6`ue@Kl^-SC{ZjOPum@<PS3smgF$nKf26Ypm?+ z7VfoU)(^j66$#Y3hZ4P~rs!nDRU!Z~39^SsTCE=M@!e<aZ&q8|SRR1)>1sB+y+TjE zGr?~&fb#|?fSsS(cYfMsxjpOT89KrdLT9JrVsrEm{&Z}HMqE*n9HpB(-8g#Laqk@o z$qY3zRr1b)<jJqKR@Ha-g;sP@lcE3~06zQ9`SK_2PfV1rn6Bi3_dd73den*=6Nw60 z51Lk%X~ty-AJd19Ut%HI3}Fl1XmlT|?>MJ-T3EjRs>|ZwCunH^6See|(Fy%hTlx2w zj<UVxvdM}yGn4>051d@^sfn*zH~{m;`)Zb)ySIJHIhj%b@1C{Yla=h<;_%jKTW%(u zo7q>t<Eyg4V4$cjk8W~uvRbY7zIHeqEiEkpmt<vS1--2>h4xlSnoNOnbKW9Ljo!Jr z%Bqr4rS~cpPLg}AT02{1VLch@@W$3N$H?Bxw!d}C_H~PE<Ou0!4IMuk-D_`j&$r_4 z-%0^&Jknm0uej9cdZXIbOYof|B*i(*fjah~ZN_m~^naUNXTNtYzF*OIq2o*U%68|{ zcXM;|AS``Hfq+`~s8q>>+i6{z#5+#)cGc(f&hrMB#ldf`vhJ^Cy_N9DI&85B|5~Wt zQf)hYiT%96nKe}M>Yr4Pjt?@YmYXHrFh{lRq_uR9>Golg@<Qe2Dr+zCn~GK0L&YCl zVBa}o`;A)s^6bPX^CSF5D0m_inz^y%srzHp9h1ofAT2E|vbMtFrNb7l&E<uP;#{Wt zY{s<F3CjR8%OAb$c<FGvw-SC~P2@j7p-7<C%usrV9GyvGsDO#TkCtfsuOFYlJU-#3 z_vKT38`^uED-?_H^aF~gALwb*JE1EP3Z0ggrqk)_>+AJ;eJJq^!%UhqDYCZ0qS5FE z5OlI4iJ>0OWhQFr@#|ZgOzzH=7s8+>@*&591q-}eQG!OQ)oPti$HZ_SYmLG6dR1=& z(*PPxL}-D~r!llvtL<Ff893DFoi*XFn<EH-u!T06OriA3eo|9YW7#+B(j)oq?_b(M zh#9?4S!8Ethcc(poiK#fYPAM~p{}m3uC6Zhs}?O<6w6+GU9c%Lg0Klg==}VAmSyYf z>rG*<vL_H?Mi@df3{zB8R8&+Hy=(}CK0?Nh1VKTBCJ2f_BeYa1;W+YR=CQ-=cGFQ_ ztsV^qjnIiog{8f>1rLP&#cH+6Bl;DT_%djO)~J&$?bcRfo6AN1V^DPAIJePcc67Ls zk_O*W=|S^PgFyZZR;-|DsZt?N(L`%kL{u3}LK6r<88kx<0wENL&;&t2geC|IA~Zoz d5TOZz;{Pq%>c_L0VG#fT002ovPDHLkV1kuEi|zmb diff --git a/docs/images/copy_data_source.png b/docs/images/copy_data_source.png new file mode 100644 index 0000000000000000000000000000000000000000..20642b4c6ad1adf58383d6ed3526df1083bd3d1e GIT binary patch literal 4618 zcmb`LS5Ol`x5rU>lhBc_C?$YMC-f=^NQcmiNDU?QAO@7KQdNpnY0?Q&qZp!qAVg{) zK>~!1fb`xkKHZmlAHJFIVgF}lch7k^W%f7w(!@xYo`#c#goK3tp`NzcmEE{no>UZ9 zdO-PW`N~`iGSk&0sT<&4At7PxdZ?{o0n1y@4-dAmEa*`~w3NPz^&tZ!Y8DGKrQ=%M z{F}bd&;V25Eb{hGyQQ&kpCRmA(>GYxK@-+UR4dG&4O`Ok;jw&$Pw~&C>py{%#p;jK z)LZ(jPIJ}nNmynyH@pGwpXQkhRklt>5YNXBFCq`aoW`)h4tT~`EH_ZOFFbrXxGJf5 zho5#D;qLU1<av6Gj~Mb+MxG%AEF)R>$1{rXrAS=!y151KLX`&wieED&w6L65@0wV( zzHFT+2|~)K=U_GcLW6mxpQrQ}yEF$IjCsT=d|@e)zaQl$yYV#tQ}~UR{<WT#@N<sb z<!=rM=_t35#$~#Ki0JP@#=*#evfNdU=9xQq74Se;xv`{c0LV@D7D)=2y!ZT(h(Fwv zIJ*-eaNEAf;J4Wy(wX0Or*GbR-4(+2HMSm*J6~?AXW?6`e-1`xj2YgXbv{PBZH+#H z|2~C3FSo)6hdZFCsWamIZuS2j6$;js`;o0X*d~|Za$b|vA2aMhQ1DmZ)3w9CJDduT z`SwkQHmkrjb;8M(=W$p%U&5pLIc__PQ#68Zt_h;x!&w-$Jpb2Fbol_hnDs^LufkH# z;t*09Z}jHT+R1Al3dXvjvLsB<*08EXqf@2NRucnik;39l--pjb3W0>fLAu1s_SXsH z!!ocllsB89@hu5Lcg3v(!5&}bKl8(^V@EOGd)car@9#E_kXU~$%JfjJ`cbTLzjQv> zQ><M)S%h8i8Wd9qL^^*+&ew_5pSrcdyE)g(WgfoWs912MOV39COe6oV1+4_}?ZBVB zkNaV=Oh8(PzK)g3QbCB1_@1Quut>L<;t!lAUu7>RZFzZUIQ%%qS2@Jo>8hgHd%-v5 zm7NBvrV0dmY6+W*8NXNlK2y$x=-zD%%t|0F*E$@1l1wgN3f=1UIk*5v-Q#?`Jli^D zx*XLsh^dh@Qayd7=57q#S!7h(1zYWYyThKqw(o&I^NDw}35VLS7V%n_Gp33Li-6u+ z;Vu+kAB&b`#jqKrI^KA-R}o*HJcf~5O8I0|hgNvmir(3p$W-Ko)xJvjNRe%qHUj;X zKc8o)O1$>%FSU`?`T?|5>}O)5&o9pTxhWpr0$T^IFW8PakKf~g<GMDF5^Go%2|JX8 z>>x|zR}f#@aLCU!f43iC;I^!HV*YyWssdCQ!0Hf8dDT^z-rlC25tS7ct3rtnZ#NpC zm)h)MsX<bwi@*Y+2?p_T&r6{@`|b5oK~c$gO~vnk$@Zpb2c1Zo2ZGyLmWhuNETa3_ z>DmT%Pjn3y6RP-0{G)EJPxUN;s1BZESZ8S*y4&B7@U8DRlnv<y=%f_~?#~YNHc8#- z>_aM3k{>^GSWRZDt(q-tQ}M~p)oF0(^c%|#ZjYfS!bECsxPbCSy+z!b%Et#kYN@gj z3%Zll^eLQsdzzI$0~JMUWr6%cSbeKkaq_%+J${x-Q~Qc+w54}KqN4R<MT2wTcdsoW zYTVaW%06_nWQa2Qi|=g{A>LPG=sdeON^vbrMw@~|wyXOxl6#>p3mESF<~8#MU=V>= zoV<{uD6={K&f~LxCX=<(-j;Iw{;3O7#H(|$X%oCVs<9ghbB$18xK}*~HaFWM4~{%_ zl=U-COy%21sLgHEy-KY4GbA2^LRee}_DYMZz4t0dRY{hM0~s0+cfKV=psr3kuR0NY zUX;YXY2;xWX;!|9-k5vBHWm-_4xtZgub7Zj>=~FNU~^!-OB}etz7l-t)*hwoS%^8$ z?@G2hu8b~XOKW83g9~Ra1NSoNO&)pj!A#kA%||Uwe(d7U1rLF==}m{zzcGc%I#-!@ zavDoG6*?$FG=?-I*FrM=sumd_-dELc@O4BF$lWyAo$M^r7;)f?86ia3Emr0LgsQa` z-0cX>Y+O{Dv%Mst)!jppxcDJuE5)aX1~E$N{9>eLTyapHzL$9u2|V9cD-@Sa8b*F2 zw>vJzhlfdA9nC^3FwoF^N<ov$I>6^CAm4UgyiWITDm6VeWZ5QCVfGsLZ_Ne784bTr zsvipInx~1TW}N*$#ndzBfiR0AwqK7*vj|`-cIYM-Lj_LkSPV3CaESPrn+~u5<L3fF zO2Ktk+qXpplHa<qT3A}XUS1cxa<H@<%M+{S7Zs@0a{HM@7b^%usrCV@Dypk>Re2p! zKYAhUjT)pz^z;LRYv>s3__G<PGswTkCPM#H8Nb2G3&Q#W$cpmEx!2DYLp1;c$~l9} zZ~ZB$qkGdpSD^v_;{U&rhJ>lsZEIzGsbi*zqoRnZQF30~_w7Zh1jf94|0lO5sE32W z`~le~+gfX}%y_3${}u2h^YBH$e1JeQ8y!VB+|F#0Q@kJ`LdKDpIqKYFBFuvIR}#uO ze<U&aa~OBukst7FUGlu_u9#nHoI3vy|AZcwyrAhiA3NBCV~``@Jo56UU;2>ES!7g9 zPu!9YBrWk^^8mEbN{qz3Vim2ZsPM#xzh)En#&4$!_}%C5!GG4=6p%8+g`oP&C+_B) zw>cm&SrX+94GnG8FQxL7Yx>SOLX+q+>;#7Y64yIqwF!<46O6Z$g|c%G5^?<LfBD#h zCiNyDK1E?8rb>dZ&Lf`jv<d`GCneGOwI90&1_ia%@VtjRckypg&dg9(RMBp7sRHss z_oMj9%KQ{;c&62aNgK>_GHuLm!tFzWbovSKy8s21^}hc2(S;>~g=2N!Prk`uCzAx! zk72?{-u<S+6DP_l6%(^-)6XZ|o0}Z17DE0!Z{ux8kVHwFchL?+zwp}9Vtp2*{1p|z z!>jhpiCVT`ysow4>oMFT2Y0fOJAXeAU=cA(7};vS=5@bbf?M3YB~e)o5Ivu<nnNGY zK<1F3C1mTTYj8e2-?9EPueG>1npM<lYpatX>>-!?1;<7{85^x<1;a@Y#ARKzIm+nT z!a=I4$Fiw4lF1u&e_j_lU-k%dj-`MtS8ttCB4VjKXI~!9Vq?4s=W20$43KsQ$U&7< z!Erbi-t3sYZNJxYcB|g`!Cng|gP0@3+F|wg^5%w|wv;}_qe9vG<Fu!ROVM%C&I`_W z{pjzWkZ)>O!goW+wCV%Pzht)59Uk(@pf?S%R>Y~Npd?kUjfjWzEpoJH8*EUJ^TBKJ zLi91<%rAp;W{IWZ#mskn4lmvceTalANiFBg`#ZKZj)+SFDI){O&w|*`!b>LP%1L;E z&!>Jsi}ih#>}k-ETFGBscV^NdGbsUE%c-4i=NKk864EICvnZ!>#X-K<=(Bq0GP=Bv z><HsvVoQ&P>!KLQZml%wOix{t@TbYb+wL89oo@XIOB<$KGBQU<pv|wpDifPmY&iKg zSv@-palz5*cLkMyulIokgPy_Z?(xv1IxwK~s#az1uj;jQ`nt}1G;qOxYC*KNV$F!+ zj+6$QFQQP_d85Ge&$Xo92}4k6Z&={%W7p>gV?fxMy6vs3<XSwz3T;P%t8>7-Af>cM zJ(cxuKNc)}w)1fE<0t~fu?p93azsBp_Z4Ny&~OYrQg8tGjKQ3zRE2A!zdmprp9lAF zAk?*u7CZz<=pBj0%0}*d%~R}Dd<fDalOy<(L{7Q~S?gYIUL(SdrPoy_N##WI@>wHn zx<gpN-w00XZ+PMvOo}eDm5+LiWC2Gp^|dBrk^u^(Z8<noQ(6q#gs{`#ik_a<H-Ll9 zsQuwLTnFPs1YB?mvL>}e;@Pn*?~y0NvoJFq{aQu?G=XGwbyXp7f$FW0@dJDauEwS_ zT|oCI2D6sZu24ePZx%%F-<5FadGl;;EOW4Tox!HEuUbbc<jGMdc#S^7@_F=Yi*m#X z=LR<-1-`aBP5_B<)$y`MqSvy$!ywi&vQi0qe_RRXxX+}c#GQ#P^^h6Zo+M=y$)lK{ zh7a}c1kV<NfyGEjbg8sH1b(;_Dyyy*nEcAGzzP4<2A)5p#(Kcg8fh<fJ`DgaFE7sr zR^F0PQsQbQ@L=QKCqGfL7;T16f^4=%P~IC-{PgWAm0vyXv|eb3Gn#bE#OC6iP!|`U z^^g846vs~T&ZcmXX-Rc%ihA3z5hThxNZ*d>{cWoLeD$zFkvJ<GT6baN>}kq@b%sK2 z3v=>L2>xlwn)Gzrn+^iay=cs?#ie^WaF7|4_C5o5GQj@4KVdc#GfEO#Q?qi+oKQ_x zJXTDhlgpav>x(kV(mMC8CFbVOd*`sVLR`kCmm2e|UY~4nYSvVNcW3epk$y5_1g3mO zGUTQkD~0UGncMqe(0D2D`=xz1C8=(_>%t9;Ikl=VNFB{7DLmEh7d4ow57a&F1KtC> zT(mR6R<O_~rk02ccVuy@yylZIILyvP#7Cv3D|acjtWbE+Kh$=u?V1;~(=&rsA&TEn z5xwHPcu?^xoWE4NeZ?6&XzKcS<q&Dt<NG?MXG}`(LE%=5DQIh!8~IopX6KgQ2as?? zWa_yaTNHWPjn_aIsm4BolT2K2X@+jd=(9RBuSs<HvM4O1ac3_b&qy0}ZV|rXW>;KJ zN|{<6>V)%T-54nXGDj)gHdKA4**u#H8Rds9AEh{@+8iAgRu1hKU7AlHd>8GU9+_El zt$#xLwceDz!uKWH<F!NNk*Z$=%s+t%>x%DO%gq?tIIX04!8z!n5sm@NcYa-ax1~DH zJRQ~x0@_On*O?T#=a{=0wX((^s)VW-wyRTjaRnF|(CNisZfBZmDn1+Q))K<4gE1^& z#)-VgN@i^8$?VfaYI(=vhIOTsj=NeJ9>=Gl8AA0!I)&zDm2ZT>sb&wWroR|*iHh$} z9qn3N_xf$+c6gU-G;qj)`CYQ>k44Y+vz8N;*;qS^8<(H2U$|D;dSFcjCS65Gi^GF_ zlu6Xk+#7i$YYefi@{U^YMQPR>c-Rc)L5DTki8`!f>*M>LrECRz6rH9bi!$~E^yLO3 zk|Na@JbV5H&Le@DiAx4vli!uf0PP>m)~*j5+)1`+(YZT+RSgKy{p08M+zZ`B2ZJe~ z2D!dZF2naEgVB?wPc`FgG1AV%zy9pd7I?K0+s+?S>@M|_cC57|6T?cA^pfl*lMq4S z%Dk&<Yo9OIi+nQ)_{y(^@)u}woR?Tmj;keZ|JP*5inuORG5?N`kZUX93Mo8ua0_;t z`Jd1?Ju8XK1M?4<-rF<U3YGTAsWE@a%sfF6;t0${8m3rnf!T(>XaK|i0@5BV5C{U@ zOzpv@iKJxg?d|z_r}Jm0@*W}^-TS>;sjyh=dr*)&fs#fudT5?kUS2*+mp8n$yqYx@ zd3$3C*)KjqyHrvVYRGchlL4hKu@Pf4;}$&1Y#9<0NpbF!j+qeX&~AvDq>>-;WP2T# zyMxgS=z_|lhpP1g$dDrFn(rx}UdjJMj6-0Ud^f7<3ZSUSwocDssOk8u!DYw+NJev$ z-wZ03OP|oGVP<C5iScW7ZT!ud0$Fif)NYKjpRBI&;`~zQ`pLT9wsw5|AfE4fy|vh* zf)h50ZfT|T)WM{t{B0@T#+%_j28(z85tV(XXR@weXPR8D_-X*#KMv|@zH;@E^O3WA zYw{ViEm-yM7HOwv`ev;agt_F$+wHj*-Hv#G{{qo^8Ts<bB@*DbPyyi>JOXGByojAP zSm4G1AY#TP@&{Ke^{6oKRq3`?3fHB?o*!qzk57qmZ?2MHPt9JeRRoi<z6}r=J(xC3 z7d}XRZ31HI9Qve8XY&UYJR17)aTH-=s%<csi%i2)D2I1&JC?soBp?V=!heV9_CN0a d@5m(sa?eNAIL0&k>T`$Wp^lMuou+g2{{Z#F29f{( literal 0 HcmV?d00001 diff --git a/docs/images/copy_report.png b/docs/images/copy_report.png new file mode 100644 index 0000000000000000000000000000000000000000..124d306bcf9a9ba99881647936add65c627b5423 GIT binary patch literal 11780 zcmbVybx@VjyDlI|2nq<&CEe19NQZzln+6H#kd{UP>5wkzbhGJh>25Y9ozmTP7QdM@ zbN{$^?wRAv7QW3_?|SQbo_7T+DM({s5Mv-9AYjRSkWfKDKx78LL(!gs|IdUiyuc4c zM-^!?gyJF6Ed&Hg1Q`iYHP>Hzscx=#kjV#Oh+Y$I>U%^af3IgLubC*rc7iZN&_WpA zl=L;)W|y=v3p`<4%B~F~Dabx}kuXXmiAW?#DuSo0q>PM+*#2q~IU0Lx%eo1oad&TC z|G|u!%8LHtM9^*Q_nG?jWX;gqi#gTk;vE5`?;;~@Q7H|G;1?wle@%2`-yo5oI1$Pq zwEy=(?1t?(5lZu!%RBE|Z(%4j3(Az7ha)>2OM|WzuZ)Kxsr)8oFmZ50$|8^^S_}nA zR1)bs0(m5AWVc_Qh4ZXNyooh032zEkOz(t&4YWcNg&V76I1u`j7=~j*B*|e|rTL~8 zYc<NFNk~CJFE*|YXE?K~lSVwl_QIYpng$2ogr1S=c_Dnv4yzC~IZs(#J>c&iZ+y~6 zI+j<@Oe=_Ddi{Gpi$&8QIO)-2qhX^VAKu(udI}+ZAD=a-`5PP(B4+#TUXQ*E!<;zR z#MWB<nW-cPF$@1YC%x_xsB5E8R`%}ME0&U}?$KRRm1x|c;9%RI&tDg$wFkZ!ud<q| zSok!~9TF<X$jVIpmJu;D*A1nA^+-uY#Yk{XY*JZoUZ7?1i>3)So=IT)_d(M_jhk(} zisNg~Y1k{4@U@s~9vT_W`vm>;$O3ooB>Ggg$47*QS_EyN6BB902Zx4=#<#9JL`6rB zS=%r&GIGEe3*oYAYIq?IE>wF;bA7fos=(r@o!r`5LI}j3pVVx=OW?gbgy-883%iKY zds9gZ?3#Mmfc(dgPERU7jKoSvO9$5J$;&G%N7>&eCK4JM8xLz2^kp+(l$DirbRbit z+18|FWo1<e>8IWyFJJQrVx0-Yo%L$Hae`R-2VZ?+VPb0AIeXpDd31Di<^qSdnO5LX zP(-ryyEZLUykZ%xK=I|3m5~XiQnR<W507_15S@C%#x{fvi;j*Kpb!V)H1hPwRWM*@ zO38LH39ar2wiCKI;^pk?4p~ZbWAkKFO<oQTj&Mt5NCyZEj{jV?Ziilmi#AJWVq)^I zJ_2rO6H5qBDxI;`EK+yU%ZI^WlQ;i40sFhiAG-(e_Q^$LnT`$f`=vBv$=GgazHcI4 zAtzo~a5O~xM!@`6@aABaa158kcGf@R3avBPn)k0(_itX)>%=-5{;Z3!HBG5gm<MNJ z7Q?hPFSQ%D!1VwlUr|^2m@}FDco+o-JZGPj746M+ZVYT@Yu#M*TQ&;f<nAda_=1$x zym1TCNuF9HeM`!(d_}4cDVd6WCCb0vxow?vWEeFnESBZW>p!dXSHz>R4CdhIc(qrv z7}i27^IAwKt=a2_H>s`=$|RKjqG6sq>dM?v15b5rl4x|iO)kbSo0lsAuZ&~7dk(*| z(M!iO*Gpiq4~6pP>fFqEh>th+Xu?PK@ONuqKg#?-qqCKu%=4txVuVfMsA0z#qa@A& z?|2PbwB?JNQErfKY9$%<{a)8w$`mP-+*5D3OAbHUSUUAHQB!A^DdI@g*v|hO&wruU z;F6n@<6opxGdW9N%zkxsb$huQ=}aDN!+861>xplkmv&Zgr9=#sb_<yX$*t$#i0h=9 zti;89oy&jw$Z2VW{X^Dq`5N%|sM2<(pg@Q;ZBpK^R=Wz&C;t0oVALvKb|#=+z?xYN zm&n#p?~bHErF{SXy>(sp>hizQ?AX}YB`1OKb#!!ee$wY-V`D>oYBr0_P0J4gq|aUU z=Yq$#Xx_e^neJM5SU$N5iijxw>}-)NrL2qth2p+^=?AyO(m6P}va8cg{76U5OyK*l ztF%-OgJH@<C!6HtQUz6Lcn9zVXpn`34%P_-?Dx`1o%%aAFr3;_EU?m|M>HVaI%wvK z{~8!6URGRcGz$L|d0obUyP3mixHrs(X}~9^i5D!7>g6QEJzA)Dc<Cld+(LTv(j<y- zvONqEENE5BR$zJfptq>@1Lr;K?Wx^y5+4i&l%<-#HkCzm$sJfM?3(0}fni}`Z@x1{ zj&F5rIXRsEQ^v)`{h+8QyptkCLzB6vUy|_2=Pqt##ptt^)Ht_*%kGC3@4IhHJss6H zf9)FPyYPY*nmoqq9L*5Q%gd)*e1x&_ctShdmRkLS!^3I0xMGaEBM?iyZd_pM%+%5m zf`u-81O#W5hgWRu<BN=#j!&ov&0pX+-1Vt?YuV_vISQmi(YWr$7;S`q9pRERwA<|C z8=La*9MIPf6UJ5>INF?bwb3R;C@E%NzZ#Z{qD3<oa0%@wR9(6BdRP)S9otd3c2=5p zpHS3~>MEknP86mJsA*<i4UVJj!M#KhVC3L@sAWxa9N$iNJe?P!SWGWhVdjJ;{D^8w z&aW5yfK6Q9rRIfRmD?GFkBlM*f&5Z1(`+YwE<J7Sm_IIKXqb8AHl8P+lb=83F6C}( z%QiYX3Tpau3ZbxhTY5IO=`t=(&O`SmObQ{=hIs{jeewpEz1M7P5fc+?<rNi2?oDCD zJQx)-15wn{VLo~N>qZ9)O-_5Wapdpamo9E*>Ky681xA;L%T7C!Sdz;84r^#nP@lu# z|D@zrkB&G%;2Rs~MJ_9@lk^)~<3>h)xs-kW<Q=2ndyaUw^jD5Y74$^2@PX6>3H!69 z!@%)`7B@FDcfmVVeDbTDMEt*kx5N0OIr&T-(=;1;I|kc!@5qHe`sDvaO~5a6-ru@E z27Y;XCy*SxIWy{%Iv996pr<^L&K|zO?qj#aGG4nnFLr;M7J6`>9Hy)IWXE^kRR5tq zw&cS9*<`L4>%gto$yulplBriGluGJ}xc{t5%GwAWR%trFecR=)<Yd{4{ds0B`T2&g z-K$4-hfAX3OD?<9tr?<zmb99I&&fE2=;`RjYQB8>NcR%Nrmrs&XcG2yy0yA&l>D#l zS31yDR8@l`B0wl|a&cKKHov#9un2&>Do+NLx2Lxk(RZfh$B!Se@$oBm{9(rgpqx+d zMG;^t$Z;QaKG1P|m2};1TU~8eCFs*pIKwJ^u=h{3-_+Eq?razKVOWjusZVsv?SmC6 zB;a0^gzoCj^0|tCq%heOZgJ;#;uLX*wpA`Gd)|fy=4-@Ze@WO%haR?jQ_#zCN0YW5 z*c#CPK$S`tT6ppaKXYfKA~kqj$G7-qX~@)jR!OSMo@XFLkOUUe@wcXv>yF|_fPvoZ zRGYAEX!lrHvFVjc$C+VQpycPbe)daAav_)YLo*Hj>!Fva!KOr5m~~&_h7~g)G3npF zO$rOcRD;k`Q~M<*CUVvLiw9t~?zvQWT-bu##T-4n@=eVvC}4nbiWdAKL`G2?=LV_C z?|sW3k~LDO!47gbXle&Vp`ns$Wpz~tk}RQ}^}A2GrK>G}yF<En?!JM+wy9_6)Th?f z#EbDFp+ZEAgYFxl&SArNrGaew1m7>~ciWp#M0E#UVv=)|7PC)Fr(ee!^j!Id?NPo| znxwR&ib<VTVvi7SG=LH#Qt=ETKBkoY-8omQqi@S8Ix0#z+W;|uw#vBsxf;Y|vDpig zn7g3M3xFn2Yw=#aasY7wI(8cr?|gqgrLvOS*dwz0CEYg}85yVT3H0>g?zHzF(-mef z04kZ@a~Uw@+D!8(rgDKoU(2Y_R@4{s6zU*>cI(+Z-v1&=sQecqbqNzI2miLCvf?uh ztjfv1+YH014~)wdpWegL<NOC%40&aA=cd}O=k<?6c-qMd_g^hWt(&zAQIsfz8@K&j z`g%F5kvs69cxi4VDgK&uR3ch@!|&Y+M~W|V=$t?~=5wodblDH3H0C&Bv@*E$#o0UV zO|U|rxmUwsFZ^r`yriG(813=sq(Pjs{qcbrRIIGw+0u~?D;>eID=K7O*Qc#pPUTip zY;3hlNMId+VWfht*cCG$t*u$;71KUH+~0wcGoC4q%E`&u)8B9W=Nsbm-uWAx?ddY= z?g%pDsbamyf}uAx)9{rY3gxt(egoj0)bon+ascYG#RH$8UB2MK0;}?v&|Q{DZJBE= z;YmXJnmTPkX|KpBD2#(9XX<?r{N6F&-W4L$H0t$`5lBbr<Q(<YtGHLG{v$MdonNpK z8&g}bByRi{M_A*$rkuKexNv7!aLRi2kU{9OcQAd?(f;9p>{2aNT9`4fC;JHQ*UNyF zI5HkBMQEbMyPZ3g-zO#>@ibs8#8Ne|NGnFnidhq6@GpM*ECrKK_>xrb->2IAt+rqK zgXn&?E$QwqKtx0&bzDI6==tFDJwHBOP(c`3S=XmN`Jni=wg`D$6PjqFJ}2z~F>!gd znf~427V7hG-=#_gFimVM9w@rK{ry%)MpdTy3BpY;om7piagCqO5<2#|V{{e(a5nAy z(PU78-O9y%I&sVRG?d<LXVSF`vvShulFMQ6EpHkV+|?a|{&1#VO17Do<mPOtOKVlY zR8OCh_<pjrqEOD&QJe_$l_|{X<9kIOjd{{(ndK^rqk*fhI2uh(WBik#I}xiDXcLEI zaR+Jg&`^~@<vq4f+k RgB1@`1&9?>BE7;n}sgkYUW;&I~Z?FL*$~>Z1UWl_YLK_ zJ04s<Z5X;=!0w)51`H0<_Y*DoNJ~539ApJoO~J4wZBqgKI$oJo8N-!7!+^zi=Wp*0 zKf|V=9pz}=r-oe764Qt9xaTBpf9@?n>=Dc~;+1id<AuB7l{T}7nZt}B(pp+Xxw*L@ zp4{PLUwTIWDD7Q99mtu-Lei<>#$}pz{G@QGo4FUnh*~E4D6(j1SRF4{a=lzl+FWAu z`)C9g2EGVPk5dXx>`<H`8TD^b`7LuM(g&9w&}W=o>hNBk?q!Tn=kAN)2m42hiu=i% z$%zkQu`I^V(m<;*b(BV6YH@3f=3a?|rqs5OlsZ@VC@2@+E$JeZ-&p#qEag%vVaUGK zvU<#9YLehem=y||p;ndQHdi!#&_{sO0XBCPB;R$p(L$LeG2&K&zy(6%O2IX}fPiu0 z=I*Gd)MlcmHb6yT+t^05*p(b&e&YHiHClZ{Zi3{E`s~p(@ZXca?4F<$S=KA5;`TlL zV$m(|G81PH;2z^^E!C#wu7e8__P5Ubq|WY)C8e-ghRhUkG|1l$fT#8i3cPiUv!PHb zHJ1jmltJ};Zw8h^UMc5Pu^{#NV6}cG2`4A+?;;2#Q*wreRDkP*n;QOLVUiy&)h>d> zD+!DXwhDwoyy!(Sn8uY-)uL&*<qIb(i(S*8L(CF|=iIS>{OOwcx*vq-=$`6L0SLjL zI;0KZ{ml$IBzQptn0n!n`o{5;j=m~%JW(JxE-Ue{kSU(d)6YuhPYIMk_Ks^IJ8Kt@ zAOV+vK%b$6E(#j{4IP@7*#XVT>97k<27QE(My74gMO97Be`<%8i3z%Jf(n7;!e9do zC3wjjsz(Cia7wZCOiSiMg`zTTme_a{`ru@INlnc&i7Dq>Xh`=CwE*<K^Ew)Lg+G~$ z36sz!*0k7l2<okItE=aMl8@eWzh^zl9lo-n(l=)tjk}ssRjCBXWH}3o<Jp{Tjq*-W zadGD2!3z?LSejmjaW&Q8<e%mcGZVPQYc3Iy@xyD+0?YA6j=ZXD20GZQXL$l)Ars(T zS98WCwUQcCHeV1FjBFk{;|o87cGIYqd?kJwxW7l!)ooP8tEhJ#0>~XrcmP^cJ@e@{ zOH41GDdG>Nw?H>A#*jx#88k(s^4F0KK#RN4v)8(^*<n%l?a9nSh$AS33h|lkN{p1F zAtOl%sp+u_X~mam7foAN%c7@K;V`l5!-FA`hA)lavcLOOn1nx)y1$MmfI@ZKq4%#U zS^uL-5DTy0gwV+Jl!s+8bSb*klrcT~ee0b$Ol4JctJrz>L$gW+8X^y&Rmpr#w-V`_ zCpG0Pj*w|NiouFXrCt;boofjZ%iVGBXx`8tsULAB1Na?H%~?IZ$KXq++WOFsG}E?Z z+)6we88Kx%HAoRB8O>=8cA>h$zy2w1Ecx+mJy&^3bIbwL3XW*P?Lzz?azPO0Y`T)> zMosiW*^%g<G>ji%P%-djZhb9;65BjlrVvR~PZ`3)lBZqv;#ryH6aTqCO2=u~%%XBM zH0o44(5wW?4k*4Vq?D6J8%b0yn~9**lj0Sv(JUI(4%2DFTe(>bWQcOJk2<~wemsOE z6S~C%PRm;1E)78|v-CqN!>yDUnqZIfN(xLVVD!z)C`ZZ7wh(}BcOPgRxeEgHHdg1j zxnWZvqV&C8Is%7`EF5myZmq6_$5qaP7@gQNNBrB8EhEoD>MEkVoKsd-7HCA;q*_{9 zfT;l>7#7V9;!4%lpJe(e_As?5&1)3`WF%3EN1l%@fh-1lJfQsl9b|3$X0*O_J*4bU zVDH{?O0)m#?~h#Ea_?UM^<V*+^`!)7TRAQ_e-%CxLjHK~@p?Z=ouu#NH&A!56h0Je z45#~a*-&_2E^9wa$*`ejVHPbkfkPpl$G!AGuCT0`sdr{h_dG@qd@<F0-6Kum)$tAS zi3DOhos_1!ulvbi`->-46*<Y#S0|gQnwmfW(e>D?ikn15LNpyrmQ+x99w~g!4=A!t zfI5a{D0pfSu}rjPy^}@U4zKmJh^=k0E31}2<>PZVe4nuy0M@d**pjZ_<o=^br{0N? zK;M-rh1UjJxu__7*YS1HOPUhfVCm2IFe0v}TE+J>&6aQ`3=E7!LAUZFq#5}X-XkCu zWbFtf=sKW-YYoF~-h;KVnuKL$WPB^uYjBu%o?5o!htBIyIX7Qzka*u;nE_}9INut0 zJlbEtlaIFY(;Y4k4QH#Yx@+xMKgi4TO!{0d2V#;75`rZRz#;*9ovzmz{B&pP#Wx<S z$)z)QB4O{wGDSeN3e}5N`j|_s$Lh_&x>s6)v!yI8nfl{dpXsEfr1XR1+WYGrX1KgN zSmL^!;s$_|vI`1=y2(9&O{Qud5x~fVKyvq>wYAUU1!*4pZ*5!dTq4Ncb$x$=nkeKY zaL{z2U)8NyWr@Gq9ci{RS%i#)xZri6NB1#N1Sm>XH-eW-egnVW#=u}KtE44O=gDP4 zfZ-mkcAu2^+=Y&fsR-Tda&?815YP>QzY<?8`UszF4sWjY#=x{H0!Y3zMun5`A=&*K zp(1x%iz*fU8ON+qZnrE3NKT=Rj`4q-1v-u6-PzR1u+M!=5|^1##mqN@pMq3W-<7ka ze%Pb~ORvv%l7xMP0I^0fSnG)zNahLkLw%v%hkElbJJNN(ZZnbZ3%PMu7^3eh_`8F~ zcB_)6F4_cLq(51bp{`f!33cbQR)vkdiyjB>ZR@w_HU^TOhybFR@X7map+?C@7s(e` z<sHE%&(>{tq1(+~4H2(3Ap(WZEk~r#HN#7~kCZ$-aUfn}J31s5z&$|1DM8{q_Ui&$ zZr9midJU7<lYs3mxbLt6{{u-+`2Hxe@v!Zg=jo_)7ztm5#wV{h_LkddnHW06Qrmd} zsc>TWni1_|q;_qd*tcWxDQRo>ZKnG?M}D#clFtml<hTXZsb(*a$&ydQ(`Cj-H+O4m zYd?SfMEFx?+}&1vw-U<!Xf&7E&{_bA?VEU?^BFUzgN1o)rf>GEUCQ8gUs~?eP2Lf( z>jVEej&O&+q*nlZ8>pPr@7^`^?E(BOmoDUmnTbUB&V}WL;32|yk#D=Z*41AYhgfUo z#6YV;2+6X$nEx7+kkAd@-e{N4e1uZGcGu+{$Si6=#sFCraNh2y+f1{%Jy7_?-wGlN zlj@7CzX`25+_gzx1)z}K@_x-~Yof>SY7f9#TVLm}7^AuM_R00)wVCaB#@_g6f4%_- zCp<`p`175h2qL|4JzOeX2c-=Vf}e;_o~jABNVYzUN=+TC7#8ZUS@aH_x9v8qSazs` zvdW~!#E9oBq_)Lt6{*`c?D4J=f0K<jh$Md>b^?*s)+R;(A{RXeM-+&Nc=|L~B#?B# z89(5b>*f7yg)R?WVnJ~kkeB7Z=oHfiK)Mgz-`!}He@4%i3eQr_S6JKF2(U?yiD^%N zxHZfBnA8TM%WvLx6No*!j)OdL5${d<@Xq)CcKf{$>S=kn`vP<@#GfWfAia)(a;Gty z4|oWq`UIpN6`-*|!yYTvdxyZ*e7Ry-(r;9@<kC>6M^PRtNUk`2oR#&O(p(fGEhB@D z07Mo9ki!A_Y3?D9k>k8GX+L9<0I&5irkWEGO)=;Q!g1Of`vF2Y35$eRjR+NKy4kA% zrv`ogC!E#?L}`=Y&2Ghq=#Ebh*Bb2@vWkkEhs*7coWlF=nn-xQ-kDlRsCmpD6ujx0 zFBBj+xcy}z5Ko~`RZ}r(X+yv(?YR2So;^$AFn;QBzMJ`Fv00PlmG|vYcj=wj_Y+aS zr$qf=bTQ4VQ*GU9R5ns(ER6uNy6$o%^a_|hQ)%%Mth960p5;f-t7eg-X&ya4S)?Nk z<es(_nRRs905<K)AfWA^++8f5OtswOXY_r%1UZ)GbMMA$JJ*x)r5XLtAm3tGp2eKc z!`<%RnimQEov0L^fABSQMZv#Z<BrZwAczk~)5<;4<9A>8KX-)@;SmsMFl)xpD|YR< zaDCm^ePl#KLmQ*g6efSGZE|vRaq#gwD=o%dL2zr;Inbh@pm?4P^08=^e&aCi^3NrO zX_ond^mzrImds<N+33mv$k+Q<p`FvFbG7!?i_K((aBT~vtl!wS85yl*Gu8l%Q&-%5 z^hcTwAM{6m>AD`LQM(*K@+Rpc#CE*olQ6maKdM)3x;-nMp<rw<=Y6yH_%I-I*>&P; zsLh)F8_9gMu<`b|ACpWV_GIW?B;SIoA4rGCCWH7BXal=56<8W&f^;##piZRCRG4wu zuSfvVh;PpB$$O7;8ZNGiK9{`Q+@tHAl1HPZBFV9kjm^F^U*+K_r2{8|^ahOq6A-T~ zhA;}@1wwmF`vtenBhX?9TOWWr=kRcM5oqJMm74&fxE%<>SOgWnaoFG5n5dmaj{|A~ zg@&I;kB5xgXp09Ewk#@w61q2ARX8=6l%5_5g;rWk6?f&Rpmj|hA0Gn>#Y5+(PQ%AX zAgTO8Ru<~e)V)e2ESwHn5w$aQM<>Cj0nia$4_i@z+EoD6Ubhbk4SkFrH_!`+ggjk~ z{;Cr&HLs~D<9(!yIf_;_)Iulz1;UaGbl?qJsV2}l<D~u$1%SG!{e`*Tp$hRb*&Jvq z;X5|GIz{9Ppnt)Obt=tqdV720IM-5esnn6aXJ|imYI3hr<zEW)C5^|jRr`*r02nEK z*(T2Rjn~M`ECTJRjkpMCzSkfaFat1&#&(PA39;}dQY^;vc(i`%j+9~&advz_DhdIa z54_M}O2CATc&;w(h&+wr8?f{d(FMHg|74u^e;f4Gt^6L_u=UHJLN_VRPQQj_TOm5h zOHQdFHFF{f@$IBpFPLH=J4<Q;V<=xgayzuwNRO6*gR@7LCNPgAngjakcM)tHfIHEU zaYVC2Az@h8l~Y^T1a&<3+U-1O$WaQ}q42jB#EPFvIJZ2D$`s+|l>t`ZUSuc+5{lvG z%+>;ws%wZ1C{pKRhu4Pf%^7KDA4lKP%Xb&HRcGVQD4>7S-*c*`-P!Gl{uhfdALAVH z%Rl`^Zw~D?<}O~Pl*+1;)Y;_a{>0Ic%=i`@{EYbUIaNSkK2g54I<IT%7emygVM|!I zkGt36^VSo$+tRy#TgG&6odTRjdzqszd6;XhKeJ;spz!WuRi9WFXVOy3v|8<A7go?q z=MOv~Gv?tv2=v`vlx4%N)Qo*@-%4^7;)I2Mxx$xpQg1%zW$}%I(}Z`|4_%x)FC2~V z-PM!QrDrdN$$t_ps2aCnhm$7YZO}DUA$}$*-leFNx^@|n2}WP|ccD+dRPIeoF3dnz zd-mGZxhc5hLX92C^raw=a;|jWmgAgjbC-BOUWniu8;Nh|;tD!Kh*%4klN|&ty-qUz z4eVH@TNwi-TCE!BclQ2LvSCA$)MHwD;&NtP_=s%V_-`yr2^yXx<|0TRJ7MS!&I^3c z*ze){RB&{6zcl#Omf-#F#l?*;-@(K(B1XT#q{@$(etab2+pVvn_Rs9+P%auv4krUL z2z4jqjL)7YX-uL#+`E(vG#xPW!|b$LGTzJ`$0xOT=KR#5;V9P_WSdO<PW)cQsd<sz zaZ?yy-8Mlcc(iWg7XtW>4s*1JJu0zEZFy7?uaOBn0ueXcioy)K$5ZL#UOi4Uqm$i( z73qn4>vK6VzB7(1xhs$D*3&e*wL1{caQr7jftZeHXp7V-*Zb{jN898GnpXP+iS%;; z5DFOmG`QK$xMcb7?y9pn-?9Y{2KwQ~W8+uaPN)*+Y%M`r_X68E%_#~UFH8^xx^`~Y zsefEO{MJH#`pcnH{orRWdz5B#XGa^my!HFnjtbg#^L!pO&o|_5T|TaKe<Z3vzy1ol z3)lK<WxjQ~huyewaNfT=7&V>1Y(M9GJvzn?35ugUR$`Go47lC;HV}8m!*-A#wKj5P zn;xc!63AL>SmGj}7o*VNYa$d>1+nkonnOj*7R7rGHVl+M?ghZwC|0^$MGKE=;%3`> zrqA=o>Qj=XQwizh(sywVsFC8x*mna2!P?zZEUgu+`-n2J-!&V&@H1j<fzipOF9{9X z{285i%LEMOF$^^=B$ejg(-e4xDQuV+V;47mWba$}YnRWnaTQB2Y^`{|zeX*}iE6U? znl)k*vpcdCm36XAmqaj?OgJgNdcTO8C;;2gRML-<aEB_~Xr}sI;qDnb7n8%vgbym= zBeYwm^3!J%o5pPDdlN>vbMj2-Z__?Ky6`x}x21t*+h3bC&gwRJ<wG6r_a*MWoD-gS zn|QzC43d>>U-j`xYSHCOutb$Oh(iZ?Tsu$%Ed~CT6op`djUs->Te{DCnA<R}Ezj_S za~Uv3K|Btny{}THn<Sqb(o#RMpFD`K%P9dvC!Ttm)~k7L_q=~B*IgYo9wxu2k#uJ@ z+6#T<(Ia?G-_g35%F#${{R2@W;6sX&VeDz=Us|o!7+{j3<MPd`pDKy-)0++Tjd9Tz zBQ<^z?tM&mXXgF3Eh0+z$HRfXd|7$zT~AsS8n1qaJ;Zizp0^vPGwk-o6E27wk6jVl zyIdM+><Qc!TL;f~`*9QpZ=&$;M{6GbKo8A2`2#_;B_T5SRNPUxBhN!_W>($taMR^K zpT%|Mi=c<=e1B3nFuVA9&inGO<&V`LxITmh_FBT>V4a7(Y+q-HdIxumrOPvL>heiY z&t^HoO3x=HJ3-*Zy82}o=IUR1)k^`-c#m!!%)CUj=lh=VdTjkf&eFz66F;bWn_n(h zAkZ1Cby@aLG+A7*2DNv%`>JljG3;n|t0-~OC|^Iz4BVVn;wNCE<O;P()%KN)JK5D0 z+P2I%38X<ZnDoU%*VK-hCS1dDUoezQ1?#MKlDwBlxtqISr*K0tvGzRrWola>f;|L= zl2)Exo~d2SO-!!viqNlk>UJkiWEisTkyY=cH^yz^E{e=i;A@<dm2A`ZTve6s;NEhQ zNaVyXDf_a@Hftp|k>f5IF6~<edU0_|a3$zinJmn2l!N!Atz1x1%%o5krVUo=9sMeh zL*@>T*g4a!Eq<Mtn3|S5ZK?cH*worf|M~MzIN!pD;k5T8wI;rk5}~-vqU~kOIz}4t zVx><ToS;uVRtIcgkxGdCsS5FIdVK)Xkd#7fQeN}bddVkb8P$B~GSD)ndThsHVrMli zcYCmNpMRNw+q5q5GKY29#)WZ2SukV-++}OAMYw#fURokWr9f;mw2IF*NEASMIOmrI zK`<(!qC!MSAC?B90RUNRKM3?Qz8+}Pq4QFMSy|)Fe#0Yrw^@+4osT3CP1&s9DPG+v zY1w%E1X6e#++CjmNyed@>}v*3w9ed5FpLA}?SrYQsUu(R3{HGphVmmx00`4|-;@xH zA;JOPPCzAh{nq4hF$VD7g3p83*!Va=ou+`U&p8gi&&kQjfHI|fofG}7`9cqrdcdK_ zi*)!_HS3EjT=ul9t!I2YjSMFWl))fXaMzP9A#}Yz!4YT&^)7pTSbTFLU@(FJW&_r9 zwNZ?!d8O7foM0fhHCaT$H}CY4HzgcUgfbI3JGHfb_7)ldw7^LdJJ*}6{dpemm}_*S z2fk?8s^6>hd@7bVStz%TAb464;89r<y)fFvk)PgBY~5W?CTTsbH_#}PI_+H>_M%z1 zmrHeQ8y-<13IrM{6JQBItq3&@AsscQuTRUvlmxAo+kVUe+Xs@k0$FvcfxhYAy=vGO z%P4rY7K4Y6Uk1o-R|HvqwN1XGmFYlYD^LY`hKK0^dHk)Ke+)=0@y-iU{y&Kv#+N|< zVPs}r1tjbdWaAN_ZkFC1_p?9VY0h;u9MJ6e#zw(3w+*8E(|ln>JFqn8yE9MzWQqq) zH+g(*D)G8m=DfdmV?%Xl!i#@8?OZvaDU=oI_d7ef*3ynth697*fi#Tz-a{3UxrV>j z`D&)LVLGvSa)M!JLxOKO2_@W@+?~3jm_!&T*ANI`+_7<@GBV@!!eHFAnq-!?JoN4$ z2WSxT?)xEt-3nbUJt2LL3PxER@!cIArS4~zD=RAqt^hwjGAbY}<N@+$p4@f1HSTu3 zmH!wq(h;PF>wR&LIR7yM5gu6ppp6G}UKjIVuxC1)Ccin9ss#uKkl6wIh||#T1uUo+ ztZ%sEr?2zQAJc`%zIBEYob*sf+U-tr<SC{P0W#DIs8*-X!-Wr*`{}=@R}{L=wxo(a z4q<=U<T>Kk2?|MtmT5U=C1hb@ixH#za%$(KC#CBKMA_G!hl5`_ZyhpX%QidtKD^rw z0kWFB@83PsR$mnBg~mFW;hVFmPmY_ZUmYGlJ4H3e4U2aXgDHGmKKGux=i|nEKpq48 zj}X$muNkLw9kIv+oDf9fO^Tb&{}a=T2i>+c#+*r)_csUjb-r-SgN6g4$IT>Q^g&M_ znC^3*q+?r$QZa)mqN_`qpzqE|!e@(VwJOgy|JjrLjR68S-es53%8y`vIY{%ITRnzx zF#R9J8@WphA4ch8uW43Jzj}$I*;(DEPSuX<OZ<}uOrYGPJo6ABF0$dFGE(<d2ivhQ zfozEK$F#!!46fk_1fiJNShYQ&`r??lxXzn{mSOPuWN`;la%(L28Q|NTih@VT6kew| zKxP7A%keRV7fU9Z-~E&U8?O|U4|eN^iHQlzVKUj@O7O0fKfin4+|)<R9d%JXb$UMO zwjFO|a_%4T>n~4V!3YOTgZtsag#6+z^9l7ee_{hmN-W_Iw%&LtE1mE?+xQBMSyGU` z4g2+lz4MRJ%xO0COJI*$2DhBWaa^9y9#B&aZHn^h`@=wCJ2*t6%yCnp`R-ij6{~hf z2tKoBnGp(|svTVykVmXE<*3Bs)g3u2rMWe*BB%?OoXBTWL!TqPDF{vVSs0HoY$9vC zmn0{+2&L-H9N~jY4L_gUuq+2k_i9XXa-ZJUdHfBRDyQp<gJGbAfFkQN?NBlm3Mvy& zZ(#aO$v{N$ZS_Otay>L~zt|s~|GGH%c;b8m9}tT`K+?%4f8GVVfVOGWqbEv(<0*y( zQgzNJ22%@l$ya_AD_eWHWE@?r+$8f2%XPqz!}~@u#gpDTno6MK{{c_$e3WKm6oq<$ zF92^_j*=!foUODFu@hPy%awE6NHXKMUqOER_O0jr)ux1`WK?{-6mTz)YStg_4<0l@ zjw$5FfGW)CysZXwge&bDpmPuoEHt`tfvdtDilTghvDstM4zTSuh=yM0qo@Q{k?^ep z4S}8$4iE^8_vh*mFiH7;f@lmL7P<)ow%d<N5|labsDm~CoFgT8d)WRlRUppoWP`)$ zpAx8Waw;m(QOQaqKmkeOw!j6fo9<El$kw&Wq1?(WvlT5|8u<Op1Z=VL$ND?<sL7%g zEYAJ8-d#)t6#^IwL<Gpwb!R<#dV18*P8FMxg`KnLloUB5qu0;7XTo#V*Vhff7Run@ z-|+Lg;=GokzuoX*6PpotbtF+8@puy;#HsDM%S2m}&OZTDHlPjV>hGC@M%i@!1q<-N z36hk-fbLZemYfPI?itOIH`A;tF$qBy)_{X<4+6n&jI#3r8A}?LD#boCHW8EvMka5K zz|F<&@)F|!{RKqwJNEy<D*tx}g}fr-30?ST$-W_X>Ol|^akPXCT2<8ty#FS$HrR%G z=-RK%wiEt)XaNn!W-@qnjf195Y_^@91+!<C3W}y2V`SHDD+iOB=G|jOR{8PdItWs$ z1(+j#h6GRi;1%m+pLG@J<t%4W)zzIN9mGY0=G4|s|4UJFe!w?^_Z1AC#nANrLn~*A zNl#~oF^h(ZjB_8KoU~t?LGCocXei%kL?zw?PqOvkqQY#LoJ!l*l$u5b2w~uhY=#gj zH3-UVXJM^2)q0DH-W1s5CqJ$P!3W~X)DDk+@A{GJp{pR6wjq80Mx~42@Z%qyZ@r2t zL)&LgL*5kNYp~aG6<=4q5Fex#nuZ&*T-eiUF&Y_+mkGz}abORvG9K6zp{%TI7_Ux= z{bqp&O)LXAr&BcTr=}LmD{I~qDkZuY#({dK$j8Cghgz^#ZOS=2#-D{jE1m;*8j@>~ zTna_Tl8XC&tolmTt+iZ}1x%a8VNi@pFyP9KtttMj#TyVHPxrFE7)-p|1U`Pqf#NcM z&&WgzhFo1*T-C4#F8skH^u?c51b)iAHP6UmFjV9R@2d&<hF1HGIQ*E}n`)GaT@8O# zZ5_u59>VxVFD{ZuZTC$kKg7B947+UwcOpJn#r#`oty)hqlpj~qo`QpT`e^~bfaZOE zmPp)-(r3t6@(`kJo;>EDi67N6oyyHnse6g9*xbDMolqvk!ahzgXzVI5H-VgKQJP~w st1_UDRA7E6-!a8zZt?54Y|jI#%L>)xb?7oN*w&38BdH)!EN0;QU%7()YybcN literal 0 HcmV?d00001 From 3e6545afe94f57bfcc52293a251cfb02ea89a399 Mon Sep 17 00:00:00 2001 From: Laurent VAYLET <laurent.vaylet@gmail.com> Date: Fri, 25 Nov 2022 09:55:57 +0000 Subject: [PATCH 105/107] reformat with black + fix undefined variable --- slo_generator/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slo_generator/report.py b/slo_generator/report.py index 84a6d3df..23b4fbae 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -223,8 +223,8 @@ def run_backend(self, config, backend, client=None, delete=False): # Set offset from class attribute if it exists in the class, otherwise # keep the value defined in config. - self.offset = max(cls.getattr('DEFAULT_OFFSET', 0), self.offset) - LOGGER.debug(f'{info} | Running with offset {self.offset}s') + self.offset = max(cls.getattr("DEFAULT_OFFSET", 0), self.offset) + LOGGER.debug(f"{self.info} | Running with offset {self.offset}s") # Substract offset from start timestamp self.timestamp = self.timestamp - self.offset From b01073b6788416f2cc2fad43006f506171b62292 Mon Sep 17 00:00:00 2001 From: Laurent VAYLET <laurent.vaylet@gmail.com> Date: Fri, 25 Nov 2022 10:05:22 +0000 Subject: [PATCH 106/107] move definition of variable with default value --- slo_generator/report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/slo_generator/report.py b/slo_generator/report.py index 23b4fbae..e643730d 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -67,7 +67,6 @@ class SLOReport: # Global (from error budget policy) timestamp: int timestamp_human: str - offset: int = 0 window: int alert: bool @@ -89,6 +88,9 @@ class SLOReport: # Data validation errors: List[str] = field(default_factory=list) + # Global (from error budget policy) + offset: int = 0 + # pylint: disable=too-many-arguments def __init__(self, config, backend, step, timestamp, client=None, delete=False): From 6b164bfaeb9adc99fdcab0d05a5139108231fce8 Mon Sep 17 00:00:00 2001 From: Laurent VAYLET <laurent.vaylet@gmail.com> Date: Fri, 25 Nov 2022 10:29:43 +0000 Subject: [PATCH 107/107] get class attribute with __dict__ --- slo_generator/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo_generator/report.py b/slo_generator/report.py index e643730d..94d09316 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -225,7 +225,7 @@ def run_backend(self, config, backend, client=None, delete=False): # Set offset from class attribute if it exists in the class, otherwise # keep the value defined in config. - self.offset = max(cls.getattr("DEFAULT_OFFSET", 0), self.offset) + self.offset = max(cls.__dict__.get("DEFAULT_OFFSET", 0), self.offset) LOGGER.debug(f"{self.info} | Running with offset {self.offset}s") # Substract offset from start timestamp

_2lW(zuPmkTz`E_q~-jkkeUAN+c&bqyp$!xA>Mcym64St0sN(;^e{BEEsR5p z*QAm=T$ziDtITC8;>C;eDJ7U53>1p_nrCHxk&cc|ad9zla(i`BQqr!y=sExTb_T1tM<65fC=JKA3x156>gw;t7i_Gp!-VfVi-_n!)%MoLSYvcsYirFD#>M>oM`tjr1TG6?4=coc+0u8i3!yI3X7UALI zQmIt%^4dJiXUu&m_j^x)#kIUkEFpETy_P1!wK-?Ll?ZNdTOT??PrnT+Saxg1pujw+ ztV}}qM;hG)y>?(pEbh!1UcczI=^nu~@R*f*%n>Yc4cuJU9B3Dtt<1dutN~!W(akt0Ph|Nw+x^ zISi4#`kBXbDF2a@KB~nB39^@kJhr~KU6kSBko^t`H^X-#b@9XLwQer*J9vC~-(a~`& zue-HXlO-fZ#>bc?M9J(Jxu}2uDGABL;9x%NJ|3R_HxBXn-1&Z4yjq;$YA*!XPM_8= zF{wgz)xN&}z=4cCk&%&yX=&{TE2YH5@LH|!-VwoW&Z}3F9*gOo!5n=T*2vLrgI>JQ z*U$*b-=luh$?0en?eQ`rQu@!9tthG=uidzMbqJtv*REYUvl3=f9E0QawDpHy^P91X zC=Rt$pHlvB1Lm_fdYFQOf;X}cQV3|Dj<$ApU!Q0pgIryu*M_g$ReSqiSonCMj$Z;r z(fbGqG-!?tvbxH?Cp4Q5|1rWvK)u>1G?4UB52qv1p04iTX*wAf_&oTwR1Z zXWN$A`2GF;ii(Pch6YiGfp6$+@9*!Wpc9&?38Hr%D7Ne41$bLu{8?F9iN-}sOWW>$ z$_|VbkPr0;tx-sLS@U1jhK`O7XxFJNl~@?@?gP13cNKYldPEcNy`BMx@fKkE0G$w+ z%(39Z^z>zpLvlVoK4!5et6q>w_?@n^#MIQvOkY=RZQ8+6 zO1b*_dH_=V@yzJxxuvQ}qmyMXmX?-2JUF0o@uFTC4FtkZA=b0eJqbq8PRMAv-@5%~ ziEPk$p78dq8?X!@jiwvx9)q{9UcG_@AborJRSEr0LIPpT#IhtNHWuBlVU_%tDrflC zxXkto86W9;Q07y@)symE%?0}cp|D=p-9dP#NU?Gl*z(T4uK$Z< zJxZo>&$XMFP1FD|r-KI%qUmWy+-y(R7_JT0Gop(Ys)w?-ckkXcKkB`E{jYv_zqHc`1rv)J9cRwt~;sv_s@xm{d7WgP|Xq&j*Ey8 zX{mLA&_s|eL=0Ra^rM^3&eOd`Q|LfxfHCtU=NkdjOt`T+a#Ad3$AD;td9iIz z_4q<^Qc}7PRH~-&g*SG6HA?7@X;;tL&aHXgzI~g9OhrXS@$r%O;|opkGC8HE@R)Rb zQ~rQ)g~#u;KaU^ppBx17U0CxpF&TVN7@J|`vH$z#(Dn?vfBUmNvAWh|weAPzjIBI2 zkDoprLJ0{?d6S-&R)Oz?FUjtEAo2?G^3JmZv2{ku^S*C3m zTnC3)FjMeVEPLSSNCRUHjXcYiB>XXCpRduPIWJ%On%StTs^V5im{BM7G@i&FPtV9m zG|tUX;|!;ggL-y;b82|l3CkBWO=CMw><0K&*VnIKySgGXBlBM5U#C+DJ}5_iOmbp) zxUJ&$3N${5ulEhyF!^`x5OCj2O!3(1x!*kRoU9pgbar+YvHy0-$cU#(cZ;8#jO-A} z?xRPK`ab>HmIgTA)!9iU2W1goY*Yc&29B6w3QRFZk=-ZW!Iu)fTn^BiJT~Fw%aLH2 zt%?3JQP$xMt+ZnMegl1dD+>!(Oe^||-zW+$QHTT5#$Ts3eSrc8Wcdx^Dkw5$&ZN|t zV`C4bRVYlN$IV8uHk1?<{XO2NmOgz-DW_yIWu^8v<|z)j&yn{O}=G+zcRvkx_qrI2W#^ zsd6v4U}OU$m`Fn;A8KvGrIb1T<;BGlzNV(8?VX(vgwT=;3k%E3%b|~!)eqwfM7*M* z@%vZ5#9i6%BO~%WCi$kH@BKv^tYB;~no*_gSr;Xm z&z3-81#jMnx(F8mh~cF&b8;e;Q{KOS?H@V42s3h>M8bx3|CF40;bQtbxIV@x^Q%#yE6hXWXeB4)}C!GLw@xW-u9F zN+u>IIVE%$F$VG?`M-LO-JT!W=l|h`lM}12yuAFMr7n|0>}sBWR!=jZ0J%6$33lpF zIS*{OECqGpOJgH66%J}r)VWh-hC^4OuG?B#J`V{AdG-t-FbicO<>gCtjtmvD`tRj? zn#H9KEcA7EOM3qilaPom%~4=%77-D#7Sw!0b%sHWl{4tc6ZFi10*m1I`1s^xR>s)e z8z4b)4=XC9Hg2|`?1SusTL3QHyZ;FPm3Mc5^@Hg}f1%t!%*)EjNwa^i$&nJ|@idyW zM2z7{T--5c-$$YA%QH5nrt)mfU0n|Hc?4BfZtm&syzA^2%hRkN1WKcma&jO(Kz=pgI%Rz|; znVFd+%QdMJQZF?PptETQ?Mo8*9=i5DCAjf`SAV+~f4T;d!mCm_#kSB?qB5A_s{ z`oRAEVd23Gbl<(27uV0CN}zPXv4{-`c|YVRASej?N(>;S=qThnT3*8=N00s-8(W5e zzemT~#wI#0PTKp=D&&>YQgJyYBO^wd`&`;!C{X+7=jVxVO-JXizy3M^4DNCUw6me9 zi3y{O3+~W8OhdCY8fgNJeR-ng!^e*&czA*+1+3do$<+~cx~4eQP49Lne*2hldnbLn zdwQ^@gCMfy=Rf1rN<1oP{tdb#L^(`WmDffw?lCKC3at425>qXKWrVxD03$ie-tVJb^vBucgSbHamI=bv-0|WRgnnR6$#dHy7!!gm1gMvV` zm)FMQ4^vZ*jE_?=iZ5ct)YQ~i3qHZyZ~a+QP*4EL1Wh}Bprk7oY8_+w~8fx0w+Avx`p}MsXQ&Ku&KD{^Bp=fgp3wPAl zb7DV;i$@t0L17RSENE}n#`o1#x5=(}6lzc;O+!OT+irefC^H@=apws@s-VY@iSjB$ zLg)*)osISNUp)o2pFWL3Gxz-c;~ZQIxF-#-cUjUhkXw9y#+FYU02FvMP`n?+;*&mu z!+J-Uut(i;#DdD4QfUGL0@`CA(u>+(h?DH8sd-S^Pn7-|RoXVlf|jRps=!=Lj#FMT zFc>It@~0HAYJE9oy&uo0vo4<6%MxM<_>0~It?o@;-U|`CUW{IJ3LBpAt~ur(q?cgU zTK(rM9+(R``@dl5dJye}Um|!N*@k;-|ALVJh9v)IoZ<61C2+l&*z+!NQ|WYdbPX#S z?1P%e)g>^$OM)I_h=v{LcnM0IqJ{8US>SU9g zZswMk#a;EXQw10l3AE15>?6=A`+qpi|DB5bV1PVb+#R_A+0~ZjM_e7pMdXC0K~^6z?UBSkWJBSKx=m!r40M+Xe_ud$@|Gv!{V!3{QIV#M` z-rnBI%F50TQmYUgE7aeDrqStXN$6rDBO`l?>`)aE6M%A1keBxpqME1YACyNhBzkQU z&$oq4Ot?p?k^W6J^Q~V6Z#o&&dcBr)@s@`xdGA-0T37J8FIre1YVVj3F|6&+bnPds8Tf6xbMfPA;|g&O4l8H}2RL_p-#j|f zAuKEyT-sbG*qp{IMJ1;xxG-xvwz2qKwD28T0?KqLEZLHh#utKV$S4?~+)Cg0ArBJ- ztp|0m(t9(Cl7*54`t{#LP5CL`Bk@>t#i_9|nC)lel+be?9H5t!k}@(h+}fP=f$fl% zl5$2&%yWBl1>Mo<)-B_5w;OhLJmE0r#7~~A>FMc#`GSN(eZ6-79(T+Z;Lw5fyI$n} zT1G~V(hWonvb9y=3B!55-*a;$6cmR8v21Usoiu}Pj+{uFve6eos;ZY)k{FfztMrdCu?*n)-*0f%fK@k<{}47_Rh$B(h5`0WoAGBBxtn0@{H zh?B@wP*}JKNHaDz=KJ&jy@+9zw-=hcA+Kvsz8Sjm+`>XuM#gzqIskRIZry^$4xtDR z6$vw5vEdsplbp>g=clop1u3x#ZmvI{g?1CKro}Y^QjGPZFMs{ zJ5|0RM=@9l+y(bf!rM=n6AuEI4Gz8-D*Sgte8`c#C=1QUvGLLm>zbcrEq`GVl%5)4 z@TpQZL%S4jh~0(RM4xVqIU~r!lMR@PTS`j0wm8v(4uf9XS8PW{LUP;}9V03#ib@W8 zURZdEZI8fR*=9gepjh<#oJ)DtA3l8O={du~LUio3qhP{;7wq1>+xNkPI7#d3)&?qDG32R(%07)0tkG{Lvos$n(2LO*0nnz4*hGw%5K?V!hkfqfTx#6 z$fd~PT7E#P0GgAm*~{7;xJ)Uxx$l4{)DfRk59CMPV7OvoQJcT}&We9!C-&j&mF0aE zo|`dEb`pic!ea;J>N)8t8|Y9o_4Pm1)^4n=ElspspKQ-?adClqgE0mPJ0fh|9&^U& z3B#Fdrlvw>_0$n&7*|a`>@5u)9br7cPW(!PXpcx11i^!H6Ak_-Q*gmFGlcl~a#B-k z8XJR*<3dCKmT{Dg{7kiUe)ju!exObONG*p4*0GZA^ANZ6UK{p6=&$x8%XsH^z{|@^ zjzr;LV|88KDIuXkNU@90o-j(lO4YfTbKYnEQgry|C%YtPB%1H;^gFgUcUydyFnMl4 zfuM-UN#8HHVx)r}-?)QC451C9Nx~cv5dnEvzF+L!$B(|mQLo(iB?0;O=SQ#u(r>)OOML6^-^bi8cBfKJDe`mvfx-t*c9Aec1_#hQ zdGdiOEose;=4S!c)?_N23~OTn85hZv>wk{=vC;JWm7?{)gE~T6efAhPLOFQhp7-ys zHbw~(0qmDAf94zbGIa2c2M!$AL@0ph*rKEkHSkrvv@aEI$G-dW<;#T6V7dDTrG&Zp zV>E3A!P7_8;v}ssE$c>aqG(;LrETB4$LmEXB@1m}z=gT1*nItsBMWN~rBRTf=9_87 zrlhbjpWx#Q9?-=0f}RSy=DKCeZt1m=Pz0)=>*?s~Mny+AJwKiNv&t~JarMq=@`U00 zOHR^*Udyz>S&DQL9h5RI69DunSD#wlt?%rFC(H7Y2tn6 z7Lk3`Vpaae7`j?op7Q^2b^8UPEEZ@KNl7fo4q2Z^jndaY8nSj$4hle;S}f=stSJNv zpv$J;t;Hs{v$G5DNGU$B$X@Y^oKf6SEC%^0+^M>XfPes?ae{5a=dCF!NWm`7&bS`Q z&B{nDEQlfu3}5nhH$Q#xqUZ5ZL9A2}43D|{N~gK!of`y|BYO#T9$^cdrzUE2d9I$`I>(rWpTn*A3N!03WXmqh zzAw$k3b-@T0*C@lk#z^85^1z% zT7*px4~IT|nu`sF6kD;wpf|E15WKGp@;5&kBPD;>IlSxfF1q$T`bYM@lQ9JFdVNnb z?rvrHi}SbMZ*Tm?Kgr12-Q~C5e}bl`|Jyer7OMEw_huHCIH9wcmM?oeAzm!v(~x83 z_SWsmqCcX+6W2Hlc|}pIMMNTu>!0FR9^Zfi+)#NvH>V-$!}-OGfwsiqw|9A?gk=9(JMssTbY_9N3{h%&o$Y^? zu4T`iVA=CO=?jXLtD0W@@VXzgvR>IjV&F4KXpWh5dK$E{YNq^nmw^&J;d_PV4K}9l zA39#j6w8uW<$wPNDYcop6pe@w3WZK)?~gt$LC9@s|9zIsAFJok#Lp|LBpH+<=2RM? zx(~E(5!`bA#X9ahu)h9*xrHwcSsMLLe2`BQ<_@)ti-I$rN0BC_vt&s0f=Z zj-Qb*AQ~XgLKR>=HR&c9_u!n>`bIs0KpG&Q1Z^_!(iiX78ieMjPo9uRHVB_S{V!75 z>p$g<%lNN|6puNeyUTIY2FiD!Wj`PJE8(-F7=x6N(Xpnd4WrhVsENFqSK(OHaP*Dl^6 z$`GZOL;D_%Q@~`OoMMBj)!0vZ$LAD^>bgh31Ha?Mp@Pu#Hk;6Fh`(u!-}uj%_s|9z zm}5QXxs2ciF5ixzI-Az|=7z-U^w;A$FaJfzM;2RFt87zjwmvp3=qoQ<>z%m%!tP^) zr)gHi@<|ZV9v(Pd7|uxOZvO9+_^)4ChP6O#KI#cwm3WJ(~+su(Q(A%ZtB4BLICPHLB(XXRBRvj*2;1AvN=`#b2{{g zSTB~56w^(mZy6Vw%9|n^IgK`>UT1e$m6rW zTm=?f<132B99zL6^U0K}=3I_Wcj!#HSUub_)Qk8IYaR?7w`AhAq$4L+4LhPP^%7?n zbVl^(gNF}yC7je4z5{d12qX=rC!PuxX|KMIz!Y=V*houoA)PEpa+lX+D=YH}VZm)R z^U8}9($RA1ZSKHLz?W!%tVB!Qp*#0Ik|R*2h>ai66=dN4{SUB^2XjnhAfN#v!#dII zK~m}ZwQCS5E6U5^h-ALHa90*V-y?Dl5gU%yEqVKPWNb`iB61G0v5SlF1~&qwswyh< zXKtRT`1s)itE8kMAWjaoY~2439n1nxgMIE?$nY_jukG8na`-~Nlr}a~2$&FwdWo9S zXntW~jD)K_1iSma3c-MoDBJMYVHrT+hNfq&eultAs&(C%{`x@tnNRIiphiPyD`lii0CS^)@LfDKMwltF%m{ zH;GO;-0vO>4sPxfzLzdvZc`CsP;+(`YM5@`32*1@&?u9lq9SC^lQs_`WIRrjKlQso zxMT3|m>qlaKg=$_5U66`?tAbNz}5j!+|W^xD}!eSkC7OqZ2eeMLySlvA`1=%UD)?2 zZiUHv^=eDfIf#p3L)a8ZlZEkxQeMN}vbI(x$;-`E?|TmB1P}{pI7YSM_R=Ka;s|VQ zL_tBZ^S>Eq{`RHB9WLu9_|g9-M2)ze>Z-*EcMhx@ur|q7-S1&vq_0~)6A$niSlhws z8>W+gAOx!`138#-q#vUM%t&2!{sVfU3oDW&@-dZ`l(3yRp_I}J$rj-^n6y2km;>h* z+*sd%l<-n6k{p%ta5gnP5p4*gKuAa^OgROnmkA6qFjGiJ4JVT;E2Wv`Pf%$bmKb8a zVroiFM@J;X2-91i#}fcrGt;_W%DBI3tT(OQ0mW=$Ax18E*Cp@Rt6=f~ioTTA40HXi zgYIr_lyb-yLZ?CWjD#66W=PElFoD&4hR0#i7O{N#6hj%%peT|Mu|Iw*O1$?EL+GZp2zFImbGoa@zttDlih z_9q_%WHvQ7SJ+0th+bi*U*t;q(|cZ~O|o9CP9@(^(c1FUgNxU%Ll}4Zy<*IK()NDQ;eA5^g9I$^m)qDH9 z)!B?RFGcjDV#Z!7xeGz=JqP&Onw$AgoEU)_JM`NP9wkmNz@15nJ8YbH?VOvN+pSy6 zFmuPg#;TB<=I4LTf3-SF*ye(&DiSP2B+RA~af<7O2j$SeILDRkfos;*bkx*^wmta> z9uZwQx4A30?)<->AWeo;bAbFjPAx!OKq_;5F|)i}5)XQmjqtfEC@MC^oC$3jhYgPK z98?;XQ>T=c)z~}}u#;I>S?Myps;V&gru(4rZO?#MfseuaqaxW&q+$E_zwFX<(D(50 zz}zDqgRIuE3S|-9xy41IfthTIDhd?{YDczVNj?&be?Qo-pL=D;#)Gpeordpsw0IEV z(0egLf(DwdD^1w^{zd48`D9yz6oDGEM{qCgvnMm^e!*1(+38{Xxlf2RgUCz+8B zA}XfsHfs(tpVK1kn!uwnTse!1(v8`wz9G>7g0J5}2uj%WXFg310G-exFI?p8D0{x~ ziyl3LP$b&6i0dK3z+tufmCby!@~t2Gf@ypP1Z#HqQu1v)!6n&{v$v#qT=ai3yr)zs z$#{~z84bH=sr|tpvG2c!Yb0s@BKSUi)+%alry8EE^SkoBLWBDt!7@)f{=RK_EuR)q zvFCa8h-(c-&%fBg3mgMNt4J-0Cd^&D+FGA`v|ty3UX_u_lB{M2f$4P|d8hAF(yQ0U zP6}I@DVgoPv!dL#N`T;QnGTs1;4%uKq=ZMQzjxs2@Mf4)iL?y=CoOFdH+YozKRa<2 zP<8_ffqcOrybpy?rIe^`e!+i?6gZPTXTA^;I668zCnv4f*R~S9s}2$j1dav6oSC$? zahE(MMIk{bcYr#4f~upl{{OQP3lxa0y8!_jt^bCcJH4P zhJ_Se#6bLi#Tmp3G;R@BSWkomv-f{4;NN$?z~TQj%md@_KRh;>N^xAeuD%}Y9)y91 zUci*3NCSq<%f@j?_@4t1juX%S?VJeDSD=;<$OY5i@62;?bNHfB@w7--M)%geAu#EVV{3k&n} zH-G+Yg-|cfyr@49gj2F}=gyj7CSul|#2sETGSD826Q^O;Akx!fC5^x^q+xjD*Wqzv z{rOU6Xs-h&tZi%{=L6jI2ZU%oMME!Odwq(pr{NQ677qU4XyGk{zf#p=*O8Ga9bo?Z z0e}5ftyvrn5O$vjx*P|KfSRs-in`wZCKQ!_e~_uuM> zLr{ZyvY2+<4#9lDahT=6du7f)pDCRXhn3Gff?uP>_&$C6mY9=7I?ANvW=d`@KQr^) zdNHfk6AH?5O3*JM&?+g_VJYo{K7ERt+ZM?}ocLoBbNr4I71U!%QttC$1a4>$@YDeY zp_GAt0EVpML;`XS7g9cd`BGS1e9Os6<-&z;Z*Qmxp-4!|1dFL!ssWnji-wpNGX&6uhL z;v`9Sc7^w*2M%uDy?YkC$un4k!%r^p5;p=nsT;a3XMCUj`_l+@v+^Qk3O%p&G&rKK z-%(Z&<`WfNfnDI70ugdyyvYP-zQD7PkNX*$a<7-0w9FnV7F-rsm}qJ+B%H5_C>Hg` zXqO^Wl^x7HV}I6rEaZG^d$ zO{!flr=K3IGSPq2aLkA_c=y_jPDL7>^GJj1L+`VCE1?c0y@$>YeYLYZWAIJ&K>r*9ar|FZ*N;y(C*FTl zG5GckNm2Y`o`>4I5D5W7<_t&V5YE96L|r_BSI3cD7!aME%Ruwm^&knj!BO~hDv@|q0S1fB6+~Sp0%1|VsVRtUmwW3$KhiO-S4i_gUyi+wq=BP=* z=}{ic-h_n&ia($+M@_KRM8Zk-xZC<7NAfAZqB;u;AIfbU(I|*xTYz4ezn$H zrSJz<#yw>;>lAJr5d2x4eO*xB#bWW>lfMgM^VZd zZNF6Tb+#hq$oc`X0H=LMZkAY!MyOVxE^c7f6@%!^u3h^vKCVcS|9LQWFz>qJFUzP$_}Y%m)U5~dK86={5pbiur=h*bPIwE4l=sBx?+GN$@*VH z%_ms}kf+#w_!si>gntC`d# zJyFWL^am4(TR$j=VvJDYIVrBW#h+So52r^{ALzD#rQ%%gUJ+Nw#|BFWU z^5DT6FwT&;ZOuxrOyRR9^Q%TtOg<=;n#-`Z`*Cb~F-u38v0X~x9=n4sk;Jnls0_}x zZjn<^L>j65W&AicNjy*v`c0&r=ikCVUaCi=ptK-`S9N;@rzpIjj(DOm;RNh~BmA-= zImFDTO~-QPPYmTLp5$u;`p6k>Eq3;wNk=>%>f=X<-O zvZH`o@p9Su>>97dQ{Og%+ybu*8u8ri{Ba)w4w@YLbjW%aC?&pNt3dV0%_SaGLo-2u z!}saEd-o8$U}a6l!6WX=#W-Q7#}YDuQ%g`Z6IDW0GjdA-p7So*T3K;~C?WjC2eDZ* zqu0=v5{DN+abaG-^;|&Z0Cq@WqVYr}Mo1b&6?H>amcVTK<6{7N+=&yW0EZ}mY`#dA z=~sS1R~JG=9;X)Dhmu}BI(vT)%*_Y>{ys|`dPsqA*5e}=<{WqQWUvc9h-iuD}7+enkyxvpQQ&y@Fx%!4$p!6 zfi7#2;~{e+9=Ou$DB*bSyMFuLFGr1qb_VEZgDmCvGLYNb%~e4ym>TZuF{c zBo<-*zcsW+jsvj(Z4HgHQazl}q7c9^q*z&nh0d7MdJmoA(abWVt7#mbqUzn_kH3b= zP*L0GOiF#)s?+Oc&g^S@*ru#c(V&-IDwWxl`DyElR(}Plt37Ky_JsAf2U$|0G`}_# z%Fm^5n%XuOmpkT7Du5xfB0LF5xy;&^Oih$bi zNJxN=d3bo1=I1pvz9Mq2L|u*-U0L z8X-pKnsS?3i){3eup)kH$-~M@Z_JOEr{~B5B2W-}t^kQ*_ET$4Tf4?j+Us>%&Cl*3 z-H16m6mq<(B6CIi{u9r&f7^2ZFI>j++101d2GDjc<*<9i$uZ-DI5oB3C2~|1!=XL}H21>Zi zC0ybr_BqIW+WVjE07He*(<|gM*^anArXTJG&Wz(DBVLwV-c4ve^0o@Bj&R{E(Vrq* zU^4ZbiyN!4Q_q`br|IxqZ=n@C%ugVAe#1%cghatB#oa#@^+jHpeO>JlfQv&g=-U5# z$8eYSr7`P?{m*%Bt!qRaX#4Rv-oW$Yh}W0rh;s5gP5phhdd)}YVqa8P*h`OlSJtdf zhd1`e;7la$Y<~VGG1ygs6U8{hI+3Z<^dZZZ>S~f_v58;kBhDB*-KE9kOnQYvp~Jbo zYPVcmj1XhO@JmTa0S&8}>51uN#mY0mmebfhU(h@=?)dXM)xZqfU*^zc z5yygSOiBFo0M>A2ny(QX%%tCoYuE<#p$BG7eYTXb*O#Ha_rSD(%R&Vq1DpghgrbNfFTAvxiy)UGf|uo z>#ujNfyVw`_4#u&#qp!K0rtBcKMdblOXzckPpb$ z7)*~J(K*S_?+Dn8YKI7t*2ZO_$LB@p+iw!b!d7PEr{XAmjWpYazH-mYX{3?x2^{{y zE2jvAav6~4#jr?7jJ}kK>d)ii;FqXv&E_;`@i236d`8bR?oJ^2FfdRFVQbK5dlQuw zg*)qfLtmlQ-`Ob^apC>DJv23><8w+k@$T~+MQT?&rnLmb&xne4;m_l4hk3{)`85$g z>t!}rp^52K%)I7IVRn9Z)9K7>d%ob~L(rhG(eari)~z=^va40qMRsIWXUGY^@YbGL zZr#`I8k--6_dG=EhX#cZ3&sBZ+7dc8fwBJ7@MKlm#t)6Am75CMWpPJ1JS%L)2=VnYo*EYBHG~x2>kfcj`vuzk2m$aL@;33gIxEV`H97wGOw_ z0fK@{(%ROB&!NBwS9|SW;4263KyVwK_2n-z0zDE?sMB8f2oNXH-B`#;6k{ug!a4Z( z29Zbt_c;g!ieMS2(8ON6n|QE4Ie8eRJi3e-4>JV^4^{jBXY$_&E)opSmV&6A{{FKNM-6ihPvwWke@z8_(zwcyKL6x^G2UqXGxXg*+OG`rX%`j6ZL z_pdGwj?=WaTrObgOFlm+i--V;q79kRQg(f5&sbX8(PQy1O{5yyCzrGN=6y?zhbD&h z)vS^%$GsY>8tc{bG>do(6bm`pJBEPSL*;cRZpUoc4rqDbqPd)di z;d}KT9Q_yT)gMv~81@dG*yfV5ru3=pC?%0P*kTo(KTS6Ni!w4dasfVNht0MJQ?+j+t8(;T%c(y$IG52 zQ~y|`){iHY-8Q@}?Qb}neC+!)#$Lz_9Y@yvig4ENR=mZEkA2q+)-TdrA_B-e`ofbV#^8s4|}bj#l3Q1I+Lz{O!Rxe z`hZ=<>nnc{pl8q#{TvWckagh51LN4Mo)aEY2c;tHns{g&7*b=U3{Is}$M^iOt(q>_ zm2|%L%P}4XtEcfzzvEk0MwxH?9RGS(V>oBxmZT$7G3{;46c77a-cfmE+9A8$AlgYon zF-b+Gc3C~o+B1LP)@Up!?V{QP&Mje~*J=Hi|4kbP$vI4O=XUL?+l&x^47;n?XUQ5YMPw=XV(}!VKCwrQF?41zxkyktv<_n@=&A2x@)NHYe6IP za}}OO`421t9#XON%tp!|YNP$cQFyM*>VtFJgC(2W zu_nx0cG25$(!zIxMfg88)fAhYIrV@{UHyA$QK{S?_1*J3unCn59XnL3jtKhj*1j;k zI$NdYb~A86wrJ(2)q}w6J+J!X9D>s&a=$)Tt4LE!eR|HI(@>mNeT-b~U^ zd$e|L)TFe-^Oc72n+BgZ{ds9iYw1p}CoOFP6eetYkdjHSfoxJt*&ez31eqU_+ z=F0YTYmk#lc-ZT++TWi{t$wxU)573JF0L5!sr&9HG5!Q)a$AGA7wKuw^S0>Te^-^; z{`z744P}hi1geHmdOpG~Gt*LU-uriJ>zCr^xvSUSK7C~0&#b-g<}w`zmY11vUxF|D)X&L^DvzIX z^=WkQgZTMB%J$h_JCav-;giQK!{3hGM^EgTnxFqW<5j@jY-44y<Hdz* zegAyQj#8Vs+cbaou}aOZG`zknQL}hcz5Yt`&oWlPeSIfBy>NN#y?th}eD_SF5b+%g z7)paPi!@xmIJHGy4%g%Zd-Le58*hF-OBDb3`umRSZ_9Z59-E!*?dVSEd@)t>nBvDf z)vfPli9J0RXr;6&kI$s9_Hn@5OE24mdr!T#j>%h}e^m!q+WSUzl^va{rt7X6n z_sq#(Up{~Pxwp6M>9Z&UtM~uzTmALq zUrVK1k?$hEObnfp@iFD{6z$gV?Kk&Zz2J-qXxO!~CgP3FHI4grLFd1Q?mzpZZt86( zCWe+K_cwK#2U^{hU!C*$uU&QDer?m!kGH)k-)^sb;o^>ix8D4`SibjmOvqL4WGzN6 zu7!E!$wz*<0A~d^1A{z0F5BXg!h6e_eSSN>aGlWyHXG(_eZaPO$q%udjGYJCuNwk8 zalkgm{tJ;e&gsWI*flr)zzO|V8!f=!FhY}UFT7^%K>m;RS$ucQT;CJUX{_c2};zwq_9q$i*$#DK%VS{$2m d;12yO?kLn!yl(Cr;JGdg44$rjF6*2UngBPyiK74j literal 0 HcmV?d00001 From 0f3744250385677383214d62c6c08838b2846954 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 29 Sep 2021 17:56:15 +0200 Subject: [PATCH 050/107] Update latency_slo_distrib_metric_cloud_monitoring.md --- docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md b/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md index 912ea470..063614cc 100644 --- a/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md +++ b/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md @@ -65,7 +65,7 @@ This looks like: {"bucketOptions": {"exponentialBuckets": {"growthFactor": 1.4142135623730951, "numFiniteBuckets": 64, "scale": 1.0}}} -\``` +``` Once we have the type of buckets (Explicit, Linear, Exponential) we can calculate the min and max values ​​of each bucket of the distribution type metric. From ec4534edf4e6f9721fed6cd120401896dc6ae67c Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 29 Sep 2021 17:58:03 +0200 Subject: [PATCH 051/107] Rename docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md to docs/practices/latency_slo_distrib_metric_cloud_monitoring.md --- .../latency_slo_distrib_metric_cloud_monitoring.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{pratices => practices}/latency_slo_distrib_metric_cloud_monitoring.md (100%) diff --git a/docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md b/docs/practices/latency_slo_distrib_metric_cloud_monitoring.md similarity index 100% rename from docs/pratices/latency_slo_distrib_metric_cloud_monitoring.md rename to docs/practices/latency_slo_distrib_metric_cloud_monitoring.md From 07aaf863971d3dc023b4b49f0feffa6ab08a3f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Labesse=20K=C3=A9vin?= Date: Wed, 3 Nov 2021 16:04:20 +0100 Subject: [PATCH 052/107] doc: missing labels key Signed-off-by: Kevin Labesse --- docs/shared/metrics.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/shared/metrics.md b/docs/shared/metrics.md index b50c95ed..1c0fefe6 100644 --- a/docs/shared/metrics.md +++ b/docs/shared/metrics.md @@ -22,9 +22,10 @@ additional labels: `good_events_count` and `bad_events_count`. For instance, if the SLO config contains the following metadata: ```yaml metadata: - env: dev - team: devrel - site: us + labels: + env: dev + team: devrel + site: us ``` The `env`, `team`, and `site` label will be available in the exported metric automatically. From 64dd45212567432e89281c253c491a8e0e3bc1d7 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 25 Nov 2021 14:43:02 +0100 Subject: [PATCH 053/107] fix: Set Python version to 3.9 (#179) --- .github/workflows/build.yml | 5 +++-- .github/workflows/test.yml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d2747cb..6f7fb090 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,6 @@ on: - deploy-* tags: - v*.*.* - pull_request: jobs: cloudbuild: runs-on: ubuntu-latest @@ -15,7 +14,9 @@ jobs: steps: - uses: actions/checkout@master - uses: actions/setup-python@v2 - + with: + python-version: '3.9' + architecture: 'x64' - name: Check release version id: check-tag run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80eae57d..2f51be87 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@master - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.9' architecture: 'x64' - name: Install dependencies @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@master - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.9' architecture: 'x64' - name: Install dependencies From d5d1516d8db7f017bae8f4817d48d020376cb503 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 9 Dec 2021 11:59:14 +0100 Subject: [PATCH 054/107] fix: bigquery exporter lint (#183) --- slo_generator/exporters/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo_generator/exporters/bigquery.py b/slo_generator/exporters/bigquery.py index c532c29c..a7e15db8 100644 --- a/slo_generator/exporters/bigquery.py +++ b/slo_generator/exporters/bigquery.py @@ -92,7 +92,7 @@ def export(self, data, **config): row_ids=[row_ids], retry=google.api_core.retry.Retry(deadline=30)) status = f' Export data to {str(table_ref)}' - if results != []: + if results: status = constants.FAIL + status raise BigQueryError(results) status = constants.SUCCESS + status From 71c082023f0e0023e3c761c181a465c06ed9ab90 Mon Sep 17 00:00:00 2001 From: Yasser Tahiri Date: Thu, 9 Dec 2021 12:39:58 +0100 Subject: [PATCH 055/107] style: formatting improvements (#178) --- slo_generator/api/main.py | 10 +++++----- slo_generator/backends/cloud_service_monitoring.py | 3 +-- slo_generator/backends/datadog.py | 3 +-- slo_generator/exporters/base.py | 2 +- slo_generator/migrations/migrator.py | 6 ++---- slo_generator/utils.py | 9 +++++---- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/slo_generator/api/main.py b/slo_generator/api/main.py index 61fe2604..d614c27b 100644 --- a/slo_generator/api/main.py +++ b/slo_generator/api/main.py @@ -49,15 +49,15 @@ def run_compute(request): list: List of SLO reports. """ # Get SLO config - if API_SIGNATURE_TYPE == 'http': - timestamp = None - data = str(request.get_data().decode('utf-8')) - LOGGER.info('Loading SLO config from Flask request') - elif API_SIGNATURE_TYPE == 'cloudevent': + if API_SIGNATURE_TYPE == 'cloudevent': timestamp = int( datetime.strptime(request["time"], TIME_FORMAT).timestamp()) data = base64.b64decode(request.data).decode('utf-8') LOGGER.info(f'Loading SLO config from Cloud Event "{request["id"]}"') + elif API_SIGNATURE_TYPE == 'http': + timestamp = None + data = str(request.get_data().decode('utf-8')) + LOGGER.info('Loading SLO config from Flask request') slo_config = load_config(data) # Get slo-generator config diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index 2c431864..e87d7aa8 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -257,8 +257,7 @@ def build_service(self, slo_config): """ service_id = self.build_service_id(slo_config) display_name = slo_config.get('service_display_name', service_id) - service = {'display_name': display_name, 'custom': {}} - return service + return {'display_name': display_name, 'custom': {}} def build_service_id(self, slo_config, dest_project_id=None, full=False): """Build service id from SLO configuration. diff --git a/slo_generator/backends/datadog.py b/slo_generator/backends/datadog.py index e9f69bd3..dbe0e97e 100644 --- a/slo_generator/backends/datadog.py +++ b/slo_generator/backends/datadog.py @@ -95,8 +95,7 @@ def query_sli(self, timestamp, window, slo_config): query = self._fmt_query(query, window) response = self.client.Metric.query(start=start, end=end, query=query) LOGGER.debug(f"Result valid: {pprint.pformat(response)}") - sli_value = DatadogBackend.count(response, average=True) - return sli_value + return DatadogBackend.count(response, average=True) def query_slo(self, timestamp, window, slo_config): """Query SLO value from a given Datadog SLO. diff --git a/slo_generator/exporters/base.py b/slo_generator/exporters/base.py index 2846fd66..a2e36813 100644 --- a/slo_generator/exporters/base.py +++ b/slo_generator/exporters/base.py @@ -183,7 +183,7 @@ def build_data_labels(data, labels): for label in nested_labels: data_labels.update({k: str(v) for k, v in data[label].items()}) for label in flat_labels: - data_labels.update({label: str(data[label])}) + data_labels[label] = str(data[label]) LOGGER.debug(f'Data labels: {data_labels}') return data_labels diff --git a/slo_generator/migrations/migrator.py b/slo_generator/migrations/migrator.py index 5be17521..91ef5a93 100644 --- a/slo_generator/migrations/migrator.py +++ b/slo_generator/migrations/migrator.py @@ -421,11 +421,9 @@ def report_v2tov1(report): # If a key in the default label mapping is passed, use the default # label mapping elif key in METRIC_LABELS_COMPAT: - mapped_report.update({METRIC_LABELS_COMPAT[key]: value}) - - # Otherwise, write the label as is + mapped_report[METRIC_LABELS_COMPAT[key]] = value else: - mapped_report.update({key: value}) + mapped_report[key] = value return mapped_report diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 40263ce9..ce6eaf11 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -89,13 +89,14 @@ def load_config(path, ctx=os.environ, kind=None): config = parse_config(content=str(path), ctx=ctx) # Filter on 'kind' - if kind: - if not isinstance(config, dict) or kind != config.get('kind', ''): - config = None + if kind and ( + not isinstance(config, dict) or kind != config.get('kind', '') + ): + config = None return config except OSError as exc: - if exc.errno == errno.ENAMETOOLONG: # filename too long, string content + if exc.errno == errno.ENAMETOOLONG: return parse_config(content=str(path), ctx=ctx) raise From c7e4649356c56c1f39261b35b1363985d4f79c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Labesse=20K=C3=A9vin?= Date: Fri, 10 Dec 2021 09:58:40 +0100 Subject: [PATCH 056/107] docs: correct bigquery.md (#182) --- docs/providers/bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/providers/bigquery.md b/docs/providers/bigquery.md index f5eceb34..fe963753 100644 --- a/docs/providers/bigquery.md +++ b/docs/providers/bigquery.md @@ -10,7 +10,7 @@ to improve on the long-run (months, years). ```yaml exporters: - - class: Bigquery + bigquery: project_id: "${BIGQUERY_HOST_PROJECT}" dataset_id: "${BIGQUERY_DATASET_ID}" table_id: "${BIGQUERY_TABLE_ID}" From f6d843a254e44ed81a9699b820fdf88bc5e3aa0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Labesse=20K=C3=A9vin?= Date: Tue, 18 Jan 2022 16:04:13 +0100 Subject: [PATCH 057/107] feat: search SLO configs in subdirectories (#181) --- slo_generator/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo_generator/utils.py b/slo_generator/utils.py index ce6eaf11..8ff97dba 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -55,7 +55,7 @@ def load_configs(path, ctx=os.environ, kind=None): """ configs = [ load_config(str(p), ctx=ctx, kind=kind) - for p in sorted(Path(path).glob('*.yaml')) + for p in sorted(Path(path).rglob('*.yaml')) ] return [cfg for cfg in configs if cfg] From 23b83cfe40909030282a7d7cf1c861f16e9b0680 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 18 Jan 2022 16:48:11 +0100 Subject: [PATCH 058/107] Revert "feat: search SLO configs in subdirectories (#181)" (#189) This reverts commit 575f2f701d43e8dfbb9505b98a81ab76dde79d50. --- slo_generator/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 8ff97dba..ce6eaf11 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -55,7 +55,7 @@ def load_configs(path, ctx=os.environ, kind=None): """ configs = [ load_config(str(p), ctx=ctx, kind=kind) - for p in sorted(Path(path).rglob('*.yaml')) + for p in sorted(Path(path).glob('*.yaml')) ] return [cfg for cfg in configs if cfg] From b8ff3c0d9b03859cec54c0f77ae608a18ec55f73 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 18 Jan 2022 17:02:53 +0100 Subject: [PATCH 059/107] fix: dependency issue with google-cloud-monitoring for the bazillionth time (#191) --- Dockerfile | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 43abdbf3..467bed97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.7-slim +FROM python:3.9-slim RUN apt-get update && \ apt-get install -y \ build-essential \ diff --git a/setup.py b/setup.py index 30a9b058..18bb5832 100644 --- a/setup.py +++ b/setup.py @@ -44,10 +44,10 @@ 'dynatrace': ['requests'], 'bigquery': ['google-api-python-client <2', 'google-cloud-bigquery <3'], 'cloud_monitoring': [ - 'google-api-python-client <2', 'google-cloud-monitoring <2' + 'google-api-python-client <2', 'google-cloud-monitoring ==1.1.0' ], 'cloud_service_monitoring': [ - 'google-api-python-client <2', 'google-cloud-monitoring <2' + 'google-api-python-client <2', 'google-cloud-monitoring ==1.1.0' ], 'cloud_storage': ['google-api-python-client <2', 'google-cloud-storage'], 'pubsub': ['google-api-python-client <2', 'google-cloud-pubsub <2'], From cd04d6ab1f28919180059453446449ee7a9c7d75 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 19 Jan 2022 10:56:55 +0100 Subject: [PATCH 060/107] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 66f26a4a..6b16eed4 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ ![test](https://github.com/google/slo-generator/workflows/test/badge.svg) ![build](https://github.com/google/slo-generator/workflows/build/badge.svg) ![deploy](https://github.com/google/slo-generator/workflows/deploy/badge.svg) - [![PyPI version](https://badge.fury.io/py/slo-generator.svg)](https://badge.fury.io/py/slo-generator) +[![Downloads](https://static.pepy.tech/personalized-badge/slo-generator?period=total&units=international_system&left_color=grey&right_color=orange&left_text=pypi%20downloads)](https://pepy.tech/project/slo-generator) `slo-generator` is a tool to compute and export **Service Level Objectives** ([SLOs](https://landing.google.com/sre/sre-book/chapters/service-level-objectives/)), **Error Budgets** and **Burn Rates**, using configurations written in YAML (or JSON) format. From 6ca4d71dd11e36d002dd2937bcb00d1b80265ab6 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 19 Jan 2022 16:04:44 +0100 Subject: [PATCH 061/107] fix: better exception handling and error logging (#187) --- .../slo_gae_app_availability.yaml | 3 +- .../slo_gae_app_availability_basic.yaml | 4 +- .../slo_gae_app_latency_basic.yaml | 4 +- ...gke_app_availability_basic_deprecated.yaml | 4 +- .../slo_gke_app_latency_basic_deprecated.yaml | 4 +- slo_generator/backends/cloud_monitoring.py | 1 - .../backends/cloud_service_monitoring.py | 1 - slo_generator/backends/datadog.py | 1 - slo_generator/compute.py | 44 ++++++------ slo_generator/exporters/base.py | 24 +------ slo_generator/report.py | 67 ++++++++++++------- slo_generator/utils.py | 11 +++ tests/unit/fixtures/exporters.yaml | 4 +- tests/unit/test_compute.py | 11 ++- 14 files changed, 99 insertions(+), 84 deletions(-) diff --git a/samples/cloud_service_monitoring/slo_gae_app_availability.yaml b/samples/cloud_service_monitoring/slo_gae_app_availability.yaml index a2e97de3..50e5e8f9 100644 --- a/samples/cloud_service_monitoring/slo_gae_app_availability.yaml +++ b/samples/cloud_service_monitoring/slo_gae_app_availability.yaml @@ -11,7 +11,8 @@ spec: error_budget_policy: cloud_service_monitoring backend: cloud_service_monitoring method: good_bad_ratio - exporters: [] + exporters: + - cloud_monitoring service_level_indicator: filter_good: > project=${GAE_PROJECT_ID} diff --git a/samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml b/samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml index dd6507ff..c857e925 100644 --- a/samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml +++ b/samples/cloud_service_monitoring/slo_gae_app_availability_basic.yaml @@ -1,11 +1,11 @@ apiVersion: sre.google.com/v2 kind: ServiceLevelObjective metadata: - name: gae-app-availability + name: gae-app-availability-basic labels: service_name: gae feature_name: app - slo_name: availability + slo_name: availability-basic spec: description: Availability of App Engine app error_budget_policy: cloud_service_monitoring diff --git a/samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml b/samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml index e648e99d..3f82c372 100644 --- a/samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml +++ b/samples/cloud_service_monitoring/slo_gae_app_latency_basic.yaml @@ -1,11 +1,11 @@ apiVersion: sre.google.com/v2 kind: ServiceLevelObjective metadata: - name: gae-app-latency724ms + name: gae-app-latency724ms-basic labels: service_name: gae feature_name: app - slo_name: latency724ms + slo_name: latency724ms-basic spec: description: Latency of App Engine app requests < 724ms error_budget_policy: cloud_service_monitoring diff --git a/samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml b/samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml index 16fc4ec0..f9564905 100644 --- a/samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml +++ b/samples/cloud_service_monitoring/slo_gke_app_availability_basic_deprecated.yaml @@ -1,11 +1,11 @@ apiVersion: sre.google.com/v2 kind: ServiceLevelObjective metadata: - name: gke-service-availability + name: gke-service-availability-deprecated labels: service_name: gke feature_name: service - slo_name: availability + slo_name: availability-deprecated spec: description: Availability of GKE service error_budget_policy: cloud_service_monitoring diff --git a/samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml b/samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml index ced3999d..85908ed1 100644 --- a/samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml +++ b/samples/cloud_service_monitoring/slo_gke_app_latency_basic_deprecated.yaml @@ -1,11 +1,11 @@ apiVersion: sre.google.com/v2 kind: ServiceLevelObjective metadata: - name: gke-service-latency724ms + name: gke-service-latency724ms-deprecated labels: service_name: gke feature_name: service - slo_name: latency724ms + slo_name: latency724ms-deprecated spec: description: Latency of GKE service requests < 724ms error_budget_policy: cloud_service_monitoring diff --git a/slo_generator/backends/cloud_monitoring.py b/slo_generator/backends/cloud_monitoring.py index 14213c3a..86df3e01 100644 --- a/slo_generator/backends/cloud_monitoring.py +++ b/slo_generator/backends/cloud_monitoring.py @@ -206,7 +206,6 @@ def count(timeseries): try: return timeseries[0].points[0].value.int64_value except (IndexError, AttributeError) as exception: - LOGGER.warning("Couldn't find any values in timeseries response") LOGGER.debug(exception, exc_info=True) return NO_DATA # no events in timeseries diff --git a/slo_generator/backends/cloud_service_monitoring.py b/slo_generator/backends/cloud_service_monitoring.py index e87d7aa8..6e4a6dba 100644 --- a/slo_generator/backends/cloud_service_monitoring.py +++ b/slo_generator/backends/cloud_service_monitoring.py @@ -235,7 +235,6 @@ def get_service(self, slo_config): sids = [service.name.split("/")[-1] for service in services] LOGGER.debug( f'List of services in workspace {self.project_id}: {sids}') - LOGGER.error(msg) raise Exception(msg) LOGGER.error(msg) return None diff --git a/slo_generator/backends/datadog.py b/slo_generator/backends/datadog.py index dbe0e97e..b9ee6a6c 100644 --- a/slo_generator/backends/datadog.py +++ b/slo_generator/backends/datadog.py @@ -182,6 +182,5 @@ def count(response, average=False): return sum(values) / len(values) return sum(values) except (IndexError, AttributeError) as exception: - LOGGER.warning("Couldn't find any values in timeseries response") LOGGER.debug(exception) return 0 # no events in timeseries diff --git a/slo_generator/compute.py b/slo_generator/compute.py index 3fd9a8a0..c70bc6f2 100644 --- a/slo_generator/compute.py +++ b/slo_generator/compute.py @@ -65,19 +65,20 @@ def compute(slo_config, timestamp=timestamp, client=client, delete=delete) + json_report = report.to_json() if not report.valid: + LOGGER.error(report) + reports.append(json_report) continue if delete: # delete mode is enabled continue LOGGER.info(report) - json_report = report.to_json() - if exporters is not None and do_export is True: - responses = export(json_report, exporters) - json_report['exporters'] = responses + errors = export(json_report, exporters) + json_report['errors'].extend(errors) reports.append(json_report) end = time.time() run_duration = round(end - start, 1) @@ -94,11 +95,14 @@ def export(data, exporters, raise_on_error=False): exporters (list): List of exporter configurations. Returns: - obj: Return values from exporters output. + list: List of export errors. """ LOGGER.debug(f'Exporters: {pprint.pformat(exporters)}') LOGGER.debug(f'Data: {pprint.pformat(data)}') - responses = [] + name = data['metadata']['name'] + ebp_step = data['error_budget_policy_step_name'] + info = f'{name :<32} | {ebp_step :<8}' + errors = [] # Convert data to export from v1 to v2 for backwards-compatible exports data = report_v2tov1(data) @@ -107,24 +111,20 @@ def export(data, exporters, raise_on_error=False): if isinstance(exporters, dict): exporters = [exporters] - for config in exporters: + for exporter in exporters: try: - exporter_class = config.get('class') - instance = utils.get_exporter_cls(exporter_class) + cls = exporter.get('class') + instance = utils.get_exporter_cls(cls) if not instance: - continue - LOGGER.info( - f'Exporting SLO report using {exporter_class}Exporter ...') - LOGGER.debug(f'Exporter config: {pprint.pformat(config)}') - response = instance().export(data, **config) - if isinstance(response, list): - for elem in response: - elem['exporter'] = exporter_class - responses.append(response) + raise ImportError(f'Exporter {cls} not found.') + LOGGER.debug(f'Exporter config: {pprint.pformat(exporter)}') + instance().export(data, **exporter) + LOGGER.info(f'{info} | SLO Report sent to {cls} successfully.') except Exception as exc: # pylint: disable=broad-except - LOGGER.critical(exc, exc_info=True) - LOGGER.error(f'{exporter_class}Exporter failed. Passing.') if raise_on_error: raise exc - responses.append(exc) - return responses + tbk = utils.fmt_traceback(exc) + error = f'{cls}Exporter failed. | {tbk}' + LOGGER.error(f'{info} | {error}') + errors.append(error) + return errors diff --git a/slo_generator/exporters/base.py b/slo_generator/exporters/base.py index a2e36813..7c5efee9 100644 --- a/slo_generator/exporters/base.py +++ b/slo_generator/exporters/base.py @@ -19,8 +19,6 @@ import warnings from abc import ABCMeta, abstractmethod -from slo_generator import constants - LOGGER = logging.getLogger(__name__) # Default metric labels exported by all metrics exporters @@ -81,7 +79,6 @@ class `export_metric` method. metrics = config.get('metrics', DEFAULT_METRICS) required_fields = getattr(self, 'REQUIRED_FIELDS', []) optional_fields = getattr(self, 'OPTIONAL_FIELDS', []) - all_data = [] LOGGER.debug( f'Exporting {len(metrics)} metrics with {self.__class__.__name__}') for metric_cfg in metrics: @@ -103,26 +100,7 @@ class `export_metric` method. } metric.update(fields) metric = self.build_metric(data, metric) - name = metric['name'] - labels = metric['labels'] - labels_str = ', '.join([f'{k}={v}' for k, v in labels.items()]) - ret = self.export_metric(metric) - metric_info = { - k: v - for k, v in metric.items() - if k in ['name', 'alias', 'description', 'labels'] - } - response = {'response': ret, 'metric': metric_info} - status = f' Export {name} {{{labels_str}}}' - if ret and 'error' in ret: - status = constants.FAIL + status - LOGGER.error(status) - LOGGER.error(response) - else: - status = constants.SUCCESS + status - LOGGER.info(status) - all_data.append(response) - return all_data + self.export_metric(metric) def build_metric(self, data, metric): """Build a metric from current data and metric configuration. diff --git a/slo_generator/report.py b/slo_generator/report.py index 9033b1e2..ec4dcc00 100644 --- a/slo_generator/report.py +++ b/slo_generator/report.py @@ -80,6 +80,7 @@ class SLOReport: # Data validation valid: bool + errors: list[str] = field(default_factory=list) def __init__(self, config, @@ -107,6 +108,7 @@ def __init__(self, step['burn_rate_threshold']) self.timestamp_human = utils.get_human_time(timestamp) self.valid = True + self.errors = [] # Get backend results data = self.run_backend(config, backend, client=client, delete=delete) @@ -225,9 +227,7 @@ def run_backend(self, config, backend, client=None, delete=False): data = method(self.timestamp, self.window, config) LOGGER.debug(f'{self.info} | Backend response: {data}') except Exception as exc: # pylint:disable=broad-except - LOGGER.exception(exc) - LOGGER.error( - f'{self.info} | Backend error occured while fetching data.') + self.errors.append(utils.fmt_traceback(exc)) return None return data @@ -265,6 +265,14 @@ def get_sli(self, data): def to_json(self): """Serialize dataclass to JSON.""" + if not self.valid: + ebp_name = self.error_budget_policy_step_name + return { + 'metadata': self.metadata, + 'errors': self.errors, + 'error_budget_policy_step_name': ebp_name, + 'valid': self.valid + } return asdict(self) # pylint: disable=too-many-return-statements @@ -284,10 +292,11 @@ def _validate(self, data): # Backend result is the wrong type if not isinstance(data, (tuple, float, int)): - LOGGER.error( - f'{self.info} | Backend method returned an object of type ' + error = ( + f'Backend method returned an object of type ' f'{type(data).__name__}. It should instead return a tuple ' '(good_count, bad_count) or a numeric SLI value (float / int).') + self.errors.append(error) return False # Good / Bad tuple @@ -295,48 +304,56 @@ def _validate(self, data): # Tuple length should be 2 if len(data) != 2: - LOGGER.error( - f'{self.info} | Backend method returned a tuple with ' - f'{len(data)} elements. Expected 2 elements.') + error = ( + f'Backend method returned a tuple with {len(data)} items.' + f'Expected 2 items.') + self.errors.append(error) return False good, bad = data # Tuple should contain only elements of type int or float if not all(isinstance(n, (float, int)) for n in data): - LOGGER.error(f'{self.info} | Backend method returned' - 'a tuple with some elements having ' - 'a type different than float / int') + error = ( + 'Backend method returned a tuple with some elements having' + ' a type different than float or int') + self.errors.append(error) return False # Tuple should not contain any element with value None. if good is None or bad is None: - LOGGER.error( - f'{self.info} | Backend method returned a valid tuple ' - f'{data} but one of the values is None.') + error = ( + f'Backend method returned a valid tuple {data} but one of ' + 'the values is None.') + self.errors.append(error) return False # Tuple should not have NO_DATA everywhere if (good + bad) == (NO_DATA, NO_DATA): - LOGGER.error(f'{self.info} | Backend method returned a valid ' - f'tuple {data} but the good and bad count ' - 'is NO_DATA (-1).') + error = ( + f'Backend method returned a valid tuple {data} but the ' + 'good and bad count is NO_DATA (-1).') + self.errors.append(error) return False # Tuple should not have elements where the sum is inferior to our # minimum valid events threshold if (good + bad) < MIN_VALID_EVENTS: - LOGGER.error(f'{self.info} | Not enough valid events found | ' - f'Minimum valid events: {MIN_VALID_EVENTS}') + error = ( + f'Not enough valid events found. Minimum valid events: ' + f'{MIN_VALID_EVENTS}') + self.errors.append(error) return False # Check backend float / int value if isinstance(data, (float, int)) and data == NO_DATA: - LOGGER.error(f'{self.info} | Backend returned NO_DATA (-1).') + error = 'Backend returned NO_DATA (-1).' + self.errors.append(error) return False # Check backend None if data is None: - LOGGER.error(f'{self.info} | Backend returned None.') + error = 'Backend returned None.' + self.errors.append(error) return False return True @@ -346,8 +363,9 @@ def _post_validate(self): # SLI measurement should be 0 <= x <= 1 if not 0 <= self.sli_measurement <= 1: - LOGGER.error(f'{self.info} | SLI is not between 0 and 1 (value = ' - f'{self.sli_measurement})') + error = ( + f'SLI is not between 0 and 1 (value = {self.sli_measurement})') + self.errors.append(error) return False return True @@ -376,6 +394,9 @@ def info(self): def __str__(self): report = self.to_json() + if not self.valid: + errors_str = ' | '.join(self.errors) + return f'{self.info} | {errors_str}' goal_per = self.goal * 100 sli_per = round(self.sli_measurement * 100, 6) gap = round(self.gap * 100, 2) diff --git a/slo_generator/utils.py b/slo_generator/utils.py index ce6eaf11..2442f2dc 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -503,3 +503,14 @@ def get_target_path(source_dir, target_dir, relative_path, mkdir=True): if mkdir: target_path.parent.mkdir(parents=True, exist_ok=True) return target_path + +def fmt_traceback(exc): + """Format exception to be human-friendly. + + Args: + exc (Exception): Exception to format. + + Returns: + str: Formatted exception. + """ + return exc.__class__.__name__ + str(exc).replace("\n", " ") diff --git a/tests/unit/fixtures/exporters.yaml b/tests/unit/fixtures/exporters.yaml index d26adf51..8d5d4140 100644 --- a/tests/unit/fixtures/exporters.yaml +++ b/tests/unit/fixtures/exporters.yaml @@ -16,7 +16,7 @@ project_id: ${PUBSUB_PROJECT_ID} topic_name: ${PUBSUB_TOPIC_NAME} - - class: Stackdriver + - class: CloudMonitoring project_id: ${STACKDRIVER_HOST_PROJECT_ID} - class: Bigquery @@ -35,7 +35,7 @@ api_url: ${DYNATRACE_API_URL} api_token: ${DYNATRACE_API_TOKEN} - - class: Stackdriver + - class: CloudMonitoring project_id: ${STACKDRIVER_HOST_PROJECT_ID} metrics: - name: error_budget_burn_rate diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index 5aa31257..83cf0d2d 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -183,8 +183,10 @@ def test_metrics_exporter_build_data_labels(self): return_value=BQ_ERROR) def test_export_multiple_error(self, *mocks): exporters = [EXPORTERS[1], EXPORTERS[2]] - results = export(SLO_REPORT, exporters) - self.assertTrue(isinstance(results[-1], BigQueryError)) + errors = export(SLO_REPORT, exporters) + self.assertEqual(len(errors), 1) + self.assertIn('BigQueryError', errors[0]) + self.assertIn('BigqueryExporter failed', errors[0]) @patch("google.api_core.grpc_helpers.create_channel", return_value=mock_sd(STEPS)) @@ -198,6 +200,11 @@ def test_export_multiple_error_raise(self, *mocks): with self.assertRaises(BigQueryError): export(SLO_REPORT, exporters, raise_on_error=True) + def test_export_wrong_class(self): + exporters = [{'class': 'Unknown'}] + with self.assertRaises(ImportError): + export(SLO_REPORT, exporters, raise_on_error=True) + if __name__ == '__main__': unittest.main() From 13082f854c057926848cf4ce8da66916d5b74ca6 Mon Sep 17 00:00:00 2001 From: SLO Generator <71889107+slo-generator-bot@users.noreply.github.com> Date: Wed, 19 Jan 2022 16:31:36 +0100 Subject: [PATCH 062/107] chore: release 2.1.0 (#188) --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3e2dbc8..ab4b368c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## [2.1.0](https://www.github.com/google/slo-generator/compare/v2.0.1...v2.1.0) (2022-01-19) + + +### Features + +* search SLO configs in subdirectories ([#181](https://www.github.com/google/slo-generator/issues/181)) ([575f2f7](https://www.github.com/google/slo-generator/commit/575f2f701d43e8dfbb9505b98a81ab76dde79d50)) + + +### Bug Fixes + +* better exception handling and error logging ([#187](https://www.github.com/google/slo-generator/issues/187)) ([0826f11](https://www.github.com/google/slo-generator/commit/0826f11533f04395b353829ea877999d390ffddf)) +* bigquery exporter lint ([#183](https://www.github.com/google/slo-generator/issues/183)) ([9710b06](https://www.github.com/google/slo-generator/commit/9710b068ea8bc419c571b90ef11ec2469c3c511a)) +* dependency issue with google-cloud-monitoring for the bazillionth time ([#191](https://www.github.com/google/slo-generator/issues/191)) ([1a613f3](https://www.github.com/google/slo-generator/commit/1a613f3112e1e243679f4c6ec2e5f62b232c3331)) +* Set Python version to 3.9 ([#179](https://www.github.com/google/slo-generator/issues/179)) ([c433fb9](https://www.github.com/google/slo-generator/commit/c433fb9588660cc4f6b1b724b0b896988a96b9c6)) + + +### Reverts + +* Revert "feat: search SLO configs in subdirectories (#181)" (#189) ([08b13e7](https://www.github.com/google/slo-generator/commit/08b13e712a6b385972f5c641ee32ad3173e73865)), closes [#181](https://www.github.com/google/slo-generator/issues/181) [#189](https://www.github.com/google/slo-generator/issues/189) + + +### Documentation + +* correct bigquery.md ([#182](https://www.github.com/google/slo-generator/issues/182)) ([3ba32a1](https://www.github.com/google/slo-generator/commit/3ba32a15923660e0f0f82e341cea782d8e300d59)) +* How to define a latency SLI-SLO from an exponential distribution metric in Cloud Monitoring ([#56](https://www.github.com/google/slo-generator/issues/56)) ([8346c47](https://www.github.com/google/slo-generator/commit/8346c479471ab57f0630255876eb7ba656544ab6)) +* improve datastudio md ([#54](https://www.github.com/google/slo-generator/issues/54)) ([4c8e8a2](https://www.github.com/google/slo-generator/commit/4c8e8a21d7e5d261a37b747b95ee7ce13032a722)) + ### [2.0.1](https://www.github.com/google/slo-generator/compare/v2.0.0...v2.0.1) (2021-09-29) diff --git a/setup.py b/setup.py index 18bb5832..6f93f9e1 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # Package metadata. name = "slo-generator" description = "SLO Generator" -version = "2.0.1" +version = "2.1.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' From 770488b059b32a5f8828dfc5a5f90a5e1c8f39e9 Mon Sep 17 00:00:00 2001 From: BatRaph <95982637+BatRaph@users.noreply.github.com> Date: Tue, 25 Jan 2022 11:39:04 +0100 Subject: [PATCH 063/107] fix: dynatrace slo import (#198) --- slo_generator/backends/dynatrace.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/slo_generator/backends/dynatrace.py b/slo_generator/backends/dynatrace.py index 3d288aa2..3d788407 100644 --- a/slo_generator/backends/dynatrace.py +++ b/slo_generator/backends/dynatrace.py @@ -51,10 +51,9 @@ def query_sli(self, timestamp, window, slo_config): Returns: float: SLI value. """ - conf = slo_config['backend'] + measurement = slo_config['spec']['service_level_indicator'] start = (timestamp - window) * 1000 end = timestamp * 1000 - measurement = conf['measurement'] slo_id = measurement['slo_id'] data = self.retrieve_slo(start, end, slo_id) LOGGER.debug(f"Result SLO: {pprint.pformat(data)}") From 7e623b0e68354740cb88c3af1d4d55ced0c9bda4 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 25 Jan 2022 12:37:42 +0100 Subject: [PATCH 064/107] fix: remove row_ids to solve de-duplication issues (#200) --- slo_generator/exporters/bigquery.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/slo_generator/exporters/bigquery.py b/slo_generator/exporters/bigquery.py index a7e15db8..7cb98468 100644 --- a/slo_generator/exporters/bigquery.py +++ b/slo_generator/exporters/bigquery.py @@ -68,8 +68,6 @@ def export(self, data, **config): dataset_id, table_id, schema=TABLE_SCHEMA) - row_ids_fmt = '{service_name}{feature_name}{slo_name}{timestamp_human}{window}' # pylint: disable=line-too-long # noqa: E501 - row_ids = row_ids_fmt.format(**data) # Format user metadata if needed json_data = {k: v for k, v in data.items() if k in schema_fields} @@ -89,7 +87,6 @@ def export(self, data, **config): results = self.client.insert_rows_json( table, json_rows=[json_data], - row_ids=[row_ids], retry=google.api_core.retry.Retry(deadline=30)) status = f' Export data to {str(table_ref)}' if results: From 1f88aaf568228e71e8bf7c80bb110d5caa17cf68 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 25 Jan 2022 14:40:57 +0100 Subject: [PATCH 065/107] fix: alerting burn rate threshold null in BQ (#201) --- slo_generator/constants.py | 2 +- slo_generator/exporters/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/slo_generator/constants.py b/slo_generator/constants.py index 58653c54..8404100d 100644 --- a/slo_generator/constants.py +++ b/slo_generator/constants.py @@ -59,7 +59,7 @@ METRIC_LABELS_COMPAT = { 'goal': 'slo_target', 'description': 'slo_description', - 'burn_rate_threshold': 'alerting_burn_rate_threshold' + 'error_budget_burn_rate_threshold': 'alerting_burn_rate_threshold' } # Fields that used to be specified in top-level of YAML config are now specified diff --git a/slo_generator/exporters/base.py b/slo_generator/exporters/base.py index 7c5efee9..0364710a 100644 --- a/slo_generator/exporters/base.py +++ b/slo_generator/exporters/base.py @@ -35,7 +35,7 @@ 'labels': DEFAULT_METRIC_LABELS }, { - 'name': 'error_budget_burn_rate_threshold', + 'name': 'alerting_burn_rate_threshold', 'description': 'Error Budget burn rate threshold.', 'labels': DEFAULT_METRIC_LABELS }, From ef76ca6acbff73cc1396b0c9dbdd3b644b2c5dca Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 25 Jan 2022 16:25:06 +0100 Subject: [PATCH 066/107] fix: custom backend path for integration tests (#203) --- samples/__init__.py | 0 samples/config.yaml | 6 +++--- samples/custom/slo_custom_app_availability_query_sli.yaml | 6 +++--- samples/custom/slo_custom_app_availability_ratio.yaml | 6 +++--- slo_generator/utils.py | 7 +++++-- 5 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 samples/__init__.py diff --git a/samples/__init__.py b/samples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/samples/config.yaml b/samples/config.yaml index ca8e7fef..a594c45b 100644 --- a/samples/config.yaml +++ b/samples/config.yaml @@ -5,7 +5,7 @@ backends: project_id: ${STACKDRIVER_HOST_PROJECT_ID} cloud_service_monitoring: project_id: ${STACKDRIVER_HOST_PROJECT_ID} - custom.custom_backend.CustomBackend: {} + samples.custom.custom_backend.CustomBackend: {} datadog: api_key: ${DATADOG_API_KEY} app_key: ${DATADOG_APP_KEY} @@ -22,8 +22,8 @@ exporters: project_id: ${STACKDRIVER_HOST_PROJECT_ID} cloud_monitoring/test: project_id: ${PUBSUB_PROJECT_ID} - custom.custom_exporter.CustomMetricExporter: {} - custom.custom_exporter.CustomSLOExporter: {} + samples.custom.custom_exporter.CustomMetricExporter: {} + samples.custom.custom_exporter.CustomSLOExporter: {} datadog: api_key: ${DATADOG_API_KEY} app_key: ${DATADOG_APP_KEY} diff --git a/samples/custom/slo_custom_app_availability_query_sli.yaml b/samples/custom/slo_custom_app_availability_query_sli.yaml index 4869144d..8fed579b 100644 --- a/samples/custom/slo_custom_app_availability_query_sli.yaml +++ b/samples/custom/slo_custom_app_availability_query_sli.yaml @@ -8,10 +8,10 @@ metadata: slo_name: availability-sli spec: description: 99.99% of fake requests to custom backends are valid - backend: custom.custom_backend.CustomBackend + backend: samples.custom.custom_backend.CustomBackend method: query_sli exporters: - - custom.custom_exporter.CustomMetricExporter - - custom.custom_exporter.CustomSLOExporter + - samples.custom.custom_exporter.CustomMetricExporter + - samples.custom.custom_exporter.CustomSLOExporter service_level_indicator: {} goal: 0.999 diff --git a/samples/custom/slo_custom_app_availability_ratio.yaml b/samples/custom/slo_custom_app_availability_ratio.yaml index 06a9c957..ba235e53 100644 --- a/samples/custom/slo_custom_app_availability_ratio.yaml +++ b/samples/custom/slo_custom_app_availability_ratio.yaml @@ -8,10 +8,10 @@ metadata: slo_name: availability-ratio spec: description: 99.99% of fake requests to custom backends are valid - backend: custom.custom_backend.CustomBackend + backend: samples.custom.custom_backend.CustomBackend method: good_bad_ratio exporters: - - custom.custom_exporter.CustomMetricExporter - - custom.custom_exporter.CustomSLOExporter + - samples.custom.custom_exporter.CustomMetricExporter + - samples.custom.custom_exporter.CustomSLOExporter service_level_indicator: {} goal: 0.999 diff --git a/slo_generator/utils.py b/slo_generator/utils.py index 2442f2dc..8739c755 100644 --- a/slo_generator/utils.py +++ b/slo_generator/utils.py @@ -249,8 +249,11 @@ def get_backend(config, spec): sys.exit(0) backend_data = all_backends[spec_backend] backend_data['name'] = spec_backend - backend_data['class'] = capitalize(snake_to_caml( - spec_backend.split('/')[0])) + if '.' in spec_backend: # custom backend + backend_data['class'] = spec_backend + else: # built-in backend + backend_data['class'] = capitalize(snake_to_caml( + spec_backend.split('/')[0])) return backend_data From feaa637dba373fb393a7fbafe634af3ed276b14e Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 1 Feb 2022 15:15:51 +0100 Subject: [PATCH 067/107] feat: add batch mode, better error reporting, cloud run docs (#204) --- Dockerfile | 2 +- Makefile | 19 +++- README.md | 25 ++-- docs/deploy/cloudfunctions.md | 2 +- docs/deploy/cloudrun.md | 46 +++++++- docs/images/export_service_split.png | Bin 0 -> 207633 bytes docs/providers/cloudevent.md | 29 +++++ docs/shared/api.md | 105 +++++++++++++++++ samples/.env.sample | 2 + samples/config.yaml | 3 + samples/config_export.yaml | 9 ++ samples/test/config.yaml | 19 ++++ samples/test/config_export.yaml | 6 + samples/test/test1.yaml | 22 ++++ samples/test/test2.yaml | 24 ++++ setup.py | 5 +- slo_generator/api/main.py | 157 ++++++++++++++++++++------ slo_generator/cli.py | 19 +++- slo_generator/compute.py | 33 ++++-- slo_generator/constants.py | 3 + slo_generator/exporters/bigquery.py | 43 +------ slo_generator/exporters/cloudevent.py | 66 +++++++++++ slo_generator/exporters/dynatrace.py | 2 - slo_generator/exporters/pubsub.py | 14 +-- slo_generator/report.py | 9 +- slo_generator/utils.py | 15 ++- tests/unit/test_compute.py | 1 - 27 files changed, 553 insertions(+), 127 deletions(-) create mode 100644 docs/images/export_service_split.png create mode 100644 docs/providers/cloudevent.md create mode 100644 docs/shared/api.md create mode 100644 samples/config_export.yaml create mode 100644 samples/test/config.yaml create mode 100644 samples/test/config_export.yaml create mode 100644 samples/test/test1.yaml create mode 100644 samples/test/test2.yaml create mode 100644 slo_generator/exporters/cloudevent.py diff --git a/Dockerfile b/Dockerfile index 467bed97..d6669577 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,6 @@ RUN apt-get update && \ ADD . /app WORKDIR /app RUN pip install -U setuptools -RUN pip install ."[api, datadog, dynatrace, prometheus, elasticsearch, pubsub, cloud_monitoring, cloud_service_monitoring, cloud_storage, bigquery, dev]" +RUN pip install ."[api, datadog, dynatrace, prometheus, elasticsearch, pubsub, cloud_monitoring, cloud_service_monitoring, cloud_storage, bigquery, cloudevent, dev]" ENTRYPOINT [ "slo-generator" ] CMD ["-v"] diff --git a/Makefile b/Makefile index a8c36570..29cab558 100644 --- a/Makefile +++ b/Makefile @@ -137,8 +137,23 @@ cloudrun: --args=api \ --args=--signature-type="${SIGNATURE_TYPE}" \ --min-instances 1 \ - --allow-unauthenticated \ - --project=${CLOUDRUN_PROJECT_ID} + --allow-unauthenticated + +# Cloudrun - export mode only +cloudrun_export_only: + gcloud run deploy slo-generator-export \ + --image gcr.io/${GCR_PROJECT_ID}/slo-generator:${VERSION} \ + --region=${REGION} \ + --platform managed \ + --set-env-vars CONFIG_PATH=${CONFIG_URL} \ + --service-account=${SERVICE_ACCOUNT} \ + --project=${CLOUDRUN_PROJECT_ID} \ + --command="slo-generator" \ + --args=api \ + --args=--signature-type="cloudevent" \ + --args=--target="run_export" \ + --min-instances 1 \ + --allow-unauthenticated gcloud_alpha: gcloud components install alpha --quiet diff --git a/README.md b/README.md index 6b16eed4..d0124e95 100644 --- a/README.md +++ b/README.md @@ -84,26 +84,22 @@ Use `slo-generator compute --help` to list all available arguments. ### API usage On top of the CLI, the `slo-generator` can also be run as an API using the Cloud -Functions Framework SDK (Flask): +Functions Framework SDK (Flask) using the `api` subcommand: ``` -slo-generator api -c +slo-generator api --config ``` where: * `` is the [Shared configuration](#shared-configuration) file path or GCS URL. -Once the API is up-and-running, you can `HTTP POST` SLO configurations (YAML or JSON) to it: +Once the API is up-and-running, you can make HTTP POST requests with your SLO +configurations (YAML or JSON) in the request body: ``` curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml localhost:8080 # yaml SLO config curl -X POST -H "Content-Type: application/json" -d @slo.json localhost:8080 # json SLO config ``` -***Notes:*** -* The API responds by default to HTTP requests. An alternative mode is to -respond to [`CloudEvents`](https://cloudevents.io/) instead, by setting -`--signature-type cloudevent`. - -* Use `--target export` to run the API in export mode only (former `slo-pipeline`). +To read more about the API and advanced usage, see [docs/shared/api.md](./docs/shared/api.md). ## Configuration @@ -165,20 +161,21 @@ as a shared config for all SLO configs. It is composed of the following fields: * [``](docs/providers/custom.md#backend) * `exporters`: A map of exporters to export results to. Each exporter is defined - as a key formatted as `/`, and a map value detailing the - exporter configuration. + as a key formatted as `/`, and a map value + detailing the exporter configuration. ```yaml exporters: bigquery/dev: project_id: proj-bq-dev-a4b7 dataset_id: my-test-dataset table_id: my-test-table - prometheus/test: + prometheus: url: ${PROMETHEUS_URL} ``` See specific providers documentation for detailed configuration: - * [`pubsub`](docs/providers/pubsub.md#exporter) to stream SLO reports. * [`bigquery`](docs/providers/bigquery.md#exporter) to export SLO reports to BigQuery for historical analysis and DataStudio reporting. + * [`cloudevent`](docs/providers/cloudevent.md#exporter) to stream SLO reports to Cloudevent receivers. + * [`pubsub`](docs/providers/pubsub.md#exporter) to stream SLO reports to Pubsub. * [`cloud_monitoring`](docs/providers/cloud_monitoring.md#exporter) to export metrics to Cloud Monitoring. * [`prometheus`](docs/providers/prometheus.md#exporter) to export metrics to Prometheus. * [`datadog`](docs/providers/datadog.md#exporter) to export metrics to Datadog. @@ -186,7 +183,7 @@ as a shared config for all SLO configs. It is composed of the following fields: * [``](docs/providers/custom.md#exporter) to export SLO data or metrics to a custom destination. * `error_budget_policies`: [**required**] A map of various error budget policies. - * ``: Name of the error budget policy. + * ``: Name of the error budget policy. * `steps`: List of error budget policy steps, each containing the following fields: * `window`: Rolling time window for this error budget. * `alerting_burn_rate_threshold`: Target burnrate threshold over which alerting is needed. diff --git a/docs/deploy/cloudfunctions.md b/docs/deploy/cloudfunctions.md index 8980a2f6..36edc390 100644 --- a/docs/deploy/cloudfunctions.md +++ b/docs/deploy/cloudfunctions.md @@ -2,7 +2,7 @@ `slo-generator` is frequently used as part of an SLO Reporting Pipeline made of: -* A **Cloud Scheduler** triggering an event every X minutes. +* A **Cloud Scheduler** triggering an SLO computation every X minutes. * A **PubSub topic**, triggered by the Cloud Scheduler event. * A **Cloud Function**, triggered by the PubSub topic, running `slo-generator`. * A **PubSub topic** to stream computation results. diff --git a/docs/deploy/cloudrun.md b/docs/deploy/cloudrun.md index 6780a8d7..9c57f632 100644 --- a/docs/deploy/cloudrun.md +++ b/docs/deploy/cloudrun.md @@ -3,7 +3,13 @@ `slo-generator` can also be deployed as a Cloud Run service by following the instructions below. -## Setup a Cloud Storage bucket +## Terraform setup + +To set up the `slo-generator` with Terraform, please look at the [terraform-google-slo](https://github.com/terraform-google-modules/terraform-google-slo#slo-generator-any-monitoring-backend). + +## Manual setup using gcloud + +### Setup a Cloud Storage bucket Create the GCS bucket that will hold our SLO configurations: @@ -19,11 +25,11 @@ gsutil cp config.yaml gs://${BUCKET_NAME}/ See sample [config.yaml](../../samples/config.yaml) -## Deploy the CloudRun service +### Deploy the CloudRun service ``` gcloud run deploy slo-generator \ - --image gcr.io/slo-generator-ci-a2b4/slo-generator:2.0.0-rc3 \ + --image gcr.io/slo-generator-ci-a2b4/slo-generator:latest \ --region=europe-west1 \ --project=${PROJECT_ID} \ --set-env-vars CONFIG_PATH=gs://${BUCKET_NAME}/config.yaml \ @@ -37,7 +43,7 @@ gcloud run deploy slo-generator \ Once the deployment is finished, get the service URL from the log output. -## [Optional] Test an SLO +### [Optional] Test an SLO ``` curl -X POST -H "Content-Type: text/x-yaml" --data-binary @slo.yaml ${SERVICE_URL} # yaml curl -X POST -H "Content-Type: application/json" -d @${SLO_PATH} ${SERVICE_URL} # json @@ -45,7 +51,7 @@ curl -X POST -H "Content-Type: application/json" -d @${SLO_PATH} ${SERVICE_URL} See sample [slo.yaml](../../samples/cloud_monitoring/slo_gae_app_availability.yaml) -## Schedule SLO reports every minute +### Schedule SLO reports every minute Upload your SLO config to the GCS bucket: ``` @@ -59,3 +65,33 @@ gcloud scheduler jobs create http slo --schedule=”* * * * */1” \ --message-body=”gs://${BUCKET_NAME}/slo.yaml” --project=${PROJECT_ID} ``` + +### [Optional] Set up the export service + +If you decide to split some of the exporters to another dedicated service, you +can deploy an export-only API to Cloud Run: + +Upload the slo-generator export config to the GCS bucket: +``` +gsutil cp config_export.yaml gs://${BUCKET_NAME}/config_export.yaml +``` + +Deploy the `slo-generator` with `--signature-type=cloudevent` and `--target=run_export`: +``` +gcloud run deploy slo-generator-export \ + --image gcr.io/slo-generator-ci-a2b4/slo-generator:latest \ + --region=europe-west1 \ + --project=${PROJECT_ID} \ + --set-env-vars CONFIG_PATH=gs://${BUCKET_NAME}/config_export.yaml \ + --platform managed \ + --command="slo-generator" \ + --args=api \ + --args=--signature-type=cloudevent \ + --args=--target=run_export \ + --min-instances 1 \ + --allow-unauthenticated +``` + +To send your SLO reports from the standard service to the export-only service, +set up a [cloudevent exporter](../providers/cloudevent.md) in the standard +service `config.yaml`. diff --git a/docs/images/export_service_split.png b/docs/images/export_service_split.png new file mode 100644 index 0000000000000000000000000000000000000000..43ec2f6f91abbe97b788b0251239b2b95a0d9c70 GIT binary patch literal 207633 zcmeFZc|6qX`#+92q(lpfkV=caWFJdZDp|tJSVt1FjC~n3(M8V zNy(gMW_o@h%KJi2%MuC{i=E%uWF z?1%PT)Mn*ZJa+No;|8{Z0$l9L)JY_b92AVicLr=|IAw({X}S)Jv8>*V*M7P;mi1|) z(ZS!U)fJi=PrTf;W#xFvy88^9!#-u;=;YGUE@Otzx0;T9YYi_k(~Qz1~oe!>ky%y z-3zz(_9PM%yX;dBzdFoX0y~}DU02Ka%Kqr$rcnDFd%Yn$kAFy9=$ySUJXl zR6pp(PSTcr&lz0LYRvYb;hZO1I$K=!k#x?K2B7D`_4};yd#}{1!1m=eNWY9e#>IP6 zTH+F$MXz#%!5s1wTk_N`@ZN#8t*n^?bX)Op3ZLX@+;u2 z=8V%x?2?+%cv64pG$rQ9bwJ_Q*y_jU{>>3^`J}o45YN zvJdPxR<0v<$38sLY}Dz0Bz7R;-Eo~e3eTeti6>pv=#KFf$3EMChdVI3!S3$iQ&+_7 zAFR}~+gH9szwCI)c4PMxd*1Gz$5nAhns3-^+7+4yB4mj|l*c?xEp`f}RsF}^_&20C zxcynw<1?DaJol!b>O3$Vv8PpHTxpcsGun%%QpsA(eXsI^)(MB@YM44sEc1a;)ATs+ zn9;c07-MB(6U%=7;^SL7o+mExZ9a6<*`Jf_`K-$bcKzk)nEkzviI=pFCsf8)roqzt zuj4KsJ>8l3?po?ebrZg`&nn`~ZWNui5!X0|Kb6=fc}L2dMKuDKWPL-^p!SNWMBSUK z#=d9VuAh|1D2p{QaDFrN2A5}2C}gJR!mb!s9c!t(kgfT;mOtn5;YYBTB^303FH*e@Zk!jgak_Jio8@gR}yOMs&U3&3!=5fd| zh*X$Ohz^@vj`)~lL*n~qDyKe)pSz-ax9EaWoIc4=-LOVSIDcX$)z>KKxJFi33|TiT z`Nh)Xdk>S#J@(}!{|U5d5v0i0#n94l@TS@o zHKXc_LE3cuUegxfscVj?8|5E$EuJ9a-R`;N&F>$ODl_%=RTWk(w$8MEGZY$V4$;#z z4twzJ+*kLnM>sBX1aL$&ibw8?JbO9n^_$zT$HSUqV-l}lkO;5$h@~xaLi4l;Xev}KZIK!w+!^uo!7ra`<~ zNqm(d&akjk#>Tu)NBJVH#63$CM7W-S0v9U3HTo3LP3S`Ph$-GWym|PJ@C9LuvRyQn*EUs82TkI>Hkq!s3Qd0Se9|l}vdjsJCL;<2RF^*VA znjxJW_&j){-(|3+Ulc1+Zdy9g5A2WYS6Fy6pD>u9az@?rsHBLfdXd`dQTWj;HD`4R zwG7{&F}F!+Urh?2731h}8(Vcuc2PD^R-^bpaYM00al(+<(D=~YP~LFIuyQ3f@RkNn zVfuTwBMUy~5 zP;BuOLi1o`9Cut%9KC*J!pz(8wZ@zF*ZEa$ix(GtGEaDDd7N*N&6A&UM%cL7&0k8l z8LF*pgGFz?-YjJI*SXf%ektMI>bt6wb#Le1MoxB3Mjg_*ZuUX7mR;-dNYxcD17fOX zMDa1Od>JulAp+f_<6#Gsd{KQKEfE*L9a7VS?ykEg)=@$@(*a96@;vaAJ|H%$7jNh^ z-05f3SA*OM+vT_`K$mglOB_il-ral9tn|p>QG8Ws>|AoX%!Ji2HnacAU~-WPi2vD}d}ezMRriE<&^bW?8 zk)Ei#MV||#)uPmsz2@6io=(_|l-HsPpCdn@=;j?LQPush*X1o`d3m5({{Ru|6o_)* z`PDm1XH!gNsB|hn&y%QcJ)^$#StqSaTk`!g!~J1VetFw^RI&EX?#?O2iw2a`ytIil zJvg`=`=TqNt3 zMqf^t6a#L*-qLS+)cij3X3D3<)^;)81h3Z2om4s{dL{kz=hK&sk}`=mUfxL0@=co$ z#f%SZ`ZTP4EgTwo7cLc?Lmh7&5eZ+)+?OUAem10Pxo8yqkus<3=rQQ@%}KN)?_($7 z3qC0@cwB^H)5qwd8BHr_jcOeV|GeE#Z)5a4hcHV@}L zVysIczkcchFLc-0S7A3*cOp`Jhn>n|H;ucNu+u@S>^iwQJ08Q+sa*2X~*l9=<(l zUv4mm>T@|GplMTNT)OgScjjLj$M5?3 zdaB9E1q1}h1}MmScst3RQ&m-!lRqza{=5wH2^pV2cV9c0jJwZ?KL+_R&LsyQdv9k? zUuO^ZW8cTMyY1oUt8x7JcS3*t{4q}lnDgJ1+lmWj#|G!%PrudJZ=70A*r+WU}KfC^;>3?p zeKr5ynm;@Lv+>W40J-l={|77n;PdZynT*!t2FU%jX`0-lF*0K;EZQvBE?qQ(vCa=$ zJaw?J*jSrqj_E3E0Q=I{09muaS$^IQ3qyW}AV;8sLuTU6@SR2H1vG6t z<85@voQ+g@b+zR=0d_uZ7S{jzD%`l_+E;!|JUKZ6U%tSecZ+9{JUpS=m#s-s7u$ta$^% z@!z?_Y(2(4<@8Ice0Ptt>WrSE%LnrRl08f&X8n@t|7GHT=lTC-;=l3d|DTe0Y~<2X zBmCb~Sb&5v<_U1a+rjO{O3NTI2-O}&f{);>6WtTZC|XD`6if1%&rPs|x`@OQUIFqK zDnFMQVt_U4VK}8?`*1Wp2HH(oR}83*E=Kj85>2425#$K#A3==e&dp>^4W zXxl{F!n$AE)};Tc5Om33HnT4X8!&7nR!Zo%omUs_Q#e@T^t4WdS+^>+QN}g8q0b3? zgl4~0x@GJ5SD0+^gcs89M!P!=g((O#+IUv0gwYo@!Wz{YRfi5G5MFOhQddIoeyg?U zVuMU0_k4yvS;4*4ZyrpU^dYK=qOPIrNt4vlxX$4OiD6rBTas5HHSnEJo2##MV8+J8 zTC4v`$h^Yvsm0cf!GylMR{BWIiBgwXtC7U5nY;6F;?~61elSicumhRtx--=5`g~#B zpKcwbHDE|uIpV5e1Q0+yrVP_@Y1>6xpdo`D&})`TMJr4?$}fLQ+Kg>UFBbE2liX#- zMvrtUd|n!NofzY=T08&ey&n)%68&du^zrO)iK@PE8F6;a z7YXrc$A1YXI8SJEg~%7L|I2*-vFxKUEaJ;KB|fIVG_9kznD(EyB?+?sR}dgVcRzD| zhy30D56C}Zl5eQ&mih%c;yUJehV5Z}bYrK%ugO37PW~`B=<>g*@xKKm zv%e-U&m_+|aUJy!-u+x7*5`2Xv59f3EB_ymzrrMc%X*gomyku}5EBCF+D+&Enta1| z@=v7jzm}5d<4h?zGe30b*W?c~$v^PQ1^-$~jK51sWXAs$(*MrXi2u)pbaH8`IH{)U zk?T)g1eI~NS0}bZ0HXgm8NvJe1Go?@m1b?`&F8nl_{+%gfptvrr>aRWky!mzn77q5 z8s&hcz-9cyjA)5~w0VV4K&>u*qAe_u)V9GGE~WwI^2r+h>Y26SGBxYe*)7;oKe+oc zv`2q61d!sE5)uUiKoV;MkZW@a7^4lbgl5g>6wAID^Rk`F40o%g%E4jh$AqVDf{fUG zYAHj8n7HW*Lt4`O=}?1JYn*IxA~dnp1=?!wC4!1$Shh;gTem9Xp)PoP0EjXO%b-7= zo+!2KYxb!@ClJh*n09PJ(Xuu8Bx%-s2rLYBeGElsXbRB$iirnMaSP9BSH*gmCd^xX zSPzizH?K>woU=jsuUIo}8r@eoqvtuGc+0nL)_%~)_8{pb!PzQy(4~+DDV0M%hW0EZ zwoX(rQ3ZPN?&GBcSSC6_BPH;*8kSp`nu4?mwE+-Gt=It@5I1d4@@ya~w%L~^OycLD z@C2r1?P;wL)94QEtG=>;eoFu*kmsbTvfRa(1{_{j`8ilL!-%Ym8G&ZF*?QEFsclXs zX`9xw3zaB8x{yeZNjoNJim1BK>U4u?VTT9bE{QC?!YM(R+zhy0JW(}S4H{W7#~H1@ z@`KJRY?#7r0C`HKTF_QknvdqjGu)NdZhf2)oe4y$PgP|N#ZPzlwK7hUT+}hK(-UjW zJ}a=Hggy$)EzSKP4PD%;IRKtNAsXBFZlV;`Zv@C>Hn!NW2qAklMVVJ?D{2d;N$)1S zLsjEM{!>-NmB|CBSPD$`_t9OEy@pH>0moEs1>^LrTLnsu7T_b3&)w37Kxvg? zeJjcyQXw9z8`gmz=h%p}ZEqn=CO})c_ zS+_oqrN_5Er&XB>XP|n+am*Mw6Xle}VJ1SE0I3I>I6dxpTsFg}$qwg7%M4^Ua2 zyvjuQ`LZdKt)@h5)^+8+I*WYLDOw__6bq9so>*B|3)FOPrMg2Ws?^cFUn@TcCBWS4 zD86;*;sio6g!wF?)hB>~Xqz^%ojWaClcA5fW_vK}GIYGE9}Ey=Zn+b7VwPsK!B3c_ zS7ty?GvZ053H?fUm1Dl-s1u8!iV^@EGRKhR(p|Z5gw{ZwdLN zlZ@V~?|W(cW^3{QNn$Ajkhfqt{aFtnN;=Ij$c!bai7}XBAq|0JO6!;unA@U&F%a$X zFAHCAy(Sjh*RtJv%Dr{lPg=H(LV@O!h%SN&&Pa(s)MV9q`CC#erWaB)LP1{>HG&9Y zC%#Uuz4IyEv4r}{VtyZ1a6~S%u15_b$QW1%LSM@=f{0-!G?@67QkmSZ=jSm^xCKbI zZl(C%=pzm$wAzofITc`$+v8Q0hGIQo$XY*ZeSjdg>EcfEqMWD^=KHC?D}qX(z-~}B z;|C38UEmH_rhv3 zgdF>oFluM78e~aLBtbs}HMurxJ!FKM@SF9kw(+)wPOSA-XAQbs%Y=(oZJEzIVK-*C zZ`-cCs>pIXn9%A65L@EUlG->(I8Ad!$89ZOKw?xns?P~`Y9lQqei%bRC$zdEduTDx z=d^2L`irhs`XR0BRhCQU_;Dg3dxK(=?^Egl%})uUjSyD)osdp(TLlIHCO7@s{HpPt zv*tLZT77^3!8HRQNK0IJ4xI=@ChFtDaao-bjJVc`>hBJn-w4d0y3TT8;v5qF&UM0H z#eBjFIX$wFYDqX!6GtfAx>LjSfvj=Z_~vPYjhqRF%{cz{NNL@AEj%F0L0uYNI^wo3Wlpk-gBLp1Y;Y%VqYc zZFQ;%r1Zm()ZYTc6*PahK&cosG728PK`pZafVsd*R zik-YQf~S2x68Ce%d7O)#5O&mYQ z{JB%{dn>Ot6LGR0^V#!jAa`*$ADh-0=fwyAWw<{m0$7;nkn`S0nP1K}f>T19>s$8Q zD&Bvmct?dWR97+msKM2bQsTc$OxHTwUe4ilv2${d ze$KPFW7*nD6gv;46sk7HO<33!Q--@MOmY1X*0gd%7R%qY7;=K6o;yT8ECXY&m3#XW z5tNKW`AG#9q+2&`eS>hn^jo0M#UN4A!8Ny4@}>)-8rtCfSN>Q=#1+2m1C{$VB{KK_ z%PK9kd$G;qc}Q7V()bVpDJ}DG8EY zL7MEKfcl3@Y9quw6z-Jw91II-`5?+3K-<^fHFJ$3a&*vNI{3^{5(0(nZ%K{rsI9D>2o3(W&x-iqwv14BRW7nattJ?1 z=l=neBG~w}>xBI@;M)Rrv6j`9>T)|yDr2Q+;_HAjV+ve*w|$Qf@za2Vu9KW)3bqh- z7!S~z=5L%(Qt)buegr{-0b6Rqs)1_Og6=KKKe)a6o!i?d##w*vF=tzF=y1&^)e2z( z-Cc%aL^tCzRXt8`!Nc#qdoUEj<*4}1CziJz!3U#H0fmpM+99C}qKg}+@8aGCxH>P) zR470chb9`_iPQUuXlNK>+~+p}1AofH8hx4fzM|WDTe%=$Qsbb`9h=&77hG&0ihwYx z3D=AK(3j3YHp-E8T+vm>KZ9ivZ)BK`XoZ$o=DvTCT=Q(p-PRZhv6eunpDH{0LDOlC zNsU=n4-|Me7{JvDL%RnQYyer*wG@sNJphUyO|}|9f%m5%^256fv%=pQGD<#WWGsYz zq=av&4Z};SoK2Nc121@4ksroNeRu~7eTvvNa$aN0Y$}<0O;P5om5rE*=72F7YPkHgja@!pI~Z z(&*!Jl*OgBdgTpc8ybvdSoy+Hy19*I2^F2>*(lb^APppI{MhrL?|Z(aF!uOAMD;JO zj=tnN^A4y&QdXJduiaA8RCevrq@hekl5!9sO4GC5H18OP>#UW;QLRJ8Ga>-BorfYj4+&8(8B95Mtqf}@pI<0gIfO(RRH z4^2rI|JF%D_WxvfQBj`1Ci5SsD%DEcXMGBzGirzWU;}x%XN;00n6)Cm zX?b|CYDCJLDU88-DPT5iRF}$h1M>d(^`lLy4ToYLU_bcrZ$~xS01pMjaHX|EQyidW zxcuf+k|5N7H4l7jNt%Ngh~{xFrvX>%&Be>&%m0kp)PE9WMv3TI`;GoZSsO_5?HaT$ zZZ=F;_4uw!0w=C>Ii>*kK)H*P)OMnfWQ02Ob9A7ns^6em4Fdq^j0y~<=_Ibyo3C+r z6eA=gZI_(ec5F)1ji|-sN1?KyZlYNY(6$aYoG;KkhWax@3{OE$_C5ge>Ct z*{o6X9n)5g>GishP{dds@JkxxLV88*A<+ODY-V0p*HxUwP`@T!VuPCOhsZ5PN%Er^Nz?rcW%cjAaJQN|eCWP?czi%Fa)VC=#0FM3 zy^D|>mFFx;u&|M(Ts@4V$J1W3{s~o%)-pqtW8uF6e%$yU;lHD%tiVYZ^6;y|G^Y6x zDe@tOY>&wD)XlFgfkIy1-O2i5Wra7XG!dyD(0|B3E)ZLjv_#%ll#}hFx!2bz00m|b z7-Smv4nw1dbGLXdFAOtGc!GQ&&zMrZf9hmB|2w7v#s+-Y|Kq4X|9;e0?ZwS8Zq0V6~(J;~FIr5EyL7v|(^TzDqu(Gnjbi`4OGvG0h4oXd3?t$V3F;Ske z#m0|s<81MVBp(<($=EZkgGNo)3nd46xwjP8eY+psqM@V`^kG_(Xxx2-H@1+AoTZNZ z!@R(jne*W#r>p;4-0|YEy*=|j)c8y_d)Ib-@fV@91E|6AZwQ`!StZFtC8^+<>guvP z3KPn1j^owQXjNVZqbN^MpOa+pVcx+d(295l|9E`n(eg%mYRA`ajpd0GnumNpfdrgLYo#}m7 zbcndzZDp^n{)%nf<%@g4DwVIY1O7* zN!z!Vv|LzD9PS%=@(p+J4BEZ^Z4|KXm!(v~e(S;)-@4rjxE95Pcxb|EUXlRQLbKavX5MV?YF_&T z;+9A}wXm_lZ;i0|kDYx{jk39v^Kv1+%~X-exYAvmDMffB`RV4faaDMn~&y0@e&koSI53V|b? zI;!%KK=U4pL1&t-y%>KXi9a{lF_ zQ&WI>;61nw`TBs)Rrpio>7?3Mh)Sv2>{G zQpLU#DkU8}F8La`OleXeLI59ozkbd|lTO8hmcWCxhT;_ttK!SU`j66QVhnz$h}+*) z#Qd%?ub-<3mWVUU$D};rNzJ`WE8B=aVPf_dDO118wS( z6#IScD(m9k>T35zcq$1Kmwz&!f9|)Z8M9x)htpmug#B|SI_96bn3Lw11Ef0lMz2k4pa>O=ZHU<9su^;NA?IC zSu9`%&?Wn)K2#%-$cJ_@BsDc865r(g6_7KjaWz^zYm44El?O$j2BzrMNEB-D!#GZ+ zlQ8&k(~lx7RNC+b%YYaWt|2ee(i_w*GObMVxu6U}o!KfZ~(4HyBQwxK+ z)A-wpPE4Rs@vO8b*~6Ksu6$K^iR=Nr)^(S8`}sFJ4WE$a*DGZh8&fT*4!H9$AN!`? zFj4pS_3Tn^P+#Nz?`!Q4?>Z#zd6;V;QDy7P192;Acz`IMDkwP06{AV$2Y-X{|3-E$ z+(?|i?3Y6}y~;`31_V?a(Q#3d`|KQY?o!~8-Sgs}IVDHP;c^X=$bjBttsPM^ny6rw zuz*h(9)@piHqF>0s%G$5DUV$q_+&-7x`~UCvv4VWrG?(wi9T7<&?Ky&a&dkUy;1Bg zRpnJ>)_tPvS5h#^E~LiZd8bU=32F8LBqgCO6uhmQ{ z%{RXK>POx$-kXFL0$H?Y(g)^r7us{n_%)=>vZ`=e37oOl72CHFG}oJ4h11Lxshpk*1ot0eBy$BCnqQc5!o8G-9%NEqJB6H$_W-Rbic* zWZlVF^_xFUHeEtE6^9YaI-&z^+PVa5350)JHWYS+gH{44B8Mw7Bg82S!h36blu?0K zaUmm%0a}@_G=F-){tg{IVv7@I3mye;jKxcd4i?dYYZw6~qQ-l-Qc%`Gu=eLtf?p?(aWn0C^h0h4|LT;F z3?$Bq=Xoz-aL$|St)iZXzAM9DI~De#W)MLJClv_iuvL~+K^9sKxHWIAR7a=uE>|A_ zlyQelWQ3ex|KyIHU8)=7f5+{;wp}ceVM8JuCM!b{Eu$I1cj5IRKCZRaz!r?3?xwv>8ki)di7L&M2#Z1@H3{zdSstW)YkqPzX6q%v!#31c z6w#avtze>7-EgKVmAht9@%h+p+u!#hIL~%@ihKH6WR9@`xu8uKNCpXW@ zhkkUv+S1n-MuUW^RFqxiG}eH;8-qxXTm9A5!EQmTc^w7@4<${}u@S3}OUs#H>-_p` zTOJL#FJhva6JJga`!***SrA+8D$)ALUn8$n<|h;YyT>tVJsLAD%|3fn`;W!2;6VC^`Vv+aOQB?#u2zx!##58ns9QwE2{5Egf;TCc&Z((x2~6dg@m z$$4fPHbUe>kb9kQ{GA`{eC#{2bb$?DZtP#WDd|hE0mGf_#n;$%uX+X z_gCOR&mucV=T_t+rb4&`2^>3;wk~fnwy+!J5A20EHhAaE0N*4kg)Kbqfbk*ZP5ALS z&y`2Mh(plV%_2PE(J9-9C25pirLVYXl-mAA_wY|o`cHJfnBZ~g+&Mv^_>#V0$uS@P%gq-ABgP?;$NWZ6Qa3q+XRoVbD&;m z&PL>e1yRzGk|b5$`?|TsU{%|Yc`JmU=ldS_JrPK@eUnpiB(!2){`F0MdrO1Z_V zd}vG7<($QKLTE2MRC#`-57)r(pg9$EMk4IWNgkk% zCuGe0tQYUf)+<>JM(^@hD6AjpBvP|n;8zre~|=9T0ODVSbMz@6W#g&513FzygKRoxS9QtS2Qvek-w zz&M?svYZpIUQ+|`93B9ElDf$V-5hPrq!W$fik@0vO*awZg>qUuHwtQjz?BNlc2zA@ zJxH7yCcu%Hx`TNGm)dJ2q61m7hX#J~7KQNAOWx~l=So7*y`g3{@DEmz%{__AjePyQ z8rx=vbu4;6aC^y zD+|y&Dru2aBNSyhwuAt$7R?1THQna;Qr)^SPRNa!*>1SW`Cw_A$|K_yoZ!7SV zt5KER7tFX2_l$edMYlB=?%9te?l5vYZzuE3?QWM1A zS9zt|!fU~wo2n4tJC7hd4wicKc?c_{V<}!(bO%-U4HsRYJpYz()2j3t>}x#wzCK4S zR0IhrMVN{0wV9)7jt zbj~L3`WnDYuY%dMEQq?&{mr@(+xY33sguFl_6E;-YQjko0m4m#%qBWjqhz;_@XD+i40bR(Au}EC)-e}ci;41_RtQU^?(mzOq zKqoCqZhLU_Y}s{=`w)k<_{f*HU|tmj3h6+n>-)LOyr?yqK;9SvCn*_}Vd_wZU1NcD zOhmPX}4AnR2*TQ=~fi&DO1+j>Z-cI`isb3YUJu6{Niel~cJ8GngWwb%d%ERVS5 z<8$I|@feJlc!}d-RY>PFc9_eDE;-MWTSWqclzs7%*A(wmRGX+2<_cW}O(PaoV}=o& zD4~584S7?eKnWML7LV|}&-)Jmi;SjE4kBeQlZv z;utTw2jX8UB4FWs;IYcivlU^z6ta1!%1B?=Q1_};bgoHTq&*a3fb3GJ8%wq|{{~#QC`h$xz#R3dN;bAZvQ$k6!9@)K}0{abxx8F*GUx~SJr>{PY6wAn6W-b4JNvcZoE z$DZq6of8C4^jqQQ9hiDk=y>(;G@XaGtoo2&sc3|cJ=#!`#9ujk3p`fX_;F*SQ{JgG zRqBNdG_pG=@4-H4MeuSe?-zvAC%lsepKNu)eM^o)yWMStyiyB5g3FuS(WSFJl@Gb^ zgbM;G?g;4Sa=U`&rS&3SMdYm^Ddd)1M=n5FanVOwF8Do3aIuila+y$8RV5G)*FrgO zm|{EO8BxKVfz8;?xxwMz{E0`KgyJos>#80jl#BH(DYyTn@>0)GyYBejo{ixSxu}F8 z(y}(w)zJ=Ry$oQw@z|K1iYD)Q4k1}~kf(X!3pYD=)xBQ03(8DGyautT5q*6L_%Nyt zz-RUa9+g0DH;EGuhrLQZ0DY(rp)c|;(7}^B>6Hcs_0qDl#|XEZ@LA4rqn#Bpk}Q10|zf2 zJtOitp&0D9{B8b;%w`v{>`ZV8oCVa|oEmKIvo8>1w}gx^KGWYTm{l&EV!NCjpz#q154=z1k8;=;HEv>~SR-79 z%df^o4ctHvnuR~niEt_yH^7o)?yXy!*umNKd5+kuPWRQ_o1t8kt~TOo3$g2 z=tF=yz(C_+q|wUVcXT4! z(d{X??7K>bW{xVek_B1W6g0oj;amlU>L3nr0v5~%C`f%|*`MvQ3%F4-o8cvH-RwOzS!L0jQe7r=ySMA`4%nE-hKMKjLCr@-c_@mYw=Ye$XZXIXV`Ui&2 zNUz9X42~96y>b)8x*{ihKB>K=63G7O!T&C`j$z$O@#IL)AL*gO-heSIs>nz(QqvYLb=6P9Oz{2~MZv2q}?8RVC|(m+f-OtO(@gPj4X)|&9XWS0p-IYB^A0n0>~SoW0_Lv|M8)i54alK)YWDjb zz8}m?VG~2hql=nZ2Vedu`}hy3X2LUAJ)7hcXJ=@k26|*yn1K8D`ud|4+70F9CT-nE^fo2J@w{^sz7)qoe<2h#EfbhotZJ z`5e@ai%z!?lYN`pJk7sxzQ1brn$`F8CGx&ehZEc`-9u(unv5@U=uNn9o&kCf5C8O@ z;dAy(Ms;A*6kZOvpT;o$Q1lVce|%&PT4(j?H9dyE9hP78nF{726b54hN7Yq=8VA+X z+&)PaoW`eubBZ&q2(P|%?~(q>fCPW#u(Gfm%Io1H(oMqCL>*njZt=urv{&KDJ`|&r z0-+#gx;(wFd_kvf1e~k!FV1p_Tm)l9y+3sLY0Dphy`%dhqKepwS|w{TNQm9w5)ivb zw6k;Y9Tl!5KhK&UUBnGdLUeZA82IidxtMX3rm9zxUmx@Fb7*Tp*`*VMV{zE$ z?G9O2gU9I(ydI|;0oW7vPRKXqa7S=zta(GxloavXJ0SrJ@dBNm+{mx;dc{#Ci!+O* zV~I|Pp1#x*o0$T2*1+1J0FJ)uow?Q9in!r9a|m}PgB}eC5#`ZUBitaDkcA&KNI~Gs z*ctJSRI86S;zJR<=(kZPoQ zaw3O+@qBlpvU_%Y;z5Ooh^RZA{szatX>aNw zHCgfOM{@U1*V0GC-4T`0z76`~F_H}yVOa8iY&zZA7;G?H>5`JUmFnC-)i9Q5S8x}2-nG+~ zhhHW^fHz@JS{g_;StrRajga=KcbFm$3n+NPEOQX0&xsu7CS>QlroMVOXBU2!habsI z#|(?7*QE%4vHS&I}8ZX(YVJY zfIj|4$$wjXPeU;_p%0X#5ksBsrl!8BG&vi1Sk0dM+8QLcrmECEvN7t5#z{@u%tTI> zxd#0DSUNMsbu*DxKJ!WUQ+c>)Zu$M$hG&VGxukaP`@qMf9<3R>wQH1hCl`Y6LmDV8 zq_+3L3?yxxdzz_8w-G)$)s{=f{nos|I^EMP^Y31*LF_c<1bW5xbBt6k@*Ckhiy6kB z4nkBl6ugF`Q!@7f-!sdqINw3O%=cje>yg_j--d8t09?zU1M#tDK8wIi>uOEJ@a8Nu zIrUlG!ObQ$yKYxO={{$di<{s2OYnYG?DIgtMimm?EJ zWs9o8td#ftN5mw9PR>(II1=n?t)S6z9+WM3OywIm32v9tI8>8s(x4V1Y8ROZ70K`) zU$5drfOPM4T((~B^4Hwkbd+JbQzKIi>aCkIU3$x>zO?U6e&@rgKp-9w7)TmaZ_+5w ztd*1}b}tgLxovpX5ZCW?@FKFUlDdUft8)EL?hQbez`Ax#3e+WJaV^1J$ROI`6&6S= z*~uqV?FHq)uE z&SWpGslHHLDv9@y8{BZ}@j_Q^Zkmq0styX${j&05)dW`S^0ForxW%(%?IRlf!3E_{ ztfs3DS6`y_5gu;y-Je(Af)oj*cpcrQgIqB`xc-w6!1q*)%F&1@?ug5Wq6I*F2sz+P z(ICi&;D}!ZH651P3%U(!k%Cck7x>ED!T#jy2Z5?ZPfb`Cvw3?$HpW_Y9u>K+imFi3 z*-VmNv>3}<*a%_y-%eQ#>U6#89aOW2GfYq@W}(Oatb#C=Q%99^Ih z&yVYxxSG#dWwrU2FYus&@9F(LBSM3-5$o}#yo!618p3X{ZuVD2&wn0>CFPXdhj#1^ zUHhP)Ggm28(t4Uy+Y!%mT3U_{+rAi|@W*&2=BwY`_;`QLEgquyAtNF-#t0k;AI zpCV&MRpwWVw;0!efPhU!?qv9wBek#ssp)IP@qruSaFg~Gq!G;o06Z4q6a4_R=n@OTf^ z-s)B2IVx(-B@)+w?6jXn7HNqMu4e7tQ6)7QwIGE{|9F+~rwsYdgnHIJ7u>kg_nHCi zgwN~oE|i#ptw>QE&GUp!KKDpxW@cCTymB$W!6KKB>v|!kZKK!NC*LNn9m6YuO2`TBw&Ahg zAqhgEW~gBcyYFv?^n|ud%=*1d35ZqRzIcySOvkdf?J{o%4KuU$p#~a!c(sa5h<862 z^^$CrU-3uX&tGZF|7C?EeqYQB6La{^Fe(>vuWDpFfTG8mFBza%H@^@Ix#H!%C6P{@ zR1Zz4f6ARsp_Mgb5;kUQyJ<#Q;a!X%QJ^qwo!1f@KHTwIa<1&uxgVv6|8iorvspP( zkF(bb2LKZUltN|gvJRTt29%%i{0t~feW0|YE7@8VK~a|Vil0re9=Jdf#Nnxw zwkN5cu`qp4)2jMc??y)0pYfE|y%pNY?M51eqij8{PN{DJ9*!z!VMFv|d1nzsbhLl1 zsaCq)zY5KdhQ*ZnQl1DPGFU4LT1#U28WwtQ{ns_^CppU6L@>WduYZ?QAuDEWd8w9G z=8tlYA3OTLMXO(2n+x~gfyl2Am3DS12UmnJV{KygmlKE(VP^Q%Rr-HkDfiDg{J%8O z)wr!qz`wZ`Oc7}{%6<*rH{u^id-7^0;dj?;^e{KGzORWl1N>*{`ah@n2m9`7M=-;k zj&)v0+4bKUx$knev69t5L4Q#=lF{c_c)R(z zMO)nm(M?Cdd& z{ntUNU*Ug0gVp#R2V-H2lQAwF@-ic|gkAh&+eZ!A#9vT7xPA4vk)vboId?_Z=@=AP z>4Os1q;w#uOlC94ul%+vN4bF(c=ew74zO#yj_R?7>cl*u`rkp+DY!ncaiCEB;9?#t zo|O_{XiB@|#~s1@_S>=JhFSZjnPR|vSBzPXjv&P5|Ksf3%D=bceTEYLQ42mYRPslz zr$XDJZUxLYZ~MnwTfy~f5Ym~TN($@J`t1St&SdEA9Jiirbmg2~Ni?2g$LY#IpViKc z{K!zIn0dIS*^Xa!%&jdaoz6Nl{-J?nbZzI8jHB=S`)_p`Za5%nP6+Fafjr=@-vb8B z|MjBBl>R_XeKPxBd&u0~$yL>~j7$uruh_iQeBG7}_$@{G8-42V_L=1mT*tcV*}hyK z@K)V%_*Qt1B6hfhNW}aWCM+uQeTY$a_Vwv-CH!1KuT-cYVCMfJ?5m@qYP+`;K@cQH zx&;)JM!J<$LO{B^Tbf}&8l(h78UzWYyJu)dq`SL2XNV!b!}G@Xd*Agvzi+LxX07=L z&fI6;dtcYJuYDi&bKxDs_#58x&4gxO1^5K^<=M4OUEwFq8D30`;~S#Kg@zcN=Cro- znT1-3w+t3OT&DaYS~OezlW#LMADvxldW=6t=w$m>+w+M)YSAE4&SAPy^e4dTU*=wfZciRt^2^Od45M*``9hHNYdl^PILUW zOYD^p#lDjC;e%_kI&vUR00S?!6>olNf$s(+Y~xq%M=UE2rNq4k%xd|ag7acK+=<<> zCYs@W>m!SsOl)D2<^+4Y6#bL$7X@eOh$l-UG-!rY-5M-`C|=}-cxU_n@K=6$a=5T+cBO>K6y&m7IPsfzXm4lGcyaFNE}{6zx34nM z@JXk;b1}GMukJmKyrZolFrXTKrm)Wh_%%G)YhqAGI@C%W-F(2b9WZ?6emQ-50j*K2 zl|PsO{5VM^L)jiP2MEM89GKp8C_@yFm#=yOQ+LmRnL>{ShuV{F-);N+(g_&wN5yfD zrmv|=kxqp%z!d5(2aWt0<=voJ>GdghC!Y=zDr#cXCex0zG~&7g=`5ZR1K&ZU?pDsb zR}7dnBc_>vd9#)By7x%JnH)ge%bgci8<@WUA7|L4&nh`E4U9E-_(TgYl)FcGmexfV z51+}2$=bz>)SP(ZZW*4UFE+!DaZ{6%r||saH7owARfB101*7@~95dC)aZmvj?E8mb zHE)=&#;BSp{#^JfUT6DZZ?iNe>{?vS7YG;}p_)b1E4U>E8wFCTPuc}3k=u)S_yO8~ z#rXegddT12zi9wxc5cqxineLONY!!*Q{_?K`zAyLm z0{%&ZkC|`l zeYUCHbibk5B*)*Xe|6fV`~&cBo0eQq2B|M+D)~Pn12pd`v8zd(7gBL;Ecg_hr^xdj z&J}DT{LHKj77ueBb$gH^EjKaWIB5^ZrE7K6C_}(r9YxjCY@X+h>ypw7X=fK<5EQYP z6hg2Rk7l#vVXMtVZ^X^h@%8m}B<*hM>rA%CB;{ha%pLY?PFgHC0t+`uh6gLqnjv`@^EA2?KPp%|IShW!Uq4W)r*H1B39~O8usrWMyyDKkviQlL4%65>d*G)nH!JE zZ56y~Gx(@Z9JwahE@V{NrNEmr`@{{@8H(;z-iw>fTN7nNHkz!znm-rYCHZ-Q7c|8$ z2q?gtJ2T#R)nnP!(bg|LXM)u$Hypgx%M2UWAtLTVmid>a5jKq$SH36fim2g4n3>^u z8>zzm!{g(RSgyo%NCaZ)mumj*a%TiGgukh^KSPufD^49oNJxlG9`Nb9IrF>a2P_b0 z>-^1AmnhW1XIo$2c^I;~GXzP~jzG$aVFelx`E{%lcyjk-D+m^rxFKbYPz zyuBnKTIjPn}H=luzkhR5r>z2M<78Vw};Y=A=_p1wn?smKX zi@_kjp6Jr9|1qWgUs2=VPO6M33HjvP9qN0toErcYQK=!ux2h~?LtxKoC8YX%2yWA7 zBj2$e&}znP&VN|J$Cp$|haZ4yd^NsB+(LmN6#uKM*ZIB9-Y*Gt2uTyYr}dnj_bB-5-=ue)FLo|G9u|=q>PQ$BU$O`2GC&s|WKqmgc z4{Rg#mxbHs+hc`DxZf2V*{$R|Jb!z0HWPER88dAfBU%$GJP4qG=m@~g?0k{qrSx~` z%pa5e`&+lZ2V?T8Vpk`KZ2jb8Y+ZZ}$*Gb|dCV4?g^)jn7{&PyE0JuLF38WSf;)S^ zb!^#JmP~km;@&Tt&$FP{#r?=aL*!AdZdb=-i66Dvge_pHX(@-SE_@DNmznw|66P@Q ztG{mAV9uW7CM=~783LHw*-6{KjMUrBzd!ZDo<*0Rk}pzsD|9;Z@ZIoeqr9Fjy-tCy zP#=_y^r}b=e7Hn5FvZVX2`HNP-AXH$02PLe z49k9h{YDRW-E>urjw9N)SI$`fX3+x z_vwvAC}&zVce@yEDQ0Wn9L48)8D(((*{QtR)Xe%`!Ui~3Uju5Q_2y9X`eFuf%nuB$jdfFmG{h!;_+>UxqK_l_W=E99_uLpf zR%k%nHaD)3X(w21#ct5oipmZxy*J~95p}YG)xh9Ig)u(bv+q# zodmf(^J^BpSdHfs5UBf6+TZ_vWa3yO;W+bpDl^!Y$KyzM{`ecU{fKbElZh|m*exeF zOxWIFu;gI616&NqRKU!Q(~4h6)8EY806GWg$>5jXg3~e1K|&nhK=vl9g(hummy4`NygqHGT+whiRyFVCPuWzjIyVl;QN3qzbQnB=2dxhJzQ6fImo(py+ zZSncwuz3%-4BS^3$ZL6+hp$}>ex68zpXaulD)!Z_7PDKQnp{};>xox*VY>>`4Bv2# zgb^;(=lT*s5W;*<m4#!r1(O5siAKARABXeu^6c#U;A1eQhz!|waa%ki zT)^Atvs3H4V}IYNDZOn*v-xt2Zcv@`8U=*YQUia}5^mOK!I}}B)=~*W$gj^@>U}sk zIOFs44y5eBc-Q{^p)YoWOZ`LOlogJCJ~X%Fl%A%FRp5 zpj@go^6}=ndTgm1KE^Z)S{Ic2+fl^(c;_O`8?qJ10&XwE4G||qR%Y#SU9q*d=?O%% zDC0S6LyTik{40Jna4@DH|VMCUT~1O4hT&Y@DTKHFXB0mKnYIW#0>zXxnvXN}QAfv3|0 zlspOPit$D^CZ&w3{7he=UeY4>larGrdE=4{e)iPVRAexRPb=&+mm_#7k9Xkcu3Y-c z%E(BlmxsjP_|u;IH)Of&M(_f$6MYvuJ{lr_d$`l<2Ust7fWVp9kV97B*ZSC(*riJQ z9e!!iFnsF3Y)8Fwwv-H+Uy!}tIaRLyz)yZPKcIQ1;6}YGHcmeg#K`j6!IBhB+-(RQ zr>GEm*wPIiDDqr8H0#KpO+j;M?+BiW_KI$Q z2crY%13+3%I^S!VTFpTs;aO>WUQ9h;%%|pwAf%O4{!?-m>Td~~Yw&*L3!>3^P zdOMDDFs`d*(L!P3jED$Qp3*^O@fX24aj9pp=;@#&a#)^YN>==GNEuXA?6u0;cI9_HVkh~JKgT`l3y_?UFD=L1HzYL-PUL* z>izU&gs@;}B%I8Co>HV;_O5Cc5_d|~B;?;N8>G0o*mLv)|Mb1S+7Nf7J#WF3kS;1( zxZd}epN(g5Su`iPVEy(=oy0NN2o!BqvX=XKYqo-lqu|11pqM zC9zglR9rj-php6CVza`7(-R=Ymew?17B*u~FcWDC^alVs4oAyvJmoF-D(*jtz<==q z;47YP%LBdJOwG8IDDXLIS*?5A$6%4OcE9*BZE!aaL#F z4P}=q0~WdPb0+w%sY#eMjtdsZ@%>k_fx{~8ymJp%;|9ET-<*XF;Lna^RN4jR;W}p+ z;N+Mrv@w}iaz2bxLZZr za|i^2V7LH{A5R>bcr#`&04*ycxKs&t)FZmu_G&&(D4ksEN!%OGmCtIcheAbpR2B;b zg8}s0%>S5aZhD$R!aQK{#cvA#E@G>}W;T9y0j22w;sO8uI_QJ;S&sM%iJZlqPLY9p z+J81eTw#+xyqNEUwFy@OI{RY}7E$rf=eY|qestm}%lag3FkM}cK(oCpaxZ=8df(3) z4{Zo)OhS2dYWn7js;!1s9vTq&iUHUFdDxC0DyO49;JfdViF1F8N&KRlQ=wwL2kN{c zLnd({3gkC?;%X1nCAJUoctN&iLL+uy{F{A;>#loV=H{J5y9JqMpNg>8&w!djhkgU3 zguOiit*qL4#9ab)p<;Qvk2KC@p?S(O3p2cb<}lDN2z2GNfbAcM$o*SGq`&FY@AusL z3N7+U{_t(7SDh4Eq~B1UoAKxzKdYVz^{1iy9U}NC)VqT2nHV!&nHZy}qoZtBf7I%3 z0p?V}xxP035CWP1Q*Payc3p?=aQ0*8dptFTc3q|gw?^kcyBDWw>SeI17Yfu=R8_Um zJ$t08I6^gx`TPg%^-svABrW!v?O+_zUEWeJMKaihfuV|et)!$RpNI%m+xo?YS7{qk zT(K-vn(lCT_ybmcruX)*J)3pUt*w##z4QYgOV30l$hImH)9(;->FYY$P07J!34xF8 zi>;xdA!d&r#DD$njEm@j*xLVYx%_)42$=cv;>#FjFwazB=3~o^$WK*{`j+} ziOG0Iqc(QC7W||kG$aTx_xf92=_f*N7H`;3&Zfw{4>soU8;aXhF^Y9;RR~gxe8-yV zl|+uD1T+r?!5q-jIO}}_Tw(}fTO9=?_7i2VLQaGMsrnLgm^#7L8i(H$Y(*HxFl)I4b!UtTqKcvglRRANpOx~)!Eq;XsDI3qQ+vb*~o>Krca(0oMM408DK z9d&)+=jUnJC*S*5asZI|c_y9XfAl&2C$@4zN9SvLa=sm@2`@F4-gwJn*TF*ypa+!gD!hzbNa)4pt8-6yZL2V;8EqCid|aDRcAzk_MP@B}N-y zHQSq_%n$%CVEa{XicOE&=R==(&WA>c4Q(W}(1@^nnzzZzlXX}5UU=b59Omxk7Hk|F zzhr&fsw?tz{IENPU(saC38K z)z3@5DP6||j;s2yp;?Ez+bqUEtudwykzN6}3%A$J$Sozm>6?qpTfx(lN^w*rGLIkn z5`DJYI5+;f94He}KpSJ0ER#?v{NL%}zm~|U17n=OucYr%q+ZPDBYSL6!MIu@pb%s_ z@>@Qbt5^O-8FO$}trjc{RYN+Bv#3Hgt<}+ORn(nQ2l{3<)SDPY$BgsOW<2al3+9b! zOD@g)#=S+#4CYP?OqC!jw&tQB4Kk7X-GrYsXryw2s!nNhmI&vjg+K#R>|jo%{Y2R5 zb725$k=pHENSW_MKE>p zrmeBIvx?0lUD59Sy#z=_e~{M#&jg8KAv4~_Q@c^kOkQXAi;gB0T@wyyZ0 zKYoz$S6+pRfcbbPrE#B+)tWyGO6z8Sec;pSL?XuP_1MuXu#>CxU_OHAqbc!P#zp-T zI-~o3HyeJe8h9+d005S5mtwiPo}PeN7+!B{D!M#qK8kXD+*;AUqt9W`LTE;RU=jQfyej_rx5QIR@^ePQZ6MU>(&ueNK;&X0exHD!_ zL*0+M`P^K)E-^og?Z0vXu0uSi=fR7;>i2*F=bh9K+@VdC#*b@qbHH@P_2opfL+0(( zM!RyiVCi77dT!yNI$T}4aY1exP6KA`9V%Q(zg<9|=DzcB^>+&B$nV`mS!2o%4= zl>pZAn<>57E}gb*D&T)O;4O{;EkM89K3=F1@#_&ze5VH5J^03$@b<-uNO$%0$Hzd^ zQ1+ADV%J5H=ZC))Nf=&yC-CenG$w`k&MV`l0=CFJ!A#cLQU=pGYHi!2wtljTyp(-g z58>@GnTqy3LezGN#mf6k1LW!Jadss&mMGmP#jdFVnxdtbKYw3t+bgAsET>{yEnKbh z&j2S?LyqG!t73ds8IY$VO1h7gc%>>ORQjlkn2q|VuN-fYj!0R*Q(5D;n}(OUhS=&2 zTN7%B-YZ?WU;sHvVC-*h{sKWj!)2uIGcJstu2Sx40e`INIejrNFq^~t7~>%u3Vb^NB(uyLo6?&lBZcf?Ms zm;)kmSEqhJi$9$L@~d&!Y9^{a6G!st?Vr*sxg^!IyOJtNFg-VCkFCd{-(Z#^tVP@) zgoG@dL6CC`*K=8lmu$Re`<7xS$()ZX1B?K$_EZqMnZ)dr>2v%}^%H7TRVnzwTOIc* z+M|iV?iL6WRjdHFn>!&=QhkR!1A^2>P550;5NyJLpI&_@2juM#?dOr=g06vlv-1}R z_FW?NrJLOW!#2Qx)3z4jV0iIIv+V+hlzp-g*q65h^U3$R)%DL%?*|GX3$NdDzr{k2 zd;QcsveP24_TGF?w7xkiv*Z1gxn^fVZk$>tF~>WqQfHHVszlAEPD=NCsbjxC-M^0( z^!n{viMLh@vMJCFKiG(itmC}v*=eLjPDb(qujBmA*?wcsnb6SI33x+~w07wSJsLe{ z)1$ZOH1h*WKS=L`2>$*O=tLis(Z;OnMiUc!Z&QYqtImivkrJk3O5+tBD*wlfwy_vT z1jbHiQ^+h~nc+X9t%RW{v|IJo=QFMRyEzV>s^6VMa9Y1vjKu@LTU3Tna%y7$zlty+)d<#M1lIwMFMov0b&XgwG2J95aNDo8ljiX9=g zo(JHC9eU28Zf5$YXziJwq!yQMAM-x=-+nZKzpBFbVkkEOD7Dm#^_m;h64sKyh(^j> zRNwPWZB9W8AA8uc>g%T_hVAX_PF;JYCM)Qo*|o}^e#sEbQ;p_k$mKpO(W`$| zRaNErmECrSM8`1kB*%`1-yyA*bcSw?TY|HV9Rf6L+f+5UNj`i~O5y2Nl^qdZ0nQ*e zAo*mPTF!>#T2B6Lo*gWH92*mqm&fQBp*DCb?Rv61Re9}A$0azG7cKo7Jp}W9cuV_= zp!HW^P5zh5T<)H6)f~FAQm7rA_z_*z2#?CJcqJZZfDIiT>2=jucDOEUN|O6Wgpi^F z&;9Xz#0!&|%+GCF1w{uolztz+Bg-eNqgibqY#8ESs6~1_g~ahCvyq@{et5%hRbk+$ za~g$M_Pspaf@L^f`DBAT48~tr{CGP1#JYF7C9LKGJNqN7&5J%`>P1erg>Tqcq;Lsk z+JQkj;e8Mp4E-4z{(@zGY{Vz5u)n{)g6@w<=rbt7+v)qV20GamzJ%wAI`L4t)BXu* zhYRU*%8->5mSBxAnzcoSCv(n!;La>)Yv{70tKR2e*yhO;@-L_&s*!VFN{`dC;-0ovSGz>H@ z$f*<#10E@})Cf)G*FPvlta+EZ5e7%aAC(AC7K!v?CXwgQVI(&Br6-a8YH?5mD=WLiq{A~;=wZOyn6*7QyqLlzKsD;$OTT9ftHlATN*`WNKFA_0Ld()N=7014cI<|628Lcqy*Kdg4QE zWIj~-CJt23uf6@|Rt}~JZTsAlu;v`jW`?^&vF!=I_X=3cN~24?29`qv3s{brjqfV| z%Okp54pp?UO!d5=ihW11YLcVcvGHE>;spu%G9_y1F}`dE$UyW!eGJlLG1xbib@<#$ z-qOn9N2lHlRvmBNZggo+Pc*k35fvT&@}p@j=XM^RxyzB zfF)6;0hS~Y?lSn|?(Y8n3gPmfF8IvZHO!~_@yX6nH-U7-y%`e;PB`S-oz#m1! zqg;fa&S<(rB;gf;##8<3sUPM4Z2eKN?0TKd0Py-N?)5fo^8VEnqNc z@Ylaw{QBezmXF*~q04`yAhHbfILDi_l-9-bpWeK>Buv@dC4)wG`wU)}!Fk0e`QPqs zk*|Dnd{#tDTI)06_WobDzu57J;8El~k@=b`(chNz)~Sk8_72bK1xRrf&>b>!zY0P4 z;tF3sbE-r@&XXRshJyTA^sq{btFYGk6_P~QnT{D%g+?F|b1p68FCA+11`~+MIz%x$ zlXX-nx~4x?=X})w{Z$=a;G;e9?^zMFBKnVfp^QX#+Ful)57w909anAjVr~qg{W+YC zrG?o?ts@GNG#77{cwk1M5E~~Sm?*y{E7#mHDvF?uZWfXcgi|=Y+tA+MKfa3a414|a zmCqw8A=ej8r^E8bC+mI23ouAl^;lDnMnPd=dzJM_+ibl{d$x4MmFIc?2Zn!^E&pT2 zPlCKPd_j*$R3hj$ITRTcSa)T1H@nvL)Y#zC7$Y4}Ng)>lM#;xD_hnM~=L{pHIS+)s z;=+47UauhP-PBFy$txi{tm&r$5$Nnf`L6Aktbc{d-|QZS(0jfw($qs!D~w|V{qcAz z^2p!+&||%a_nm_9H9UntsEf*o!CsUzO<&59K(KY(;@1^H>kmJNwxrLTE*}|t(>2H$ zqoOQ(b}kss`FMGQq2RL+Q+RYRFjX@+Ik`!Hn8anO$0H@M>bc&Vd?P6MV1EM^h>3p* z9J&s0`ggfK~YKefb#ozTJ{D<2_T%V~M9S zCROoUpdAA6$J?QPr@?F+-8?gigk0r)YTQ|K*$?$I!#TmNFRS{suAjd?2*9t;xYEgR zh)#Bbx;gUk=*VrZde&2?`lU(V}P7~>u3%}{*eoS3}kfp$cTG9`KK_9eFfAS#pX_Lj3t};v2H-mPm~a;I1N_2UR_V%lfXY*!vR0S;9! zHUr@mu#jn$h%hVuBT`H4h(7Rf(tG)M`|P?gmfE=LH`HQ20zmXJUM-V0W8sUPf2ZZ{ zpMdVW(bzY-Z1k7ej4>~S5s5i1zV-wf)@0u}4ChUSs-$z+(N;rnDMh?%WrLrH6LdDX z?P!pGrS&~uIs{baM%4Rk&2Xo=Dr?oV5G|iMw~1huk~(^Y6rPu)&|wICQ^d;J`caI? zaTrdErh%7v9b#Pf0X^rkm#l24qm$%SCbwn(W>AjyKM_L9J9>iLA)TH`Ziq?lR1dX- zuscWNkh-y-Dc>$oaAT6ogkXGx1ze1cB_MNMQxiK}P@DdSei-$xplkSuzA)H}P77x1kdT6oTR~!9_Lz7`S`7Eq67TEZu?V ze?n~kA=#&&v+POg2Tgv(D&kbVHKOP4qczi&*9|Tu-o?uXg}`}uGj$6eLD_0(bSkCa z#N!E@67nm~qVxNhJj{BG6TtE^(C?zhFyxM%v0vTvG+)UJ>=aeD)j6+-AYKK~|H_^y z)+L4AAWFfmFSn9__V1?cpSW*=Bp-MQtv?gY?oCFL%Fhy}%Sj!Dik;Fmul%K|_1PK_ z=Dknfw*C&{0&#*q`ymPqfmYrU4$Vu2hV2OUhTl60lVf4UT_@hVpjmj11`lVX94`M0{!u8yNFDVQVDnk=kG`%}s1knA; z99at{H8|uA1X=lAtgpF1&b;nw%# zPe*b-<%t}&JfIo7oEq?TACy|0M84htUEo@zY=4;EE(^2E&IR?n%ujdM5kT!$f{#{o zKD2DOyv+T|bvo8>RK8b8tN*Vm_mN&%Qx&wHiq&MVBt2O*v1?vQ9KY`Dmd8z1_H}Te zd75Tjp?~rfGcucmF>kCV>ATQ;wsU?oepa`Eh*@_ml1|0{-E|}etwLBo+uExk1}_$} z&!k_-hams4qWN*3D!9|+ zF1L061wH=4Pa!Y;H>oJ1+bwqms(bTESnFE(qo;P8|_U`{mzG2Ljc#{5)5>qF_i4`&TePJr+NE zF@Rv7n@k=0J5n(S!t>C^iQIMt2m3AU;GC=b2iqQ)|s$Lf;1)_lnbL~$P}Xt zyu1rkdDX*$>g7A)Le|umVRTLEtV6*2bc>WI{GKx<0NMtl<4E7&9 zC-ODg93IWtGepMId>p-P{flRr#(Ly9b+0S;us=D;YSj|x1q4R%nJm?frx~cLKc6LY z6bc8U3kS&|zm}WAy+>bU8^hWVF8zY=iFs*s86ukA;mJudjZy<@z=`&Y({{Y*aNna@ zI1NyfG7N+a#dEkaxJ+fVD$L$_0zNHTU)0RP!!GGn7V9jWJ_bYGG|4on)n`K9wc$Up z>WUe2UXhIVJ#QmlN_GpyA?{Ea*Ro*8Dz~eWRpZ_&GX9R_1Z?28S!E$Goa z6aDG_Tw^;+Y16XF{@0hPJ|b1^M*tRTdWk-^k?A)dyhCc}GxWp1{XamS7$(Q`4&)G^TBV=Ahrjm#G_gzDPm5Eiy4 z$wHS{(Yq69YX|@ivk#L7qwslS%7V5Yzthm` zVCp|Itt4h?ksw{7Mm2zW8Jz+YL&Q51YVn`c???3O{v+ZxGD7=sJtJIhxlqA%oj&0S zJG>IUo)Y;_*1zJ|+0+=AXF^O%Mp2t*W<*pP0{$jn+3@Itq&nad7#l+s<~?md>+hWh zO(z&<>|nr)NYK==mL z?_mAXov!P7z7*OXh%ZL=UL1HkPXJa#Sj5}QM8Au~rRq|M{XAoMkU3;c_@If~XG3_w zxs^mr0Oiy{mjY-b!#`3uikJrPVpm zk0ssS*?}r{*q|>e`79I8&$k8RK=GxTO4T93kKw%n`GWhkk2h(wd1-1WWjV{ zF3{(9nGiU&N6?*UeCB-Q6CYgP9w-F-gWEOR*W9#esa0pW9FXn_3fNxFj8DJc5vdY-kP@ z3I5ckweR`#%Z6`?GmZBc<}Z^VhghR@R5tw~_9PfhLsHxA;~wlFR|h+1ANR&m2A`Dh z7Obf;1*;@tJ8rub%S;I0tS@TMxF|f2OR;8PB!dw7xQ4lgo8$3uHZ{s;nVDnldJ$9H z33b{<>G(KN-+k%}sC9p}^R1okt5PH3<6$O_EjqD@iHXK*_i=4Ygy(&y-w$&_0AcA* z-0J~DUvohzT)F?K5>iIb2%MZpZ_a%Qn9N}09#sj$&!$cpV+yfG*eU|tdYl5B#oHWB zU4o^}TtlvFs*@$du);oIpEY7pwF}GDZ#^4-)5~B&hwT^UERLEQXam^H2B`TWB%tgq zk5gcH#H^tLXTOJExr-DV0fzh=PHES^4Jr9rP`Y+xFyeFgTSZx$oP&41GEL7vd~*x- zJ4sy#;ecBrH4XHXBY<)wuXKbHwag_ej*(YB-IVYPfE zX3;oL^Yr=@wzmLSH|M}54Ci?`C&s;4r#|rc%kf(89KHqNpw&{yp6$|Ig@o*ft@k8y zcwpKzP(w)m0WUbYtfo-y8-uB_R}?fNPz zpSl7!W8I#lm$8rJ{Z=v!ALZqIwL-;p-?Os#L6foN_yoHHuS#9vx0rOHUdJ?P;el76 zWVa~zR8NAr4YtP#yX!qoJ4S=5cHd=bUb_t|OOr`>?DeZpNCQE7^-S77 zV{jA1GK*a3;gJ!!>SYU)3=g^S3@maEE0~bfz22CL4c|Li2UCAZF zl%?`tg5X#2`?Cx=#$8-|S{QJW=$}=AEOTNL&MC1K4mE;>PJf~bpBL+vFut!S|Msx- zRXlDw;~=%jIC;vY*26?TXLq`Lnp1BCg<%AOnk9N$TU)T(>w{3#e&Zc1Haz@|nWmTX zx!o(Mg@S<}m^LNCfUE0ek{KtewV%#u8jFNga@mks>iJ;@FfSi$h@A-<9wc25cE72z zS5%COsMgeT4xRv706@;@BPq|PCL<4{Ve%Z%(O@n5 zk06nye%|RYmmC^;D+mON(?4MVE|x{l^qBX`Sfy*b1bmr-D}Nwy67 zG%%R~-WMc|C#U^Y9{zRRH-kw+YzJe+P{BT(Bb)LF87h8de2%Pb-aC0?BqPNb%#Ft0 z_Ue6C=G7|G&D{4=0n%ttl@|%tQ0C2PzTlclTUicnSL+F5MH(Wl86X&qE&XTF8^8Vq z{TX8KA@|Q4K6Cb<4|B7PxrDxK0Y^U_MzW?`{N5L%YU3_-^(1mim`YIHGndYGu?~zf z9iNe&RGdzwVpyi%jO(v2mXGLl*YMO$`vElz-+Ml@)R0{5phPS34OjQftQ}`YxQ%g! zzk1rc(}KyxkdCB&DAn3{tI~jBt=z zFySzDG{%&f`PyJsQoog#Iy-K<71zohHrAxj9TAEzdirH18NeDaj)k^Ka#9X-8WB&< zBE#nSL;^Y-TvKz?=us7bRFjDUlrAR2vN-wk|CD%JUNU?#`b>yzNfi4aa1G3vsUTp{ z|4UhKDRGdk=wGYf8edj0eZqj;RbSE5iL532>T=CEams~o~i@v!h$pa z&-Yn=*jw4;;`C0eky)hy450f-PlD|`rdGA?3a-ACYga7m>HV2R1MhAOODQuc zyX>E)S^WVRI6k;$!_p_zO#n+@G2a7<#a{$j>WR@*Z1@Uuy99V+>!auI_f->?-c6GiJc*~JWDww0Hq<%Q+p zq2Shx_L8=|9a3&wxZ;lD_+x#TCSIh_cj zr}|X=KWBAxNwkd^BE^2JI>Y>BY1N8iB=hQ_+;){j-o=WdGja9SrHSD>35x%k+3`QpV`ht?W{qQuj%j)cmMaq`r4u=J2i8fBWFRH%+YN zLJFTtUAVIFNLnLow9a%3x$B>YVzS!v4ABUn{gUUw`1RfS^v=O~x;gJ-|Fr+6JM$_$ ze-YJdDMt8q0H@~^W1 z`g2J#9)Q@S&{Q`u@29E-l=n@NfPMqR5RV5$P9ihu=^3>>Nvc98adu7x&P8UZVx{D6 zw>6uOy#Wm4Uh#Fc7l4EX4y%^~V+$7Jw%HCHma{LMSbs69o>GDVyJR{%U&;^Xi)=2z}$^6I&SQ4eZv`kfeP4X6{@K;ze4oBn% ze20rJGI#+LE0P2d^ksYE$3_0R75}jz7m1!cEy()Q_K-1#$%M`{`oTDF4=&cmhhdPq zRnnHwIv7iqZQ1yY?!-MAYgSU_J+Fje2775VZ}0Ga(Mm$N1|yrRTddG1l4KYE{`aaL ztv}GkqSn}ez(*TcvG=gtrD{v`8~J>WA`Evk+U^EkJZiq+;fE^F$ts=UGHn@skER-z zGQHN9+P`zg84qPkdr`lwiV*;g>wm&{{i^K+-ZQEXSig%rAMDhGXjDInwgp5AvQ$JAHg$f;{=Vgme(dqqKr=UV3aA4)2LKS<^tIJT-A ztNg^$qAe>=!OD843X-S=Oa#?zgCa=gEBGlXbXgIVkILBW6O6lZQBXQrXz!& zzw`8pRS@3G>#t~f$C^};z=%HL*C3Ueav5oP3zENY&!H|>+ZVJbN=;L}>n=nbPdzQW z8&!h`290!0+Q{d-J{8fuwC2W?QFv*g6q|s7%A2#3)XKrhX8e^>?nhuuNkQSQ)_NoH ztn1-IxoId)H|v6bRA+?eq%P|Z5QarXlT_*>T>}5FsIn=mMy*ljDyUUrB3(FESDu1J z)Ek8^jkQ_8Msf1FWwx@@AQEV_0LdFb3R)xsz$QZOO$hgls z(Zsu?Aay1lrl!Xx%r2|7*m|3k2D1Z>@pZlL>_SCnts}fAa{#)xv;M6p-xNd^g?@a+ z%}K#!)&C#*+*35kcTX?B5D$Q>Y@8*1m z>w{~X*4Xy#!a;RRLfbDM592K}NAq>5$r(3sEz^@@%uGOM+f!PIKCF}<=_`;HE?Ao3(r~t`7nYef%>EFr z_xumPOb4T7IGNXg=viG~=%=Z{!9fW5OgJf_*TwPu9s!r#BnWu}_A)2m6TTPjBobUT zv=Ua4r%}sXe5~%oNY)x{u~fdA2)6JSpRwfqPHBIb{R)iop3wKeFc8TRwQF!~HP8d7 z)2+i|Uzn!YUoHw+SFcQW9R{&wbFP>u zOfH!fFG=^lh}$6a*FH;nt~R;gxe{f-FNzN#@gI_VO@C+wIZmnl56LV|3QeF`tJdQz zE81x_DZFJ}O-QAWh)F9Vx8L-do9cb%^olooHa%%Q_{95Qe9sL7`hON9ega@0HO!yeG`29R- zg3bzen2gFqAnWa!On863h&njEtz5>H8H4Uy@t1biXomfIOeS^L(bfm~M0iyu!ADEy zKOD0Kox+A3S*w~|w}u@^j(ogn;h=Q8t(^zgks}0_^rkCb&n9$f-lj?^`vJ`qC(K!V z-@^x_{-A);=KTgI|L}H!+pxnH=eKI|T&-SDuXkwvDPq3hM;qel*rhN`eUz_M?dD-` zj=9@Ms<4U}?c=lBmKG+7<+bJG9%@TvW3DY1q;%gct$Mrs$EhJVfw90dl1*9yGQDQd zoGJb*ZsV);kGOt~sk270Om_-RX7# zU2xqrQ3!XQW=Obcbbej`ghvKr&o&Vssje6$28z%*uIdk<3cEKxq!iL81+(SxM{^8x zzM~JTXu`^7QmJhE5`HZe#1h*nTm&)ry-I;}er5BG&yxS@~c*&s08~CQRN1I=a{D99zClYDAcQ!$YL@j5+jS! zVOIXO=|#7EFxIHlp#gZW+LwAJ*7;c2AJ?C{k+@tadI8hauTJJRY~WBzIfl=*=m$_b zHSS91PW(jEf0oW?PmFMl&}Y&yE3c#H+4vNOC+Vh_AF-_4W#5`=#QYAXAar2q16k(# zMC_&Fw^6%bQdDM5eOPg;pIBbohnz-%f_GI~^m!XuzE0i8Bcl@Db&g!*X{95gW>6uD zYVZ0bWwR-~U&jCtGjV4oBFAfd4Roe!`s@!*W=DDArgFkA%bv?(c`fbtrQZp6nLs5_ zn?=m7kg@^pf!_ji?o>D+Tu7fVhH1-G)9p;0Ro3_F~fdGp2S#@8nA!A zx|`PV!*}*x*Y$9C3i-d$k7jB6k835 zqob2ATa>3~n9+rUkI*_Qo&$fFkB_s4*!|f=9mf==+i>s>R*Rik=R274JN**w&{NO& zJ!Q<;Z$i5(kX=xrDI=X-E1=GJJ|sI=d5fXAz}h3h!$r@DNMhPMPJK4rxuLb5biz94 zqOjI`wTW*`-Wcnar2Qv|uRQDJY!A2OYl{USeNw;KB|eo!mwNgZeLV3Wkn7){M!z2| zYE+QUwyT_coI1ERh%WJe+yc=>V&U!cbg zUS)fq(8??THqjAIQ_1l(Rh)u#D5n_+_&V{GlaS**Pm|3h>TpbxE#e3vjA0Y{CRd;7 zn;jUv`G`tU>LDp5ceD`w6Q^S?@ic?208UzJ!!29M>*S43gm|(Pc#8Bdl4in5#%)a7 zo&R{?Kg@iSQHoyZz+ z*Q8I)KKIh-`{TCfJvm<*C@3lElCtm<+s?~-%8uTO`kc*qoRN7)COuC9a-KIOGmok;*9w20!?5ZNP_v-2xe#*-A1jFbuEI^P?WGjUqpr2Q9*QS2-oQBq0 z+2u#3;qGGYJSwe+EBv_hTvO(C{m{t-DW!ItJZodIn*6^{M??fqhqJyXT7O!3+1+ZE z;8eq_L$Jjs4-RFp9oCA-cA`t#3`=_(8Mymhu91=CXA_4i{}n1IDMHeHhW^Dh1nSsJg7kq0ZB ze@Lw(S6GmtVUPppnKw}@YT?$`<9KnBIhRwD>Maqynlp(sosgIK^%4~WnpuqCmXjXH z`gIkW2s`xjg9nVxt|G)C(&~K;D8}^C>cj4kX5{)?p%ari)$$m6PM4<@82Nf!AhHd; zRq7x7Wc(%^%R&R8mKdbyjzJz7F9*J1McbCtGd2Ojl*bY2P_pU2J9xb4y(FC$(m&H3 zmiGkH)oD5#9e|g!m9gvqFYIFNNq>ez&zg#{wBDgLauA(D-Pjz9@L~HiGQL{Odq2r7 z)ay+w>Q~IYS&I;Ul}1Gp=!_NEz+J=!&Ketl_FyByZO%QOb3ONd?)UeP zuZ@nI*=xVoTJQJsex7IXNAuATc@a&9QcNMdznQ~q<*KY0$kI53$*Wl1L^1h~iS6=l z9+O`(mYKp?Z&m@|8Oly&E$QNQC;EF$mYaH|@qx)AKYOyVC47g^4e#GP8eP^a*tDDJB0W!iLv>W6olFOWKW$*?&gC-s{pJOG%-OwF|<;Si?s}Umj|l zyT`XtIu2X%wz!o@nY&*hF7{oF=c${;TW|bD)Q8%}`j2DEDch)lGQYC3vT2L)x({3y zKaFW1wh&X(17qe=-3qi(*^-2?24^m_Ffk?v4^3S7jR8Tt!jP9RM?u3PsVe?sGE*Tt zNaXROVjzUd<_FyjjZQi@l+&`{n{pmsgl~vD0xftc1+;q0OuMkfHtDvB3i&!I4TVoj z)`wu-_!J7lc?5z0XVV@M2OjmD|B$c}yJ^%3;T^o9l&@*%902^>f2WkMviUET3^~m( zVTcGTR_ks%ZR@#z4a9+VA~2%+Dk+|u_=4gswkAFp(iMHVB9YtkL8s~04t#;Oj>j`8dztU% z;#-wN_t*M1h*BIv3X+-klM~wW*&5H+-+C2Tp<`hgSMmq?*XN1V^&J^D&~69N!s_{9 zku+T+fNXh~-FclkA5SY_L;DjDLL7YP2k1)MoI7#3)?GpL3lUI!6Q%o-c+Sk!$%5YL zDtMdFQD4)3grIlAl2CeK{yJM-62dgyNn(+Hma6Fv1=*nMSYDKey4No%0T-4Mz8cIS z1Ggm}{Wy5V=tI&Qw{Pvnfp4O_#9bkaIIFg?2C?-R?dxZDBS=l9xLd>$p z2+lI%(|y4;^Q5QbiQG>on29O$3GY#gQ+dVS7582?uhW7l!ZdC3Ii`+TF*-fj?NqUh z%iq2H8d~L%LoiUMlVh~bRBsxyZ)(Wd>R8vi1YHn<8W+LYE9!4nnKix`wwxJeeIsH) z8L`F7gMPGXxX!nHe*U6=S?;qvdSZTd|J zYb*Vlho=|)HIh9-Cby9i-EEh9!)=~{K~~5DU9YR=vU}6Tv_l)d$mx0Wnu}`>jU~ZA z>y=8edXcU1C~FDKe8=G>)671b^o@W+q|tZN({^?#@Rc=BtuxLFn1D9@;gn*82;_m^ z$;Vkw7J-E$pi45pz53Nmx+9uOI`-P3M(xJ?kM;Pu0pL51cjsS1w31!tl5rM9O*#5l z@cKBVNVqNl^}(I*`$Xg;sJ^@H){&#y+8 zk4^zWHG}WjTO-d(?ASpGoJ;{{y@nUAr@#$p*(iftOlBOu$zO+jaE#v1fZGYz%q_K_ zee#^hX-*XT3qR1qHI^oh1*n81>RIHnb^sYR>m{zfn|tt9mHm#P%87SRuEU*P)86r- z4lZ>8yN{8R+_{e%PU&i2WbTg}zAWLNbX%y2a0r#>$m08oHq(#Ct*c;r82mMDqV0To z+(Cj+=?Ozu#`v(=(6~93d*)QS3cgzKZc4@Nr%MXM7WCf?%Dl>Ka+OlO*P76dy(BW67q zj`>$==#w6_ETfNAey|yPm%+;9E2G^6vKCK~y#YT?=>k@}cSwDC7{%ai@FEBv32S6? zkAN0{(yc1XKpZsa&sl!M-60P!uel5DhHi(wbb{aFl3)saNs=Yj}Bd3@GX z8o@pTKublPwXedLr|-?lxLC8uvqtPzHdDImRn z?8W#l5wHbYuP-dAh!vcE2U>O0H_Wg!f{jOn*~*iPpXJH|f-m|Q>K|81P=~yzBu{oa ztyl*2yAvv6U4sV!0_q|Y%TQgbFt6~|g8WkFXfazD^1)!AcpqP=NW+?1U&v0!^zkL5 ztMH;iq~BZs&UfkP91~rZt_>_^tU1C7f-~ zm0X}j8s{+&^k>f>1L=K^?!TD-GFS=%*M11Gh8zU*C5F}rS7({ZZkAjhFKerM%7)EQ zWo0n3Xs%2Wk)kh3BYc9oc>dar%Lna;Mp<`0d=eb)N@g-~s+w#)(By0m zP)P4#vK3Qfe(L+rm0?j{s+Ako(X>0W*fPfDDQxpMMxa$4!7;jc*66Rsvb?Wzy7Ctn zHyhv96XZ@O^cO}jcO;_iVV~L++I*K;KR3dRZSQT`ZL>@Aw_58(M){CVz%`hw2FX(_ zKNSYiJt^U(Xn(f8zre7l9!(}*{Su#HvLmrb8l7Hgjk<=?L@A;wP+LUs6w6eT&7cb^ zxpLTcm-8C@d_JTvMZogK`ATn^$i+?oN9cjmodfNw*i-{z^EsOs^FLxU5@UvDGMA(? zL-T2tvm>gO(4cdGbc4*ssm7o!AQsi@P2ZA8+$Xw_%uQlmSas6IpGHJL(QcS4?)Y*p z`wZx}tOU~K()CfNbD#ho@(wP-73RL+U^3GcN6M|-i5_{vG)TUpc^B&JK#t<(@|nwq zOz#4amEx?aB-&>_gd$Y>!tZCl_MJsu6-A5ox{hN|>XWCguf+Rvhazg3r_kjKxyV*B zHvT4HXS@YUUJFcChY1mIwD$r6WNsAK2eD7Jl|6Op3Hm^2ET; zS3~mXVa(3;S^TjD@3z;S$4l`Vp%*(%vql8X<5U-ZZ8S4;3u3tf{ZIdnv;eE8CKh{v z^ObvKCf;fVMW&hJLNtcsN%;E57Hw6R;|{yUo#)it6*uZO%RT&*Z=p1|u(w-M88K@{ zWMh?;8;lDZz6j+F!i@F3T+_o}36V+BuYY~`Naypj%~>KtQCnR-TT!xLUDWxG`AlUC2;Nqezs)1RK=*Yi-Wka5$=&>^3P|Dag`7>jHXK-ATc zvKsjt=!uYF!i0MFun(B{7AGF7t6W$NAdbhqF}F!6ZPb}YuE7r&PM;FoO0BqU=)@iM zp0H}sTOw%(>SI&_oJ&Rxu+h*P1}gLiDa8G(C$)DvJBOb3-*(l(=^>*q_}Ke)njo3p zd53rB@!Q^MhXmIaU8h*w41iZP@3Z5TrO5rVisBljB0)46tq2UnNU$)bEcOhFt_?~K z`$C~Ic#o{V-~<|0Bckg-J&%EmErJdc34s+Ij~(#Ch>s91GEk`jS*(5WnLUD-Io81I z0mL{|m%M(<>ZzNuAsf(vIXPfs7T%g;Oyh%xil0)lLoCHwuluGqoh_%eT@@52Xh~%> z%4qoqV8AC@4{Yv7+TIr4sNHm(((M)LpS9x6HP}UFIRudNE4o$|bH4!9C3qvQjxdJ~ zXX0J;%=+te$Kv_o_IYfY^jERcSwrw;>Wzgf{j6IR^c#XY;N}*u??mgh+w4zSO8-S4 z^am~3YC##3M^XZxv1u0zJ)*ql_T!d*(ut1nrnG5n3tQ`Ux#a6y8ZKGN$#~Oh*YRcx zzl#*B?#DIL5uOnU@xI)sqOtR~p_1t#)m_$o_jGxtp?)lV9FLNxr9uNqjsP;&hcz#n zNPV6g@>RGnV)w$;Zay4vC)(drZ?KP%9x z^|uSUOsOi0lVe^_BK;!nw(0QkmKV{-;?ls3U@n#RyEbF?Q#AuyP}_X77Pe0A4#PHK zO9_M0jl57|Dk+6mz1Q%<_bf z@^dFXG@-qFPfj^Ht-mrxP04R8V^ihqvbh1Z>wU`A_T5AYoB6z@YDtur;HvtZ!FusS zcGKsn<-g~HSJ&BT80tBwMhjhe9bRU5U!VcYHE{EHUWVtsk=9jiW2DY(o}WbwFRKF& zINeDzw0tLiI%IAn=&+x>UYRw^7&DyG7VxCqQO&&LLc76sg0gpt0U2bh9ezS6FSB%q zH+Qax^o}^DJ$rqKSmGEBY`ejZtA7`f4K*ja>zGS=M*_ZH2J`_b-sLF4K9IdYfNc(6fRHG#em)dZX+;mtwlO>{W5hZXAn1p>9e{v6Ge&?}qa6GqYpN{l? zn@L!5D*3}j*YyKcNAhyA+tBYsvuF0a48pYOnpn%advu;lv5E%iib@JSMZR@&WwusU zt5Utrku$CI@~eokMutJa6_-)ab0a48xfLdLD$fRFjWkxK`H{l)OTiC z$|TdKRZyFqs>!s7`d})%C;X#ks%IVC1@H7G(Yg~W$`*MVY~Lzh*J$^;u?{93`GdpP zCVy-Qzp=hK>6eZ>bfDTSpKX6YAtT=Q7^yCN!1O$Pt1^wKzC}y!2^MmsG8N8+@u3fr z@%AoeZ8ceJ_3QW-#FIEQ2nynHIJ3zU%`x~+>?QU**##)=HR)u`Mc!qtNgH;*_W{tF zt3#*?r;p+A+#8gF2w>i+GOS~S@aR|d*qH~9vS)gd6LJn> zg?}qhwlziv5UDa;az(lZ;P%2Wnez|Qu;*(_XL&ET6p!SMS-=2CBQCRW`4?~LUr?l+ zDn?Q7L$7h4dFw}+QUog+F2X2wS>qDXWc1?$Y!64xq#U%D3Z0wZM$&}6(b4YDx%y{9 ziu&=QQ>k{$)TsvpJ}r8d+IOQEF6JLP+@HswMlf429&Em4c)v&~rh6ktkPbCD+vu41 z4OzDJTpZ=gr_k}<(a&fU&fpl+tRi?*T`}DX$~>eAG4NZ>36og=gNmb+=nZkO71%yA zlvgyB006%~MO!i^nD+8+DRCBVUJ1_&#z%#YP0HW(do7n32vY~o7AD$42W`8hDs`6L zjTgQXF2mq zP;PW8mtO#=nmapTV%9)05Lk3mSp0~P7CPs)Q0jMS(=RKR7Ws+-Gljxg0V{fuviRLj z>=%o-XbKU4&GdG7SLm#=8ren@aZ1OQ{xtlEQZfG%{hN}P^>-U(mm;8aS$jYb1h)Zl6Wn>RVz|Hc|V z)0U#*NBQRcpRf5A*|kk&Of~_FAb42(E|oY5jkiHY!4rDR+_Lck*fn^OQ)Xa|3JY(F zXHDswJG~XvNpH3Gjm|CKRwPMSYP`Rtvf<35Acd-xl&J154|lsVz!y&l^Dq+baP%$S{lls%U1Dv9gHwfOCwhbsP{u6hZ@c5Z+G z(W(-5=W)rdmq#su390z4SaMVX#R^e`ja}1Z@Y8cM6>`H?xtfNhIF))rTS3@k;app= zqHb$YEw3T$NtMfT@)+9+Pt6==Q<&1sqqmRGs9UFytH@&2k3AOyiVR1NY@H#eFZx7R zPRGI{EY$r@+ZZJ7SNGX=!AGzcc$_WQz*x3e=6G9QaNw4-$}^7MYm%3Ie%nzSw|R8R z=JFe5Qt9KRkTSI-FF^*|8cK`5fl&ec5PvyNybN=}l5(2+8t!W2_!`%V1MD%OQ$#PG z16o0cNxSM0ef!!DIwW*oNkxge1)!l1$(xSEHUjcLIbhd=aM$6k{{k-5i37N=lb~L4 zB`uE=)VKTWpz^o0T;#WRp3l2uaAvHi_XkHra<-DeSlvUn0F0(L39&W~iA9sBet&AE z^673$n&Y*U;lb;xb(U>Gr{NSMMC!@{fHqU)qYTmxcKH0R0s9*=CeoHfm(IIUv3=>X zK3Jbe0oLV=y3^UKG4hj+;ztmqvSgG)j}r}$cI7lS&KT8An#=NIIyWYfQ4!`^t5Wrz z*Ni!Vx=3OA{2v(=Iw|&vOkZc&$BQyrzz<9=_}C0FVc%KT6MdnCB+JoCQQ0(|9hsse z_^OjVDSM3DRjHspQU2aCdVW*B#fT8EaARMY4BX|85tJ#Uo7<*a9w#V?~P)u_ne@hDM8=nx_{2?b< z&WFa&Fk=;YJIJ!Heo;;B2OIB)<#mq|*)02pBg)KtA3>XPZueBWBH-wBVlUvpFHr25 z5UeP{?WB>K6Q_2c#Eok|yM({-UygT>fZ7-KvhbwCP8xBan} zEKs@5c(U~sj_#@*hx4krkwZMg#n{YUH%Vxs5BtVkV07_=DRs8id(@gqXC%`-9#%MP zk(_;G^niQ?zE?O`rmKqVvDEE%_te z^VK&F=au&sTxaO zsHtUOqTow#wuMnJml1O!L087R9{OY6us$v6#4!S&0qlk6QaK_c{nbTLQ1=Oo?=WrV z?@sR?C`ac#$XN26h}7%{ZNqLCb$m1PxuW#}?NRq%v+U33&1S&zKxL6S>8f0!GyA(* zpL@sE)BSv>f$w|T=~M1@ehS-2aPq6`m(7CfJ7&G@)Za}v=@lXv(@bV^M&l|NW325z zgc;WmF%|&O?-qXjNcVE$jnD`{#lx{hshL-*TthB2A@^Dv29pLpSQVJ zVzSa=YbA2sbH3-ZLt}Uo3?qv#kAHYPwLPz)GKKf5raRZGg3&R5fnYMnl`~;TpG-O1 zhk?K885zx%a*XCvMr0$$kl89JQa*K)&WJY9>yGwO(^hA{OcOr7dg-lpzKYpeXUnlN zlc1u;WqJ8_7vC&0lGKZ#r1gkqC6;WZx!BW(LhQc*>y((7piH{uDQlsAfa==!Wol$C zEANv{<4xqPV$j!D8T3reD+b|O)>EAaXUFV%MF|%NM`=3-ws)!(581vwMfNroaJf%e zXKDJUJj-Z!ApO2?Fn;{8P4xT4gZ|N~zHk{&Vwuy9*W`51l*3FtqjmC#e*ybYGOV(f zofG`;4{W36`uf%%So6qAUEcpFpKj5*@A=BaQ_ehD=r#Xvs!}AKWUAd_$u+12&lr=b z)cI;&;}rHWxhxlFyeDoIvyVOE8SKuqX;jfavDUwUV=`ZwDaBSDQDBgnaMsvqK&8T5z8*i3}nA)~(57*Wl@} zO*KE2v(W3Gjtno_s7(;ri(LE|iVxh2H z=EqX`XGRn&l)^$PxF&KY!f%bg94-R^yD`iNw@FU^$=09r@8(ZT{vY)1j(rQpe}GzC zAAM~stx96DRR0`2uDzK)-kzuXJxYV-F`}WQtJ@dwAf5+OnWG~=upr=)@3T0eCNjO0 zyS3C6k1Py|wxgJY4-H5y@PM0x$iC^9^taTbRl+B$>~3mEotce&uz508=W3IDrJ$Sy ztzJ!UA7GoC5@G)O@BJ3wdKDtWvYAuZSLUf9EK%Li7W~CP&+4JE;H+}hkI}o%=RN5U z=QUfVLX{-aRT7--JS)n6DhbyE1V-WQZIGh=UH*wgsW#NtX9N-Q0s`|8@l5d#PEd^} z@ZZdW0^)GXZs9duLl5gLmEty4-L)^~W1s(&nT;fgTHIXgJ1(IADFtDK@ zn+I5bi+Q4eOe}=QR@qA(NT9wdhJ8g8NHi)B!#2XWXa{5c9N9Y|)UtEV0 zr*sU{%}6SN*5TLMPsdoqL=;xbL%BzBY`CTu8wh5ZJ<v7`-sMpK&M|bd&;!!gD4!v%fBnKf54M343ZnQ2JOrhB}&2BJL*VxI=Rp+vrS# z*EBeY#AcyAyzSXUPY?oggaQtyGoW>9v!idrtHZwY3vA>y#U0bQM&V2LUH>~+G4_e< zb4T^Aw_|HCO?VShKQ1y_K3fnNZI#7*y?-AW;F?G#B&s7??c{^V_m4BDfL(T$lD`XQ zeiU0jd(6;DXMe$Pxa-A4QP}vC$=VGmljjrClMKf&sh8({iPKIvIz~QO>l>h2+pVf% z0aL4~3Jc0UGU4chf+(A3&+q`!7XcvOSJEjnz6S+-Aj1kgKBrgPjuStAr$=^$opU#y zF8KCew|uCgf~;VPxL!DL@5aYZaYxgn?;PGV9jzPv+qP&7C4NaKNt>052JGPnN=(uv+^xAjF0V;g=Ri+4qM7o&_Gzk1bGVJ^1! z-UNJk4tO4isHF*q$b}Jv03IElz$xN3D7Nu*QD^VocYq@KBo!pPGLWYz8JOtWO>{3z zKRmV>Gy&L2>2r0uOW!=P9V=r3G$c}G?`dekLEtaCqjrGqbiQ&({pL!hmp!G3 zy@9m;6W8A&b1*DZ_0|B+5b5Xlht-l-(7T`_z5rmA>){kE2l``VcBgFy+e z+Z%m9khLF10Kso<-!b%2yMC?_n0x$qYjOpX!+HdvC+U4!hRiLk-@gxaT2j)fv(-(O zer_1U@fho2t`*?iExG?nX<=T+Lobq!iIT6n>A-&)sI8x3%v9N>DPqK{r@FX;TEyMA z5AhQbZ6%h%fJ&IYSm`a`Vq#D6wKjCM@AB`Qff3W^#q;*BajexBtr^G-o%g)E0 z#w#v>SN`qS>0kyhk8(vQNR=*l0r>mLjsZtDqKfV3{6d1RXA%b56=&2V$5RV+wmm_M zT6hp>E??|$^{nqlWRqp4XJHXKlti=HK-yzB+nLQ(cmNH1pco514#7~>+X93hOn|G~ z^If4tR?MV*(ucn{EU!W;n+KRQ7m)mozn}GwCks;W7wk!BA-?~#s@uA>HWc~&U76wA zyXFU#dnvqdz`cMv&2f)>D~4l^S&htOXWn)UFn$W2>wCEcf9c3way_!9Cv)#KIU;iL zp#g;d)u%^T$}q!0sZzij%)nzlg#H?Bhrw}vghB!=on??r^8irIzC)smOgH5QEJ;nk zE*JU;O17xut9o4a6tK)`Wlb_59%G-BX9+vH(uS3O1YCPH2pb&cTVQ18uF(r&cEIp@Fy<|7AWE9ll~-g!m*L>hG@x{f|uf~=0_`^4?<)n z>otIST9l-4tg))DwL@R;9cuW?*Ob1g8ZkAHlqqM7iq3kXxxX@3?~7(u?6~9fcIES6 zkm~sg@JiUsY_qRqaYgKX)#_V7f&Vm}Mb86ZdSqsLbVQI{Cl64*Hw`%D#Fl~bK-q^X z=WE`qg{cjU;uV4?RWs`o8EU+Kuzuy)qhpGl8qNh7LY z$HZK{HRL>ow7?>>16QiPu9ExCXuR(04c9@*{$#$k5LZaZm5~6&be>XwU2CK};NCK0 z*m(A`k-BhqUM#i^aUQZvtZf+INV8mWb_<@{XwrHK<8byXmww-pRBWN68ME^ESw8>e z-V_!4$$@t=1$<6oYd`Q>Kl7~nwZ2HEv*{de=K{%JzD}UnFiEaSbU*?`qeC1SAB>LIb_ti4 z;8&pvAhdRCUcoSo(FyOdBWeQ5U%gtYgl5ywg0U90xf( z?T>@U@^yMVCm@*As8^CJO5;70L~?mDDfZ%$Sxc7f%2=xBitXwA<`MmOblK^d)d2-@ zm~)i-tMC#;k|u^cjBO$HlTBZzYeywbT0fRJt&STlxr1vX?UUAdN3aNomyg5Z9#^scLn!th9oyH;ixjf zLN*dMX4xXl2k|FFV2Oc2*yv-vn%3)w08`Gt{WfgNj9Ky{H&O%f3>E{agDkMhS9@O% zrU<07j;@VSdNd((c(;IY_sUWec`%SBF`_23>NvkKWg<-?r>Jr5x4PkR+pjvC2ddznEODSINtncXs88*+vCs zQlU8vGVU=TIyj3Q`)q)iDnWQm>D!AJ&S{GtypBm%%tPNWw|if9$u6hM06fg-3dWu0{K-=PEO0kocKLF zyWr`i6L3qCXL(D|3i^bQyB^kDq2K6TTg0z1vy!nI^9vLGOOh!g60q3x@}tJuKZ-~h zEbJtVjg7}C#$vvzY^s&-!a$M!)JG#6h5MqJ8ks7$?!Zm$qiz(tA?x-sVHqA)6A1-g zRp*J9)q}8ivfn(+p3jRJ?7cn40oR~?AHAvo(GIaOqnV1;0WZcIA7pPqV|Eo?IFG>K zT6Uw)GcGg;e}o%sRqFe0TcaCyE>o+KEz;HL%43v*vX%i%t_h9EL8KvKVX?$H5g1Az z*JLHJ)F7r{0{FdEvsM+s4bCtH~b z0pk6Yn~mhF!z=IslgsD9DoKm$&mWhKbWn%pkl{~x0GK4N)5vfb1Xa83uq;OxJi**WE4FOm zu(<)K?Xdq6@hLy-mj}YHQu3-=`R~1P-cMLB$6oGDaPc_nD_WMByMJ_@4RCw4R#H}F zD>+9@lTB|b=rv^TMez|$sb=)~ju_i>W2wUa6ka_O>-SsPf#Tg2j4y~&Xhu`zB{ozJ zm&5`&^~)JKmPyRubXf3qeV8Fo>IsJn%0GzI%tMsZG9NO5ApDR+CVQ<_aRs0973ao1 z$a@H*Kvup?6g0BtKFc~e^1eKW*{%HM=s-4EK?rNE8S1RvlNC6~=f3IS?IEeyS{M&= zIbcL2AzFH$U)%TzpY>pGzX5x{s_9B6jr zGC7y2QG{8fgFXZBl4A{kP<|Y8Igo@gE??X`NFW2L2{Jyf+J#3}p5;PEbEg*T- z1$|$fG}tA)Ia=25xBU1jpP{&FA-`M={(U+%8sMK>0A&uC9}N(_!Qud zj!LtcCU87Ne=&~zgyeM&!y5#*gIAPh0>Yw$2N;hIRq(QXU1XxDOl=UdToaRk(hz3HGmzPQFYXtw)P8%4i$+ zI@bUvzx8K4oGOc)l}ERpEUy)15yY#tFZ%``dt|J1)(}lDQg?!eBo>#T4FbR9g#Q-* zAmV@{*1vOrKmLW%2@@m=s=A=A$i$=WJ%q^{V^5oCezaS)7}4RQOFRmS5q1xi468_N ztZnUVu3eI>{AvZ+n`z6)>B8EW0>ulh?w!^QROt|Ulr6{E`0W0Ka6@)cig#)5w?u*- zv_HYpgegEohSp-ke*l;b>u9*F`~wzh;Y5z}j{eJO<`PIcF%y!qi*j(_LuB~FHc%C) zR1>kKVFY>yql!;VDSnh|748%r$aTq2Fd|{`9#LRT-f<3Bg*EXLo-$5BxZ8MiiC(1K z+zh9Tj;$ZoAaA@5LL){?+R{i7;AddZ6b)6ID64LF?7d`^pgs$a?dxt&ES>Yn5Y(WE z1o%9~ozx#rm+L0lpJ}zl;=Zj&$@V6dul3aqrjKUK4%ZeRPOvMMJG%KML8W`{Y#LPI zm!ABu)!sOt2W#%t4{Z@3Ct2S5CYIJz_w11r`UvI|FB#pgz7N1{p*l_Xk9qT%^ z9vTv7XJ@Magpx|c#xplfO?770%9SW!l2ZG}W6^H!HT8tfJu$NNj(KU31SggOkv3N2K+Z8W<> z?vrNSQPVyC=Ks!A;>l{JwkLfHK2w5ZF3kd+>BIxlVA*y_m`rfJ>5Y5wFSwUe%EFQX z-g+Ymr(^V?k!mo|RaXQv#s`}DT@+=7hGJ8ATI1w6Aq#{JKf2i&PtS%i_>QIJ(}P<= z+NaE#5N{%Hf)iK5Wc2Swq@6toYjs9I9OBqrp$sdTn>;g}b@h={j+sjOnU=Olc(5Mi zEjTdV&N*s_eBi5ftmrtQk2Av-R^9^)ni`_-u%=`SyU&W5bJyJ0U9l_!Gxrj|v> ziuDPKz_gP5qrnmo_Hgc~1^t{orq$T!q~TVWUZ(4aZe!maOe0j_7u9w?O#WO`6mj1D zwVSc1(ogfIcC4iX(u*<=@(Cw+9@d2VE6sM~+Qy%>aVty=QouEwbJVn;%=r*3=G6B9 zVV>C*{8)4puxqL8J&QJyNvq@xAl7W*l zbS2%{04v(oa2nNC8%Hko)VPJbW9sl|$r25FS|ea+NNt=MnS!cV-iaW)V<5wGEVU2VmLnA7EY03VtLFMV5gjnqeWRlY`O>-Cc^=QoOU z%N;(C$Z1Ykd4~yi4Sl4hdV*eSIF9xp$XFU3gtk$_~1r>JV0KD zK~}mY4s?#B14Mi=pT>1^{liyvi#39k(J9>NmbXA39PQ>@=(h7>Y6hg?J8yaEVTSOX`vl@bj2lVWqcawP3)Gm&IHr941bdkr_Q6}SH`Iso80lBw z+N*{WtoVW(RQ;Kf;w1oXF+4hYQNK{(us3N4LdZJTaiLY3U^fp`4_(>`(S zeP!9a!l_*t3>>L-j>I{eFjXDSD~stoJ0BMPz4QWHSl(epG@&K+UR%xlBLqWmlaB*` zEKuJPQ&mTHecd@vmwH-*(spKz=4s#&q+Zv2>CF}7Qp`qIN$0THk&C2*eV=lzGjN0H zLYB&zFq?o9i+hBOngOy+u7Y3hd-;OGxmG%{k|XWAY6Eb(({$c5_nn!ofr73L6h4+Q zB$S((3*+2Mm1y_OQ<4&}{EC%i-P_9ArDlHk6uw^<$ifKJPTy zmVK7bZfKSOi2&qyel(e$%TzneM$;bAqKsyVgBn^>Fd2n(w%lY-$RMDFYQ!Xb&)f`I zfJmGuQic#h1ZcjmC2{rA#U}6EfU8u;6syciJg=eWM@c-LlOg3Z6$<3rYTv;*$k%{b zWa^XFQ)#?Fno3+8QV#j7SNxvW0Twg=UCC<9h!qDZqH|Zyn$9ijSULOoJ@CV-0wFpl~i7CSxG$>q07uBd{$z2OsMmY=~nR z;shb*SD%>wab^D3@hG_6Z2q9$^8jRaz6FTBW^88StVd z@Wz!7?+n2~K11-tr&!6$7UGtShuyqPb!7Jh$k7oV_wQ|5WI9Hs=W(C(rU-O0$An8M zFv_cj$W2}O^zH&h^u;A`;J~5pIdk;>w0wfOePyr5TG;WT2f{y;wV#xU)1C{7BSXs*{}jhd+VKB@7#>_l8`x6+Om^kva6rmCS3JxKHL%F zmo4LI*uV+5xG#G6y}RHct#O8L3oHCQjZ%u30Wpj#l1y8t6A79sY37-7N1mpuKCZUeaG@JI?=n@v19d z9lok>l%>L@n!qm0W(v`)ED`nE>!A3!N;6r65Ag(SVy(|2jX%-zyfTNpUP+R0?gVPqTyd}5bOh*o4Z-k;*XEPl${C=EqC(bx_(#-=7 zRgMGkq7Xyob;t_^R9Y3)AiT{(yu)qq#?YQjDpPPZAig6#5z4Pb`O8?|G$EL;4kyad?rKZ33yqf^sso$jU-SlZSSQKEFDEY4D~dSPZsPO zTz30LM~yhv&%V%pK@IlUPwwNj^R`Tu7K$ec__$0E)Fr%9t9>`Oe5b%%<6I*56JWwX zWM99hK!G1#1p_H4A!BPmNg%snqgp5Pnz_K629z+C$QyhFC@?B|^=m$nl4QFKOr>2m zR| zVo~l;ZC?e>4qlG!no_LfqFJ~Nhgn(Jg5DI6p38Xz(!zRxWrOai zYq|1Bx|z$jpXBvgqn+Pb&RCwKAOl%qAB0TP&%5#}0CL+v0-f8=sCZR`c zfg=xt^)3ob^u`glts=JyNr;;7y4~8Ig#E=^c*w-m`XP* zBBO*6_tpBDB@0hee2x8A?>8aRL76=<9G0-h>tzGn5T1qG7x`mqUin6$Lck9^i_Ca5 zOi7+h-jH<2`+eK7Oz-_Jhq3mr(`-KmaqNiCkTPBYd~Z=_Y&TtvLsp}*(_=(@cqz{1 zZ>@Z6F-Rr}*}3V{Ot(kjnBhigRL)DjjOR|dCwtBAi_~bJ_>KevPOj9}gRs4h)`~!q z12YL2=@7Q8NQh;yPVY%r)L$~DFBAdo(n3CuzW&h`qpZS;*8ke(hDUu$L{wbHU?l1H z;b=fvUzmit;#DT5n$a66{1g0%;Z&6!& zzuQfudDfu6>Ly>nm@mgtaK@>2kP!hbyWhQbn_D7=F`b(SuRJtLvZl19rxoj>mX4Mj z>kP}^WgZbVBv;&Q4xgmNJrGvw9=#QMedY59q81f1c5p6G$3hyJDw+D2CYU}K++1Ke zVa$YJY4K1^Q*R8p87wABYNA0kaMjb2staggX+DT@Cq&@#3r-;OiB@sH529#F@#AoT zA3D?aPSrg+m?Ap8L`B>&vL~qjY#H2{@I1!&{>RETw9fozC?%n-RhpYgIAtn#Py7YZ zA{F}$6Dp>}6(W_*JI9Jom*pfUqHW`D)lg`PMc;{Nh~#`4Q+IgHc^T7rY=HX!(Vt=y z8Bk@6PUIdok(weTTt}}|%)+K;XIzttK3uWfB%h_b^rLu-prcDVHPOkv2J>AWao!7; zZJzxVdPl4AIUjP_(k*kPZqVOmPm&3_)aU0O0zVTNFIV01bFz-ioH=c(JnmDq;?J^7 z+w!#!P_fKJ-}K?buPPs<9uwrsC6m? z**2U~u^F;8$WIq#%))H??0aBIImLFT#e`hJ*2Tp}_Q{=5W=tg8?}a^tmlO+;&&cZX zN*?RvX`E|Z=hXPWdPj27MA&Ryfu>2ABj<-$*#&88wPB zESMFg9IEYNRXRl2@qks7w-?rD59!PHBlu9Rk<>N*swCU%If4<^%6lXMB#utmA$u%4 zmgxv!Sc|mqju4gRi-z|bz2HVDX)L+r1dF&$0V(bFAuT*RrZR0xLQ~s1AFMIi{>*cj z!GQsSSc9S+4#`j^=@q!QsopPCuHxGX89LM%+}ik+pt0cYKF9lcY&%^gsY)q5^7E;R z;V9~gd}2rtwNqQ39~l9?_olf$3x2#Zey&_v>7A(fVowo_yu`D6c%Ck|; zD(hA{ynP&x<3~J!bS-ls#bvtWS|rizlQqJm#)-jywXFjN?lu3=-Df4fy=DG3O5YHr z%GDq+7D+&y`c0YDU({|^vYZXoNzM$8OdCH2XST!q-3+>6apsPcf z_RUtX$L4kxkN|L4tpbY!C&Xb~UGMt8 zo)`GMSx+**ogSsg4_S)2xYD|u4p(c}>(PK#Vd*+iX@^gpx&*_?+C~>ZT~Hp;U`TTE z<8(cL=FVt$_?2xG{?Rc@BB@dOQq)spU+>)e+!-Ym=NjL+M-TBA7ZYn+eo8m5qhgsX zAw^=LUd}4`?6#~CxMl6wG_7mL-if||_T;O@{)>I+xbO<+8Cr8SbBa##XQGzKNccr2 z*OC-#DQgUT+IyuYQHnvj)2CkAXnN^G3$$<4(zbf+gAV&_=EY{Oir-##VO3vaSe09R z1c>qqhoSxl4)Z4(Lv$@SRC&JMY5)>*%nI=U7=WFlm{o!Upbk(I@WV|Y?cP42?qL^{ znf~)slQ$rjR8S8wHXJ>igDgfgD>F5hi`rGybq4glGuilI3mw|btF3t3+Kx+hue%~M zK`~oB(w~Q>O&{kh!3#|73Unu9>J;DO{JI?! z#N=e}{P?`Ohn7qQs&NN67zJl0VEga?7QoLUbK6QwW~U4^7ZgQjasc$3~Uf^M=$!0=|ur zJ>f-~SP<6@+Gt_Rw!)`$CMPW=$mYZXgFnP@>bHSlVJE8=`_H-mH?sG~P*Eyly}2Xz z$t}3d&PZ4g0Ps*^!NT3xm&xz&zklvK_Q$x3@4+JJ*b7Y8zt0NW{W&g9!l7L8rmqPx z^=!88#XG|{BGh#)+a_b~H^4Wj?vW~zeuFMzI@4xQb$(mDmFG;GZIX8)Q=1Y$PqWO!&=+{Tl~ICvgH z5YsW!{DCOM2Fx0%V;c*68Qv@`(twrts?oUr{AR!I1$=emE+VT!3o+u@=8I?{ZH?k4 zU94q|gv=KLTM{}NlgGn@yE@+QAp*GQCgPXP^+dbumWI4u;Q#B9|8?U;;(+vJQEV4R zt$Wgk9+JrVubkJC9pB^Lu()VZIA1>2qXu04S*xywEtwrNqWJ)8qWk|C!#}@inG5LU zLdwH_9Kw-7>Ft;XKer2(8n`faWA|)~%o9he$cq^QCZ~AHJdgV=Py$#*+3m z{{OybSpx1u=FJl;M4gdeej-6IWqs|>!vSKlDTGfz41vf{UM%O%b?4x>|e;*kF{F0TedrkX^ieQQhYHI-%~p2Vp`G0S49{!0cj+)e&4@b17?GhIICz>yv)L8X zy9jrO)wG7{sOd#^EWNarhFmOloiG(qy-PL}cxl;pfN^mDW{1-LZ}aHym*+xrO-o!v z_J*&OJJ&SLKm*Oio!8H??{`l;2xK4}I7MAQ5)>Qh;mrMLXM3!ONvxxXWwAR{S&6-A z^*^TR)$N;WiHa4hcr=@0Hs2J8GrTSO@j$n5oZvKJS>&*wQUo>^woo5?7gK{o%=@!$ z<5fsFYy%_QFY93qa6kCMWw6 z&YK4gngBOCkdXm<&@+1B{_pukE^;Jt_{Sgr2SNF`Y%D-?M8H}A*(3_kzgrA={J|p*w zp?jWx$%Ms#bmL$P$;?d20_N(9q73--`>AKuf2?|czb)Wn-ToK@^=)8W?~^_R7QLFa zdA>)cn#s;w?4CuPyCTO$q=MkH`KW3sqed#_M#N*alC<%^Tl8HR@4E zJp2+hHk`Cr!THdVXHF3(%~}Ey=p{$m2Pvb$>c`G=Wn~PYB7T6C$;9zi`L`$iXW*87 z2Qdzh$Q}8(_(@JT6=kL#S-eRPa?sE9?PQMd*o6tu(Lc)%*8{cO7V6$n`G#QSCB*t} zor3epQ1*Y$uYe9^jJ+zV+@08*7h=4_s+XwR=U|t9bJ(3clp(fVOTmzF3bMU17#7T~o8}M_JT8SG2p$b$IvS zJ`Ytn0SuG4<|Wob5&HM#<*plM&8^$ZMEaZ613y%N_ns_c{yBb%rRUSH&&iAu8As4Z zU3wAH+vHC(VftM_aY_TASo;*rJ&NSAT8Y~Dczui zbeA;3(p|E2mqNqh;p~e!)!}id*kb+B zj+!uBrW2jaiY3&DR+wPx1q2s)th6onkxzc$a%33y4eg2N4@P%&MfWkiJNpC+i|bXX}-_?k}s#azk=q61N=o&-7Q^Sr>?c74*VvESqI~ zKIyhJX`aI?2A&O-g03&QsQ~;~m8Uw>=kLUl-fwr}ALO;l?eH+i_gk>Ny%97k^Jy}~ zE@CvL!g5*CYBWilFbx zC?>`7{I0RS+N06YZLIec6w9ib*FF@ac`E}=dbM9(ZkWD@^5p#IOzqfYcvb5B3GBe$ zOs=R?Q@47IvE(?3^JAGp3deDE<@wlnk^Piz^|+bmyw0WhjSIsJen%63rh`$11bW^#VG%%IE9_Z2phw4)PRES`itwq}~iVX=eL zqRYZ7=JN$(dm6e*DefEdqS!Q(OO8;Tx}${Vw0e+reC8|j*{p3b-& zBp!aL)Y;q|t1{ZFt9X_&XtyoDk>uh5D?fHT8(D^3DeNmft+xnb&p48NNdj@tVVWpJ zD3voGD(dANszU0jNl>k5?|o`XdYxRn>6AmWY5hwc7*JEOg$HB*{{LSZoe*bjeOFjUP<$Vn@d3;~RJKd;|v|-n3Fwu(5t5OR831 zd0WT#w}0Er-~<%R%4`gjfegr<8u-Kg4vbO$8?IGC*`%t(HN4s|X zyB1W_Lk*RonF@TdN`S%41BhRE2sd0iTt2fz_)$u+wyJEW(g;Lb$7vF$H*U&{Jp6yW zDyE9d|8ch4W#*8lHfi`|t;c(k5!e%1yg*L@%|9*qtn?>beLTEMqbXQ*Torj?PDWYh zRG5>9$dvqA6TyEdzDIbwsr|{dPS3AvT7m>I-EYMkLHAu;u}6mI7L6V`)hEJ!E!lgZv zs+8S-c;U?w2i+B4$tr&TN4*;udQn%v><#lPvSb;P7nub?Ss|DcG;F`N4?MB+ zn<)MDqaFAL#KZd2me5nB)7u|SBzxhnzv;k5DM?JSZ%UJzcnbMTu=CQ|5zdNz`N!&c zp~W%Y_&kntZzgcH_f%P~{oERa_o8whG!u<}`-zmG2-KNiUPea-7 zz=d|Oz(}zl81;}tR7*Cd)@X-sP#TMQU8Q$mi!{)TV%O1c47sOB)i-8PTfCuQ(%uvU z651h0_+0&&4$!Bk3fCP}?_%mEipJ0hJg3ubjVbHG$deZDF`c*hq0kQv!U3cL_7%iw z(-^-r@;^$68UqJ^=3DqLw1x_xARRHJj>h#9r^CSEXeK z(<%VaU3Ojvo&Mhw>sjN7O>Es8b$nBr0)v&Om{M;XCN1g~j_d@x4TVED);bR*zrCe_ zOB>TVTZfG6T*EEEsj)ur<~oZ$5!V^3`rRV_1?QvDzFG!-BM)=;z?OWqvnnxhPl4~n zAl1gt@JW2Y+q^grt$250eUZ4)`^P~&C`|(Gv*t8Ey1vi2Ya+nAYb+^}wSc3s;q8om zyW#wv$)lrt-bB<+J5Rapb=+*I@wuaA?5KBzF|V!H84qBLxuf|Z00#cPO=@pZg(Gjf zxYb8)yP89`0JBPZGBB6LeA1C>NKHXWVV?O8u@=;|FqaZzAovTsUYl4QkDYw~GwdAe z3MBl-HBBv7-f@>2mvGIO6bEv%@7#l2TchtsUTs!Upq2RvoM)3hN%?T*D|BFt5WFU<3T)bqZY*kEnc2AAvo*&8#;C+-wTFYJv2@T0R?iAz% zIcl^4Cz*Fy?_h5LQ@im$v znwDt4esTI@Ie1-m^~!K_+oHael}XjvPD4-nwxxb7e}v};E?BUXRa$8fF749Zj2)pI zJa7`lJqVK|S#Os5K|iA*ptD0}lFfw82fz7c(`0$dRHHhdo=5!6uV~uuVgZ;QQot&& zICPSi{Clb1tkg}#4V&V3J-XXtD~u%T`Cr^reBrx<<3>q|h2Tp$6+f7*$d5{y(N(yK zD49GA8N^G&Ez#9hIB+TsK+2s8O?}vk8k)X${jdQSo(6E?7bm7Z|NDbCFytllgrKwi zq^@vHn4>*`t?cTI&WS6N$^4?r$- z0xWh(d|yQqE$fOp7)RY{{;d_RP*MIBk8LE|Yh63Gx&$#UxWLiT3pg#|fze`>N{ywc zxMgn5Xv&il(@dAS&vj8Rr5@}`Qd8*G2|)Ffg{$-HsUP5x&xD9VQLD|`v_o2puOTpC zgTKcBVcMjn3Xbc`PXK%4n1fA%Kq^J-onvs|L#1YHs(%ar3uxV?>qb$?eXdrvsXPnn6(lX5Z{SR#gkOjRW zYt}sA;lFoXj{{6aZb$1o=MBr#>GXcI#3p&eDe9f)bsiT(CBG6WuO;g1o3%!P3cdJ+ z9j`&31OacE?AKQjwrhsv-`3<8r+a6K1N7z1$aYfyo4)@T3k;wFXgTHiw~YLMzEbK7 zs-fS-txLqZ0z1DE)1*xEH0Z_)!($si!e=C*F%vWs7C!XONY#H21ODS{5Dz{fB*Oi^C~X7s8A;NFCu`yMkp&O zJ$WY)YPb3h&*XAQN~1A9s<`nppLSUueUZ9)H)>hZe=LnZ8Q|S@d7=No)4vC4i+#$` zkTqQHw{s0_+L?W+R5GcousQx>xP7w#e2>h!`MdA7&zDz(*XR61iqSUk{GC8Ow|m{}DO=?{oGpe(XXkRiTJnEL;HUyRh7PTvIa z^y8bG%fIHdg?go`|DTl%P8G{imK9>gRWn74z{-y`y6`Zp8~YHu3w~an?N$7=$n+~~ zr7LWv6R+`m-jKPkVBd}dpC{pcZ(}D%Bn6-8LGBizi+@)k-CqKwMuC=1M)Zg?u5-&F z=3hM6REzh9nmFTOb8XreAG=h=QoRg86~#5W`UU!w8|hsq%2@@Qu|9_z<6Y0Hh+&%R&NssbLAB0@pM4%5hGI@yO-fBHc<$}1 zE`NdDms%%!5b@LM=o)mYJ{w3|Chb#G>}m1Y0DG;i~#H$LZM}o z^^VPwkBjQRZUuhPorlLyF)2QG*&LoZ7h5oluWA|eTS5lzl`fkEd{{U71OR_>;d)AYYIBmw0O=mPmPWte+ufwOlp>f56^3CRWZ z8B$o6fJ!&>Kw17j8eJ|K<;!{LPB`6;a^rBA5^x9T33vvvCQcQJ4wU&hWin~i`HfM$ zg?ZVdOb_$e&N?oB4qOT&-v7mZ-g|WH`1o#fTN8ir05j+Yr=S`EI&f0zGxV^DM^45! z`ttuC`&tZT`V%e_C68BpbH76%;8)k<$3Kzgxk@%5SCqvrKQPvxrlrvUcnFBvV45xKI4Q!~#HF3K1pB6We-J5yP!doE1%3D-w6v!VhaAg!a`x$7vN7i}Z*q0NdIZNu(& z*nSL)u*vX;Cv{HlJn+XK@3>65VUXospDYOvpvku7yX>{~J$RdjF|xr}q@%7d=<(#z zB%Tuo$^C%hcO&`N$s_?d#0}T)Pt`dMDm}2G<>g!=UaTfQy}^XyIIaCz^!jJMT3p@X zY=uowUe}!bvY$YW^w`M%xG3>1#|=Cf`O_Z#OJDr6$7&C+fMZXx9ohA7|N9XBMkqjz zK&kG{>%{!IZeX}vAYfKm1NGP;P?>McxTUeawVwiZ-2KmQU zdYfE9DQ-QXKiD2q?h`KKC$q|!?fGdL%7&aciV006n&ti^0vCPP@1uW$$ytRanT2hb z=%M1j(N@!E+OHup^B1ruE@NXUcI!Pc#SNJscuR`1J(M4R#B`bTU?Z#fgTJq^&?K?? zAJ^FgUSdUa@&&3d;Bu+8kgMApwuMl$d}q-Gf5M1Ew;lRXMztxzWT) zBUWK$r9`V;Q5^Nvtss)3jxXJ7@8j!^mc?rl3LLzV1OR4vD+-yeProOHvKa5~eVTKz z(&IWTHc0Bh_Tc)M-~@mK2+Y|*``XsnQ0Z-~uDp#W&$zw;eYD;iTD(S@f1KF$xql5P zEd}_3Fs@Pc-?sNp#{D&#JqH-QdOs)hI->ed2~bl5LJak#XAO1NX-x7K%ZgHF>y2No zos`cTN^lE{|1aUrrv3yHx|h)IL=+$+Oa`fx;#$PX^N-G?7ornhWN)^Hv(O4!roY?g zy6B9Lc5%r*AujonTQKX>=kdGup{I_!Fwarx@auh&^9nXMg!CYFt43?O(NY!{|_?t>huo)?o}*KN-eBq_spqF^xGKH zc=C>>f6a>K;OHfM3O}~O<=< z?_+|U>c`{*!ujAA%~B*3G?1Fa_R)I+`F;D0@?`>DO zyP(-R>6gv*bxq}|`n0IAk}ZYv^XR-Z!cngug=hofRK^1kGXm#9mR}qC_A%(&{M2sF z&ybRRxATaKXv&JLJ8qA+6GMN=NK-8UphfsBwQUL3Q>AV9t!TDHw3NPxYu)=o!yGPy zPM!z%?_@$G*>Jz*dD)|699Tgj+V3>1H>)8OU!_RA8^h6x=dbJmbuU&4OIB`N>7QuJ zRBPCKDx4hXp~MsGFQrTpytZIe{(-mfiHRhXE&fnL@hIzmj0gbkCqxQ{85t&a+;{WM z>nmUE7NHSv0NoxOn-||zRv3=UoJ>?n)iE4oNocBO+9Rw!PL*ECVx@yS2`cl(dN4lL zd7(||FHUQ^Mcs60hnBsNlevEcjd2ko!ItHBGLnPkq&ygP?C^{l<3c9&jX)~ z7`k&v^-Mr%D5C8 zH5+~ed=2T+MO!!^<;&W%bW^T#*{n>&2(I{xDcGygcJ0FyUcYFguZ5f?7VN#WM+&r3 z23c@m?xj;Sj&^potP*R^7loP!n5oLYZ;tlnvYmF+{QLg24@iRAHGgCOY*vbh5bzSo z@7!>jACdy)_M6w=$Mo~0`)MV)5$nn6|IMWbG!cK~1o}+B#KAq)HCQhKmg-%LArb=H z%dlv3J{v*>F39)|PsDR2QFH_%;Q3F@#|uLm+?UHty;e;{jL30cP(AC}2eX&Yhc|ZZjZz<&^&6=LGK=ue z+`TlSLA814+$%W6KmUA~tG<5pZ!JQ86>MmO&!SuV+zrrkc>wV|qBP5VN1cS*J#8)m zRu5)FqoI*r@3_MYkD#hm4nKb<>*xc7SfMz>9eSW9_4`<0qQ`N$3y9o#Su`Wb3t2G# z=jTP+CH|2y@Q#ULLc}5bV6?)Sg=s$DM3P8xq`e7fO%rQhSlaunLnZh4uD+8~VI-D( zP!si~lxuWep+MrJ-;}-gFoxc`#Z)QptWB>9d}Gv1v1H+|^d7bQXklcOvNW&p57XDm zlCL5Hfa~`SC4Tqgk|6m#%JrcxMSgDO?r?jik0$5&K*aPRsxD+Li_h%?;*uB-DA<=n z@{9$@`MWPN7CDhpGMD?!0Tf4_Al3MN=`>}+@CpdYd0*Mrx?t#6WhwC_7F2d|*4wu! zD_MOy`H4Yk_7X%an=}vO$Dz{(BR_zQhf^9CB%&F|!_bdpJuyH6`FiKhRL_>AwvUr) zTdI*JgJmsNM=uoA6Ik17c))I>45ba|BJJoui4CnVv7ptyFTNY6Flx7c8%;kX2j_i; z6216LQKzVRfT6wCx)ZN|x;4~PSwiVB;{&UXZnsHJul@4p-?!=|sy>50lVU_i`(I!& zI0_~5jnG4UZJywtx2H@1-);z}X>?M0SN?3~r&t?p6d@&J`mSE4y^gH`=kTKcC>V(g zQ>>c&IKY>9(>fbfx2rS|M~Gmk38~4ZquT$h**rK0xBd+b;&5eWnxd!|8kq@(ilA+; z8{gBYFn$X6_v%e)vPK5s9-?81z0B9Hplcfs?9v98tP9_ zUNZPbRr4LMm8#poyne;`(Xu~gTzhxr+70)GI(mpCk3ZZS$5)k7;%;%kf|i1+<%Vp$ zMPWw<{Y&_Z;T3D)m$SkY(@ted?5hNtfq{CENLj!3hmWRU3}~-Ao}Y9a{H0L-g7jNx zrAhz3!*5Ci?L*P-wXmvVB1D%Q$-@s~kg{N(?iRJGA)2p#p9tIvli`BPC9mY_Ux3fV z58ImGS}LAs)4$V}qRXFD3bq>l8;>U_U1hS$o(dKF35LsP11}J5vHb%zgO7+9io{u% zCfRj(J09}*ssPR;<*lWibJtCiQ|~|+dC=$^{gDv~=OX{Y_s5a9CdT|VPN$4g?~g-9 z%bf&AZ{I@@M>eCSg+NVLY3P-w>;W%XY9GP#(G|`DZm(ZSpU|X9_u=C!P?s^%H4I_u zQ0~cGP^jQsSCJWnDV25(gTY3IC4T&^YW~3;|M)9< zd3a62{@AfUg3JvA=J7orV7o<{=XE-`bkdya7wIT8EDmf2iwav1<1IopGu@=nbckWS zbGcj;s}#Zh9j38w3^@~wY@xSoS*A7&j%K224v<8M;HAq}!m0Ur(N&QS+8aHOuf#Cr zN(^h;p|fHBM)nJ8nfHT)-6nGg`}Z@Oy--NeOC@BQ0SnnP5$xy)Jr}vZv_o^cv91PP z`~3c%gujdW-OR|mDBX_KGcu+Hr^g?U-SfB)$a0Jk361pwe!R|LPZ?=#5i|cGI|k?` z9<|7hFlUNoT|IpbtzY9mwE$)vEOa%#M2*^x&WMP!pWdjx6H^iLNr-55OAt5Oo=D~9 zH&gYtTi$p>*03I+y5Vqu$bQ;BaydLl74|cB-H**ppVvi0U{F=p$R?RaIxN$5yO!P^ zsK2fN6fBe-1S+u)m!}*Qj!X+WioPK7(nYoq!sB84Su@*mx+ieE z%f#zrnpXWc=vU|KSgv3VfIhz~s;dhb>cj|31_G@?7|@;_M(0+Tq=n-6sY*YIR1mmj z_XC6Q(qevI275a%2VDY1gs8>3>J{DVDImp-xk7H%DB3pN_r~Y+lBh91v0e|t0>fLR zJL=8;7PyB`6{!{O7~lP5NE3bfI^yB2*#rlUnD^xG5cg*lD`oIPyIdb^819ci?j77y z-Oo}eL*0(TDFlZeNehbv=V&<@9gsik?X=mqF|p4Vj~4}7)LG5LDvSqHfT-gKN_xd4 z879?Yz~3tw7|H?4kIBVrJE(hT1uovj!pgqt)waGEK&g1u%kP~li*B{1q+Y_-L>W!P zj{8opJWz@(pR8Rd-h89%^8!#JUSY?nQxFam&W%YqsmRF!9)h~Cvg!YSTr6#Q6gERo z{s#!xDA_U+SkDin`MT|)nZGUP&JHO|0d00&Tu|mr`Z=O!H!G@p7j39U$J{b7 zhb$bAS{^%q!wwP_J(VcT9mwCImfn^ZU9h%vzD?6#vHXTb*pWNSsy3T1J&wtw)f8U* z+^{i3<{tT$t&Rv^s#-_#$Bh$Fd8$Ads3i4tW)=Qr?x5K2s_gRE4!!7c_xcK(r?bms zuTEUb<=JY^JyPC2=lnNtrQvX8r}=)r`z)n2JbPt^or>9hDVrjAzzcz$x&4?NAzB5f zokO*4;wEA`<$v)0e-0^dSKYEYK?-4SVKGUgFDG$;D0e?=$yhuh3(j2#!f_VNvh6tN z@n}LmQ-zYd6}R6DaK$5Be~MI+Htel0+jU<;*QPwZi%lio>0I=fC)s7OY!Xa`Lw8Cf zk!)N547gk@uVfZQMYuEoyqb@K}8ry@XP> zTSJYbr8sJp9VxgU4YBp$2{VmfQnn}YQFpyvbkaIji~QEZ1h6HH4|>$x36>Bq%9M6q z5FTX8Wst+U4f{%cSV*aIQc{WtGNDrokM~&_KORvO<#ZYhE^6!dl@hXxbDT35n7*MC zUBdPsx*3^G^7uGUH2|lcT3N009cY!NY=`4ntH2`x)~{T&%GKc+_VTm)E^pp2%zB(| z31lJAfu34+^X^AkOeOW4Ipoxz@vZlo(JAE}XRM{`E-y}pf$Fov1IdEJ1-j=OT2`!` zEv9tJDP$sU{3YfahocJn;lk0A8S&jn@q>7?h`5j49wJ9?QhHs@{y)O9DuqHx3mb#& zhGDne9CG0N=C72(fRvhyvoiOI8346JpQ5H}n0@xZstRJzQx;!#EDF%&I#tT~ux!sV z<>JOZMbkwG)vr3x{1VmRlcAg=mAdx6#OU!tW;n+o>cB-~O(bE@Ab+r>y8R7MDG|_V zjzUKFY>9CA;{0^D+O4-bb5&?nU>zv!s_Ql9B#Y4}3PeYbjXe}q+ZM#m8W?WqR0`(TDxCEl(KEwFzGc{*)d!HUX}m(FK7!=Kag&EUR3?GE7-yJm`H z$e%$f;I6t|3`NOMH?T7E?RVl!vqSAFyOc+-1j#}f^r|{(KHZ|+vh(X&dU5{a-b|Md z+eQDFxZSSWk7P$~)HfRlA#gZiB7*30qi1&#A`j$p0F=p z^{m76jCNDpgRd)B@s+dNvD^B-deDgN+LG$Tv9`IoOe3Q#l7e0aSIEI0Vb$occVHKJ zFoV|1q}WlT1iBwqk3XyiXM{3|pEZ16GOcqnRGO+GX??(Ft?2pHetr5qPCB2|bB@U7 zcnkB)U~!x|cLoY+lJzFNOy`cLExU`p48k0_%hWjYc&7whB9QQEXd_M?x_D67{3;no zy$jXK^Ks~7M^}Apox=)R)P$cvQ~IrSLD;e&9B9g5`=L}*87G?aWH0s7?WEK5tg`}m zZMYBgzhgV-d|0`DXO+X5Ix33SxyryKPl>c>tMBFgl^M?4ZCmhdyK_s?f!a)?#kwS= zL|Ia1BYtA-0Yc*soWm}|Ngh4=QxBK@1l&b5bp^E>D;_Vew)}`uJeK-*yzg&ywong> zxCc(9fxNB6RlnD2<@~;yac<$=YtaR;vbZ$PK)%!9OV^x{j~nfoxk8N0G-TDmP0w3% z+7KBuP_$*Pd#P3|RlUI|q&|8$KHceWoPAI{svJ_+%-s`}DtOPC++wwYUF}#s#@97H zmZF!8na3GDo|(QaBoqC(Q#O(_c;D&kDpxGrBTsjOpvp}|W!_jI-gscO`FM30+bu_E zU?c#pT8x?T(u0gC>n(F$v$N-rB;m>eGh9ODNwSS%&Z;e96>e$rWu~(w(D=lS8rGP6 zvcmzmr)*I>D0?D}iv_Aw6CK!NB#(43XdWCcg^qM%Z<}O(0n%#`I9?nOmlox~aGP$E zjMh);<)7)b9c7j35I=Fe_>R7<=^viZzUV%VdHE6Trk~6*lem_VZg4O7-O>2)O;_a~ zN;8467SSe8b32_8ALD7_s-?f#c)F2t*ad=&JPkVtfQX*LZ1ladz4eu|B)KO+0TeC_ zMH-b4F&hILGvn6wGX~0l?yIclyV#d+U4ZUcaA}dBk^3bVqm4c53G`H2jM{9u|@XOq{8e0kH(+U zc}nQXeLA~jrUldh)(oYev(esIE^kxqe(9F=R>aWw^3_@Yt9+s9%HCR`@XnOpB&AcN z0??HH>j}c7gqVKyZRi-w4a7&zlwQ^^%+*7A*qusAGU&@170ph>fWQ_CF@yxUKAGR` z{+V)r$SrJ#fFQY8WvUHbu`$=S9bdJt3mwalBCr9O#pnliuLW8%A7BzTiRius96LfX z#Edpd#?8}ol4sSlZynhoK1VuvMxP zO1B8Q=yz4K?!K!RdLbocUyyUmtks(@v2Rdlw(DOIRO$lbA+cl zZz&pE&+Z%YlZ&Ll4Tmmc&kyQPEVbVlf<}@SMgC0i9V^k~o6Y(O4M@9su=4#r zQ9T|0?c2A`^?ZQMPl>4lD(hH~qGMCYyB_}NUZq}uv(Y^n=oHgi$_ia?Q=n5n)LWRU zXy<3oXRZAfWnPajktyF)M6RFyZECvOY7mdDV=g-N?3^~({l%tz|I=Dvye*dM~FC zFfpQJQI7dS;4Bj4UnIU^UaX4UxLE~SOS3rh1j^_4_)vRjqluoqN8Xs+u&N6jPSnDF zjME5#eN7g0OxZ^ica)IA)Z7Y?YKTfWVMp6s<^IjZ&y`IpWt|kgBFdWBc5MsYq_rm7Tx@A-Ma?>-pBmg_MY+0$jIy5y7U6XKWHQ90|FmX4%5ozw+ zQH{xV(&T>CiCl1wbRdHbX<8ewAG?P*ZCOtobTe}4QAx2;Uhu=b7;8aKDM&0Jph%f|Sdo(qmv;NY?uYKaB-nWraq5Nbp^cnB zzCa`|oQ;X6u91#+#LEj;+^G3Va1_eX_?@R@h;8Q1z&PPm65OqJ*F};}Co!n-+rr50 zAke}VI*CA~jg=Y+5snbMmdtS9Tq7TJeHtt#q}I2>==N6`u8rlBs6zy&iCtqJ+j&w4B7 zW#(t_r4)a7KRzvV$P@X(>=@gucR7%?+ zA|_&sx!k3^kT?lSMFt7UR)@VCWOt~tn2sBCAOAAfo@V~^EjgWJ&*0_R;OfON9n*jY zW+zSr@Y|)((WQcwi^|C)@Zm5ByMJfuS$AsY5da<5?^!uXoTO2nqzNSsK1XBqCpM|8 z#k!4uYk8A&9x3I~?|v~me?GepwCNTlMe!pntU0oB-_*37)ji45?U8D^JWIXgO6^R& ziX%8Tu0^A}{dc|`dkNaEz*6rIdfXaEc(pn;rCopzTb!_W&}}+QL#_>_)zTlBm#ZIk z{c!a|JjOwdru3I|7m85B9w(%%S=lkr*f?Ao4e=UGEdQJx-2TcsbME(L zhv{`kk5xywG0TpnD)w3U8@T~bVzzScN9ySVj z2=Q8yZl@0}IU@8}B!pvMEMR{Z3wQ7|4loj=5T7{fQH4ca>?>VzpRed%ye@wCw3lbM z7Pi7-)h#k~y8VUiynp^o;iO&ntUc**lvvb~?Y9R|jk8?J^EK(=!Lz=>Zw|ZkTn+p@wcDY+l7xEbBj{Wi2wUq~)25w+PyItm9yYT8HCYh zrTB=GtUwuy4?gfNFyETWLt@VZ=-9yB!kOoLr=I#{=&X}EkMKc$iVXEEUcpzo3}0h2 z?$Y`8xz>aIb*qq7VfHcqh>+YbjE6f78%rs6_Cl=cgi^9@89%L@wY<#qK9zD>Y6sSh zOOc0sGTq-S08)N~)(k!CbExFXb$1tWe1br8_fIiOND_c-wCV9~0>bZX1yu?66DAme zt{AQEgUtPw^Q)1~K)Ap8J^Q$vA1=jV;XqUXb+E8TcQkwK9R zDs&O^rw#LRppd}KgM>9p*zxyzmwc^vhY4NmR4`;an~}t#=tbvc!fDW=7yH?^W22yI z<8qkbXMTA*n}S8(q4tABlwg#Qw5AREDWuju;+%uSAOU>P<9q5uSpUh6TGV!{5gFXb z&zQ~`7TGv zloKS<1=?K%l<$V2ggpBhVk|^s>&E{w*qwvl9cX6Wce}gftfwSXwSEcTL|dzbuUKx; z1@w(5>hdSsWz-IS=ap}2yiFu&{2nK#_`l)YLJOEdq!WUMhioa6P233D{BCIdmK&<) zyPyGT553qGU71oEVJhje4Aiy##CIrNJR8qZ?Se!0j)38Z9K!Hs>~cG=<_b~|xnd?tU}FVeZB@^6X{cw;c+0P zJ@(i?Sdn-jGr(Tp z3$W!nHY#5_=eWnJ;C$k{q&21s>oYjg7bD$3-n_HRG00HQ|;J)XeX zevYW{>0mF9dt8$r9gs$qhN0_q$N^=4mfl@>lKk(SIS!9&H6M*CP!c9@1aybCvcMyr!pLCdL2%4_M@t>qjc(WikuFvcj z$yGTO`e;@s;dx@_#jxjzlCy!aJw61I7s8>O5Fs1X{I2dko&V1|dp9C(-P5UY+@VHL zX<>cKfTxG2v`VA2%+A)0zmz!wC-u8*c^By(c=NF zIE!u}5uEDJ&?Q#iQJ(4XLe$6)r^aEtq|EXI@@Mi6C4ui84$#SfcMN$iSj^hVwJ%HBlT#SWHt?W$< zhUrEtx0QzYK6-3K=_qMzKwQ5WJm8dEPB?Z@BEfRut)f<*TO z8}|flvqjQH5ot?!k}?vY2MfLn#Hf#CZVBOmi9{cm`0dp14KTlKL2cZ?1a1y~l(b$3 zBm};ZZes>lBaIODQ4nHJqk=$Zx;UgHxKLI!qy&5eZwNn?)kFL&4mxLJfog;!$Vg^& z^cq4y6L>LZb^cOOU;h67UdmB@QfyRRyHW@nF)Jt=`E>E+iRM`*(nanqcwEx1Lpl_x zunYEi2s2O=K}(5$xXKVQPO2R#6Q%=4aCNzi&Dq5Yep96pcSos?dg`0_z)kICqY z3|q%T8)GjZb8uDEBAmCBp@I4nVC#^C)3It8&%C1>tfE=i8Mqutc3(z&0;auZsBDN= z-_`(ml3=C+%7@6HQwqAXc|+Z#M8TbxEY3=HuA;+m)=LyZ4*8V11ck<5_%ftam{i;xE?d3SuT{&KEc;z z^)Ya5Ha_X#gK^|oe`0x-u=YV@5jm^D3#{<7V|%dCbV26^%TO$@VPr6H8NVJQ)?( z#qcy^M8ijr>&d}@DzOL07fmo8doFu$Btk-^M%vAj0W>XC6OegCCl)l0#@3wbs{TaX zNvwh$6Xy*cj2Z0>TgZ(?id2zwZVv}ocoEEyKh8(=GPcAmOPv}MCw28It+Ad)^b#|y z&9vMdc$a)RBtu9BG3DSvnlwMndoGcwF-XzW5Se~Fk_!|&ePof!?X}e3JzwV39M8c0 z-Oju;5q7o!1&USWr@E@>y8L|M9_jPwWxb>)!^&!M1G8mKAs^beKqHZta-Ct)@1p}> zW$(nK$kf>|ha{aIa<_ak%{(aJ|Jin2t4w(Lxan28nAYp4*Cpt5g33^uh$vK_2J{4E z0*?C1h=@Ly5FI6NR1SM2J9X8arsH?WR{N`vfn^QCzy9;xMBxdnD)CT3d^=@H$|e5s#x-+#caAURhllar}m z&3fGOzFpn67UgnX;kR^l+4vZWqH+B~q9=t!ne5*kB=8)FZi2dqVt&h>_$~brc-GJa zYhr2%QZK^El4Zqz(?RSrjNa#yC&!WL5W5h#-lPzTX!Z{v%Cp$15Teosevjo4L7G>H2O9sughs#x%(R*^X@Wc9mddcU<+*J!q9&>Q4 zL1gBn9I>vstG6Lj@5@wrgAGE7H|sBfk%x=Jh z1qfTY(nJiSyGVYNhN|<|t;8wl}>I7Qro8prBAPaRq&eUVmmVydMaKF&`G| zjcPJxKB2BV`Y=I*%gF~nlDl{O(fl!6UYVse27acc<+&}-fd$+$<3U*QEYyQK+uDt@C{oKfg9nd8 z#tZqG{BitmoI=@O(rt5cR3E+@C%Q+(xw||A_i$~f_0YY`a~I{VpEA4ogq06$m-t`} z?STuTAK|K-M^~_h~EFY;Oh+izhE7A7i z&QK8&9f9#V++y-}DQK)BH1++8#bNrK>Z%3Yn<#xJ@s@Bx2T~a437Wg4A}7Y0-gYEu?l>^;Tkvc0BDEI;l&N zm`B`*e)92+AzUclD^EE`2=9dgw+%Qb2)a7=G<6ffFus%ozqEl3;wD&KrSB-52^9yV zu~XdcD3fVELzYR+Wu#NYi78!JCvrbHSq`(Dn(<6ohsLga z93lle|jes9wfQz4|)oD%=m++%-#(8L+ZhndzmSwbsRUv56Zp=%^o#SxN^e8 zkzYiQucZExMe@YY<~ULLDWNI3P2b@Q^-Wj0(J|Sw1FjGZI<=D)5i%m;6^$(S0hV+I z`$P~&UqVveJ8!R{W`6YpKaQb1_8@~&eP(uj+l$&b&XmL?-3^L6H0ha{-|Lm!mhc2{ z>q0UrZn1^DGti^LHT+sZFpx$ewPaT?qOp1UXCZ-KkBKfY7Q%nrOQjuZx|!Cx82_MY zr2-cp960&i^1iOo2xj$a#f7ReA0Cm!N2J|gdp|yFwH6)3>-$@@VVrw?hPG(WEwzM+ zo!((-Z6uGaN;;uDj(8*2rdm}kvF{3Wb&ttV?Piy$f{J`$bt^0of4~fDOT@z0rG*LI z5w`*FJW37^k6H-B-FknC?MdmO!zuE)k^*9HMjgDQIrlTp4K!2@svO4XUzu4lX4!H6 zuCb3_hOf(XEA_$j;i4x?i`fO$#aG?49@;Oj5bB(YXQ^8Uz0O;MTGc9P!cZf$muFK* z*U6(e*QaMSS^?v3QV}@IKB@@Y>T1maGf|znWjx~_$fTf=l^ZI)s#PkP3ni?6oAw6k zB0Kx?l)Jin+B7)9RI~{SszipHjy_%t+tT3j-h=6CqNHNsaJdC>;6Xbq!fv(3;?Rc( zD36s9DU!&b2Oa)A!Q?S*W5?>MFmu0UgBI-mg^KpXdqpK^U>m5CrR$oRGMrKfqIvJm z^u+YdhMt;5pxuzFROXIe;in7_@G_CSQl{)AslidP!~sq)O_5!0mr6fd*VlogbddL; z+jlP86Jt7jI(gno&rQww5A@=^wZ^EgZS=&2Nup`|Z$*|52W1;?;t2^%&C-?t*y!bCyO6&-C4Rd~Z!`qaE z*a{kBwU;@65JM4Q8}%s(j~bN*4f|VQf{N)kJ|~~I=m$XF0^Jl@C~v1km#{J6vkJ+Y zLA45N2DjD>6@y(#Fn&EEm6(#@$fr~O&bZOX zW@U{M&55|dsnki8#ERa)t<>`$#(P|hCE%TIOWEnSBB|n@J?;=2LQxN}U5;7&X~+)o zyuf2gq3S(JdDi*R^(jOR8tYZb9ZUr$tm*LiBeL_}{E}9BgLdUYAsW*-+$w1ubqA9l zx7=1Fg-u%XZ{d1puz}#YmHXCqRxB2Ql3WEXMC#xU8MgJjoH{9uHxRdy_ZJjc&rX}3 z7Q%x#+R38sLCkZ7!T0L-^k}r*!OMo3v)T#IdwrXsBOkVJaFiBOiGyCh07vt8G+)I{ z_OHSz>Th0$>(Qap-5by!eNO-X=z0sFHruFM8<#?HD}>_i#ez$NQ)r>MI}{66+@Uy> z0tJe-XmNK9?(UT0?k*wlhxhy5PtJeN+%v#1L&79ao_pVGU3+aOCr0`dlt!H5_|f`t zE0hBY$Cfj5MLb6NvJaU}3AW3UOJoV+ z{_x}W9z&caWgCu#IUXOEKi!vGJyD4drJ8SaX=y)%_Vl^uJJsJqPiNM#=sRx~DycZ3 zed|jcM`y3x24}%$@p~2UE9+8U@r;ASpY_A|;g0uF?VRO`^8wkQZCQkWqF>D$7T;e~ zd94fA5swvuXQ~g~s+-1)K4Z6jO5S<6HA)6$13RU9WOatswW_6k`01FlcWQ@H57iRT z>bsnkhmnpYpCh!?aFQ)+Gzvv`ENjH}eqq0Td(lF4s=77cmfHprP%s zPZMy{CNjOumc8#i_QHT9x$0m{vzcx!MHZe~WX{_L*Nls5hRurrdhw=cWMA&(;r4ml zgyBUyRhL1;NnKDbk@&0Ov#eWikqFW8Tj2^qh9-h}Ene}gHbb|f0HbR3D@NRIQ$`!m zCfsY=CW3h4X_ig?kG)X0ts3u)OJ*_HxhW)ow-gL=OM*W^8q2j!c>4b1;r zYxy(m2uAWCQl} zOB8%aQn+HK&k8&^!~z3ytJ>vTXN}cXOWV0t-sP|RVxr=3Ir8UiI-beXUe+TE?D*u- zp$V>^PCk1?Rv+r7Gmc+Xmftd*4XsCss=tcaHQN7hx}$d5Zb->eJob?cd56YQUEWAk zPlAK}^nzA6`Le6U?{3H_Q(irAz!av<>blX#HirFUAWg1 zMb8&@V5IsCZ8>Z|(<)Q8h8I5oKd1j5-=RsaIdY1dJj~@zF}Auy9NM?PW-pY3#Hn@* zkCmGJ8nrudhr0Q21vXNTZ+%6)kb~>142q5zt+radkqx_0wCs?L-1Api)%9p-#89M! z06ASWIJ6aO_;-+gr< z-?N-Jabbg67va%J1{C}2U5mVBJ0^{2{^-Os$2OrXtekJod9825>q|~MZ&{1W6)73b zJRoRgR*SqTn01G!5Kk+%t=Eurg*0i2s6h9+3z+`@UF35jq|SS@fbFfI>VXC!Sz$nA z%Z(|MAyO{ps?Lz)PtRb(jNB`>bn--VD7t3M_oceKa|UsnaxWboiRt6EJBd9ME}y*3 z`27A1@eu+a=OPFjYJX57K^*v|{UV4}T(-O89~w&HE#ny5Df7+IbS|0w2PTx2>voeq6-_`x?0H>g&U4X>+`x1y6X7Bh|?Kag&31!j>O zIW@PZY~1cDY{WQvE!EiziC7ibIz>Z+8hu3DBG!U-9Z#tVs6Mp z`Rm^eZRPzwSydPh#3OvAu*MVK)6m9~9bnd@C785JGR^C4V65-vZGt4_^DrJJo+zWkSi}oEB`h4=uQeU!wn@BK z&Ib*v!Re|brYsdg(LOw@r**DJS_y9E?IP~inn{&S*c%7WBJTO7`0@*(-m~$yxEX3; zn2EkZ^TmNRb4GWAS}No!M=*ND6tUOxg)xCs!Zcl5+jHk+o@XXznHaoD( zXy9P)1tJ%0;&lJi|GXnkB$;8Ix@znae5-OJBOv2b5%z_4$@DU6dLpWldKZ5IdKTv| zYIRSyUt`xE=KnQ-*e3)&4rxnag(V0+P0FK=hi*3B@&~i#gV!N%GaP7+Em*UXiH2L? z%~SQhOZ8i@Fe^HgDJ`E=A|%_>@^xGXuG`ll5%BYAgnw;KaQS)JJQL|zvw9%1+o@1; zNkYbc#t={T*MF=fqm$=7Be(gu>0r~a=jdonLe|)7nBmiBM!&^L9Odu_S-T!9C6;yH z!Y*DYgKA&USR><2I{KsDZQ}K9y7V~yqpi>Cp%kof%-3O2lKCDK@&*xscO&#Gw#h zvvI*HlwJB%WaVx&1Q*YUKJWJ98OwSa`#Npbnn=0=WXEI4zCQ2Sfb@)Ndj<&JYVoPhNC_L2Ls$^sz=yY6wc+?_yw*qO zAvdWEZ_8MuCBH@YTSa%KKlXmyRILKWJQ1&0pXoS@BPyB~(#uDDJ~JEDpjjuD;PEpA zy+?7)4va|Ty!paIGf0Qvzs;Ma;D3tl|5fJmy+UdsUi5q$oNrIRr=UqiM+6Za;~vpG z{(UBsO@aP+{%`=Eq)!_tSwS2p_u7*_YOu%goso&=!zlp?+tcX`SKxTbmI4^wb-;9kX51->U@>W0JN--y`i%bL|B|+y zhB7g|yw0Z{K~31@dR*t)R@eq%#0d6@lPNbSiz~a9M0+nS{rz4B|D^t-Xi<V$udiN8)Tm6LwB`M=2CPVIOrZ#m|@<5{y34yQ8~1g$ob?;VX*sRf;6c;j>TXT}!H;VipSu{f3+v z_a!LtJnz8!JhMB2pQO(y+b|1G$!v8btNNWMM4H;!BpN@dvbgYwa>8Hb(`JsQ_v0+~7p`**9cU);kFZJIN2LULI9rM3DUl4A_9(wso}x)cw2U{Xhl<*O+rt8bTB=P zT*mM`^v6BO-?u;AJXzADTsOYXa*@EB$3JLH)_;2bWhUL=nrXabJzQWoo)wvz5uM6o6#YuxCU|vJ{j*^N0)sAibgbPAYYVixp*DBWTnL<8a-VZjV&`~ zpcE(LiJn9ezg-Cr*L~qlj(0Q^9MM?v#WaX@o$cx$Y^Rlmq$;^K*(TUSRtZ zv|XZ{c>~z{o~6XcH0f37K>4x<^A*sLRws3M@qkBxa4otr`1GEl{fuJaJy1s%4qT#_{n|71NLl%7Vlldx7eC*nhPmKKpFZ}GxL;c=w>PVi2bOh@_DLRl1kJsWDeCTZy3b>DV{ z_)onP%%gfbqf&*l-tX>9SME!FsCa}2|MruSmfUxCtUa2f)aAo|z605-nVGeWY(+Lt zs{TL9gzXebAbZ530j1wyP9^b?>tFc?cQdqokrm|G&mu+HGhAb zL#{7%cl<+$#xWop_{zNaju=Q8zppmgqha{V%Jk0lq$vD8&gFhr#V%mX$L^==l}Ei; z$)5Ib$GG9}to!ML^Y{OdZcsA(dqtWKlq`U_s831hqr@?#xPErvz_@&qRgmtfM%%td z`Cyv{l5szd{9R$zec@#`X)7~Sz>4B*p!W1e9J(+zDkXa2{{&UN=Y2&5dsu3RErGuD zD{(f`-y}>=_r&S+cDdl|Vd_eeg<-vV1^2Amd86hoQ_tR?MiGn$Q#FRq)+iVggzl)! z^;7jy`4r~Wv~80~bBP;Fs=MIAdwtyVBpx>;R93DGR+8Y2P}=v9`8FsV&x4k{(qn7Z zG-A2QSErGDAt7=^a5Vyxx@5w8@x+b9yLDCW?;G2j_bxl(?1o@CI1cdm$3Ocwz6(pP zm*vB)%ULULEP-oC1uS2%l?B>y*7MZK1#jr=ntI)`iyZ0%$vBZI=E@u+6O8W#(|P>P zpmJj;_#aK3kPlG`KJ1Ui+BL_}A@L^LE^aCLqgjIXc+c_`Iwon1*uKTO9-e3unUX4a zx&jl*x)-|vn*e6-5ctq7Zla3pg()>H{TFmX3_@(SzLXMqNFcYDLdm6yp17!iec^uiEdxTd@SfV`Uc(pv)+`HOIs8m2>Ku1z1(3&FWS2U$VH1gZ< z3w+%gc{Af*a`z6V{AX`%;Fct{azl#C z29I2D4_an%-S%%Kf&Y;nP!WRpu~V**6|}0cRxte8o-B7h_b7BszE)IpR@n@{B6x-5 z$&zU)$;?M!T?qdt&LVsrAhEnAZQrO}8>467gKc|VqE;sE%(Wi7HB#MD zW48TT^*6^p^6m8>ZivcC7C4x6nu39f@x`bL?-VF))YC zU6KZ-N5_Ph)I0-c)N!@ur6I^|rrEATwY#`CI1<7OY`KZBj^E04r+XOml5*vYQfR!NdF``wTb3hI(yhq>U);N;xCmK( zO2^AwS!J6H##$Ry{cftu9>!+q^|9*H?M;?^xz_^LRE(g$#7T}2zBf?x+C$9uq1bFM zt+M{Oi5tXRn{p`tVvS#_yUs5#noBNguh|JwUO(LP-g!&7hbtlU?Yi%vmS%P||CHfv zm+wsdWdZ4*$z+Gc$%L4~Nwm(>x6n^}Lrb6F&Cp(lIy^QQGzf$=EuOyot>?3Npz=M_ zyk>06{a5h>U4d4NSd3KJ&Z+t-@+laeBi$XFV+Yp}t$Yd;>8c{i<{00U4V8Z?2c(aA z8`hIBg0ShMF&H^P8jK~^2d8&yU12Uum%*P!Fq4(H_1i_K1Yn)a#-jRJ2gU$!#&yu{ zkWjQ4_oxWop2{wvXv1r>pN<&YM2lo{dmxMp9t-kC$G}6t1b?A#2q#idpH`8nVP^Tp&|{&ad8qA7BBC&0 zArf_%$m(ROMXP1%S)6+~XxU7}u@%xR_l#oQbe!F3^%6?HY z#31$bfP{6Y2Wcl6_4;r@4|dll@u76-^xJ-(;Iej1cif>uVDZHez=Ch51F2#^XC;oR z3fHIj{+G5?o9kYc%g@koG?uei&8m*!Z&w!1;4@$b;pgb!j{zK&#Ii6Sh zx;G~Klo6kG-+n9d2cUw_#Nt-fi2oFh)!l4SR`xf_%i{yi+Y`!Tu8Zs|=fz2JoDY;L z8ap0}IPzEN^)sdQQwKsSlVJ`GZ|^rWgb4GTNZfqp4BR!I)+8Rn_NgAb{O1S9EHh@T zNPViW!HKtu@HxxdX0z`=tdGg>q9b5UXA^coSQaAo~;f0dekY@x6OUs z4M{B)AlfxCPe+W6RK9BgzM^yjLuSV4TR!`+Nr|ZY)FDgW2Bn{IxOx`9)VJGboXRrK z)0_Vaejd{IDBDRnDC!#pezLEI+%Y!z+B9*&1M>HOV3>A=5)1K?tXjM}6ww_yp~7sF zaabJ~ID5KzSLVRtOTvNY?@fT9FhP9fx8a2bxSrH-D2gud#AffPwNh$Lu>7EyI`DS1|FSSV;b`L5vNYGzSwRyukkyessQT_TVr_zO+&`g^!fnCqMz@Bpn zjs08?raL3re|bA}HequGe?=D~98;w23NVnFBi)y|?ODaI*_8vFbITwod< zV|qYGvsoqeuz>fe_2RY}+@K3MV|+=K69cQnQ2Q7vwy-VR-O0C!!Ij&m*Q@(qfBO6o zV2Cz?M1-*zo`>rRWWI>A>dkAjw{eJibF@)LtIMd>eQGcE?p&HgYG>xk&cm{ociiK8 zItNrF9*^V$R!O~A5|XL^%>wwbx_Db^7FOrw#LMYI`uFgmY4 z=dPj~G}Z&FQA}n?yP&U*yU$>Zm+^gf>S}O z^Q`NqYqOH{HsIEP{I;ikYAK(Rk_ik3ClEHtyv&-g=6h_*TpfRR zlXOv7Fa8+H$!lS=VSBR~W;4NK(VAF{sWxZbU%ZGeoZ2Z@&r~MsuuQgfGZ7(&w&hvl(-+VH}#)UM00>1``9^wRU#sX_?h`Y3j8)$FA*JIHVcS<50aZ3(5nj z4_60UE!xJ6+Q$2j*&`tnqsEj|<}dATYr?IUH<^!|+w%rPlKBWB0L)#{8qb>hzf*KG z!D+HnwexMJ&eHY2@fTc!YOsunhBMCi7?RFZ4SuW~+zfhFO8&3aQB#RvYUl9KH2)b` z_W{G%W-L%!l0TWZpObxg|*3$Uc~z-sm|NKgc&#NR3-|VQ<{ojIX*;YrJHJ~ z@HqzD9cX>Iu)9oqcOcCbF_Ed3J7w{1Qk4=YZ;!~{p<44s`?rj$t z`|xqhC2-k;E0+WAgY4yhf{%A)L9O~d3lE>rfRE=^y?RzfkJO*gc~ z>GMXEff`BZv2W)otGCV-Y(!<&9}wz(L>M`j|=e0o%sO1QKR-)TjraCV6wbpK6t; z$WV;G24uj1Dq6v6?b<*8*IHH9Hm1hN_gqU)GQ@EZAXEZu0%tl#<0 zoQL|(RLuw5K@ zC#!uop0)^DIu^tG&@mRLDS`_q{SG4HF1zutvCTPJJgbd$r*}V&xXim5?9X?BHD2`OLBBG1M*jrK>sX^_f=b=m5S!z#kdAT$8S4oQhL1g z=6)I+k+{@qq&U0%e)00sJH3J9Q8I2eV=wW$j{=JSDhhG^A2(^B774-$S|91j2&Swp ze70>TmS&RLn5l$E(p&cxNF_*~11k(%a%D@rNhDVhhZ+|aBt#LzH)WRNTbPj~RkOgR z-3^YfHbaCaEfeMv3%N4;1rV|Pujc@cu_H7)#3}QQ9lr>A!)+&+i@CxX{&Acw8PuU< zUWDHHbgv@(KBn!)c3 z9iFy<>+tb_akKmEtHyur_7ahUQ6gsJ+iFzHVJtWx#dyt{_^3KSj;!GmrA+WrJl7R3ye$t5HtEm9*C-j|mCBu&W+^a1^!_1scDpe^ zG8(1dKtdMt^h)MS`)McEOuCTEWZUP^iWcxn*qd|Hz~^-_fRx@m7K(gZxT*kElgIk# zm5m}=${AI{pKSCet4=fmh2I8oGlF5mOaD)sRf(4uRQpOAT!G}!=`Wgx^dpZteca6B zPSR~;5kCE4LDJ10glJSHdisXR4f;O+W89>6Gpgmt4~ zhCBMY(&UO%{dT}^JHB#7&`caPjzxlxH22z61$5VrdU)M`#v`>~k%CrAtQ+puBM#@g zFaf!%%;iq<>q*%zqbo9%2t~j^)g6|^rvH6cu!0~3E`K8gl5lRz_*c-gr-^T{YCqI7 zkpN;9Wq1SoUw3`Leqy!JBYN*-1Wd__6Rb-fh)NOeCR#v0H5Ki>aQbgYgaBAq7Ck9KjS;ROd@anK69 zCW2Vi*l(EH1c(pS^p*6w3Ljura%MDUVJ=6Dr!0R0s@E)2iL|4u50K(24uS@iI{_c>FR zBPQSd=%bH2T%^FS@un+%Emx7~$#l#zLEV#`S>rW^`qT5YFn)h2j;ipruS3mZB^O{o z>ZgoxPhz+w4qAf_+2}#>YLxfqvp+ZZ=?Tj)WC(6|HFgJM*d{^b4h;V}(- z^`Cc-(NCPkb>9^W0p9!Kq>?CD-x(i(wTkw72#HqiF`CJ;gK<(5ogAGIw*AlUAVa&@ z{s_4&%EDtmyP77e3u}Hd=I*$gLIm2rCLDecSGmEgkpcc{@MCx*juR=0G0^ws%Z8U0 zFv}Mh7>AFs@iv8GbS0|Ovn1pHC^>qNrsk0l=Y-=sHuh4RJL_{;=teTNF@U5(#IGC| zQ5=X+MJ3L+MqUAwQrtytlucW4UQ10ca5WHWIC;=dp7jXOo|-wK)$3f|+FoA~Phk2h z$p`|0R}#2+FU7&DlKa|CtAT->xKNK7M%X(E=*@}hQB>LQ8{TcX_wwyj8e;uzzR>j> zMfUgMHNK0&sjjw9tB~NCcseF^(Q+n+kLP3P zs)>872B{0Xdr$weuF#pV1OKdaKHr!UEyvXiq}j0E?`PBgw?O$PQ9Qk(?h)px8D3x0 zge}E?pUl%bu<$9#|24UdFg7MIg;F(&(9K49-4{@ZNPDg?E+{!LE3a`rKRfZsQ_RBf zo#!o*LLG0w@Y}V^R_H|hzMg9IN2m<0Ei)}QLU`fV64KT=&b{*4o}vc+A<&ztb(soY zg8TWn`OMz*4*Aw};rBxEG~uG&tuz;r{~_P}_g7^{tnmE|!a^g#lMc1dj z_-5~==X1Vj(RX+;CSha`&mY!yx4Z#_@=~J z_?=zn8=$yxxbQmuz$Oe?O+r=t&ivC7@H>az_uiiPCU|w6iz~;DZk3kvUgWS7@nKqB zHBX?n((7tpu~opg`hr)ko3+oT#MjLKey>11vE%?As8{%%)8Gf-XhABiZ(Q_Ka>8`A z=kg8M( zG^nO){To+hr!cSQ+37GT#id)roZ;OIm~)$;S_5yFSW&2$!xpFX=HBBG% zHp5@sSHyso2eqdDdONrV^#Demc$()nS($wWoWZPX;uSypC1=&i;Mp${sfm)sE*;Rm ztkJMmBJF-n#Q(Ev&D%P+I_s&@SN2t3{kryUO2an|+-4K3Wjwf9{3 zSu;(||LXVXz~}zGyB;pTO0S2rzJ#vC>ICFjZJ$dB&mA0gU=gzL^s4p)e=+$lrmyq* zew+&w9(AU=aqdtfgXVvcwo(GmzIbF^nB(2deY~G*HxxVTCR895@Ep?n`_0Z6#xEGVe1&l~sLzlVj94;h*->H_~d=GLDA{XQM zm1D2LyhQbi?nusO;~uhq?}j|nG!J#x0S27)DG+h~`_N#>vvOrM⪙F^`QG2!ORnc zB0(Cmu$FR%j%-g|ca19PDKHr+T5VRN+wDnSJONjLP}t;K>xF28atc|I_zW*wGU4rG zq+h`hhz47yiCD|0pG|93f%}m&$PppP->Rq|LAIjRzBP;F-h@;UXV~4^Z#>8&ptij^ z;XCP~Z?zq+TZD9N9Ftmb!zhs8t@Bw5!Js0VXgrDD8e>Qh=x;jCntNUE*s)MmH}IpF3k!T^{@&;+}X+R5b5`;dv7wB0}~{ z0Ow_P^f)kv^EC_tkFmH~053-5ZZ(7yKSKSVPHdyQXpB5ybR(p?*uvte2180g@Kx2A) zdK$d(AJ*)8Q0nMw>&*W6j3;~xPNlni?&I&2NKfMyT3g-vD98ze1^b`k5~!H(Jg1?~ zJOim8x1ZzGSfAi6Tfe+cyjS#W+V@bt)YtSB!c-pzBot!JaXzSQpi}u&-0y3(|5&^w zeOfpA_?*T0&4pxpsB_{U;^iMz>ErV$UY+mc`RT=%>#U{q{GpCdN0%8ndfrca60m|z zZMXQ1@4<$?qbBFSaDAC$b|Trz0QBp?Fq(jNH74H%*VKGmxcBjl1cM1xf;O0wOO_<#AaukWa$llem1j_=hF&@cdX`x zYH&nfedibtYJN%7TQ+F%0qyxskrEmm%*B7OO`Al#PQ0o+bxq4ekPN#%Y_5ol-M(C? zwM*;>x%3hY04JGk>>!B}5M&R7hQ#L8U0eZu^Ji%~H|%YUJf9IcT?cYoaoCRgeaObQA~BP^bnMMY?I zvi{lRJYrn?X+Tn!LF;7aJ)U1P?aRw4#Im}cO8@jL-fX_iTdQ}unuz9>GTyuzc!u5a zbn)Tue;53+MW&R;rYBb$6q1(tRJXSspp5)ul?nk>CnT2_2Sid;uA=+yJ#4mQVReY#%u=l#rgt^pwr$PEnXP_Zfv!IPiO6pjz}ojdbtx6YQp7+UGfdM*R}i z)SWe?6yKFr7S!NDnl{0tKA~!p^Y6YaS|KSZkTNq{FvA}=Li~9&$16Q-nrlb0=xKhTB<3z$Xi)e@kHitAv`6( z8D^qZ_HW4!gOJckhJ&YAkz;v_=!`NFB<3&>pXFF@gOJ|Zt$^>pE{A{T4h&Ne@EHSpLb|MI*4zdA`IYA?rE5~4^HoHEyywT{XS zQHy0s%pkPgnt3Pt)1)_J=GeMi$)h5R9V z!XJrUU1k;``|Ds3dUP8#{e?ews!IdT>!DjtsTC^9u3OVMnv=NZo+~%FfCE>t@5R#y`3`_uA}qhOWHC;@ zBj9lXc8AY-PV&MHZB8>6CKL>+J{5}1-U~w|qw5(dFCxJ>?T3*NSD0;F?8i8B;=gEvztHpBZ3vubVUG!N#=RRs#45(cPQ{?wS2^NhBo zGWJTjqu`!9SzB9AZoF5`4HRq7oqKCY)BaUilA;qGG4X~gjgHsgG@JhsF<+l!|gdN_+0*OofU6xF@bk*Gg_@ftE+H>{7W`f!87sMkims z@Ji%01aUlV?jx*pir>Va6$D=47>6wV4X(@rPbZewVTnM-MO zy}yW8WU&zs)o*<@LHvfvZ=I>=ADVSk3MH=5kcdOB!deGEkHp6bbj`XbB@Z(JA#~Bh z@v`>#^x><19bYg)WC`)r-OA&VztU|aa=BKZ6Hs|gI%+yxJP#w^#gLj=_)42jIQ*OZHMD@S?RmnEyC%e zc2Rr`nTSjHM~XCMlt3OXl8_HF7*W3&PkeSnircU1A2lO;w-NG-tu|UO9$Fl;6Iz*i zi!($srzIaY3}iN)iJo@j9<2wb&FnIDjs=rB`uA!VgcpvsOT6!G7avjbz0nGL`YN zE)-paj49n7@UCTAfO9EC8=>EFA)+k(K7Fks6+7HRLZHe6@0k6$g5T!~2IbwF_HOuy+LdP zg&O~*$(BfV-zScQqKJh)p`Wx()3GY@^WYJEB7Pp zbMSSdp>~edI^}mTYvwA>+lk~7%HL%%t^R`DPDkgfdch*Zd^uScI>WGA=JcQJ!6ai4 zC!t8vwges&zn&!sZiO=Cjg!}U?JBv2z*T^H^Iq>Ys?5d(kJnF7Q(cE^?m5! zHG6+1m~Pmrl2)WMb#T^xLCGjj`PtRDtdev>QZ3)%`6ckmCuNh%aGh|Xqd87<3Hd!B z6hj&zGG|R@8BnOSJ@N`O?95mbURwHxN`9d2s^JAWW2$Z!pe6a^C_FX^x}bTlB$;#^S{q8+BUB zu;A(RL3sqpewQ4mE2_9_PQlHV@@@*`r`i9Io)dHs>Z(`4b=Ir5Jtpxh6TQN$WRC0z ztacu!@B3)j8C=sk@~h2W=mG^Nl?&wmenvlyaHIPKysr%%4D70GU?)*klxk(I)5LF# z2OoPRABknnuPe8j;*4S82=SXRtF+wz!BGXXJ7*G@lLNfo*ae1558J;$XQ||lyQtX< zaK&BYgF8^^KLHilUxeI}LQg=RU;;sCE|VPHBrYeB32Ue33oEa>h=cTQ&tLkCO^Gtu z1J0dytkPG{mE=)?N(ZOQoUL<3=jj6a|WuuCF6XqUTkSfe@T(hna_ zboZBN4p^$NNFkhl_zoRh*3}D`vO8Gtl1sEV_4>Sla2A6l${LJ`xk zzdONJlvt}jxJ{Kp0}^H?SKsFa)omHp_rw-`dP~yH6of2B5Z{BLa$8TjM}eLXzHC{5 z%_z;`eyrCQNXo%p;PlQrF3=)}_G} zNrrTWE{ww@)UuEzuq9@@1@aCO?`BG$6DxpW3F2;tpk8IRZDS%J#b9T1M#ZUlO|F-s z{V}DUDwW2P^%Du4HsE82o7*pgheOISYI`owiRqc%ZP27 zJ&vs-$Kitk5L}kc1B}yY-IF=7zcD`Ex-hTQ_|!z95sC$Ub}S z`Ik=FRwit*kfASMilR&*87`yZ=I_g+hJZXAU>|XL;fb2X5u6=3c6aOiSuriTB*9r* zfV4ZIW&jYEY8RAtv18@8ZKdWK^xF*hqf6Il8B9e945AcDb`jBG8nw-q@F2{$TL(8S zdUI@WxB&(5t?qt&#xyP6BU2Gvr!vp5;~$)*d;h@rTO?E{_^4PJ_vZ3f?8#!j70Jy( zHh`%RW2MRO>dlpl7xf*`(B#uf1hzOy(f*LjIVbp1MYe@7lVgeCUing`8d^d=C_de(fC`DL!EyGQMh9{$d2b?t+)(O(VMR!HeKg86V?|vMj^=!w$c#5>BDTh zcn|*BoZULoygJsfta;j?o5A%<5nNswz{B0~68{Xk_%e7#$rN{0DR`;Y_l04@=t&)) z(_j4{e7Ml@_{l6+;>(SBYNa-Qe$fUu@Z|ZVth(UOgH}=vdVytbz&l<^G=00XAQY#{cU-juS&ziCh-}Ohg&|HGXisS6SPpZIA z^}VdMl!&Z2Xm|rZH^5)5HOs~Da15XY5uMs3qdNEy2&F4nj_5OPe4RN&((P=r?b8fg zcAF&5nHgF8?Q3!}G2JG%7>nYCXdHlT1FJ#yV2^;uiI#;6lfIPQ-){S4g78y7s)Z8! zwA!=J`l>^nZeXH?ZBmjBwc5IwYw`69xpU&;cxTPJt!CbxaI$YmdLm= zBG?1=fw0hArqxk`{kOhFc99A9CWM%Kx<-yV8;UIFOZ8Ze%hbN#Ej!Cm227KMf7t%@ zBcUYW5^HO}UoxEnXKS|P6K;aY4G0UP=o1KZU+g*H6LXiqd}IuUU$TEvN?hIbN@gCK zC@F{G_Xb8&P(;auO^}Gys-9eE!2K3O*mHXDK778A+LhzuNV~$@q?m2uat|5Z8tb!c z=kyMt8Aq;b?kpGjd+JE`PkiO>gcogZy6_DKKm5+TTS4D5$JOa33jAPq2-+_zjH_L#fnJ-+t=-7Z$YyZcjCb=S5V&7NNSj)&3aLA1TZs5M$3W`YcK z+d0Zz;}n75l+_B$q&dAQlxM^jW0Nir@O57DivknvSeu>5EUPr?!+;;Wubkn)-G6y7 z<|G|@$I(Yu=K4fGo!K)?OKJZSP&J7*j1%K`AjN$q!~)dIcrRtQWit%`1o91ri>#k`=_dEm;Mn=CgC2VE6lB2B7{RtGHT?l#^Vo&vF&LCIwB-LErzh< ziqW%pOaSlplDJ~S!_wc-BgQ#h;sXqTU}*7Ll18ogA^=m?-q9neG-!$V4-$HrhBf|h zJYam1pDSYQVIG~XmXup|`b*o}P_u7fb)SSF#{(?0nfffU(zZ&T^mh}F0DJ)22A&)~ zrF`p9wm*&oeO7*NRIc+Bj4Giz=>dbdcrPq?{##Lry)++5F0ow2^aP`ZdGr9d3duTL zzT*7dlVqd^ z#~73{U?`IC|1tF!eog=H|2{k#q(R9Mf`Ukcz-UlX2?3=$1f*+p3{bj6qy{J{-3_BV zM%U*q2#wAU7>D19?)(z5<{`4jtOSq z=K5GlsOpq#SM8(R0sLN-9DRP4mf&_I~j0Sn7UF84A?PhbIU#thU>%gRYn zDvxjhojX*Au$T9iz}p(1OG>*9QAXuN_*}Ecd#C$K64_3B9aLG#KfAAFEF4m3EBRxf zK{Iha+GdzvI>G73afkWdv~r^Uv|6*iyV0^{gLPu7spHSXdZh;5dt$oE*6%z(0wZoc zNw~$S8XYMNxR3fCYk~4GY)Q_iXB9*dvsAX6=0CY<_e_5lIXzK6%hl+~dU6=I7i<6P zR2)%|8a0RfY4iUMs9osfI?Ij${R?f7ej0wksTJm3#&Hfp=8`8kM?YGl_!!$8vI!&T zap_&hFJ{2pbMI_ZHWB@9y4&+Xi8xF)YbRJ~uH&@^3`ENV*o7}ouQh7ACgkj2Ft}iB z&5|y*WvoDBUiD8SLwc6Kk{M|z#i2}Rh z{Q3Hg4r=a!6wq)R_|}86nvWETjQ%GDVm5l7`H|#lg{B6>1ck!qL3AE z6J2U!I_1$M!qnR=M-oOXH&p^NQLD+s;L`Nn;H>-9;|uPtY;fmGuU>TcFGOcmi#mDL zA}n-`1mG@nKO)*meeN0{UzhI7wRON^=zx#W-At#PqRdf!R6RB%wl=1cxfXYQD-rAc zWd1>G#7=;ey3L&BnY3pdi1vDUW=@x)pFP7-5!;bEYQ2^`!>mmGGGNCS7!onI^!fC& zWafyrP^g&Uq&A6r)epIi6YY%}mWxSkVE*U8r4M7TQYMpr$(v*SHa;N>`}*UZit-C+ zz(u`mbP@@FKCfDukE%|5wiQ}orBaD2Ba2dejv_e;x)gHQ$~EdlXc&IZ~G12P= z7(KTCJlhZj2hIhjnvK=Ic|PlDdCOvL=ltq5F$uL%X{LlPfCqpFnb@mcq>w*Gjku@! zVGP3iMl1}ocpvAecFbf%JmlsuIc9?QCoc!T2i4h@e8*6)vLH}jLxb3++Z8uCg-c<- zC)KWZ5J6@rPnfVuMT?`+Tby%`QKQMn2#R_R8J6nNzWp%h zwj1jNGjS93ptoU!m_@{czV8+$ywGsar1CTb1<0E0b?nX}Y&yi#D|uopB4WTkNNa{Xvl2jQvdc}D%ZIXEGb=hrlaYVF28UwW!$bE7clf<&7 zj(=NIOtpZR0yyb^{vn0!v*0j=^=;DC9OD;Jk9#&3d(rK$>fdBK#dYm6i>?l!He@(5 z58axc7Y`e@2bwL5Q~z9v)+|_=lo%B zhmyOz&*Z`5F}$nlm2}Ps&}Y5la{TlpLuJTulm=KniEqu(%<1saGPXvAdOv}K51j_O z<<8)n$(H+@J`NXpu*XDum7cNb;V17jXEz#cMhvn700xBeb>S3=9k+^f+G%t7we3R< zAWoS&yJ%c(w$>&$P1=X?d?(Ac^$J(9|xs&9Ci<@PMw zNNv;yWG4HtL!R;-&*5CAdmktg8LP;s2ok%x2RvI2948$qbHI*``(_BOQEw#$6teXv z?J3V`qP)l|6~aPldKNf)jC-K8bF;zTckv(q8cA5}_Gi#rG;#CHhE`(6@3&_BQu$nf zKvfptdB#3b)kqM=bVzaW@;lr5JUb#?BE|MN{K56>Xk4~JK!O{w+9q`+C&KY3GkUeK zn3zu_bRwNkc!6H8qGM#_Bhw{7Y})y0WX&%41PWms-i70s$QxZkULA&SVE`-2FCm6H z4!-2)QM9=#Du!ui>d9{?)e_wddG=<=a&20H*~Dxc`wDAc9xbleg6;( zV+}Gf)0h^8GXL3GsmF+uGELQJq)dCC{J&v%r5&0-A2E*KEJO0A+jD)IbrN9w!%vZm z3$1fo8^6u#h#meFMU~G~&ou3RL>w^}>?_R26Q!6@kbaRgxkP4)M_c5l+I={AyZmXx z1pZ~#tJ!`F`TUNkr$Bait0U`f@?;d&+AF;$%MdEzn@;>AEho7*kaDoOQ#=W~rVh0t zR1e-he7HZ%D}KK$e|pk~=?9VaHRi%^Kqi&zFkpYg*8IkOPJGkG!!sv?p3#bXL;=JM zAtWTMMq1k*B{ScC>WSfzGxWfD_q8;tGesd$1QbiV+9M^KGW{Of!13J3oXcxqtg^dZDH4@5Q1Y zZ{DUIOG(!%v5{;0U5#MtT|Gb?r z|LIexEZJ~H-T6-w=GvVg^|sS~Pk8!0KaB$A47OOz3yzMM5YJ$jd@QAe{NZ!BLUnos zoLrB%{l?#l+qJPi5^xIb+VIErk%gra!)#HP9YbW%tAe9A^Ij-*63bxWf{0+dX#40j zBrHWlj1rALQ0%7$@fNsXtkZ4?e72HBoK(L^X|IY+2NVP@v2?T0(DwX=mRlAffwkeK z7e2nzID^}@m`zA`bop|a@q1tv_`ftBGzGQ$yPDVotJt710CVJ5E8V8G&+ld~-N!Gq zl^NKlu9;IPgf%m8NMZqtF6HpUI7f7cm}Wojn4+h<==wz@$0Yc0QSU>T0+?;75bzC; zgl?Q27_xG^K-M7MY**!VQeTct21s0VNaapqk>VOX$J#d$BO6%zg?(J}s&y+xVMxdU zEPttCdV!=%%W;VZX7+V>!0ulR;WqLp7F5AtJ57-e-UA?V$(`>m7BS7uA|PdVYP@Wj8DyywuB1kp~Mnb z#$Kh73of~msWz$c3HQVNe^PJ2XB<_@876lq?L;$i(Iq(TDfY_LcDwvrXHs-Fe>}Icgs%9wBQygT5)t~{nic6@xw7*R-^ zXE#dmi5EW(_ulsX+Ht*rOx*o{U-8X)080>A62AQcaDJg7RHFI5LT7T!(LNV6gWq){ zYG3gVqE?H;S}%mIWjq#4x+|qte`WuCwcl}yD{6bbY0@xt($1;EW=Xr+SJkmPBvbmL zsCw&-K{)w-QK(>PxEtb42jwl~qzb5^ro})R+Ez>eH zl2=ezPVmS?)GTo0ILkwG#xV7$K#HoSKid4FmHNMZ5J=#0rj|duKR@*Z(E8_^1Lr6= z4!JmEIgPBqAnhBW1hjpvQVDjz`Sg*(v8-R!yu>PG;U?x2yqTo_CJ`$j?2e37?^GfUb#kA9yARV+Pe4yB|7&8)UiytpSd8~7biQ;pEg5uZS zh;p+cj8dZk)F(tBk`>RJ{n|X-a3x(uI|2@;Naqt}Ouo4@-ZmF|3_?!<`Wqzi@(C|X zhW&cBmH9g3G9x+_+v5v+upZYV^!JzAN9YaS2pX!fK)@5Z00Ibue@%fzdC#L{-tnN@ zrhka`;Wb>w3a!$OS)&NZ3h~1W=r1Q^oJ}QM`J(SS#$7R3iJ?FWb?`e)C8ruZ-w89v zl7Zy)&DB|7)@;pl#J70Y#^z@Jm9@dJbzR@Xxd^=^ah$XVS9xouOE+`F!%nG4_dF{F zQe83NIw8&+|4rADh$wg>V!D@Z#UbC)7x~nE4|jT9NvNz8Nks6?R7+*cTkRJ5AzwXx z68G1w0{HN+jVn)emer|kV)ZIJ3K2^TLD=`Yof{Z@Q=OJC%8;XfXTC@42a#^USF-7p zsqIV+JinF~`+U%1)eo1S?t2?QLH=ysNrAg|ORpd5QoxxpGAY~`FdwMKA;Q1)zWx@K zCOb`%@I=j%%_Gr*THl=GelvSU!@6ltMSj5yEGWZg{VlkQKF}t5L@JH)Wz22M8uLU9 z@2OoBwp#1IehyQy5P=;ge>D_!h z3XhR=tMWo7&t&ii@={=4)O*N+n9SIPhgCp|zbNF1#)yz2EgYCZ5|l&;@p4TzQONE# z@fXUMoLQ^kts$AYoZ6Ar)t`^D&<(&cudVEa2Ff=eNrH);hbU2YtqgL@#;mZVdcf?D zo_)+KT&`==-PII-9a+%x0D#_zn z}j|^cUtlrekMn*lj=in!WxR!YhgTKWszmRugU~|be<~{3YDX-9tF?m*uWoW)K?EEbyh;{ac# zU?Gu##%nVhz1VzbpBY$t3M}!LpCgbL7{h0ULj>w{l_S$GeBcIj)q0q7dOUB%OK}qK zcW8nMFK2i5^$jjl+T#ZA14hm5=51~GmUa%4M`kMvl6fyR?>5_uP5BA~*_&_3)))-2Wf< z!%I$d8L*d}j%8iM-Ar`WgdYqzgS>d%Z%C#sD7twk5fJb4v|7U05+0hByuB3o?h=XK z<4J)T{SD^^of3}E+U(vV4qq}=ydm~SAx(=U9<|TAp+ncAe^jZx?;n%O`Qn`n98g}D z2JQa_+W(aZ`0&l=F&}L4x3-#7GurhuFP*c(4q4kfW|kG9)b=MKsl-nE#xR*55-_&iYq z7aoVrZ0BrlF=hn}%Y<_E1Dx*-BEW>2#h^4y(`B4gF)Oa6UEW>eurx3A5gt>((S7mXTI3-r z)m%BvsA}Z`dKaEpP&yAIFPX~xmKi)cM6idxF%J8>UR$SzQK2l$iLQ)_@MKRaQgMz1 zQ^$xg>Pjyikn%@_;Rr`ETK%2F0`+KP9KF|gbf|i=5RDWIQ=pjzBC}pf*7oiw;%$Xjs1oKleR*&^P2=&GNnA@ z*+GxI$Tbl`_xn1=C3M#xemA4&`{u$q0|M7v3s_WC zL;-jtNw1!?`C>9#@_uxB-Oh^(7%>tBOm{1EF}+8lwtNQVQ;>;+LBnP>71)=*!EECc zOLFN0LMB}LpBMWei9WAMIif2l_qBGNju^4*{v-?tj&z3Hwn>a>|N&LKh^I$S)I_7fZY9_|dl^^@ZIS*Hji@A~D*YInNb9+Gq z*V6ymGsC~IIJ{^HV%;oA-X#u4e{+VQemFiygvRIdp%D3k1k^`o_}_3F)0~IzA57zv ziY|zg#=~&!mvbs}PO0}_?j@IBl3S2wniK+w*@>2kuzN}T~?SWQ*Iw*qY2_q z3#wnbZv5*n!ZLU@21T_H++G&6RLNg!nbd=9*{<#B`?e* ze;p;)^mey8I3ga~BDsf;WB=rX!hwrsTqQJ@=L`Qq*DgSgvdJ_V<(sww&{oXh-nti; zD=FI+=Crh}+$|{nn%)A+b{Yn#>Lu>Z;?GpYrjH(WZ@F365lKBSa5<<%kjg!E3%qSa z85Gqi7_J%7I4`mrtq7<+vH`)YCUZhE*a>~OTYHXT5fMFlW^9o;;)y*`5+WO=} z_t~RLPMMA1+j-K#l%B(-%zbE)&{ZHEE&g8;e<@N+?~eS}X}#ZP*(xQGdoz|6MmRH$-NPB;xqnZcdzo|3E1dZ8PRmlcb!1Xmf>;X zivYte^R>Ge&JD>5S2ellRl|PB@rXaH00v~J<+TtpVBM4uX=NY4lK^8TYycqE-c()o z3RJ+aBVU9!o2LlXVapQ_1R&)m{(excNw!@&=S{v9cCW_ziG~Jk6n^#uJ;H8g-jGZ< zbHc@#drdMq4a6Jg8xV7%V`MgcDO0$_XZ*F2oDA+3%ziRbe^OsmjuzW1DBti>G z)tOIC>iqiuHp#|wI>tat9WhG1z#QBl%2ji3j$eqAcB^V%OVBU$0jDRWp;g`kOzIwr zv67Tsm$HzMSlM2do>K_n>R=kJ$7K6kHS#YNZ&WAfp@*9ZPYHqMP~U{{z;5!q=1&8e z-TyKnL9w(1&?np}U==NA+EHKKB9AwYXK~EFam~2}T;Ty9tVvDQl3ziyfK?qG9g~mV z0WF4ytDbTB*A`TYXZF7yDb7ly&5mCTvfBoR+ak42d{2cVS>B$%{->IKJ_Fg6(Zo3__tqio~yd9r=F5N6@J+2KE z&L0)HE1HMeP}0!`!d+OTWX*AN)!0W=_=v$=z&sgd5Vw9d?N~6TavnGE;Z= zr_Uh>f5AOxomg)s*He!WR3Ip7<%s2xLXra9qtNqGU2fi7Y<2x*(UT;31R>7e*K#(9 zn{%vO!r^&M>;D{j=HBUI{7&z;ho4`TV*`2g8VbJKykiw-qj7SBtx98aDl9ng|IR2~ z=<$^R0!&pb0{rx{0x;%M?f$YArjJa@+?L!A4A;pYkgpaTR{9h@)or44$YAn%joP3d zmg(JkPGtvZZe&;^M&6o1YG~(Wd7X%)$Cw4a$;Ax(%`gJfFX4%or8)K-_Ph3zQTYm0 zpxJa2@>#j;;BHBDvwsj}!b=5`VVN8J+H;rZ`a#vp8X$Bb_go;vQ6IeBu25GSc{;Zz z#(oCLH&J$DWj-pY*{z!v9{g^!z!0&yi385WfKIZMR4O({)i^Jdw5jgl6}AlySBKRG zkDDk^&9uKqxSK~e>UonJI7C+){-WQrLu0(JR*oFaGlmK7K5km$E2VCXe)d(Xz0bhH zUna|D_4L`bI+u$5PFov;)i&~cj7%GsE+G8qd^i*Gx4+^e{f1P0cGv4dhg1I3-{lKoZWVj|{*K2R zVRQ@@k!Y@fR&#;T^~hJ~UzD0yBTPa_TFB__z8=`$fU&w{ zYhX{tTg%glR7NC(GQ~9yyr?zvkt%Qvj|9e>jDe@*+XZ&*q*$9M|yW6fA% z5iRbJi@4W;7-O*7QbUB10WI65W(c!~)udGQ>*za3>Yl(C8-nxDwJYnb zc7kgmQHUs80_W>DziZH3p~W>4(h_n!ZvBSintZuwGTqPDK@prFRCu@;>@aP59UdtV z`thb-R-|Pl>>yKU^Zf=P`{mR`)hjG{?a2pJwNqkf6l%R`H`A$qkAhT-nY;$cfS->3 z@#KPIXj4S2g7iscgy~#hTUM}XoGp5#2$?|3pR5k3vI6<0G{;U%eEqEa>zNM8qu>h@ z{D{8nF8(YW3jp z3a?&8wts#HTYg;UoL_6BIK~A-IOQuY&c^j`a&_ zYsf!can^8 zdN=VLxox8skp0}4bu5hlr%l*W(?8r%M~p#uPD`B1n>ON>F4U#?pF%#B%iHY2=)0!$ zl7i0>OmA5v#&t;OLtl_TbSQ={oeuX5u)*qgkXa8Th*%7A8m>Z;``73}A`iC+HA{Q(xgXM3bMS^zhYnV-I zSlQIlz%kaoaQ$O8&X2poDj0(=_YV=MA7JyHx^E^*rzR278g-%j!K@&v<0=f7KrcGm zPQ_GrS!(!Fls|*Q$8Z*G!4I zxjir;{j;do;S*G68075%(POL+OAj~x<5a?Lgbf>ahW_kf#J9@u@Z9FAfWHdvhEJ50 zmi_GMoJqAfaf{Y7*ZWoDd{936C?c4kpj*~mI}>e_Z?3WIU^{fjbZSZEIquGwp!ti&r5hW?b|tMYy>;8Wfhfwz#F=llbWIl@^<6JRvVPCEox%ks<7Pfn`$)V z&hQ=<&V=4C>?IB){(p((9lh31lH6s8rr&-dGXgx*kaDDGO!Q|}8O@fi6)J-_r3EFC zb}EK6(L?Oa%+JGlmTk}-J-T)Xj7;KHUO2M8bXEF*=l&&4TKb-iXg=Y?D)|++WiWL} zSP%6-g`?s|rsLV@nddfIpag+0_LX&~D1+)G!cb7O=3DZ!StpSz0qVTsKWnziP@;|1Q;w!vuKc~r-4;47 z5}2Rm(D*M_@c$3WQ;-ec|CRZIDUcFt=TqW4=^qC4DpQFiv_&O&N7XE2l{2b=Md1lw zHP?!x*SbTCd_kFkLi>?F+si#2kDL?(qa)0=K8Qs417i%_#$=JTrmq!qqUjUR1)(jc z*M;nw7px0=-Z*dSr*`tWFSKa?8iGAp)0W&bqR&F(YIJE@mHs9_CF+JbiZZ#K{Wia7o~ap|lKEu2h#sBfFTixWZHgQ(a|pG7KoL9&*P8BqGr`D9fN%|;->|`2uV7K) zc#YpqGUUYx!S+jl1Kq2s-LG)*2KVuD#Q8iCLcx+>jkexa;%j`ggewJvsba%??NmuY zuYMjZOo2bhf!|473k%(jSqj*M!ICgUPgky9h)M{-KrGXzd&vWJsinfDMwe-I8M~Wi z4ZMEh8PhRCB17K)yzRL!y(_$iPU`7;^lPVW`a?QRZi`NRvF`@T!$?jbCS#_J^k+gj zbLz}?$)pK#L~{Mxt)6iCq@y^-nRJgYJonw$PEXnIG&z3pagAkSVat3B^NC{1+Jbp~ z+=_fwUIxE->B_Lp@L|nvpTn>)j*2#9_fKdO9o?Gw;NkkQKxSX4`i{(cd%a^tgXLQ> z)`6e4TI~n&%u&%>HB1ybxv0}F6=G7190>)dmKk>k9T7p%;m_0guy>{uo9%1)N-XuW z#83TH^u$Kr_JqyZG_smfVt*@LZdH0Z)dTRKjee(Vc0=C5c79X8P*i$&z}&s8z!x`+ z{Acc+`^4ZrIdWeNzrByZawmw6u2Hdcf9txw2`p^ajB%Y3xqj>&B4!5CY_t>95JW7F zI^b)U72Bo~A=j_%F5!7LcPj$aKwM|n+n$CN>7EGu#cw`EOU0BWiG9tF z(+~d{o;G;vAotgu4>sbT4C z=|3tK<8SynO^WLGe8CUrj=FT|d$(2Lw!VBfT^W0d_)SISR zdTDl5BFu*|EzYPnqlJMOO|PO)F$N0O9AGV$I|ivN+|d1lJ;Z>?05yk(seHBeAVZA~ z8H9Z|(CFDeAA_Uf-JqpGeVcK|qZZE7bYi`|P^Cz3v;Bv_Rv2wyVKd4tZ%8A}TXEm} zYWyOxaCY(AJsm}RX?sCUV{3Vh#UJZA>(ffOrqT6SA{i`z)P>}2CG>DnaRz5GkoPFG zt~Tt2%WtHGJ$qsZ#Q2Xrdg7{xQ%paZ2o*aJpS)fkXBr_xd(~opi9l8v3?#ia`(S3m z?J-O`TyvLxxEFsusoCvMz7wF=$-A26!k{efEIPEi;r-YRnb7-vy6cQYaB`-j;cF|i zu4=u&LgUd-rIgMp+`7~fDq;9*NYiRMnW^3N$-5%+sn6Krvy0Nu1e;y=NKMp!Q)L4n z41hxzs{<;2E(*^HOadfT#_EY|0{usXl&?(}o$c={QV1D!fsCpJPj7Rz}JBW2{NBBw~Hs?&VGGtMYNk%;GH{&@xqhVtsnXdgmf^Ga?Z5!z)?o zJ<7rAvmH0^g$^XWKmJ|GwMFlBgFb9JYYWpAy{Hu^>h8_3-I`7v>Ey+3LvS`+xSfJY zNAfzQz0#g}tZJ)atH?IBZcF%UqkT(F#lIA1+l}V2+T;3Pbkp_v5;PeXsWtFRRI|!B zSC{dguLoA@Vp@6wL;h2M^0a47Z^$E*=+_EBp-Fs&{0d>p}8a zW8eC#D2UW=1G6SdG2A}7Ts@I6T2X)5{MjTsjrN;wz461#(Fr2pOapI1?&$6CyO3*< zE8aO!$&gu&>YDj{$$f2&Z+{J7P+f+B8n1|UMi-MH@<2z$b1Cl+xb!or>g>*pOSNGqd9m_Sy{Bg8|V)bz!QBmTt&w zUW~!%^wJ!0B}wq94i}%|L%zg(dKb5O^?e4pj;s!vbhO2(6$AD9MZ7d=tD4la{1D}G zm#F+;%SiZ!z(M-CZ8ML@``Zm9u4$af@z^p)l$o-5;7aQ^Nd#uzSIOleQr94xW{y2a z&xvM1z8ihXH`)gDcFRJIpB!vX0%6#!y&ORffdhi}^GOvC8-RpdC&xf&9xVogAIj`n zP||!c7ccUD5@ICNxVm@*hGx{U`{--v_Y66;J4a8|z8RPy&70vgopqZ}2J;=t;eeDL zJoJDZbU*}VZGW8pmK35tZBLgLr-jGNpf46HrAP$$hND`qJ&|EUvw>NW(DF&Wu zr;1R&`>!NfX$GNQ))W=}o^;Us;)J7DLXDBVeW_MR>iu;PgVd9Mt{O$JFgUoPZvZ+P zw>p#E{f_-y5mrBf(0cdOj?MQtn_dFKn_maC1fd0FFLf9(-~nC`RzE}0+)u6~?>(67 z&X`xRk`p!ix)J6}Mmxu`2{vo*gj7u8Xu_V3cY{_H^G3V(>Yh6b>*B9+k5R}w=z+1y zDX?Uw07mc7prF9K#w1)w2wkzxv~uI}OfhWsP?M`UNBUjU6A+IuMljf##kl`(U=2_Y60DPvokfSB4s{< zf9Y}JX3yZn6eBGf&b=#kHMyJ(#|cStR8^?8O9vod-LkMSrMRTJy8CJIV|%`*KWj!F z;*ES;p9pdFIrqLkiWD9djw8m@wj+lNNN(LmBKjvwcS1_Kx5Q;S(c4_Dp8UR^F^kD@ zhbrf`*9=MYUz$Q(EFf9(H-!t#9}4mWyXSSgGHGtOo+kgqIzlKUV66jZFC1z#EmsAxs6MN9#N{T~dqFJI$@mEYmU&;V-E?`!Dqq08;GV5Zon&nw^VBwU zkUOz7`q>F&e-(b5?+(gmbqoLYt|gLsMJ^7exFzRO5*8&TcYv>Ls>b+);ytP}!_imX z1X_kii+}A-s0obnW4RO9`CRyHL0iW21>h7CdQxZQ#NkIUC>p~|#jd*^sRxjImUeWC zV)IUN*3PRpZ*V#&UcJxe*2m%Z-A(r{DD^!r#7k|L<}vOLvD9xHlP+(~*JbqcKAdU! zd%c;PH5r{%6U)neZKJhrs|#@FV+XdjeS}W?sX6?zc5!l)ap@2i8ZG=aaB%`mu}EF= zw$3u#blsgsq-+D}tEGDsppoI2668syOjyIK;)ivzo^Wi2(3@T3vOc!oDxHuQAxA$z zq`4!M$pl2QEG-U_WcV};wE3G@Fe8Q=Y$JLUP}A_u$6_M^Dt(g>)yQoBEu|(6dDGh) zref3Ky3ksEAe6<=`IT$)!59ior5#-Vx~!&nH}D|!C~ahMv3I@lr5}DLyaV|iccT3B z(szM0K~t?Jesm3ZGAZ1#=ZXc$n#b)ELhil&Cxon|<5&Re@~RbsID47MC!I0X9_adqqr0WmRXo&05J1=8G4(-Y`qpx*%(C}qbeT;*HH z3X-CosV)g}MwdFki;OU`bM`;TC|SOG1X*i))vZ+hmGk33k08d9c~UqA~P` zI^3y980W1LlEr#Cm_H_HCi|;tm1@$bks&-7A_; zTg{3Q`_bBVy7D`RT;3(UZl`UfS9nl|0JY}X{+*#P553RY(H|C}W)I2ry8M>qE*(is zHIBV9gEQN(oAPghk@4wsb@$i1<5X3pE#stXlQB-)8T@+h-|mmab6s0X;$t`AyAFx` zJxoJu6(hcxLo;q=VXeP`E;5+}l-dho4|7Fb!o9DgIG&W%^3vuX(%=+#p)bCZ8ag+` z1Z^ZGmLRU^{cn%YY>QBe?i&#y2@&*x)~w_FX&cGrZS{Us@$&;da6bZf=>GGhyUgb243_jkVuYV2S4! zT)obfl0FO+04@W@w@o?A9#~YKJWWT97YzC7cs@mbG2_|?&SO3EAzKnQ#zng1aAT{_ z37N7K-z8*+o3NHG5Pxr{93H^u_x{(a_nquvpei=36ul?u&iJXMo?$qU11_aF(waYfiXSnynz{CRv*p+m z+hy>_;lK3K=~98xExvk~8JCg6QuXzvzC_~Cl1uXK0=k!(NQ{pP?OMNLJ)-&E#5ZxUK0wb8CNA31x-0ZVRls-8=k zqO7oCH|-1cF?yY0*0ikOYYRmor@%uRi3ECd1=vM(=}Q<9PJx{Wu5O!wNCts>qTAkv zf)UuvoyZc6p-!Ud3%Fn{H0l{8J0sSXg6fr+OJqr2{RBhee-rYp`Ns#xFoU$i6UJna zJX+|#xHxo1H0|t`HL+*D#--J$tM^z4B;G3WlSanS71~llhsX0%8XAav zzPfsN%`0E_H>7@aIJ4K+sx;32#WQD$zb1m25T8+iymQ?) zfhLcgr@cN=%h95T7SPK31ehd~!?ps%c!4xkpT>@jg-FAQKRV`G<173dw3LWv;yFhX zb9@q>xiQlG5)dy@;i5^~KXTe&wMHAaxyJ6kkRTOr!?ItLp@=XivJyzkbfrA+BkfcC zfU_1Gt$G6pdgj`SZov1k&n#)X_cO$dTz`9{!eC zupZS{h$pO6uIl*S%9J_cg8o!CzDycBe$M1U?0(aw_pU68FI~$ZvI9Lp3wDywx6{I# z>4qj0t#Mb07mJL~sLdiq(6XjFgZ%|EbtEq9;W?in(7$sRM4p=67>?!dPKD7Nr*YI8 zgwoj=yLU1)*8UO?j=f9WGiy?IQck`Wzb+^LeT`yjzDCmxm~goq)^W>q>>|7TkVP95#L6~=gJ6!Dd%;(Gf`~n9%dgWPld)&aC3oKG@+Kbdo z`@Fa3BGq>&e45WqM|J>W9n zRGbV#^_Q>=Yta|YJsE^OH@(|ZOi#4jr&`l^E}T~ewOiZkTaKgMKMi20KA$uu#OGKc zpE<-}xRPpz^;SL7IPpTD)XMKs_+aG~VfOT2bktU&eTe7JVFh1{TUsW1b95C{aI%Pa9LOo<9Rz|F1|9tP5^ zfxL7C+k{6=TR73>V5fra!S7QYFK=mBDfkYE25ZGB_~zQp++OtRk$dcX1bq{dk?KEo zSe>nolq}}ykc}HUvPhJD;R3Rmmh0b|B*hzavLWc>`l-aqI_+TMSj8Ro*OD#!I2EKP zm@Gridbp0x4iV}I!!slSK^{8YSMi-xJYEAS4G9vhr5W#ui9>?=10DLC3|0rfy0CNE zd3b;okC4hDpwOres&G>8ke7ha17I>R3EJjr zZe~{=@6sANBrCf0YI#2QMqO;YCtO16iH9kx-FC?MiCd|+!I*%&gbgA@d=6Sz?<>C# zaE8_(CuCdF$y3o)p;MaY|JtOhBvFKzv)Yuh`yw$*MzuGNHBKsa#U(YuCOa8fQI!Vs zthP9g=hD z&;n3`tkV#9cWI|hnkGzd6$fnO_L&2>$;o#)N`c?~^t<-cD(l}~d-YyXJ18oM1os|Y zAw40^QT#C}uW}0fvSCVJ8gFDA(DRwFo-TCt$ zQsh}(EPKN55+E5S+gCDD&T*AiXdqYAHE;5YNU9Fx#Deab69^RN!vwOe4N#nd7weXn_ioTSvMV-s9e>#Q?eWXr;^eNzthjN_erwlG9t&|@kT_)F zoHgWqexu|FqA)%nzUAbssL6YOHh|V78Ti3N!{VY*f{zx)abIFk>8E*qn_EAbd4J_^ z`&u+D_~fkeZhp1bs_wZd?osjZR#(P?PMCRdvs3dn6=8o}ok&FohtZ6+R=_|#&C&}c zUgq%Cu|*XBn8ltTu^TM~gKaR5Z+oQ~D)rRUok z>%p$iaDOU*l2SgY2za;qBdrz1h0Swh^9jG~P+VO&0?(0Cy^YCT4oi4`-Er*fH=aoT zi_Y;oaEU1kQP=n-_583KQtMh2(_+{yycgL)&eKle?!QJGW@DjM)4o%l;N@@i zHi-QE`$-$&SnXnl3(y!*sgTh9&Q`iDa)~C-?B9K zsWF^TbuF-x>GJrOWv!;a+g4sdZOgISmzzX_hD{FZ8#X53`2*uYbQq_IT z2>R$>!j?y;(vwScFg}hfCHNSr>bq|R7dJzvs2asovTN9|nG=!|8S4lUM`JTHm%p=` z&Py#L6h#yFEON~*ytXZB<_L&`6#>lk;NgTR4c-#ayzbs%=;Arp*}L^OpX+ps;H>(? z+{K?3Cjr@6{fsDIp#xhp_^j{s7g`&k2>14xO1d3mV7KzL7>nI_5N8!}nU}rhVSZMV z!mhqV?cra$3^9qrmg7<0n4zlxxwotr0FH#y2ef6wY}8~?$M`b6c~V~DV(05(KYKx8 zNxj74e#Kvt3OET4r9as!dD)v`lR<)y{EN0X=>r>l*Fzi7>CzjO`BEbXKL_?T-Cs2) zxamRCnRXGMhL&#cHXd%P@Pd{X8r0wQe^Uo1@VK3XPKl+@#Y%=VCrp zMbn~)p`eOKNnUdlvPt|;$F$0B@!~BvKfGsJTQ{>sk;3;o>~>$xZRL5=B~CdH*K+f1 zBWHcG`h{ve}pPBBuNaXY5uIoBEi+aeQzUSs?Tc#ggTW&*U4(@kd%ITJ@ zrYO5Bh_OcABU4ym{HfRcQNKvLa8x#|{#vOWLwUG%I8JyJ#A1T=FpOpJ2N@{s)|1EE zDO1<`X;YR-G9H+U`U^Zvs@S1NU#%r(SsX=VjkPTdu8L98-`v{gwU;fT|Li4pr=8nb zws-p2%UY>2u+C9W4V8Dk`k;@zj{o;C%$#qAehW%8GPdlno&uKQdAmonS>j#ieB5GT zI%?GYdsD7g)~$WKhd&z%lOD!rlI}>oz2P|hU0UHV9Xk8%k#_PW`}Au{o}owAP5xKh zhZfa$@#IAjI~VU=nUO{JYCXvO%t*ci_Pv90PE&fp80u+b^5=mzcz=_7}eU&C0^ z_ng}*g6bn62=E}sUZ%}ieVl&*Pw#7fyC9t(@A6E4&gk8`yUwu{U3i^Ynxb$|*gH}o(vy1_ISY`4`h%B1W?TsVj1Z5W`P`8)| zEu?>^lbU;^T%=a836c>A9`1)EZ&sK+a(p9-j8OKl@!C`%p8cq6c&9yo!L#~wv3=jM zp_iw!bYySuzEhMis3TW;pv-Js`r1UoL^3jXc$4@Cvn^4I+OGvp=jzR3x$@nl=04^a zT?KfwwP18WXa%En#x95Omj7`P>{-u)fq|>GaW30Bf7o;xP4Vav($^s}|CadpsMatp zJm&dm?@KBYni-B{1Y=&mLD2@yw}c0q$|9p( z0Y+v`R6a9#46G8~Yatfpavafh?{jwVY{rVwLu?Jw$EsF$wO04m5+t=(be@VZw4F~b za*zwq_g;Gh^X(&$NBAOl&$*dg#=cb`ZN$FR1aH8~gMSQq6It0Gm=i72q%@m{c`2tH z^I#AnoGF*a8Oefn?$E^dm)dnEr!U$X3ZVoCrsu-_uHl!Jwl(PVjB8_flK&~~TDV7o zl=qHp+iqr}xi+4OnaA)pcfbZmU%I$Ma>Z>vK2$i0H0I1&%*osjG~&483_o#Ml;M!ghW#92Y{l zZ?k(XC*Duc(DiEbWMf%XPPvvoQY5`)5%c5TPMv5nD-mg!1tz=od)Dp-OUfFo(?%qH zY}|~!7olvlCI#PA%_ug$s4hVCBO78SLT^bW#uDUKpaFw*hHygP#;IPBC?aIto~ZuL_GJ(so?t_jk1Rwy_nz^$iTH! zI8PbC937eBTHi0JxjZ3ifT=z>>mOe>5FbB=CQNxWc>AAD8CZLXTy-c4>}WvnE_4(w zrruu`@remK6cUH&z7xsmNqs}U(x_2xjycf{LJHv=kfjogg59%w<$Cly^ZWdEr4M*|GDo(dvz!M^K< zf6n;&{EX5M<7Vuf8&R0!+&>i)y!`6j1f#i=3RB>kcHe1X-1L_+gUYR6t@~XQM^0BT z^R-ma!5@B9rOJlG^5&%Xuxa`dxkH@?D-!cMDaax7PB3!yiwDiFHMIR}(9N=eB?3ra zA?q!X0tPvRZk-SQ3v1ngU1E-5+tFbgv_)k|IpbR+H9-!IlCcR6U?_&fzBiL?mNhFO zFGG6SgP;*x+c2h%$oQ(-$goMyQp1jFaImgdl+8DS;7x6MI{3H^-_l}>SN_YL?S9)d zGAuA~thnNypo>_r{gZBarGCJ1pzW_jpS{%__f?^yOcl+N;w>|Os!`l9Ae_Qz9(Sk{ zOe&s-y-u%Q<~?aYA#6=-HJ>-x0_gRwd;Pj%d4+N4FJ)0*rT2E~+}}%+t&M6ywHw$3 zpMf6ZGI6;+J*4>CNB;eTX%M)AzN}orP^R?kd7JO^V;1r$7U{Vn&>9H5$F!!7n*&?3 zd^IRohhg|}$BOdXVJQ7KEI+Ddm)0Zr3Lh%M7v}ecnllOP@WHE_VpNS-pXL1lKg`qA z(aq_R<7jbdyMzvxt%1&BYAcRS1i?UTZ(^rZPjGML1F7DA9lP~$p~dN@diKS#1~-QI zawB&sSvAMt1Vv5v`xHg$%SFP7k5*SF{UmiXcqYl-~L|3kag?e;6%9|;afJ!j>=z4hkZ0)Qm8 z2-ogw{dopBY4nTC{?RxI`3L3+$`%`P@8dKJ>#n}<#P}(et2Vmg#C|lI!|%^I!y1=l zg!*wd;p(*oQ);*C+u_9+>+dd(SjRfyOdTl_vdxMk&n@05@IquXV9dswRxZG|n`q|f*BBSi5y^aHc zv9CO8DPnwu>t71T9*`wJlBT1lZv`k~=xrM>kDA8c6H@aNwGUg-gEn6<{wWw9+?7~i zPz$FymKJr>4~8Duxc;31(=Pp#LHb1@a~8Ahaf*XZXu^o&pe{7HCz>ynDq73ID~>~lhK&3kb3%#z-p_GQ zc|D4Y;wY_RP2Ouh5qpDeviX|a2kck5`5tOo;V^6vAH!^%-WY2`w?VL^FN|IM#ZB@l zk7tFh!Me@;6=&Ty;nk9D&M~E@uk9JebUpjECj@eyn0{|5p)H@y3>p{oaaT{YU>c`^ z;A@QnXsI`~E>O3)9+)?}xsT^maJ90L-kjz&P1(OcUrE z4j&8lc6s%EfGLjIwt6uhG3|XGm?UBu|C?ygcCtzca&y2ZD4h=n;VcaYfnDkc+6F92Yi6yAClq^{X^i~d)EfUFm|J>UQaXS; z%iuz_%Rs#U{e5LDoMlZ*n*Hq@*y9&# zaDb%aQh<bbZ0-*XtIIq7ZpM$ow-XfAMc%ZPe&>5R;X)oE!!f0h% zH5+64|TE=LJTBo;&bxVWR`_?z)UP)=@MTfdz&r_lW^QSew z>QDZ$1&rkHegZ*Q>{9OYVY9LPdGt?%Sd_Dt=Xz^;+Ax&+?;o`eB@4ox-|PoQN+N%s6%w5|MXW@@UGFMz z!%Z{`jKZ_50)Qgf;sQ*KfEbC%In+YfIOdj;A6RF}U#w(D2a#(?u>J&yBn!{Ot5&gC zODihG;7?HJhkl4$Oc_oy6;|bE7qSG=kUDZsjR*ySECH+-8au1HKxWuN%+8EIX@^x_ zG-U4=*H2Xt6G$_7jq64Z{)xWzeXHSHaAx!LMna2`AX>5YPORZ;9l=tuBae+knmiAS z0z*83`~Of1fV-e}V{w`qYGehJY5i|wv?Z4@J08m}+6C8k6$g=(br z?*_6Y3LM6NK2=1E>idkk+UeohIdOac}No z<$=Sf1q>M4=lkM*i2h?i^}oVCctWdKIf=*HS30J-_LEs`@=S&t_-7dIWl_cEu)^uQ z_h9Vz{rmO*)Ue{YvP6~D{qh-n)(hhW}SYnv7F8pTgaGWUKWkDV{ zk_hUNYDGMVtk=RKO&D-yRTJ5T<-?jb?9k3A@kc8Z{ERzmMMhCiO z91;l@2A5G|V}K%#FR!>(~;oNP2PpJ52!J;TD+}V9z3X z!85>wA`I6za#B*0kEMRG9L=q|g3zYF_O z@U1!i(o9jcqH@;&=bR0R25Xz{OqtvNP!(i;&5u~FwfAPMGTyTNx!eD??PZ=eiN>!8 zbtPMYl51`5EPS<<&A#+dOPR1RsHLx*7^<>(7NC`GpbK@Mq%p0GhGj(10q{blkEUWq z2s4?P$cbbdsTFvYW?5a;SlO!3uFi;UBCKAmJfAe$K!TUom`;&=M}^cb=}a(E?#&Sx zcUQUoa9Foh_I!gpN^0#Jm)=r&QDGr};q8cEp52i`efm7{qpB~U(70|2^g=N(=UdRE zEu>;KmN2xd)^$5-ovPGD2>mLWSx&aFHia*NJ0YoruJwTeul4p)Xw5{WUG!pUkdQ-X z^lH(rJ^wuh4_#Gg_5ef7*mx544x(e98tdf7;9*15ATu$&Wdmijm@;vCG8lLv0|gfK z`g0b5=U(SZx%#CsKIx#4mw1){(VK}xN40ZeR~j(2afX=4Vfmj%Pu|xm{fCY3)U)l% zHLHKvF~4RtdA1#+hT8Wje_6)z+hl^q#T}6QS)5ASaic}5&9fht)8Q-SwHSyWIKcFA zn?6w#sMUF?OBw_5&5IO(P>NcC{dTzw8gscJHD8t9^y!*P#K-QF&TZR@IL#4C4c1je zr)iUuq#4p%s24_J8ne}XIU1iTEv9CjEpi7ugmYkOoUPx~I9VkMn?I#6BYZ_w?6Ngk z?Le%n__z(A9EZd_5FmvZa+7YG!V{Enl_QiW%!&28knB{X^FMb&XF}iA0PKu{c9=Q@ zK#xXZeOHSETD{vrpS{xQ^gATm>XIxC9I6w0JR8g$h0Tj;a+`f2YtEj@=5Bd=sZNYi z2HC2|(r+=0)j>{eJR##6N|FNbjLVcr9*vqxBUt*P7^O`vLxZ2y+0V9c8EIJ-vJSNl zPv9PVY*lmKLB_RH4287Xw~5eVxH-F?{av;+$+&kK>w`E*%isRP3B#iYoHGB-sP8iW zvSTdRUJ98Zctf0rtKAAIL>(^sAsyDPQ^W8TSol@SyQb%Yc~PyK)K4hXqbT$WACr*t3vZ zA)JCOX2rh2>pyi$v+{m=?94&KjNaa7p<-_^0I2(BfxK#1`Zu?)Y`pAGF#tAXHrcw6 zK=zNrtMlz?jZbh$+x(BB1}XPKwqZ5~OB`&jb^7@y;fV06H1`_WyW+A5TtqoCI zl!1#dv?3O5yfdK^z*HS{o=P$Gwux*=Bd)|VypP8*DonK1369LBCYS90rVnm}C}>0g z1VM;O4NB_21VLneS5IygO{IA#jGC+;*-j3Z$l6O|jehTSaD3(Xt<)-My_ekWF8bkw zds85`L7f_3eY4^sR64o!ovQZz%<`|jtkqB;boqP=B}kqx z)AIFE)Ph+oCihxc&@2geLf~pe5w3hK_pmv%I7>6a630Bre8I2n8cvt&zYrioN;%Wr zOypP0GiqjLJyvrQl!a)7?h5GOT4pfOfhmL%OuKUUb&TUaNCqR++~;5n%I)>rVh{Jx$;JLT;3^_6Q~o|+~fxT>jl(CSa6 z`={_|6alZ6G%F<^{!gFbZ{dXDU3nX&JEomla9fNG+fM8et*MFiYxeo7j;kbLxRja= zrEjUjJ3{P&)kfoU{{35+ck5Vo#W+`$ng*VOZtp5Uy<+OeW{x3{t~wv>o&w@<-I42d zoUxL3+i4pF_+!ILMghtJii#@KYT*wkXJa(~@Z(Mf)9-39+F1|XfQ6plvY<01T?hq* zlJ!VchgAKvTxUZdv!mMAbXKG`&7@X=ZeWq&45=CC2$kYW`~e!{RKE3Pmp^9myOASj zN+C)i7I&b-i%Rw#=d@TfcGC)*8)~}sn6khlMz4c`8%sIBo_MjQ4&szQ3aTAh zF(pSmEy+AMsJxawrIl*`O24#03AeD+DR4-4PDI_Q;7g}_tJd0&mXB7yjoa(S5%w0) zL>qHfCQ?c)njFw)wX_~8ln(Bw$wzi8DS9%$B)sJO;AqLE#1*e_aR)jwPq2AX3(ePf zXD{uaZ@_ty8uQLB87_0TJpm)#e(2bllK&Ykrp37?@5SuC@^$5*t`%${cqTkSU^9>p zONiz<4N9-=U0v30y6nYH>%3Fn6nE2SfyDt5_$jYH(urN$`&HzvI`4rPWTnKKxK`w!zf4+H@^1S!n5tV6nQq zp|I*Dr!gK@oK?VjP78M1gmdik+J>>&AsKW7^wA3nvHA>F>(?4-&nQVmc8{BYu+hUr zJ=G=IWN(n_z6T;rGq6AX9sS3JAY%HtIOI>99=bM|`3DkKm!pj_A=#FDA32uy=d@pF z?=_21`Naz-21%uGI*_TJl0}=!e!+#C!|UBfOZo)YNiA`-9?nDSJ}h=|h0GUpJGkCn zi$Dr7k0)wSZkoI4?#ja(WLf0d5affe|1F*dQWtofer)jg<}su4r?~E6^69f&s84Yn z-CG`0kTknXPZ!-4cX!OOSISaMdiR7TYB86`NhY$}n=abx+H5`-33DhpeM#5&(tS^L zKIhc32v+B`e$JFIe~_AiT1t`OkK^#bpmUcAH<#NB7E+f}ralK}>|fl0qTL#{Cv0Q` z`<55gCq)~OozlyT3;O$xffea?F6+COp82NHNHO}6B1&n)mE`3F1IuYSZ<#5oiRHI+ z5X0#Yy~q1qzYLD+AnKUn<=Bp0N2bR9ia}AU3sF>8&$ZRl&2V3kyzln2?B1!m*0yO* zHN=!z3luY0-`2YCy0AkAk18#5v7&>z?epRN$I&ugN2$!`!}BnvRR`B?%Wk!3 z^reY&XE;*w+AU2c-L7;w{L8FEIpZj9jU8yOX(@9&Ts{G0B+;@n=VLZ?@^SXAbhaIL z+x5&{&aDZtNb<kne6^dqct49+SUksO}qDs1kdFYYRWA1uK1nJr}59CET z3^co4Y#Sn1W8K?vZ(p(cDDT z&~9598Qp05V-a)9ea(3|%xwQjvLz`G;#+@d}Y&N{Uj}PfU41X z8l5x2ONy%IgIcFEMVB>fo3}X@Bw3)#^{op^M@BM~=9X`xr1^fNine_v`nU`}-h>#u z1mSthC=bd3E`m9DhG+7|8cmOx8Ps~TT9DfXmwS&45+(C8X+78`AHgZSW2%yjZ|Wx? z1+yI=rr0vHR`?p)^~%%7k!}771m3qNS~T|ukf*q-d+vI56HKbS(UJW9_=o7X$msGkhdREgH>~lFO3JE8E{8y7m5T5`?%(lV}6EIq!Fx6+ng;%a=4%k zWb4@XK4xtH1gW{N^@;WXv76BwgwaD!*Zz2b7CbP;wuW1SJaRNr5E$BLz<8ND@;e&M zcZ?U4Q9}{FJi_+l>qX$>Su4Xoby21vIFotx66M0^cIJrjvD%y%$a~@e8_OU+@EVlO z5*PKrM9T>5gXboKv_n%hL**#`Jr3rfB|* z;WH0+R5-Lk!zz^qg+8lzpkCJ@_vG=!qH}lmbgWs?DQp@JozXM1L8au)rzWgA)ax}a zElb+gT0nPaP?5{}YdZ0KJ)Oi21&TAxuA0BIWLbP?L-H{@o=qzYK4#~c_wT>>+bw4j z;+}X!v8HjWfk`L;9ekg&|J^tnriQ*E0MT0kv%cIaXU@y+QF03cFb5U^uobG5j9=cD zwqqEE*K)?_(Jfn&a@z~50y;TI9g(UI+!^lqA1&a0>y{F);6R<&_brJrd?uyWzJ+xU z{MnW6V;31V${y*cmJ^v`VXE8mGNjGB;wl;=+s|sQD#_;=eYC=F7m`Q&%7-PcveNNf zWJW;gGCT!MKKbNJeR{CTs$T4}_{y*D2{LYYf2o%cn1Vdu`$$$+NyJ1Dtu*a(ed)KF zn8Z~3=GPA#sI_&*VjA9@NbUm#(e0wTgEl+W436QySlj~NMUkCF*+LhOOwZu~lrXqlr?F46d>^O(w?ovPDSZ)VQm>M17r zJGYeY9nJ&=o6O}RcSRd=>uCgjF+VcFOn>N$N#|oeaq7@YC~J-6QLsr)zibbuMjq)q z|9&;Qlwud9`~^t+xILI3?q2q-SgQA!i6&z%Dp#FiGeVPe@wBnWDhA)g4SzrS?Fe>K zQX4cSy?%R7fL^KL;?ZC+OLQL>jcylV4fDRL{Nk%@)0!msFdR(Jncy412kYZh#)wQYDYeU}zpL#B1<^zN$Rr68dmy?iedH1NsuRr(C zfffu!O_P8(T|Ib%Ggc8Sz0UAx4!&k3y(3WCiK^8kZx3LnPj;{*{EB<5CY7?(~OlzUR2<53FA&%K3QfN~0@EwxP}>28OuK z5{uw;su%qD@HgHwZuOE{Sm3*TJf_R9mgqw)I z>;CAt@f{yxhY*}KOXAwqB~LhgQ;&#{S|Hg&Vo;c4Snuvkp#gS4%j<&cI5kpf zWHP`x;AFHI*>y8Mb3}&8I$qxC>k`eWd}?Q-@9)|y;DPWyK%}ARtqUr;^#+KhdGYQr z?rOXG!`HC$*B@{wwExuUK-N3cux+N^*Ke8hpjy_7 znIoVR=1v2iXSe@Vf{>TH=c`=XIW&f!VROf$C%m%eRn;Pd8RD;l6~9BZAhUz!_qZ^f_5d@VBVRCXkC^>WF5Zj;Dp$r(;Q~J&CmMat<&a zoNx9R(RU4`x2qG{a+D=53(QXhXI&o_`2?;P*78>nI?y#I#B70nh)>!-syfDa%vYxi zyC6C)y1wvaClYIoH=orbIPDq55LxPzgo+LUHe1A5f~)Vkhw`NjI%AA#U`8 z;`r5+<>U}u`*`v2=$YEqrAY6cQ20(id`WO{P`GzLN9eb4XHPO>5aLTcc7Tdq*Nsq& zo1feL)BwIs7cE5WK*@UIi$fBEEilJrP~rv`_7Y|jWU2rasnM!m_>;;?@aoryx=NY1 z4gE~Oo8vG?9Pwe=)B<~)IzTCI?QHMo{5+P^T$P26fI>r*19!K-rheqRaKWccA&sDK zHk}D)tPvkli9z*cq3a=al~XRdsqV?{tTiw2>*6}`i!7;|N>3@I;Zwi>Oj18OQk%7h zDd1^R$WD`lxmb(HM?q)vJJ3S*i7Kx)xLvv6dErcYY@vhxnqov(J9KMEe*H#boH zH{`q;LGtp}5u^I7kZz;vdp*o=4dJwx&@bvpUCN`< z)s%$fTb@k~<;GD3_>i8y<=c*D#L`gpSl$`TZ-@;Z>f^alXPPr~1s92d0y zTyQS+7Sjm32uo{SKhQzBzdmSD(o@26nX-U7kgrM2FDN%FW}24%@`6nhSF$7N#dIHvA<>yDAAPR+WwXub*DQ) zfE19l0fn=cUz~)StiancE?a^6gTWQ`h${7o$I;rGPdL*kP=;=^fzNl8$@s-td`pLS z#>=he#kyH~gN1dNtG<3^bw3oQ6hHJAN^cCA3TF85_~0#&DoV-TQN1d|0nOs4+DUCH zl<75|q9V6dFPk#L-ZCTiLPq=~K?CWAp9BxSHsj8Csh^~^yzCeA+D`<(8gQiU{O(U- z!zngiHh2sqqVe{5({0gm%w}`DpObzYe^tMRpmB}hK835F5GbL8^E)g#b6#Pt0$Tj+ zM02#^Az>@O4thbcZcBa14|u{pjVZILeP%)#%&~rhH@B-QYSeuI+wa;ckHr(v>7!?ZkX)+KT*p>tz0Kj$0!^mo{gXZPRa4>LuS-#`hn5`)J%c zyi`SHf60|1JFd6@6f3hqkWRP{6USjCOXUXvmXCX^ED>{7Pvr&qsPty0>>4kzGfc4j z?h3rdHn*9m;42JF;e2Va|8PMu(fq!-ET{^%ty|QKdzLE3ZLGvxh!2AZ$q&gY^Y^_K zKC1U5(>A1`Jz(0#8?@!QhYf9cloIWzfQ!0Q4mh5hpV3q_^GbTW{RV>;Xlm7L>;JxU zoD9H+^z$M8js0)Q!4*er4;%1wc?y(u#`yPaKkSt;CT{H{$=INFUgcGAj1Fpwin#Dm zJ@||I#2q)Pk;GAt9dEIMB7$GUKz;Wen$!A?(VMTj1T62_D2~iBiEve^4SD#w+gJPG zc@XMXwVzdT4+}-T|42P^l)ewGoC6XN|E7fj?JRerol!3(0o>ES{;TN|`!iFf26CH7 z%%i2@fZ_`8ZL|HeGzqxWmiAipgc;k81bI{f>drt44XFv_Bf)21kEMY%;)h_r($cRy zR`orYci9QQzd&8f%6S(51)%unM>Nf_06WWmQlnyG*q~?CgSvAMd@7GGR zt_z6oS!fyRAG`_zE!&E^gfv1lemW2TqWb=P0y83{}0*4vH0$+ z&)J23Thvnq#UI;A=1{2o?+e=_cxO`X$uGt8r-%IKcKDC${O6B2pMk}K1)F&JUzy~8 z6qx>*JpJ>lkYu1{_+?xzz6!59r?>m|M$r~peE3_oK^bI ztNj0XA)}aCU0odv@Oc8iMD~pm@81^v|8)~9+}P^s>a##;f*?iQj)Yw$ib%ww2gI%% zv64R*R|G6i@Fz|L6VCtNAM^8mWVDT`H%6t)nEL=|ARM-Ah3h|>WG>eYujw#*@s$CN zC>e2^$*SV(*RLOt`#n^UPjB*HQ2tH}TPjgy-oV!mRVNpv@?PsmRdhekNJU)k~G4Zouhn$xSez zgKK6mZU4v^0a9m~mNy_P+QKg9kh^XdyU13+teGPH69k$Am8Z{Og}LXg!K9=B@&Tj; zS0niSJ235F^#F3Z5L=w8U+WU+bu`AQmMqF0!6cK+8|;+Xij)}zD6oBdx29^@=aZ9@ zM^F770Q0+Hc5-=(#^Fh+?Np6~iT%aN?yl_6@87T59M6w79iEbRO;tNu_9OsPvImXR zBpgiYkjYCqY(M>l_ka@J+ z01Wgv4!liBRaROVpXTPP3(*4Mk; zryG`X^cs*|#n2{YhKIyQjhCBttvB;iNseb=C=UaP!y&C!tGqO-+v3rG13z3Ijs&B8 zuJ)hn`7BFM(;Do2q^77RO^@eeq)}W1&?W+&z$zh4(09?P+V7J_b2;3bn*biNJ#hUU zeY7oQ3e(`3bRnfr)3xbU%tTLjE@($%NFYWv^y#qz-n-$(16fM55h8t{U}kT(F_hkP zj8^CW-Y5P0xPyLsb{-QGEp@Q{xuO2C&&AtD{*pUQIITcEQ9*Omt;GE3a98g76t#1) zEGPFGCuIqUe-LLg*l#^J`sVFHCd8P;^^9utOQ6VUSEgWDPbX&NIw2XTag|IZ` ze3FWXvC`pIZiNJ&otId<#;GEFO}^i)!&0rkAd8(mr)CWm%~pI#7qLMVBXPpMef7G zFI%2~W;&mfR^}FU@hJ)_em$D>{nX_$dbg%&Jb3+AwJU?2v<~(+|EJ2A@zc_f{DT3= zemE@EA8EghLd*;8Gr3Sv)1uvEAIr&%WE}An7`}h>wEe^QODhAfPQMzLKZtTFTyoQ% z@?h(;vV=QouW%QD%s@TqGSXo(pk`25G1{0Ziy^}m1Dgf7^E}*ERSoH)fXWcUZFg?v z5c&AQoJKlC5|D8ALNcWqUmXzo22nSjw9i^;dc1v8VWUG4Bm9b1IJj;DY;0G%S+Ubh z>MdAtelrytC?2xUz2tHVAX}he43@tbi1rrMvv0ob=g}Tal0~?!mao5hzzuX{6S_`& zxyqX@?Uy9f^SWDzNX1-`B^aJ|X?Iv*g7amwJ-!`kn7^FM7Ck{*e%H z!z8#Xy_(Hg`{OA@%k9h?U=Sp{-etZ9r|(8smUSmvnqFTXlQN%=BIiis0IXWuL-tL! z)&EQJv)mvnrQByWp5HgxZzk|Bw=5={lW)D{NeVn=Omj6}Efdb6SH4W&ZZjriiVVpZ zpWfPllx))uu!#5MS&H-L`v4^8gaiyxiDhx^AB+Id=HXf!pUb_qJs_;=kl1>&(089f zi)r^)LVk(NB%qdKh!>3vH73>^?m%_=`XrxOVjtP;`8s3v{Nwh2u}3RzThD><(|HX+ z9um(D8JMclIlDZ(F6bhX^1-#@p1;sV-~^A3`k96omV1j#&!Pa72CygY`i1WgyaIm< zxs`TDc1&>GV31Vi;2DMAE}6`10D0E39cH|-ZV0ktvfRGvXKLVEw~#1R@rFe|Io@k< zpY^2tAa!|p#B+VQ`^h>i&uZ-W1B4VMA|UJVSnEEc)oOqYP~qEc75Z2kO|l?C>}F61 zl!Y%2^5i0QrPS2gQ#5|BAF81#Ck#t1)K6wfeQ0hQ;d;XllvDFsKjTS31At*XhO^y< zS07SsleypD-~1_&AZmA%_QS4pL{cN8;K2Q>Se`koA(j24-dIjO13sG^EE<)eIv=0X zm}j1%z72C@DDah=zzb%(}$_W8xrn^MJqb~s|@SkS~zni;9y zCc>T3x}US?;XY{AedIZKVj$iJ?s~Gl3sF}~ywLhKc`wZ6e|tii6QHUo_{*SCXGEtS z(1JA!Fx6eRc%N@l8|){Kw3&}X_+T=C`kEb76^JBZm-sB`%j`hS8T+@m%f2ntbaT9X zws!i`mO>vanY0<1pOA5NkQbb*n&2M4SpoVQk}QsKGu5jgiJ`)B1W+miOc7J|i(Owf zH=dZuCu@4|#TFB#s**((HJq)zYlSR)dSp_)(A(n(XnVQwuC`{sIE01myt2tcGW&Smq}~=z_cN{`!uMz%APG|wgwp8xp5wvb%_?!lg?tX7$PfuBB`3VZ z@kBl}w@9S3*fY!x*kiqui$B0x>P4G=gC{IlgM@v5r^_sk`Xg<21C!@Y^DnM)7Ggd?FOtwgI zZtdV&Y#PIV!pW65qudk7$rjz1$cn54QjKqmNWF9FDM9%yt8yli{JudIX3%djE8NSb zBbAu(B}wB*7ax{r0P;y$zE76pEwSw0r}HUKJ@@IdcyFEl5c|||8zxpQJ{0VMtXBy* zme+lUgW5eiTyHH+Km2);dIO~>_E%qyvJA4M0*bcN=v%IX`wVEjbS0_U)%u0%EaJFk z=e5UDQu#Ul_M_K7v6iDp8V8^-SntJ1O*WZs~$B&lo}yh$f@bc#K3jV z#jLCx9#u=p)>#F(x&x-sGE|%Vqs|Z`Qg)dmeG9g_lezHA7fnlnj07sIdnkTJ<0*M4Zh~dc2g}iV#E6l~eNL8z&!lW3yogx1-(s(Z^kx|s z#8me~EwcmS_H0b|hHDu$J>Ah$!}MqgP#3}jor0lA`}(5DpwNRwV2e^r(k4>wYB!qQ zb>me&C?f3+e25Y=ih4nxxts@pXk@Sj zP)uxR?Z5_~a4NA941AJTxz$V+6B>ofhOPx@D@K9LMRbCUfJqzI-r3}`EH!!$Rhn_R zaiWk*{?J{DdfWTBc0$o83f#A1gXq{q0@qPd1}e7mTGutLr*&3hx^bbTLYg8zndZ6A0uJ}mG*r}&sKW)40504sR5{YJI3>e=p#s$vEy2Z`L#!g!;0HJ5P0;XJvKv2VZ z%4ESpjOK$a_*gW}sU(7o0&Q=l+*aQ`+Ctq7&gQ~J@77|JNxWwX4MVKzi3%H6lmx;y zW*;p_H94wzRgEm5>REiEWM=f$FE6|uG6ZppNf)a`wSucwR^lH;4`IM89Qtj1 zXeRnHRMWF1TZz=G4caz&<3Shm>E)>5neE7Aqx&AKCc0(n;aTjcp~1`P>=LaJBob-b z*sy)c%LDYh>!&D5*naj5z}fuzK-qEzTaI53cd7QVKJ64J`k?l4y8Y~_=>nHOJ%n*& z3xKNb-0(MERk1Q(iS1nOHQ#A74*MXrKS}5fI*sPO^4?*D+x|2smL?6sASZb7i122% zvEJqVRt(> ziGJwYe)P0S2d|c0?8yf2C|seyPabl?xej68siRnA`TSY{|S2CM6y+ z<7BUw`Luh_OX`jj$I2ua7*A8w=H!I8k{OeLkE_upC2O@4rRrWdB+JOOg~dJvvP^YQ zmL@f`(@rK;U^ZueGYESK;T31;x}2kK1j(ItYR-vZ(~tMcERO3~rcvf)c>`mLn507> zJQ$#BKEpMgjMxx}(Av&hOZR}_T}s0|GO}18!YU0Gxpc2Jq!)|bo*K^LzE=x-ddbSK{ks1Awo+Pf(!oWD(GC_HFLR^oBs=`< zstI3~tkgU?i9H%3d}q;<^c^*u$VXo5hv(6F63t}I>275S1Bp&>~& z3FJ2Nb@*$(iQ$aIwRc@OZSIfyeWQ2bcm35I70B1ouDOR~y-iOj#>n7Q6%-lgD|6=| zt;bM)UOF9O*Vr`M%c?0TuotOh1lP2PZ?MU%6RoNzv?LvmM&q>X`T(0U7$7>i4in>v zj=m&aW@6wbYb%BsH+~S?^88pz$dFXfmnMS1(xR&1OI1P)w2Vr+(b^Piff2f=L?3g4 zpVJ|XW2mvtX<+UigKpyhNOk40Je$1=hZH-u*V87$ekT?(Be$5j+$Sm@qp$^J3_i8( zQ=7jvhG@|Swuk&U2AUn7kDA`2&hA>t>9$|`Rc$7s-vtlWAQoCmXG@+@w^fwx_}TLIC%Tlm>3wn$v>e*ND~38RGC0 z5cMASu+^*-BQxCh3fLJmrI&$I+3Y|4e?1tFmI#)=PMN;o5H>nsOw$Aaoe+pGfm;up4ra zHU|?DMr2`hNwK9e?!dv^xV^F8GxK#Vn+Ik<`$>K|Y443)lE3qfmTlsp>|dnA9d;d_1-O*<7_MLF^d7vl8u@?TW@% zc3_hd3A+nUv|iAh>SSi6c|8<)b!5O)7|ClJhvr#P!klIrzj&WonGow|Desl|)+vNTc|AxAR)kW(@VE`%c zY@tyP(y2U1UD&ZJs5hQ0ECS$J)9`PRUJrO~d87`-CvKTqw%`TzMqeJ#Z3eJbs9A(( zZ;f{BeCiXHH*23y8sjeJF(GZW06@1RI31%wFDCe0?x#Nb#Y`_4MU(F}fQIX?g-#Jh zg&;>4$fBkU0{4TZIR=8t^sQw7W6dl3QYS{|C?xKIt;X_WWth@fjxO7ggkQi=HCfrGTy5r1hauD!SrAX znSLF6R9h)ZR=QzGCwzE9RTl)Z3bDvJv`)>7O&I>-$Rwc|jKfN?tt$0gwt#Ss9{O-4 z%T&Q*C6S`5d!M&FSJsIJEVpVj$o)izQg!>dl!4St%GRyx?5y1F1G-v$f_!dnuXaYW?zhij#Hk`18OQr(2z*v!5;f4+J$cVnz8z zF6mH^X`II64G)#Ouglmsf4$J0OEo?`6QDl~<+O8o`G*2Omhl%X3S4HHt{Hq(hT{UM zH7N$XEIkYT+JKwR1A!&E<8RwkiFBbSuyOVj0xUD}waA*Ml#Ecpbu^u!$Ife)8W8vr zYpP?Pn!o0L(TM1pO4u{X>UFZ~H3L&`?qzM!YIDjxYM*w&}u=kEUa z*n!~<&~)^CBk0S-XT-EoQ8otjT>=aIj?e{e(-fszyOY_0g=Lv`Q{JQJ8}FO!Tcw_j zA`aajO+uuYw|hjV*cyNQ6kR~*mj*#RoHd#&y{HGAspw6650hkrb6^Xe0>JlW#{nHM zQ0Q_3X&_dWOVVQM!cGYp9}ebmT67r%e0zC|gHLOkt-;;C8T2reTNM-%7$MOeCLQ8k zcOa7^U@t}W)BkB^Y-|{;Et|bVQVJXq@kq3f5M<$$TBpv0Twg_w@i9^jcly$&5!G?h z9xknOlV2a8HKQF8-&XZMuCZxoj{?85!bc}H-S3tX;M?5Mu<8;J;9p+hV%-wn6lO|O z!;jt6YYx+3Sf4Dx2mm1hE{Bf~6WRWrRHYMFb)K4#fB#cU8oz<|7gE-fq>5frhF0=T z2PE9bseMNG?~*6<70ao#q3|HKunwz)@OCSPbXqvd_{_LY6Xp#oEJdj#bJ#I&?Z~_3 zX-rB}&yIaa4Qqlhhs;7WiJr|c#mb&Du<40>k6F&ps2t%*@o}L`v5_50`mho=4J6aI zSpArJV4Jz~ThsrKwzm$8dh7l`i2(*tU_b=v5)`Bnq!9!}QbD9sKuS_-NNGuxZlt8U zhZ1RNsiAY|?(%-eb38}!ec#``f86Ks@%ie^?C;uZuf6u#D?W?Ejh*^40t55{9)<~z z6c%5$jTUYa96Mv5zEI?aw>A)Lpo*0#SSBpyx&G%6$cp=j0RWb8&|qJS6QacSsaaEJmmJx<*x6oYgxPzt#<6w{3vhv@({4>~ zM?14*EDw|}%Or;2Diz|BVEDb1tQEvfZP?fi7eW%1gf%(Psq0CBC zX|*G-4$`58#Usa^zz36oPz1&ln&g(mJcSsz@5 zhfj>f828*XGQAsI(}mLJ06}&LH1=vFv4bCbij9@X3R&{cWhU6GuZ`>mZBZ=7yJrSO zx#U;BQV#V zjqt0P1~+h^Z6o7TkR?HY32R^2jJas5Wfm&IGIXdU0dC;BCe-og1c%RkmI0!lf)_R8?AF=)0(QL!VUZ zOa!MGg)Lh0eXil3l64D#)~vtUJi0w!Mb5TD1%x;>0Vl3)(NY?C4-f4^jO7`bb~wy` zyW0ScCmFX*U>`bQe!{IYC8bHnqEGWV&cWzy`s*j5ArDtZPXQa5ajPANmDN}Pee&J3 z&*rJqvP^`=6ZFM4$7j+lJjdhB}6=3P;}=Hw##pWCSZ3vPDK({ zDu6a05y>>2Zy(9F${F9*&a;&k z>S2`~TRu#UUyn7;zs@GetnS7(Xh_?puS7Typ$ z-}kfKIS(|<$6M#Ke1>5U__O=#Wky{(gfiq@qK;5K!9jjJQ*(m(^X;~ix)hFRCP%nN zNuiHdzv!ZUF_z4^IPP)BMfyt&E^=+pT*!2-N}X=dVeX#a;r@7!p%iXy2yK-Ph>Bw)T9!mew8oP$MGmY6ejwP)?b@Pn7`&ewXE z96ORrCcKNZbLqG0?^A$ zZkt909V3gV;$IJ`ap1r=?!H}6TOmC@ zs%736@(qR>fAEFM`NPz4vPUBE!-plGK-nRierj>+nYa{Fqi2jR% zu^!T};XNIAj+u#X)F=)%!W1``M7DBk0C6!JJ@-f=7;Q$~G`OjB#3UcFPLxwUBmWnV@}9qbk%0 zkIb-z5|RsO442-Wk_tC&7l>zl1teH4`8V9iq#eoNVN*y&dIJB6wUDrb9FMDq=@TF@ zQJi4@PN)f2|Jyqr+n5Y217rvv?z~N|mEd7$ZsAIV2q&EoM0!i@)wgWYcBNWBkG*SQDZ{fARCc|FFZqdM;;mol z+#+HLG&|EC&Hy6bI&+?kw1LgFoTt@}E}QQ3Z{MAkc#K%Sp%LmWpYPDye`Gg=pI(Ik zk(^u?m>3O^>D+Ven4x6<7-~$j>Ev;9-;BFJlYepYz|>Cq_)>#9-ITxsHJwDqZAq2)fjb&i{E;&dY# zGq%CimhBX<-Op?etMJaO^b$xi+!p=t65LeX=75khvH*YXq!&CZ`@Cxk_s=r9ADJFN zhQz{1Z?WMV$rccHy;V3)gf`nMd{FdK#HCeW5+WgKBo=2@f=Hi)jks8yG>l@jV&rGq zE007N<;lV5jz-i}vg!FAuC^Ezjuk_!Nfj^&6l9=GWTD2})b#%9mr|5=(Sp73R!+$4tO17WNCKKCc%huprqV#mI|MI;Y9+(&P06B*VSg+ zKA~@TmlFL<1^P9+W(gY&oO!qBS(}LuXOW@q;b#?Z*AMqRAow?7-yg(HTt%Rn`vSgp zV|1}C9iVmO624bk47{_@R+smS0m&)pF-@sHuQjx<3Xw>r%8vtuSQqAn@gl&+Nv`(S z4w$@h*FT~PQ{429`(vca*C&=U+rMJb){*W#nMT}5$TXv7Z{nv zZ!fa!m)vDE%b0c3_4 z>OG~0EKPwl_w-`b4`t5*L~}CjgW{;NocnAxlEpN8j#jAtxPh7H&1`NE?Va~gj^X7j zi4j>#5m)l;*q3Dcq;<8f&C)zhpCwyfe%Mzv9qaH0er4$k?(x7YZ_W|17iL{PT^ z54G+Dkd|*vpBMJ6_a_zr6=jqEV5w(!Fct{}2m<3+m~;l}oXu|KS-LXI#{?qiKWj0) z{cyuN@`)SO#g?CeN)S)l?(7-Pf&#)1cV@eliVE-@>onhShD`R$<7viBo@9!go9rFA z0)AQYkm53pD#i~QLZ!?2vrlBCcxoBH;e=D0tzH!rV@G3E!lip_ z%Qj#I)PUIW2cWanhcNQ>+$AR|djvrLr&)Lf`U!SloV^b@$nZ2!=&^k)1nC=q#QOJ~ zC2xhf8o|EWs1bzP)4-gKqaZo5HSyQJ`A~^c_HU|}QvxsjLe%L z7-QI;#Yacej)MLW=AtZQErzG(DKcUoZ*uLF@}NJEt?qte>rDhQC&;d)tvoZ{0RB11 z8|lYy@*&nH!CD`9MK1M8s>ov^VE;9;#@pi>i80CwHXS^Km{tDnyqxF=!_?%Jk%MA+ z+EQ$R{HfH*?7$9)XQL<*wxKf6)C*inAi8Jp?FJ>(-wm0a^3ltp}%rAxT?j}cLqZ?TQ(M)pWVJDz>WO(^M@<(xL#325{?{YRr z6`DF;8G^;+Y5!DYC^mWSVJrJsRhm)&i6jDjQwDwrAFksm-vs(D%A1o>NPN-dziW5H zL|{PlJCH9Fy1-t>PMhRFFxQUuy_Zsw=qcmFR~fVETHuXusMgmN)cIW;YZIMUvWn0j z^Mm;Lz5OHnzlFEA2`6H~KI-|l-q%h5@n@G}H*yiR;3(&5$T!6o<2#SG3dDu5k1vty z7$q3FFGi{wmo^T4S~!02qE(3qH5#vnphpPNj_$dKOq|i6C`ou2QWXlqWazEBWx2sO z%lVNB?RHP2rQ2>CNt7|TFUIrw*_LjDCl*K(jw-;`S+8-X+7K7BDp-)EyShB;?6N`8 zgGcx1X2fGZ@+V0LeGnsjtf@Jw^Cc+`K7E#{`h3g_Nymwq5R|Un(5>#N8;K%-6$~hc z!2LOamMJpe6!p9iZ@4IpAm5*GH_eItE5Y>z_7~J(9)^f1@}GV~_HWo}=shtVH{*zS z^3As%oFP5EO+f!batkkHZvPR|-=A6s9MO0^E{j~L!C{0Wh9LqQl`o$kCw)IoddUby zCJC)mjYy0$93Zjv;k%QklZP!g35zhQ@&aHw`XpQSfnmk{r_@Gt-}*6Ee5lWXJfYGwu&nk--Lx2~ zd7A$~VN+xP#aj<0v=D|YFBC+Pj-CU9F0&0(MX>J#&*Fh9gKnt&DbENU_Q=b6{oGowc`iFkM1i8sqy)K5GRFl@U1spw z$}DuVLIPOo!f_+3BjiNB94~8c=Vn}gK7gOp5&JQ)+6#Sg#!vALf>Lg+TaZ?OT9Byk4H;m``(Nb!b2Q`m*RyDOcE0C_8%ps!|vPJl~}{jTTxI+`T}0S z4L6#RNr<=K#C2OfM{6{*JDObno=^$n8}FjeORov? z7CHgP1VwE!#QRPZvuGzaim`+L}4>dQq z^qG3VA9;Gl-|_T*P7;83;(-!H@WA=st`%$p>5FXgYIex5`-7x|xqIhj1+FaR-vI|D zw&t9h^P}CzW57-3##?kRWNOX!CYH#{qsAsYy2M*6J-afx$ygSLh+B!@khexxnTCg6 z6ao%@WvfR}O(LuT5TTD(Lg?7jvR%izTXjI)woudrB3e`v&5*Evjynm+MbdsgWgJJm zvm>)BcZid&u)28KJZrT3l@gKOl}LB3aZD|DJThA0T;r3V$mUq)Q^D7p{kEg$=GA-m zC)gtLNA|gs#7;-|$O)r*JF)_doj2J9EM)9KamT5q3ZJsdJG(gu>segiOMUJ6F1`A> z%G@~=p;6y$t)8)&xqTL&DIL0dAiZimdU$^gI$fP~Sh>t`I(mO}6)3Xa=dN2&PjOX1 z47>oH`_ojZ9_9gz`3O>=1o&gGn( zZ1L26?6~#D#ly&-_&--)qk$803p$|?r~wTbR(z$e9M})%py;{o>gP!>SNxe_^kcoa z2WzmY4C9s%FRnLZp9&ZBq8caP2!S*ppKsX|&iYX(NjxUa)qn7)>#X zn*-xwj09EsQuvg9HELsAC4XZKVil0aST0!TSL8zTwMcIc z{EoP=UN4Rgh7j5e;3)HPiUbB2$$97k8(WCe-i z`YXX6K_G+c(-X~GyB2}D3xOT_{tb4O%S%GMW&wH?l7|0nq+j9oyEajkiOxdEsGnXp>a)Fut@=4P^MU>|3n zxT?gvyDdt(B4zxp9LS!F*VhnqU%pB^*+{j`bItAy|IV_e)1=+m(%$TOr!ZohUNU@7 z)bAu*Nj^ZSx3^KwYbsEmqRnR3G0~~z?!y-=`}q#F%#z|(U6>D{U)oBW^ zXNb^DhQ?-)NZLRnrNw6%^*m*lM?QhD-89=idgw$CbkamSUM5t?qI%zM{zJ?DuDMX% zX3751sD*Ra;ixlZU6(+Vc4tjDpBRsJ=V+w+d939*Y3pd1VN~r#>nN-kF|b;;kh0xK%>0{`)7($40xK_x7 zw7{!%*UW8gc?5u2=)|U++U;0eZ=GMT9&iz&K)kQ)lIt4E%)zE>m3w*fWmJpIfZ0z7 z0ADa_q()Q5vsYGRU=7StIv#?jaoDXR=tX_H3f54FJ-N4Xh;>@DfWs@z@~cofv2=j~ z-JHf7-de4lbP8h5K@@m3g!P@8pq9y5o>X&}5bOR1qm|+Qv)T4}s*cvIWT0woF_eO& zMsI~a>|xKuEq8BSxW83M?=|$bxL2F)Ip_yM@%MJtvaUazz_(J%U?=l4yT390TDEO6 z&awZZF$LpEth27y${M^qF$uu^4#!p=C9e6HbORcj-87UzdRn9vaiM+l+f~17IoP3j zPJ)^{h``Tx6hwILM%PazzbYUdJ_jucPTy(A3F<~;bA6bx78w5IIanH2S=H@-TRX%= zbZa(UHgRp&#&`vi1Rt7vN=cO=t!E0rCppT$-z*!}Ces;mWVvVn6&lAHza+)4edHx| zD}mhIN$JKc`#>~)G{a%(fV|9ymUjk|+Wfe(37uemkYNArPR6)h8u9i<#XElL>k%~XnTIjNBHCKUNKC1C{i7aziXqZc%X%>tgI`n9`aeprn{L_Bvo zYfDUc&#I=vB}NMo+EtZNT{Yi0)|ZlshdyKo2*!ZmCoI?}uDeSw2KaoiOd!h;m6(l~ zd$|y5Y{9A0h@G>lg_!MABDztLZ}03Ch40?>a#q9=9yi;#m3!Y;$f=)l1%o)zcBR!; zJfSkv(W=SYU}tf3YZGFkCrjopKJ8YxIVG_odhsoo{~hn$<)z&OBe=y%v(pBa<|ndc zS6#H@9an0h9)d$p{;&|^5c@D+g@c+NE-;>>wZcv!_KNuNmz1vgIaJp1=ZS=HN$pZB z-=O)lM3=UxliQ#gDUBw}?i%W^+J*GwR=qpDm8~#GkR-_Y9g2HgIehAE7a8xDdb$gT z{EyxIcq}?5QyKdTJw))sBTdFMcS^xvOAY%&2_K}B_6nXDv40>Cq=6u#xo;G(Qhqgh>uQc}r$_OXwI19!3MIpw_WA(R7tcCEV zGE4?s^NmA=f-M8J4$bP;P2IGO>}4*z!*x1iIxgf^r8|)3fjtXZK{09R7uIm8l-*Iz zE6pXldwDCqq2SjCV>Ii;MkT%0Jp_e1UV3)>lkBkSto7BYl3PrN7or{oMh3R8NA!4l zgm};;o_~L_@5i%W!N0{D(xc@t3Hh9FF%#YN9Bz4Zqej*}CufC=8(UZ8bBfnZqMPwO zk*F6VdW8@4$fVcx7oL9Z7K0}`S9}nm+Cfz48e7~eeTR3DlL%&S%h8y0ilqCCCMpV6`2`Y2FD?;KZ!%1+aIFsb+K1}Jct0hD?2DG)Z*^52`318t- z4b8FoMNTs`^c=1hq{VDv{A6Hz;aifVQLe;br|ki~nT5B};(k7^-t^i_AlP#NPjuUZ z9gI{&Ta`XnNYo6IRVN@d_!T7tOQ1(&=uFy|f`acdbI8h#vD4SQcFm5i$|? z$i*hauKCRrR~;P~;exLwEpuEN*l^?BxKP+JrFDpLy6V;$MD|#bw5pP42+K_3Ow9`ztZmmpl)? zy5rkLZNER;+365Sa>yqsGS}*q){&3bH#{kOdK_N8aU5Q<-VG~Wc+InEUcF{rPAn}6t%u$-sivR9O;SIHFPEjbU03?PfKZke@|S6Yc5CF9 zo@q!YnpE2ybavH@J1&IB5N9f1q~{|R+N+SGrr0G2B`qdxIPlbSCm;}Gd+PS}s_|mO zxno3Fbp79d0Xx=0gSi*Q&5$(QAR|g)J_y?=#<@ZJ;9}wlrp4N00i3~}9^&E$Hk2Fp z^>w{IUo_izdpp#gPFNhs_DE>}SKAI`*-ucEexPV5h|1`<^;uC6ZuUkz?raYXO;*i4gK@{Fia(b{VGU;$nu@U**sI$- z?#*gBjWUi1VSc@5Mp^MKk|%<2XJ`Mx6{lgro~N_Lt(~1syNCDF%blFU3EXiOVxpe& zyehiCP%aRciM)V1zw?Wu98H6C>x}KI49fGGFqgBDc=0tYm_is{QQvy z@%`v0uR;S1``ewDw+R71##o&9By)bJiQgANr23`dU#5GS^%9`oh|KKfS)?o z4>3IfVPYan%;to%B54(QP+_kNU^^XFKsi|J9W6BhbebsKt4f{PK#ukd&>h)q56#FT zD7TeLNkZq9{^rVH&K!T{#mD9k5Y(7^hHAWCrH-#6uQY#2e`-?WT+7y(u_2`8_Izja zF+6{gSU~{`FZ6|A#;Ah8XhN2balfNhhqQ(~c@JC7`eXH;93&rQ$Pf%}t-*I+dm5moo)XX;4H9Jq>y9by$~U_;2X0cMMeMKHb~+)uhhl zxq@@cf&J+T)G0?L@2yPh6NQX7!pNexMrM^SY6mCKiDx2^r5+@zHlilAIzW^WJZdt);n<&g?x3%fKO9QS4H<>Sh z1lug@Vu5SL&j4iYBafJ6)x~wdg=(hDBM~{v%u}Psree`c?1R!%T@N^qPE)}>uG9Au z8f|*T711S<{g8E%G=&{1pp&;Kw3|pbLg8o0$J?I;ftYDi6P^gUlpG6y=#Wo{Ob-^& z?r}cqis$2k9W}Gz0tf@4CflTSh(zeek-()+(xx8A7A~~O=82-AhdLZjt6UX4j0@`O zgzo~-1Nl*%9%!#~PcP~|Vg5~T?10saGY)!euJ8);eKHf zrv`;~!`*;Ppt5foz>j5YjbSdjxdS;(MC23rB85+UPBsU z>jam-7Uticj8a1#fmpjv!knzr202>E2?uPhd4`yMV32!{vQ}8SPT&b(@<{dUAblIi zvMga8tq#S2zvSm%a{j60O8x-iKDT9Td@at&NB$c9p&<<#A2Jz7wC@gtz5TgR|E$>M2So^Km$4#*RpQ2fbO!(VpPmBT4e%VV+6V=Irr8$c=;M!~xOG4ka(3y@BXNX@zYQ@}Uz2&Kp8-5}iT=No9=g zPCCbz!i1(}Kc)MhYFAMwfS#yU>7qXw?;Im)AqS2t=l035oT3ynn?R=XWzGUSzISAVyw**}w%>oNm@e*f+q-9a9ck)u zSNLBtEj@7C92ERlSO1I^2L8d={M|{FIe-Wz2r#8u1iNKy$xg(rS8sWvi5^smXulnO?Z-Jns16;iG$h zhN^%j6Xu%#0C@j81;C<|#h57N^)4feZCWrp@U_sepnR$1ma``EpY_sb&^)$gba4K! zgZ%ZpM+3A5b1^f^`Fyr~Z|2&;>g(^zgL*@0vFtXL(v7iKs_{01-u|gPmo@>-^4!Ib zdlshu;_**4>=p0fjOBHPnd1bg16N7i0+y4`Z#`HnpRVB?C8ovl92AfnIP7hpfVpa4`3UGIwl z7v(WK%CexPzinKSFp%jH*(>*+xb;oG8yv##y-pE%;b6^0*O`=jDqi7Sr?UWdb5)C% z=H_>~(m(zKD{xf>t*op}Zy=*`cn5BmX1w)k8I7lLHri zAQa7R6z-%ovBeJUjxV%cJt#ch7&J_Kdi|%4^K6t)uFmFPn=D}r%BG@{3?beBJ}Q8B zzovupyGVts0>g*N57J_fs6DP{(Na@0C!<|ilIoKXIgl9E>XfRebhlrp8hcwa%ar^E zO+{G|UjqMuGn=QP7?|F3FEOTy>pS95e$vs##(1&&MMpx_lvGdr!O$j@l@D>&y$cik z7QBa=>5lKw5Qg82gCQn@hwaSdg94&PK?U!Td4_CZKW~YRZBd76fw`MgT|RTpL4+?Z@6_;wx-_tQ(jJqEAhtg@iN~b!!1QR>>sZbO%?+m*^Y}Pael0>hF@{( zX)w_82@;s_{)-|}6f}Gf6p*uoXyW^_i|w`WWiIpC@T`OsA;%s#voAfX4jNy5B=36L zNO6>adDqZ-x!vhYfoi9WuCe-|426zUta%vA?Ww+0Ls#1AD7(6BCfQVY=GhC<>u3O7 zxHr55EFK^S_lSVQC`MOWsXlhBR#nU?bD@{f0kjLq$$@NGYetr%-KBi7~$ctS+zXCM7M7w4`{Dkj{)hBAuK%G38T5 zET~ck!uOORCv(kGv0|gT`eS~`vD2{EswJj{BuTHGUV5E9q*p+Ng#;u?j#{5PriIT% z{AFhp*Q~8vBO|p-VS@4BxQ-;@-cEYVty%-P$5yRKN5|@+|n}t1=4~ii5V2ymB`)KRJ?wGCn0+o)iwV zXRMfna>cW(*!)*74|j_Im=@%E`!CDz<5AZN`^5xNWwXRIR+4E6G^%Wr+4x0XQOZyY z_Y0QP`+Z-$DaTP$cz!JHzr^8_@PN*0O#kKn--1T1m$d2tRMvH2sG)WE`u6^+`TdRBnMXkZI(z0N1MB}2`!pFlM6#r{` z{KFGbHRR&2&ZJoXvG}4=H8}Mf`vPvVLoUwK)8`sida7vbAq^2wtzO&9(_4x`p|$D> zdW3qJSn&r9K04TSc@~fA#66SvY5v|bKN+igM6ji+PI6gzelqv>ZwDdwAe|xoGS>gA z?=yOUzZ(8N!harDR2Kx%m7j)iN84fi&e4WP^kMr|o`$hERc%2Amso^kqMZT?q!MVf z`?o&)XM9f@TI%(R35lslu}qY^9FJmeEXH$^#(?9_(ISQ6@``#iSkZcb1>+?H%f)^d8j=T~>yvzB zjAs*r1^1UNTe5Tb#YIG_vUQ1tz8Pi8e}x}OJH-jzGJdu}6mO!GQJ=nf_aog`vfL^! zqQ|6Ozv)FJ3zcaB!XJ~rKDo4mldpo?K2((DF*|4nRNXW)Sud=1Y0(~yF52F|xqHVm zZ!mOyq)*SnC#@rHeB*i)k^KLMv332ykD?+p$7Z!*Jq0hz#wbT1W6f za$^E(%g@$_rf&|vGN}-toY0erDuaJgKirdBZrwTSs?pUP3oE0$p*#v(b|QXzc-!81 zIrpDE_0J6Hq5y@~Mxo(+X$vQ>fgie0z-sm$gD(cSFlwyRbE^1fK5o8RY-Ko7T17iV z0pI{4RZ5^qY$G@<7*OeyHc`0Y`<@-c~=~ zrR7|ZPA=y+_|USj_60pVFROeovcRN*-wEA8uixL6^&}*(>*%RqUrFNktz|WVnFi80 zej4Mq+!PXsS-$&!${I-wsDDZ=^s9D1^QHIT-)2Tzr>v=J@tJlto;Q!}Ijm_x98HiZ z-Cio%;1INz?QRL`Pz}cUOaXm9KgsO}yCnA1D4sO80I^Fu$|C=LD~cKt^t#mTU?%e3 zvn*j)%q`CVx;X3DX#~{f2_J4=Xm{t{(TA{=0)xONvaJ%s%Ac& zc+I7?53)lOm~dNNE#{eM)^*x^OuiA^?ymY~vt!-p18E75gnlvc!`n-fiu5d(=?vmV zsyK^>%Xq(k;;APl>Rj+}N}6??l3j@S;k1efwZi1nyy@ed9V4998WcfRGXp7yx9$E7 z=>+r?zyBB&9P|)5thMqo;qRY#&WMUK6|yO{ESNX$!3O6!hgME&V|t$3qK57qd*ek% zRBX%f0a1&1JXMK5)t~2Mw61P|zcb(CUB1LRK!ehKKyDlI=gt_vD>>Z}d@C$nvrk>{I<6g_3B>-y zq)Ii_CS0Y2gY|+nHl`xu$tb(XR^jsfYnRnAD)=fspk+|eWQs1Yq9Up18|bcz6A8lA zMSFT#?&jFIin?&R`i4pCkg=v{x6iQ$&U!lH)~(+;2au+l1_%{p7M)D}r8!(BM5EX^ z($F&GaTV)s88%*TrqIitex<>tqlaB5rO*a{*b}uI;60H-@~1WwZNR2Rv%UGxLGm{X z8o~%(I&f5U7>FtCDfZ5ha`9xhIA6EBMPA`phn~4$e!!h{eS~P*{_u9oC1Q{x0kTi} zhwRfZP%rZ!l5X>UGcsR`j%OiH;&NCk+u+tp7}H)V-F}BT|?@ATNQkc@0`_O9ntlL~qsU=p@*g{#0 z!4WqVl{mHe56kP*Lo{J)f=enGc*ku#w80HjO`VB5E7c5|-_)-d&5?2rt?i-?e>2MK z67q$31b&>QzP#$nw^2Voc)rvyU~x(FEj$#Z&5x`$C3Yk$;c3N~lup&0e%;7x!M46s zD9wsiZ}g;0jy$dm?E`8ir05rRvU`I#DF+!+MUY02Ck;KI{JgiaNPky;C?I>rupT4x zQXJT6SNB-1wc1#Jc}=J#e&D&Z{resaSkE3T*g8d3FYUMKhvRrsT+kTgY#2DyN3cXK z_Thq&*C%SU=WU_`a+fC1FgPAzRZP;GqTb_^Al;GA%*)}@(La&6{LMrPR z44mAv7bf6VBx-Q*KkLKf&*I%kuo*nZp8?Y*l%KhZ@w10l=xpZ2fJS>)#(BU+KwtEBU!0fdkoUiSU$V zcWkRM3xc+=UJ9p^2QP=$L4U}oQZ)WGFOJxQsAbXqyZ`V*Q4h4^HlY4Ip@=Dq)orsQ zzJbUdwO0!qP6Ox=PV(!qpns ztyP6mH}J2!q#=wDn2hQ-0x?m%W5-?;uxP_oNOaDhav#{xP5e@69Oo!m&1v_}8;kQ0 zT(eIc0xu-H8G<`~6``?~a%^z+f9U?#*0l$0Uzh;i!DU^b>yQ0tIHHmnXtD9ZcIaDJ zQC^)$HiN-wciUKWA77H~wZpeGlnmK^zb_cTNCrUBw|I#UTpCcB2^G6>a05-Snw3z{ zy|ofWbMg<_^PnheLO}F8 zDm90j;YDg=4k|SkQ+;~tksML(=ZCFQBt+Jb-gX)q>hp@!RCfB;O1~T9y&`~|p8CF`|YMAKPBO9j}e8b(<_ub<&49z5m5nUTe(s^R?wPy5544(p~rflg?mz9JRq zHtLEoel|FGLAG|#Iuzsgw@bqlA}=~cwL&%>&oi5<{;_AKFlpG;RDSBpsYnukhO3-q ze?nQ=v0q4{8>0Tp#H|H#@ z#ZbxB4|(vH2Jt0EJvSSf70dhBUs`H6`C;SC*mk(SzTZ{jtf~1P*_cJAeaSZKh3U!r zh@7#zNxQa*mKCQoY@&xHOqq=c!K|$9N4zP$GtIt3z{PJt4!GdLTK$Ip9vrCv06o~H zr2g!gVTo~9sRWRhs>WrcZl+;&vpk&Pf$lv1R+1L`WN>giM>^wz1hu^F);wXL#bE3; zy$-LfA_=E^ABO(k0GJ6LSZ|7>cw=?c2MNJTaj^4&`*D<0sNz$pD((%US8Do75&k=(U|64%JpU3 znD=ET>OMgS1KR!_jmQ`55d+7@1+6sSFFKD%LX-`S-(@yKtTAcrZ#~%nrb%`m=6A#p z5|jM!>IaKN-SHLaKRsdn)*TyNrZ#8G;~-npm9LOho%J?cY9@ivxh^2f7CtdMlY)@% zfYY2Rk5+Q<9fCKmQfblO@%!=e7!R?*KTY@F4=@0S1nmT+vSVUdvWjtQghMS(kjad| z>HNEE`8sfnwP4KNF(}Ef8dzt-N1kK7JxD!v^LFzuGVzN_`}xRT3|6kehkcJ{5_t8h z+ul+1Hr~}iW2jq=noomt-@=d4Q&5 zTO`Kr@WmlIBg=7__Zppejbi4t8$QfG8xJp8VR|^-QCGbr1i}mfE)fky=`a5Gh`xND z)RQOV^<}IhL-U;FDT6|u=embn=$oU<@*JaU)60BuoOg^<$I0$*G6jpt|6t@v9AMYD z6Q&FP@1Z4eQHf1%Wd!vL))Cs+_X|6b&7r*t^z#Akf>71Z+h4cU%t+UFQFD4Hnc8c! z!b$-yNC2l#{?bh5?k|$^A0F+bsPC^-tL8X)G6r{!LU6Jz#Kc>8D(v?jH(5MWH7R%r zAnPVBr6e!N1rb|0%=2NPg4eDAGFY1;rTxEYW_lwU2fg5_KB}4e8Xx>xVfr5B4gOUo znXGP@#yC;swAhvy9+uF6kM2!cD+x`0I|4LuF%*d7qryuk-M^N|->cUY-~k$|UI~&j z?Wkt7gTlXCkhNA(DHlKpJif>i?hljkBKD#fZ)0kI>z5ky32A) zBqpj~u(%z8FX(38ZB&In&H$+d5jkpT_;qdVo%VEV)G-Me()cek0FoPulw1gd z#Q!0=HrS6g2m9|cnSGEl7C`K?q|}g&ea+xDvbX?l$wM!4wFuFEhyo^H;%=dQPJj8o z*$Rk)k9)Tf7f)*w|E36wnZ|h2^T`IG(b+Bvp1Mgva=XbDP0+!z=MP1I{utn>==Rh5 z|9j?iJ|1EsHJIiDjPvFPY4HoTb#vc09}hfp7O}u`zP;2=Ddv2(BpZ!a8IIpi=X3KM zxc~Y30@U_eWvcvdYJG2_2Grhs{`wx9m4WzBMsTP4>q$YF?RH4%k#!f{S%!GF`OsG; z<|AP=D{B>`33dYbxvakLq;dc6aV77gI-#3sPm3ENEHt%R?61FdaE$W{jNGaT_47;= zWF^-0g?I**%-#mpsUEpbQH<&T$2xt+z77}2ZET+M;nD)-_ly`3LkAd&p3i)YJ3G4$ zNNUS4G~+Qsp9;85iGByJ!5dT|;b@m-=(imv>VvO%gs6^{1!HM#rljb+nkboL*`N?? zN1#j&`^**vtL5z5ae7Y+sab;nDRl?@D(5}n1izSzzv!!m@CCW(N1BFCFd^BgbmhNulyyYfxx=!zRRHeKi20eYCzphhfOVZ z0ZXawt}**0;>xa22@P0TQ$}$6$c3A>zBP?CYlhx6fP=cpNXPYcwT|CS@SijO>21*c zLJ+1?ASM`U_H`7nm^&A=Y_?sU&{vQ8Hc$oi%)2EAPxwT9sJ8rTuP>2+I^F|@1Bjw z8+CMP7S81x^>3tlLLP`#=L@>6ntf&9UNE0pW<41)G|%b5h~J-6(nO(r7S)XI_%Mhk z!J}b|4jAyPCFsVl*6bg5H)=%&5dBez*;hD?Fx%D_%o42fO2M`7A`L>GnF?QhECf9} zUlhK?1TpyoQ^vT)qxHY}nCy;SS{4UAGcuo4Ciaw)vESb6trIb}>1N=0?Tkivt=c_C zi4@p3T%ImH(tk!_E{l_5kO#~qXMCojXtH?ImS|EnJIET#|ZGMX^vn0sfIn)I=??AndPvQz`OLSkU+tW#c~+FiyN-UL=P6~SSp zlW5jRs>3<{#n4mh1lcL2+M=N7%#ph8=DBRTj6wU8%7X#Khtm$az~lFZj1>Y(uP=gI z=lBx`>T)xSt7S6J+IOu5BI729tz$wMPf5pmC#s?|RY@siB8Lav_$`aI6F%HpmELHr zOIv*sJ71(fme6X$dNic$FtXSF%>FUZwHOawOJkKlmceY)ym(T7tlc16DbieNTw$1s zVB?Ii)BpHAB3$7{PqR&bYXk$OOvZ5e-Ne8J`cM;r318LF?)LuU!1c^dR+WqnCVRof z;Dr`D_0IR}0b}iD{q&uWUaiL8RkO@xq+FHVuHd$=SFMX?9C{}kHyN`3#ik!Ia9X6W zJlB4%YLqJy!&pVJ3z+p5d4@%73DD-l*Q|?y%kNv?8?L)eevjdFefi|}mW0H6**jx^ zGgp!J+JEOiKR++<$sL+VTQAFxI>x zm$}^K_Ix2dH1qJ8RFQ$u;JOn%T4$D9{|N4(`d)AEvmKNJ9c7)4RWpIdFO3WG&l5XV zSC12;?OqEz2?C?pUmbQ?u4_H73^{+idARTR_4!J5jI(jEbZ*H2^kk_yX7R1^4#1)%E<>1E!(qr@=ghC>`wnAk`0<5wgb zp!xl=6*Mce{E#f16ZU9If0SRnNS96p&$sl`8-dtprNew!L*jyhtjdpD)iGZGB zlA{n^;x!2fIGnvuVwir_^u1J|D;sB{>Pv+bdFoJZ`oq}Vk+4TMZ9RK)`J;;9D?tkj z+$*hT-wiFsDl99Am{mr~Eyi-w)0;~RtaG!oi-GVw-Wj*OO<4}jidPAFY9$QpdHMO}eSnMU zXnW4$+ovb#*CG9FgifnNdF3w0HeE%IWy!rDsm|S{hW@byL9nW&iYjl)M zGr#Ok?m^kd$X4H194>antv3_vTwO0HD5x+Q$cV{zJ-_jXJ8%RFumRc=M(`^e*porr z5^;NzKvqS$y=_e`tf+~{a&v2|%8i}Ig(=}-?_PCgOFB@t7I|0ZltXKLi2qJ?#_lrg zu2x7$b19&VHCFX+?7o0KOd@LAp~R>v;xQX&G0ZCxHXAfNv!hFSPCK{TBVzen+bMm* ze$1kA_t~35^RI1U5f1beuN{Yq9Q{7V2~&Gk2QQ~@S*jr2yZ;H7u0ddnl9 zTAZKk+XEfk;K&{rg~DrvO>00Whq97x{?5goBpp+Z2VqfCJJ0KOh-%JuIsuM$BYW_S zAd!j%Rz9UZW;JFfTrqe9^y_T^Fj7ow_WEVkfwd|{)_*s~@t4zSL)BXq#}=$6V8BQA z+s(vdK)vVdQ$P13EC5?+*Bt1%;@@CBt$OK@ACW;bJd1t729>n`bu$IwqxCy4V4Ur= z*lWOGU5Ty$)If!u>d=1u?Vymb{DYC;+fLb21jP(N-SCfUd}GFMN4;Cjry&D-sw335 z6h{_=$#~&HhvNqIH5bk`Kr%?pMm6)d3jm8@Xx^f_Lx^t^Xd7xJySzYiyvSS6=Pf~* zW`yf>?@98Sg*bh_S>pWCgA)Lra14o3B>;5tfj$ctdxGr4c+W|_6X@nh=&cu_)6MW| z=S}1?Tslthl_hLjl91R3t^(c$s%h+sA52Bvt_3=2C>97-bv}ruv;!;u>)-z&IJzi6 zU?C<%Tg+J8Thk6^N>Wv7Bz?EHD^rutZy++qAU>Txt$~Zbu(V~Z318)GqP5-z z`d8!w&^00@$TWP%PaT~OKnE6RP@;F?V9s-_f@ycF#tpKmBj4Ih*G-(1fWt@Qf#?L1 zsS&jPEv4v1lwswSU~)PAEdZ8i4S<&@<+DY=g<0P}mfdW>9-^+F_1UTuM>TTr%fK!5 zVx5ksOvVc`nJi^@0~%~={~xyAGAzorYx_q;36&T{5M&5xq#FqZlrD*(8|luWR0I*} z?v@<7VP@!1Qo6evhED(I-2eA^Zm;WozqqmS1!2zfIM&+N+V@|$b^})|afgkMO9UGn z=6&eF<vSX33Z9(+OvR)xDfvL57B`_2@;H4??+Gm{}$wtAShQ#$p&NU58AI$5-|W`pcK zRHT|E6|!n2YP=eIkX|)$`hJ4;s2trn{Z;E^JjY$Y)eXxQPyYLCj-VwR?MoGhpq@;F zI=WRc#wMh7uZPobpI{+@?A&R5!IcQYQ;l_6#T+D|iHAYoEc5Vnm9~qneo9V~+zE6lkM@qYRS^eOV4NwR(ma|aoADBu&b*poEYXNlkWwbyzupWj!cUwzb6~!N^!e+sHE3H_Cp+SmVG|73gd=J(mC&0})HaZ(c)uAO^0ntjBKF zbE__t(I?+V0jrEn?S*ciRX9SmV>b=HzxO^x&382KHdtllC6Ry=>FN#HzITVFk6d`Q zB#;UghxO+DGPN6CAgd@D?EKE_l^;%3#n#kGqJ$9{Of{e|Vm$4|HuZjhs7mUuS*}5r zIK;G4w}v&AW^i*91^VR8WPzod9y_MQa0b5U$%AUdMw{63>#HAwKc>3`+ek z!USk={6IfO@0V}F3T8nW)zTZ$ILw2pG{urVq#!}PM#%-h8Webu6#i;YAez6X9Sz9{ zpZ(qpdNNv?4uk=H&cV_Qs%KVODw;Z&9Pk|j!v6XFp`r;}fNim%~)X=^t+|sJ-i^ z07lu7Sf&z4nofLs3Jx%F-1_2+IF{2dNAglf}*Q_blSD-qwXO(n#XNKMd}{4 z;LUVYMsYZ23G!4M4y1kgS-i~Dy%`qp0pii|Z_rE?3F!m$ZZa+4rt{Zg= zY6oeXf#ezB=J}lpT^-c6-7h%RXy91BxZf&(cxncR7)|CoN6crEK;BK$^H$DAt*A?Fg`Rne~CgF&Toy_uJepHCd903)^@zP0)-C38A1?WAHu0)mW)2(dWud?Wm&4{t7fELAMgMa?|hLXfl7j z#($S&)E#)r_o**o&nJQ3?gQwCuA=)m@>AT#;q&QYXU_1N>`m|M;u(jwM=fwI+hHkO zP{-(A6992>TMp&FPp#`JGvU)`tzQXKVwMB9fx79aEpwC6D4oM>Y5 zGY3Dhf8Bv!iGYvNh)0u$_-~b|E#UQgA(-j@>IxzswH8}z;ZA=jv* z>L$d$kf6L&KA4EXc&Q|t3cQ*`r%Bc&no%jU>cb)Lj&?W*gc zd~X1r6B>es(+D{QX#ln9M7_%LP&sikB2-rOJF4O%V#yRrp}Tn=5d#2Fu%mHPLm&FG9d z^w~)(h<{EQ@_BlO+96NXFtOHqA&FtX&*GOu^P#3EvbJ>?=o)u`GtZgu6EAi-LX}s^ z?!9;11dEZvZaqLM+?~JOo%iD)f4hLRmDq_Ewp=x0YEd)JY{gD@?0m*&-k*{{Tht-F zRO{b>EY@-EC^W#Pa8V}IQ2&h8>JFT0@X=@PPuR4@i-lhgxN}0l?N(n9=Gm{G4R=-x z`_SdfV#F;l8mVAbwD(zTg$patA?Bg$si#1;9rd?*W`m^<4e20?d7xp!`pngjP0=$X z7|n z&Ik8>fihp>r`3r;uj$lSAP(r08LAAT)BfV3#Sal3E48_D_%it9E5+^Kr4mpzk$%NG z_PqZuS)mMqMT=oRdtrx4DC$d2sT6Dsp`>Gf>)c{p`W;-8NJsPbjbf_wgO4|}oB^Kt z6{a$T57lcVU$e^G(d=5KW1`jo=W3hcX(0i=Dxs4AQ&>Q>&-Du<75!f*fjBDGJDSE0 zQ?%fBrOU_lb34JM;sn;k5mO-g?5_NFS=@HZMHS;xVNYJ{GI7?PFPh*ARBx!bam6Fq zKO?G2yLgAJk!m5)m}Nt-N2uOdLzv)KGvgvRX-UQ8DY|m~CGV+r`Nf7+5f*zr=)<$l zsI1B}bjBG#Fq}wAPL`-MfYM44X*zZ;a!Mm9{MZc$ja_FqO$!q!b3%9K0 zOI@`WM#>+N^Z9f9w2E(4kbAN#n*e#WdQnuLY7c}Ab98q2+Xjt+s_E{ZjgDxrqU!dT zB^x@0^dV&YedV@bey`DQLaP)%~LNUs6;@^kF4Em6$CCWzr8Ke(b&ksQD=jcgHzf9*H-H%-nV}GoYamP z%rOFt_7Gf8kZ#^NqE+RZMI%r0ZBPQ9N)c3f6;Nw*+2Zo}yVPoht;|A0L*KaUeGhD} zg*24qySm%%mE1CB$Tknt5NSG7_OwBqCq$2bOS~afdj`UxQ=b3$=Y~3UO;Lkf-M!bK zs2KFBL)H&Ynz!DjP^*NQgHRd`16s{MLyoyd) z@SJ1+ifQ9YO-ay%hV6f5QGYc$Utir>!vgQUkiPo6F#3&+S;BBnrRMjZk*W{g;Ep0E zf8Jv{Z9-R(3xv$%!U^0^yT6mql(n>?kehE5`3dR4QdgGyKEBl2om$S>HcqnzM>_5a zC>Z+b)SQHFNv2m-Ko@Eq1{P_^H@7TOZB6tShCb->)(e7iD%p=Q%lTEw)0jW8435oo z<{cfK&1H%!o!|`mRH&1+*BbD^9U9RctYJdM2D*B3sp;I@t$GH$+K>E7P;aTrHNEKY zWi7^Olnh6yGizB?n>rvsA0qC+Sq{N8E596CJzYO-&+2XmfvK#iTPV4EcR2Of{ zZJLrTGp2QNMQ`tQ%jRy|>4dcaLZK zX|1mZNEtrAB#|@>YGSvytBvC$Nezw>1QEO?czdE?Jo1i!L3m1n#+@NVhBoFbt6eZh zHD8BSFJL`%CWepgP*Z7}agMl$WWW{jPRUYj&oFuF$u!-e`q63klRfRl1Umh0>r0@+ zF6#n&z4UJ%oB0XiTis;IRF{L;vUe8di57}Fyy1oHT5k3MzfI@%YD}g*UvPER zxI;?%W0OkYPHOKAx>cH@dN1F8!aB}+21ZSae4@jUo0zSeMT*tBP2iJw z=JC`w>oQPJbQsn0LBeX;PLvT-`Gmu|*-1TW&cCd{lcQi+4>bVkj8}RUD0ba~&Pl$v z{}x)ne0<=K`?w*8o|Eu>-tn?!eU;%doBA+>GJQ9%x5s}&u#u~**#xX zXSahL#O}WH1$au=EO4M0?`qi_e_q3VH8vuP&XO8YNtqxbYK`67J1u^O0p{-(@zEom z+Y(2a3{+1(I@>RsrnIl7W3QR?B=Ludh#rR=ib4)-M&m1Svny+g#j8khP0^SM!fZQ0 z@m~mN4>yeS*iK&6h84ILh)+-V;QKXzD(|wb$4mrd4t8X+ z1$_eb7*AUviWoJR^*GVek_v5OcBR-r)&MSeKF^?Kf8LPTlCRK zke=watPQ`?Buuaq?6GhM-CyqXzuJH3cK}Fmw(BX*VaO%o(v86$p)ug+7SXa57OnQg zcd5P|2Ypiy@6`8><=>>x=NZO$zN0u3M^fQ$TOQRxIH$)eVNxP0BRTFpBoaWeUzl=7 zWN$oI((vZ?nA*=*2^eL_#{)4leC?WybH2S0!nt)-*0?ic4&Bmo4#EqyH`NOH$LkM>Pn$&^i!*Et&s zan*n!T+d|At3TJtx?6L8!3W0!e)=ul0}eNhRL&)b^Yr&_~K>pJ`24m z3MWF&w>(@!K7Q-Hi6(g|PO^lL*nvrg*G6WbU#;2u7`gXx3(VzPi-DB*fpmPb888@3 zj-I7~wZ$5Ua({#43q+Ji2%;ulegK(3N^~u2=RBO=t`JvpM;tS#nKU&Oh(bk?c84N6MzSSHYYd$p=8#v6|{L-p^@BPT3;_{8c)i? zzKxb3T3F5j6{VU2;L*Y_r&K9G!n`C0RhGk;VgWI6@%Eh2>y`fYq4hR^gqpkk#`wli z=gC<|L&AE+8ebeqy?kp;>xZ7@+60;F*a+XYNBk2&&N>NVWZ6~q<&E<5$6nPky{n&D zqZvFPCOOu+bFGI$@&-K;gVx*`MB97eD{zp4O zg5=W?kLDWhK}_gI_>_K5W*(v3ex@LJN6lx0$B|zvDH?xzP1s!L7qO^;KQvgTefCKJ)#G0yw)pi-S)YQPg}`I{Ayh_L%p#0y;8Azqs|=1 zwveJ)AF#w?rTv8f7%-CM#1L z^pbB!I^1+a8i&v@vt+CTbhI~hh=Xuxb-NH0Bz4H@X-q$S+*Kpl51_aCj}L(s4|W4^ zQgJ(EdD36G@39@i^BQ=b%2GZH&L)+6sFd99jXRI3iDL?n%OsS~&L|CkxZ;}12kn!c z5j~wSX%qfMrV72HSHP?g4*^K5N2h4i-(n1w3u}(|kBFDn8MUsqg-T1s^BM9qWu4Z5 zD7e@lSM#O|jdcFaPZ6F4<$5&zcX5W^gd+buQhU=@d{IU6cf(G|81Si8U|rX_xU^@= zuYe8JAY~W*pgAecJ$PUlUfj`^*&q!Q^zP!9&F`Bao%glSN*1Z9n8jl|mgSfmKz<}} zA#$(qx(m5bYtqpXvMJBIfO*}rX^q+?*Lsm^*efRwFd?iY;38}5F^Wq^DZij3zOxUY zx?^t>5bV#qoYY%1?5*}wK28jq3b`)6Z{V>R`k8!oeKbIj`QhVnHsvlzl#X7uF4lIs zh7~6hRLIt}gr~EawFEFgO_K=f__K_V4@!Z8+=aL}A2Xu$F6O z#qU@fTxlOr;lFzIM>=Hmc?(xi0-7%37ntwA6c-owTLbFfO|F)EY6!>SK47sdQQz!y)#6g?r{(TqV8b4gBojPq8N3!|%@7b2) zB^MnG{W(_pV~QpKe90@&l$KPl--;J5qx3mefuBxU-`z*E8Y}J+2s@Buf8H`LZ?w=g z&Ns(}qvZ^V7Z69R#hxkTVPP{ypUKm>zc896RUe2@Jf+;g;Yx=l2*>|54x zd_TCtZEbd+__=!LdXa@YIcjBQ>UirLLZ6B41+6m8EVUj=^FYJF(3Qv*xU%zU_b|kG zI#TesbiCbIUX&EdQ*nl|mU2-LwfI>Q_F@w77iQD!Rf zEhG(&q8Ss0fiMSzwu`7rWTfT4l4XC6G2Og{4&A3~LzP;lXdX1+H*mc&ivkgaGTIMphlE#Y{{taCwBZpy3Et z0Fm8B+q!=qJ>uihN$UftxQd{4qlfx!%nY{WN=ev4vvDp{q)yt1VapCZ@DfUceXLY7 z4&@37yUg+3GQsmXI6%+#!^CdS^(aNJff!=^xyyaead@ll6sW;$^R)f)-veV(HsGEJ z?2#T+{O^?xtb|9t4#f5@ab*V#ojYKzSLWRGrV5McwbgasuAExzPN3fDUiGu|+X$t* zg$@Hr40YZ27B1Mx_<2ojF0{qO=pT!}Ua$O+JhOaedA5Wc?U?T}I3WDZh5_#+@8wN2 zgbvIvo)pba*wwOIJ}}DDv#zk5ObJZvpV$eqlIpL4lOjfXwD-l!P@UJQIS>CdfzoNe z0#n!0N8kj|j2+^zlLJzO#TFEgYKUE+kJMaR0;Z4eq8%o6q7Pmfwfa*Xbl=tTOGT&? zf7f!&XX~?uog#qq=8^4tkwI0Su{@^{FjPWxF|8Q{Ud<^w?e1}1Y7aX7%8#JNX~rXZ z>8LDj%K7SYBl}Bjfxl3Wx`2LAE(n*784&#cI4{sRYSB1oyRd3}un#Zz2x?fpVIEFt zw2B`?@^VIg^PVsY45&}*P!-*W1Kp$~<@yV?*i>a8r zcvowBhu!K)S*TRjP(mc$fA1GVQ=*91HI6qzUTcTgyBRkm6FejY_lIU8$7*&gYNdp$ zP<`RG4QC}@jn$qW<|HO??x2DpP9g8ab>~B{0E|1P65+k-dNuN$5AG!cdr);xXH$Gv z^Xy5QSjYNyYGmb&Nb%akF$5CtSuw|eYsur)nN$dMM!I)q-=1gp`nBHF2X;8WW6QMW z&c$m2a;w1Y#u~$l;9h5uCnvgRLN4JZeQyv(a&VDt%!3yLBs{ZlL)Jm<4_yrGId1r)M~@cI6F5y| zK}%}6SZ1+IZ&VjUJLvk9)@gkUeI1p_Mq{5`(Y}VUGt2e1zoW^zjOF9hm?a~KWg_@s zyWK|fWZ4{m7CA7u!YdhWbB=sVS}MALL$FaVkx%`?5= zR6^9WYb{fElEDS+j0+kIc9|t^*fjDkY4-K>OvsdO+?MQ_#&W8+h9|y%nPOm*Ev<;v zZz6y;_Hds+FgO=K&{uMjkK-MxP1i>KKpqT{C0G8T)=<~mru-1{o8yt*!yg|TO&?AA zjhap`i%IGISUHfU$RJeu!tKA*pFqBgLvT-)DA0`{m^24o=-qHg1lL<&~9DKyegEDRd=YzGdp_QKfK z|Iu{!;;!g&~BJpw&?4Q+ZBth?=z>9czl7c zHm)}RGlXmPUk!cBu@KeF>=I}Py+DnZ(i=aoK6dkGjH7XSu=dO3r8%&uhOWI7i zfqFsS+F6^L?|+Us?#j&!ffO)H+TY>YqpX57iqiHj^Q@(^ytF-(;LcZqjb_38w9%2LACv8SC7BBvmN!&mieUS8LB--1vs&4(XxW#OCp>CW% ztKqzCGopKGh;aDXZUyh|B5bHvQGtCqtNVdy`2H?i3StG^6}TT=c`=L{Q`-kOzw!>7 zkH>Qx2MxeyIA$EUR313%|*jkUV@VLOdNWAn1_(zc*yd;M+zf!?%(=%;a zG)M19{9=87=-7)q3dU9K2qRvmA9S^s$E*-`KO3Wc61TMq=Xa&6r>(tu|DoU_I_v87 zSat>6TGf5sbGq|A*6!SCHT=}oolQ@P$&${q=dL!mIru3nUK${ZhOYjKJ|EMuf0VX_ z_n1vZZ=C60*6#nOhm-m1r(_ZX;A}2+$N6e+TfcdDHoJDL zjosdvV18ZC6geF*ljk$Lbmi+2UtN!F=atzO2X{Zv5F4MGvV-xvoa;7EQuT_$D+~$5 z)-ew|@vR;A7bVB^%v`E7`RZSku^wFnY{T&8ZbK-z21NWsc&ZSg>z6LYsSWmltokZt z>TKfX4F#`-@zSyfr#DqB0&`+%j87n4r!^(WKFX^>gVlCax24GQ%du&`Q$8QT(7-7a zuuJPNSiZUnQyvBRt!~1?$z0@@4pQCcEYV}`|7`vc09YYv`5NK>%X!-d#Pb|DfJLV8 z|0|^_15Q?d4Jg7az*N*hdu)y8LKvC!g`XAEojL!FKbRj;9V(P%o$1(wZF0ywFi zZ!f}dj^5@;q{VAe{5$q325o?loyNSWo$%o4qFJN|@f~e(E6(V1W`gD-ulr3q)G7RSdpOi7 zt2g9T>x|t`duIB|f3potudU|Kyh4arNmPp^%3Ji_EB74i4e2{N7=0zI5dUaKD{w*$ zF6ZJ?Og2+M;I1l?2Y40mUQo3_jx<@0FNq<#MnKE%n2@fW)~r)&R}J9N_y`4;L%4T* z_Px^k61f?ADfNsJHrW1t@2NO20Rbtz`Av@8Uk&ez;7|UlL8%}iYnU?|hCAKt-rPRb zk}T;7g1vs>a-p*LG+|&BJ|tqZ?WUydqBL4JXQMM~;t|ZQuU1yamQ>*X#R@4dSnIhs zP$%V}rw*7fS4O{lNLFwkSiPU#1%v{u&vnX9B7+om%n3uzY8+=FU0Q%aB!TGX`cX^I z+0FjPrc|HP$r`6sS@g2k0sh$Yap(2^N6u>DlwzVD7~`X{D1$De`=8hTPtZ_z$fq=O z7Xkw&4a=M27{{D+LQ;Hm72U=-`0l?|^ylw%gl4}PQ=amem+r@8-8I)GR-d8(3wizu zz6aktM{K=cDgFH2&{KYdIa|$qfZtpg^Ze-F<^5nz078&Zcw90(+JoMCYWm(&Q;*{ zpXcE`{Xa*t&x_xC&Yc#_U(@7)oo#Cvu*=3&#QR7>LgLHl$Vf~~QywZ1<=sL#!DlmJ z_{wdktgx`~pgeKxKQ5I_D!^3oWTfEnf6GM-Oe7T*b?w4OJ}-gwM>?F`~1)vK9B{$8=`oo>II(dp{R^#^HU zIWIg!r1n2$y)UaW63c8HQNKbo1Benvaq+Vw9F^x-@4$Bg0^+Eii#W+bRYXvXzh9vL z(5r7>Q}XX!HgiIk^K7+H$;}=`NAm0Qsb>Lj{Ep*}A!e0-S)l7E$GhXJJanxS$1|;5 z?1fVl{?u%}fL&f*Mm3b%%?iQMYe9exTMa#0n*??hZTkB9=+@Z_qNt^-%zB=}*OIJv zI?3&;fQ7Odw7fLE@+&$pgRZSoHfZCL(XcMDlC1wN8+h^ZjIa!Q4V7f&IUxuPtvq9uqM> z`_Gf{_7jOS6V(p-sUpg-ha|F~r>l=rdV@IZi0U>F~_ z4T$%UYYi{mfD}u9Qg= z_VM9{ki%Q;y7?1==SOVUiZ;CyRgPN80z%fjqp6fGTi$|+dwp~x-N9!7FR>ONfc|1b z9JFRB2m`I>F0d_K`&dz+1z%_*g9@KPtTW1x+3`L7zEV&gXyhxrfM6?GKFec)E#UUOM?V*kb1`z+RreF{ipV^nsiM>*$epM#GO9nBEyMmMw9FFMcH_{EUq!uzH#V z*hWetBk%lu2z-%yT}oJGDtvs%*!0D|5QrlnSkaqco#l~mfF`eDX{?k^%_dE{!cHn2 zLcZj?(|f+jEt}-(dUD!^|M9O6s{8wEIcur&y2HgB+tY;#KQr#wZQSSkE+@9@A%}vm z%6RuxQ%zs4pVf{w2(n4&)H~(eERi&o0gG@wCBQ$wTeHK2x&io-STcu^9}?JYdqZ`* zZScoRTNKdblf`K%0j0{wcg8dxb)g6q_e;ZCDiOs{SbK&)t+KP%quX3vovI`O@?Cv) z1?-#M+t-BNmBZQlMJp?tKaOax0G5UBh1){Sy4+6GI+at#s|l}4E`x*M4K*&aX#W78 z`<^a9##{Cq-&NM zs}5H53?DE)ubAAheg z;Y!QsyxjH$>r&*{E%4u-#ur$?Ab;gngsx~(s~}e>Cv_d#Mj3}-V@pVYG-8ZKHEmC_ z%~W73ty>++>0dq6z0DstuSd}hDGXXvRT3fgp8=Ml;XiEf;=QkSYo}-3=P8DOx?*kx zIZOW7gcbGs>2liTFvT+ImX2Be%UG>m7u{BS&O^v1FCXXgDoKq1(0Joifz#;My{XCR z?RT!7z6CRyHB*@tE^Mb#CSuNOy=9CBoWbK7mkmR_wC{vUP($m%vBcp_=})WQ+Dfmn zj-4iiyP6Rx|KRzjEW~Jn>B;(3KNOmeFl>ArW+_4{SJYH_%J%0gAJZ;XRl#Fvn{Gw6 zLa%kuK-phQC5-HtZ^Eac!>6_II+WG2a#g1pJtVc;9gieP0fl+H1!=Mh1bgNq0l|$o zfN|`3WruuP6UbK#6&XGe@{C&^oYoV^$`V%v>B@>fM6LF;U6+ZN#TnN2Ax-GWWF5A< zX_I~h^L-CisoG{nhRNy1b{+MwHzBM=O>iAmICj|VmDWeVL8*A`*S*{W6NTz9aZOu_ z{eB)r&^DSl<{$CPn0=ec;HB!OMyWod+hkHJXFI3P(*&x89}afq3}q|0MQygX3eAsmK!jES`cE`72FQ}U1e$|cUk>~ z67T4+8)h;tNb%fodMd8`GASgKGPYXCz?!YPiP!KG3wybT$VjPXISqpFciZ_$fvRn~ z&neSIM}$6TW7SU|Z-XvG-XL+?Oj%D)@7HR~#Ed9|>ijXL{O-tu#{n?ai3TTLWQvLf z67cwZt=~SKbvIbsV;*nG()&-1&Ow8FUVJx>hKl}<+_Q?v2--|l<7k`Hoyj)~vCR{) z3rwY*K(TZ$^ZY-B!1}9sT<5y2#&Vc26D{10u8s)-@B(IaN*yD)_S3=}=400L{WpMd z%{`@My+k)VX~W@}_J&gQ+bAKu_P));u+1+D1X zd3kvJYZ%jOmc0>Prh6TCW>w`(EH3~kkt__K?LN~NS=r=Qe(A8)Uf_CTsr5kj=>nn0|JX{0J`6Cpv&Wmg3 z$Pj(Jw+-oH8yR2Rj=n1B^OM1Nt;ev!we+|f+*VrH-l3F%dE#gwrSHU8u?|VQN43rk zA$f6!cG`i0g2E!1GCW2f^8_vLlIF{c*l&2MP*&{i3iqXB6N?Jr)fqF`k+GMLRc1|C zz$a3MIMapONAXW&Y*jw;{HZJ0mdwB$9Y?H=9-Xvdf#-T1&2rhJ?+`>Qy1b&gZ`H(~dnF5X*wQ~oFV~A( zn68Qe{yo3=H32Ic4F70@M*B)={ql0djRnonGNM87Rof+SwwM=(D#ob@`++mTD7uEA(Yd5c zHaF(0ujF=Rz>zisEEIk@pHrXoJoQ-kjP0ikMeqK4>Ef2pCSI}{sk(XtelI+k(S2YD z$$?Wk=mnSuZ|1*j6f8H`E2319fR}DsVs60qwDR*RdOoE+@&R}75`#PGTt+R{gS?cw zT0;B}Fk$PDkT;=Lfh>RQqHy=GI41H#g+Ak^*L~nV2Q@zz&{$?WwE|EPv~#x?10QS9 zq~ABXV~#;vsEAlimRH*f_7wyM4xcF;+ljn%`9bEuZ2J$t@H}FfVjpLzsHEiw5*X0p z!$FQ{uap;>d4AgQUc64eiDEQw{*!>u!d*(7Yg&(O?ctMrK1&0)Iq3SN&K&Bg*CDKdVBu|uhX!}yRi7svX1@>w*#x>xuqDpBx>?CiKYWK9057QF(mHr)Q zLv@MT zpU*tYWz%3KFoZ7mr)+@h=lWA%V|Q6Qx9H?0Y{x!?0}<>=NA&b_A&N%cpCXTQJwz4Nk{7#W&yg#GauHP-g(jfZ zRlN-My%SeN!b5cf+Sl9nq8h8lA*Y9{s)9~A3jZs880O<{NQmNQT;{8smF0Y|2j$Z` zZKlbc?IUk+lrb03Aj~U3ouJ&~q`PfA47Gol7TUlbw*Vbghuph!3ixTVWQx21RB%Hw zwi(KoyFT?p;gwrJS==ue7i)qHV{CqYjWkAFKK&bez(&;)?EEW=Zc+biDVQDqY;P0g z1!rI5)DES=`1M%RC~38NdNeS@vp)iZ>5aB+v?k@-8RHY!%eQO|D^YSg&goaW=bGxR zx&AiLU8`R8ee}yp6(L2S>LZ!x9XBYjeerHU2<=o*SSs54BX;pi>8q$TVx+PenEgLV z0v)H8<)SxRGzH-FzWEvRyWk?sRWIL`%pPi!m(994DT7%e+8JB- z@(?l}nx?+w>bx2YnR>z2@4NWG0H?LGB>6{eQH^@7aMXUob_ta-EVWQ63zlX%f}NN~ z$McR0>IWkj9AWptV0mLO*UUff?ObPf;^<(wN0c_^X>gqRrE~W?O9{^Np|sttKvA~e z=kM;E15Q+*jb!29tB$Olmm=-+iBqvQayE~*yMc&lU9DvW`@f$z>d#a)m+4=HaEqT= zAD`63DH6=yA^Ds--f6pg+B)UtwNlTgGo)X(Md)7t_=ni=6dy_UdIN@cudg~^4Yl~gB#;rHlXkg%bvo11It?bf2fV$*SjIQutRVS0lU7j=*0hTr2e z%o1{fNmF!@+U)%+C&|8?`SHG!wknm1*{cltCns%X< zZEVjye$kh1#?;L&emzyw=A^mUb)c%XLmY}}ExPbtEbFmYkg^X^mdie)k^v9}d_6qV z3hb9D5;C~%)8A3m56??sAbD}~KUo04!qqi;vHqST;NjSl3!b$;bNTaZpvyHXf3J(q z;dX^Vto&$p`xroAK3J*|eHk-B|3-DLMC@i$Ndj)Nv19rSIXQMyDX9Wo{a)X$2|d}; z%Brs)hV$vAO)(NF=W=x6jr;6YkLO13PQ_ak5l&cD`kO&9(*d^B8@15{nQu>*$B}!k zpAXS4Tq?IMebGKhYqlt!`YYF_KS4^~GzThzO9dQ*|IzP={{Sr_&cGv7vqI94jC`yz z?6&8Lot9s|uD6s?97R{qVpp+JZ?Zs-S&O14C+pdiHx>swarvo@c7XrN{+^2Z zcyI}MHT$WY?O@jFw#i*RfcmL5jN14DIgC0la_g)qwA~Cha+QA?;3dK$^DsW*rJ`)9 zNrSI1fc~D2Z#<@h13srO9;N0ioUtMK3YYy#xw|x(e&50LcmXm|9a~p$;M-6g_omne95eMa!2yOqlb)Q(H@MT{Z z&SFH#(+TV~#2IUq^e9zxjabhv=ylGhJ5c3i7!DD;q%2 z1{Uk06{mpd$m1p3zXOL}z|w&|<71eg4PTaLnv zBW0pi2Is33sa;s5cfMQvY6g1J@s=ig&m7L>cm*i?ln7noJuUG(np4{S>SW;NK44;3 zg^J5X#FG|VQ+c*_qImDyqX>IYPQ=-! zr-54nI|Umift@vWZB64Ne{kcn9H~gm_PwwAj2o+3C_1?c7v#oZFfQ_Xu!j$cp^Vd* z23|Q?l40+ulC~M`dFsPk%Ki&}$))GYi0mE@rbqr5eDDbOH;PeNup7p8EM73tLz^?& zgBz5FmzrC8sGioYk8J-fM~^$rEO&)xnH$f}tg4T1i7!;jSD~w9AUSyi~POkMCC2S}O4b3U>(;8V%z@(No$L zSg0)f*R5dvjIDsg%i?(drR;|Gmt*Qh2LCI`7&_pbvx;-PolBgFIV7N{tU>0q#TBzn zG_d-nFHCQG#kE*Y{vONnBA(^99PDywAR)KFZ#awE$nmNH5Z|{32P0~)hL$R)UJR<9 zsmbPfY3-iHZ;C4GBOUuS0&2#543A|M#0d~hDL8Kha_-hsE{P$(hMT1w*zPLX^Sgsjr+YnhP8uR_M5@rFdX>|{ARxpUSCZ?U%FX^EzTRNi%khkiQiYmE4BDe?G_wP==*75Ky0s1lNVbEimH- z0U+RV3qSkErc&pt7ZHEfW^M)YJ81rg3HASfA&8ZL4Xf%{J}sFln=jEj_{EvEpU4n= zg#}3p;0SD;!%7e~Z2cZ|w_Qlk0rYYS(pvRSX2OI? zdcv-UPg|m5ck4JL+U0wEPDUZig`V_snvP*CJl)2v!MHf-pR)RkY8WW}RS8Cd)Q{!m z4=7kL>9OlO+d;dpvFNBinLOjP9xLMTy*~d$EIbcRhX_ttqd;)sp+af0^HxSeoNMKT9R2 z>2GFs;BsCFgMZ0T7fl97(AA9-BA+?EIXC6+I=BjaQ5(%jiSK1Nh z)-6tI8hg<*r3P&BDJ8&pUE9ZTC~D;W9S1FS1)`l`i>(jXzL~goo`-M`FZld4mZVHS z@@@7Iw$Eg+ga#U2hmztMc;t!m415V*!D-u^nCr>aeOk>$&p=fI4tYJfvisSs{w+_4 zCRWnina9W%H^RVAR~GZu4oNf6Nc{8L_c z_n%@dtsp--guBOoQBgv@XM6-;)#S1%NAZaMmlg(mF=3DCIMgFrcG{5?KOfrSYco4& zcv3dKMzB?W>a$BRt)IV})G=vhhpO4FiZ>0O9-Qa;`E!r^jfN3rO7z)LCyngQ1NL2c zsxqr|8EQ#sf0=gqup-{Oj)_G^6}X)7xFlGR(dDfAzEmX4WIUgGLorE$zxbrZ1rTc;-v zJnu;e;goli_bf|}4da22*T`_!-Vh;qRPAT6v7@eh3VOI&WI zFQpI4?fLYbeTVfsvxIv>sDTp+@ogfwma4NRM?ZlB*~X>_xSWOY@W`n-PdV9#hBO`4hQj8&{DM z&3WcwDP*wRe;r!bu^7%zUiG#bMc3EqGF6?y@=qwAzt9-aOl=<@cDDtJYdRb|C-z%OUBKgG@ z(}{;e1MPPn6Mg$CpR!Q+lAR;dZ=u^q|G3=B&=1pyZ$BU{i2h|Tku(usVn&3oq__t?QAZ-xhZ%xPUN}x5tw50Yd_>Z zY+WI@&Fpu`lu!46L2yYZXL4$jO<=Tn^;&g5XUsrJaK?HoF^omab$S8#Mo8nLwc zFL;t4jRowkku{{W?@|F+Z45Kv9^+S?W^7gz-`BVs-(t|IK_Kg4pMztQ@hmn4N;Q+j z&q$SMyt$vAIeuusDOVOMq5Nhc?K!1lg8jLib1$Kw3oRvmbbfRvp(9zBfr_g^T4DC+ zGRV5_%^H;{iF;@8FYJfUB)CiH$|AUkKi3avPIS%73?UnH_qnjA*uBgU>r$692l$b3 z@g(WimLfyzoy*bKHy#;2GP4J!lIe$g(Ikv4rTqvv71mK0mvQ?yl=#|`Y6IYq zo3@{at^VWjBLvniN-KkkE?`z|i1@A6u~gSVnu)nplL@2N{Z3m~qgdRQQX7+s9=89F zu&;oMa{bl^9AFRyMp}B%0j0aUR9XavE-7h|79^!Xq(hKKK{|#;7`nSVr5mLDf8+0* zbMNu~*S%}GTr()}&G+s1efNI$v!916-3)TB-C~`b2;TepE!dZh&=lkm818G2pRAJn z#@cj)y8~47SXY*B#*QuR2HFq0MaYTJmKB#-F~tOIV_CDB{c(t;RY0n+DU_~LtL&18 z{v#n+M{$yju*`%AFdva=#RYQ&v%cA_493PF$)h*#$5XHDN+}gl5rc^$-rLJpE3C0E z`nZM=^zgvwiv62jdv|^=-(87P%3@cQ_{@rY$Oh{r5P|p&mn#&a{ElxIDSQ}+c$Iui z&+7A(0T4qHcLN5vGtWaFV#VP#_|A9dP|dWMQejg*ia(|f6dvBYI&E@-b8)=)4$Edg z`Ft~Si*F3v6!cxenS*>8%06B~)D}*0@Dx{hb9u)US`l3W0D*GN(J-t_|D^b$vcFw9 z#gA}Crsl2N66h~p_^0yj7Vu;{|9pRe2xP;hVA|euFKL>nEZaeUyX}3*J_lM@^kkC6 zcf<(~q$H7a=EhwVo$YG&P9?sp?`Rz+WNxYmFRf(N#v$#mhj+^LiL||eOL49>Q|$xz zNsJI{Z)MN{5iVG(qn!+Iyl%HrfP(M3H=#Fxv+Ywro3&-qv5*Nn{^Wt<_qvbk4~TCC z_fLekMzC_O;pFS$=5m3TtV$zoS>pb!o zS^K9iPd?d*{TznP$?GzkT_utBZ#9d|sXjNFyKFx?a%jvIpjzrVyI0q1pp@V}W|2lFl7(mVmqKHAz z98aa>}E<=KSbuVZJg-SS_h#K2H#vn{9out@`QBw+L8XXQMgt2_h z&(AywERt1b=omUIolh*@M<{sje+qs;Dia(TuwzY_E6@T?=Mk#*54Dn3hdC8nx845E z@StAd(A+mfI0#qFJhts%&&~Ysv$e)%**m)_F?J&i!Z|Pk-n{+nlsoeVymL$w?L-Ju z2y%Kd9kLPXs@P~$kB-~*;-BQif*k$E9Xf=!t4_k=dU?XkmyRU)UjKrEWcsGw%D@Ib z(~;3$=dD4;;@x}b((~b&Bc`5FV|R;fx1Ym73Ng=GZ`+}04d)x~bT@If%5}z2eMjd+ zpS4$n!B+S0@a)*R>EmQOgdaS{U4E;8tRxMiNWL9Leo*lGsF$yOhkX~o*4pZ=L$gh{ z-|x#9T93Za3Nj807Rg#Jd_&pc8j8@BFN$U*KJuKoU!!tDo5Pqk*Gu`w?b&9pC<) z3lAg5LlsKESh>Lb?|kGxiOH0J&+X)^`!luBH@sE7nb?}rXMKno6X=76)Ll2Z)fXF{ zqjOaz3oG-%{rXLxY;JN4@Tc?hyJjr1%b4yyo$6&EWZG@UjUwIBgV_`loZ#^2A17`Y z`(k!$fs`~39jWL>889gC6XFDYP_U8x@jii@fpyddA zas8C`B+znPSf~*P5_&V7OBh=7AM|#vV!M$7DlE)U2Vijp!tK zJ_f%jT5WLAlV2n?5pPQhD*}T*glQhkJ@3JYVUZ{BA*cD|v0Lq7BR518|AvD$gKJr{ zt3P6k4EJ@KAfyx*ZTK9yYM2tQ&p>7JlIB;A3E$Ep@jBy?wO>^{z)KLV1E9BuouWR7 zk?A9&MT5!|&wCXM7;a@M9|MBw$GQB|`fcj1-J@*+SL_2Xt8bOu>Je43 z?@WX*Z+1sx5IZ~j$BMC;m)VNqPB5ekY|9+4v-fP0h2zA@pF^o1u)U=$;QqS_` ztk(7h)H}1(KfPnSivz!NY1aU~D3!C1^^~w?m0xmCI<+?&^66yjQ-IR7IdepjR`mxV zs1s(4ABrx)FkS+TiIfjZ0gYo3$1UXa3H#G4$dsd2mDM$I7NqoD^;q2E3^c;}P7p z>3dm$^a`_z`n;^nLF-Hx*P_=7It#c~Q_u614L46tjs^d*|K?UiYte6un-z)+7`(-# zfK~{WOWdDsCE*#H#6U*C)=IPO7?Q73ZFEu>YRK*QPIJd4Tf8fJpkbV(0}L9;Xx1c6 zBE6W$N`#K2jk=^cEHhN8a5zO=Q4ubi4QHr?yeU?@s`c({h47tn%$TsmNfOL-*GVi2 zsVFBLY!b$$PEb6Ot!`fu)eFlY2wIP=Yl2(!r3fsd+91at4^OKaJ!yP+**+OaS=9Z= zNp41IN$n3v0h}2bMj;CLL{LyLZ5KwJRbh3GY5|OOnf*|^-rb4lilFG=_#+kxy(mDG z3Xd!N!JoiEgOOfa;&r|lxa>^Jd~Q+@k9BAiN`f?&m9px z%p}(#qwY9BL@HNx#74=hoI`KBqmtRRmYl;FDcWgC>iH9B(ulcJ8&8uUPsi|%vvH3b zKJx$uHLBaFhS+|-!S&Y-$Ck@zF+S_r*8sS^+nQN1J@&$_?$-iF$~q-SYAZ3^)4bJn z?#NXRn$_>1_H)bY$34cAM#b!D$5)AeCmg?R)hPj|yb7xAozJ(*bZ!Uwi@)0G?7B3u z_FV;pR>Ptd6_`0n+OmkogStU?2s!*25lI_KSo)g~MTkmp{{uDWd$i&GF0iM9iLN2` zA@D>}f`<;PpYF11yw2J76Zj(TIYUgbv)}p2n(B%#+99aSR%CnpT46Rr<SnLMjkQqbHP+eWhaZ1RrXhALG~d ze0A+@o7z9VO4{F;N!ky=sioT9yME6?r=VvonpX0l(zhKy|Mn-Ac?lROev;??i)og5 zO&cpBf1_qEJc;}Fss@$YxUC%zDy!;Q-X4S++(F5+EuF_Yu%x~u}E z>Um zU>pSn4om+nUR|j&W2$ZgX+lFQWP4wTq z>g6dPwVe8xRP1E#CwDJ86yG*;THLmrB2=v{Kg`4dKg>P|+) z?C9wv=~Drcg>-MA`*bN@KyNo>>Cxo{bi33(a&P4T&|5^5Fb|nZ(Kupz&I-*zhTj1_ z(V$ePVp>zmNN}gIQh_&W5Su3gv&@Zh!>+9!zW1CI!X?KQ) zrN7K!>&KXzY|4wI8x&P>+X^`f(A9Wu9|SJf`&=C84w%=;eFBDQ9VB4a;f(#MoilP- z)WWXpieb1QSJ-v+qTVgOfn#EgS^v6q*qntFjjMbOK)YOS6sp27FKNwF%`|>T8+62+ zwflWqWCI_Ce->QxH~p<{368!2`l7rvj#O~NhyhR8K{HBY(8h9>ZLze^Rbhe-|zze;gB z;>N8m^J0|jeoow>J196&Gm*{CVK>9zoVa_62%uL)-D`oB?KT}=a@#ZbYR=YD*`UK>a&WjighLoQr5+v zVcz}z+#MGAfz<4N^=a$V{2kuCkGXSg2!oT&e zJOyTnZE`#n@Hn1=8tW?8Vh6Ryi+f}Ta)0G9TsLPKRk9poV@GpS-u@>wRV2(S)w^$h zAl!_cH&IyG8ScG_nA%?=CbUT7lb?|@-@|jXQJ&}U+iI~iJMg&Lf980pOH$)vTRdla z8GYeClt^nm?!1g~-@(OF(bZi-F`==BcbhxkU^vtO*P|gB+j5!)!Mm+JQ(VRVt|5}@ zg{513z@kRl;b9j%{(d*%_y8h@SC2nSt3K{W)MdC#7&t9Vk9@Cg*QqEJO1?zB%q^YA z*Z8y7;mR+Yxfvh{pNQTyvqm<{o!)M1&o;4z0u--y*R~UiL+x57(@EuIr&4Q1;cay$sMBGTdv_GVc6`uB)-N{#>0=n=b8$I%hw-W#JqK-S(;a_xb~3PSDYV5C{uO# znb*vC(LX=3#lEFmcw~nu?@B@oNX?5xF_^B#uAdB7ALBd$v0jU6;QkSkQ?%&%ZHcZx z%^Lp_4qMQOR6o--U&@8sOC&zU(r9Ia7t<=$(H?V3e7IZC$AGuW`E%&S&EzrmMRIT` zcNU?Q0?{BIOf;d#2(xZ)FE+=>*N=CAWD0IW_&{NxJi96D4&es{V-|x3;lh>Nccp~h z$Ap9*ka8V;O74}}ykA)VicNb24sFgo!DK-pB(q&4Z@2#GN^9@Yl;&3RO#G|nqtI8) z#*Sx{3jTU#B-N(Qb9MEs3j8%)AQK&LbB)^3>&F?y`hNB%O9vMh7U&97m@u@mH1;kV zvxy-Ugs1UB)%O`i`ui^Bi6_sd1JB7E@6HbEDx1XW8rT)4v&?m}=0VZBZWGTUAZ-DpRA;80CRghiI@zC@=oG4C?kMovU= zJh)E0ZKnS|t&`N}4kVfb=1Xh(k=iHm;K5H-Jd&ajA>u<}n!Zuqo0kQFJYRL#qf>cn z;hun2tL#Ax=k3+#wL~v5fJ~EN-_1kAVAehezezRM=mb=s%@njPm^)pM0iL1#c@wrM z0jB0A+@dF#J#_zxJ=62`WluGWgNIOVjgf{Yqi$_IAMp|T0Q+!IkdpUCRPr(}IW*xx zdQYAA=a7krjYz{Cpto6SBlKbYJugZmJ_Lj5BxmP~#g0>^0{<`#c-8;-+W!pAUk_w@ z{vqD(=dzpqdu#^+2N8X0yPi`DU9Mf(2|+g_UQz}oD&GrQ-fv|o~g)9^|-(~5(W%8{Y0h{I*dMs2NMpJ)zD~c zKa~|qd~5R2zcZqeC?P=t+1~82Ir62%F7d6=sMt=HSVFNRt?X4`D9mi?IsVtbLhYMG zQ~3uHtg^mu%zXKChwQdbWXQ&*H3BCA|Fl`wK_4p>;BD6qu zvs68$Rm|I*TXPlyGxfs7@9e)ls@T4~^sSC;!#oxy^d3*Nu+kzS^8UhpGH2<=T2@XK zpJ-)KYc^y@R^1MD?NCL$=XEs2J<^<>Fg4-2pA`P_n#$42g9iNK%aee-P~7c^R@yIe4=u{aYk69C($Np0nYBgboc0J`vw9ljJ<_zmpG-t%2bD_!q6DOuQ#bk^+{$h?sC?S+CrM&7}n9 z?iYDah6xxkqzXhF@H2fxa70 z_Kj(J_B>BIqb)qR<9_KjzT_FwBpJSTL(w^=(&qjy0ak2I!#bENgwf)2^Ko^RH|Rj{4aEx(FsFz zYTr@cLovFdG1_^e+U~5By_wltE{S4et`eWUc{k*wV(_C&lGd8-ypeGXP7~66sCI0( zNLLf6KJL{y=;%$n+)a<%T~`II2QuGrX&5Tr*V#J0M9TAd*hDSBZM<|o4v zO*#@9%*&kM;K^s!li&&SWF&F>!t!Y*l_M^rinqIn@d}?#$&5CsTBc(-iE& z|8+<5p5?+6Fx^Rcx$J{Y9dy7LS~-NL%V2!cQ}GaHox7U*bEEbz&?V*pItAWNWqLC& zMTYf^rAgvpeY>1T#iVT_dT}_q!b_l^*tQLE=aCnCGH(tt+j>b#yV7kW|;bxz#w#f5D0Y z9caMWw>^9Z=s6>PDZdNZV*@%pjB#g7bD4<9aR%un6!5O=wi@d|V@-m|RBnfA61x1# zKlvwt{LhQP4`>>NYB>tQ%{05-v#r|N4}2F^;|-`=<(YjP)Z>U*Ai-KYy{~Kw0wE5= z{BPT-$e0$Z124^yig=(G{$|WdUzAg)MY^QD&C6Qj9 zK42_}VGIM-nDa;3iGo_d3|_ZzM2ur8gfvqvc~7*-gG}|QtJ&9yMu3@C(yW%TRXPl9l&Q# zSGK@j;%?^N@#e@7Ha?oEcBIg|XLdAT_xDPFh~W`tA!W{0O(leY9+!!&>e%$p)isV< z*6#F|vAA>L>P+{$7#1B444z`@n=fr$sheQ}04MI)$fU85&ar85-5q5E^RKMNJMzg} z%~*uSs?4<~vXcvBpDBo~b$ZMd zKZ-~#yd&iiA7G<_2d%gp{QB0CV{G-8wpkvHKJ`n%fKJ8SP~fTT_@V7RcXijcXh)5g z09D?+|5j0>=b4R1NIgLtjLC$H z`>!yPBoPK4gR>AXV1{}jt*pMbYPLhgO{)^&+$MZI%8VnjA`wF)NSko((VG(yVs>e@Id z2NSKp&ET=W>l2kt5P?u_0$9i_NUIaC{Mu2~GRjuqz~|Zh(GjeQ@*ZX{a*CK_9G0<4 z_|Psly2~V!lhNkVRB^}Yb?a>DkLGHPx|Qc;U4^?mtp$WbM4|B|y@jJqZmtwgJI^YP$VC--*!u&`a|rU(N~{MGtwSP-i=j++50NfmExm1z0n6#rsrQajdWhQ;tV0@&cACH06SVgd_#KSN z0NTbeBk~dnjN~m>O1Z+8n&=7lyqRUy3DCh$Kx2;oU~qrGc=)Vombt28ts+Z5Xu?Ld zgSCyQdleIx1Syg-USvR{r(lxZ`_&U4n-(%@Bl6W)@aH z&{O*vO}d+EP%QFZXevyoPrmJLKKPcqWWq?v*C8fGkn{x z)HQ4vP?aeVoU;2ni69(1l8ggtJL)%WE-b{eD)6QJN1l@#P};An4)Xl_`TlOOP2gi} z$A^WQ&xn-G?`6n@N&-!rH<`U(-=0e813W?^Z_jzd-6K69qJCpr&px3U|L9A)C^n1U z8LZ7_wVn=G%-8DNg*>U=)3$7J**2kIHmRDtsmGV<^Uwk`BZ#zBfRdgmxe{M1|D_91etNX?(TubZ! zlW9|u_~xssCxz&FsiuYfdmEA_s?C;(bYi2Sl1jGX>W{lBx2%4C$diU#wH*uP-#u6BOH<_u*AG?w+rQy_QL9a2r}t z)f(qVxU)NC=1>9FGUC53Ye5{SyG2!ep8CooXeyWE9}b%0&NAxY*CvRXT375X&%T|cu5+}$V#28KK1Vf?^)7! zXefud{hpS*byEH-x*lL`k^jRH_^%f}8nm?N2x8iN@P9TH`e?MN^(d_)nAL2G^C&}9 z(!KCxW);X^S8s@q?)*UIzrOqLKJvf5;-f*gAen2G=4Bh(JgrTa*cQJN)uLvE_@=IP z0{Q(N{G8y9d*>Y+45QEgT_5}#?I#Zm zvQAW3X%=XztE&rFWqtalD#vy!2hgQB$SWwgLupe{sBiuWfJ3w+f$C+<hJi{yJufgH3VA7teH-2Qa3d^i6|ZDKgEO*-803CPxMiBNy#M( z}%#q&0g3~fUAFMw&X_kd`b5aqk810%q?E*((7 zWVvTQ4$Rq710AnoKr4XpK_oD67)B*j$-={xQf-( z4?goa+oh5jKlZZn$~Dj^)Fr3(JRaMN4}8>c!Vln?HlZh@g#X`g{m&PD zp3{0KOm!CMMN+{n3nRNFBX?JghIwc1jLVf&EM|(!KMgaUdeITCNwTTXf$VZ=otk35 z%&t)}YlH*@D=m1)9d!z?Mac9r1E79I{a@6ZW~Ge zI%F@fli8T87On}%5e?KPQ_i0ranTiX*&1UBsr0-!4kq~yM)VRplwh>Ia$4{EguWOf zEaeGi`WXU&xg3t(F1)9{a#VAGC6@?x+u!A=TB`y;i-E9*IvQ zDh%XWkBpkU(bLPcn7gUo|PVY+H zC7u{lX5!+RP@TnGFi~8a&hI}VVk#Hk^Qo&j&u7&vN{1Zl>=C{?Y#`}uLpJ?fWpwa( zbYKv9pajD@qB=k%c)iWr987sza(k}c!)2nz$qL@+<@ts%oWEzTp%nvzy)bLCv63Y; zCa5Xs3#ydp{V`EuLgZIw({i5-a%qo^JdRcpe57rzrWsM$b7p_^{3mo^4v1k}JCVZE z|IZfx&vxHr1RE&+WwZ6{)3hQ;zofE9x}LKPG)Ir-$V;+%>LX(t$-0_ur(A}$t7qha zO=I@Nroa_s=z<LHSfyRN3IKvr#$+Bn4>%(!;0xobXR2s9HR=Oa@XVAN4K&8x}o?Mz;Bvy zFHryZKhE%fZVvFG$%xjQX(MUM&^Gv1SyMZ~x!wFfIIcL_NeXFqzj1JN4 z^uxxF@Lfydv1-2hdDQZOPU?0@6F3PRxSd{W+8t*<z=|aN zuv*{D*1F^(JbfLrZ(@|Siqr4f`(rlqpL{l8479rEdnWM(7do(s$|D3{6EpsQ4(xwU zZs2m3Dm_*wP7cmSjUkqq^@cP*%Aog!HA4$?Az}b{vB7RT zgFvtqtuI@~Gi%*T1w7^2JUt;!)s?ohHH4qBWnGnZ2(J2(S0iYF2_eJEX;1Vt0e05ze#t zLwq0^?U6JznC?s28T?Keb0~XJKxjMYM0{T+Cg<)5X8cH&kp$Y$rs%h2KV2{1kf1{> zL;zGlC$i$%y?^h{7IL5jGF4P!;nSkn^v(Y2En+M45Y?<$P`SrG%d&f2J3J%0SE=YV ziOr8i=~0uFkEW0jVB(O&Y(1w>Gghx{cX>1>!$D?VKO;vgE4^zd zn!%*_NLInt4&5lakpZ035{Mr;uUPGximW=dbNh^T#knJ2MZPJj0J@9)s^uYv= zNUEU# z-A2HQSCv}^2{Qnx7iiUuSS4B zRI*dPe7+nR2^GX(CPDU#oaqvoEeZgM0pnM9Dgy0A{}YZK39_8j5$veg3l%X}x?Sk@ zYy7CmTQ(B>vUzedts$h4v<9iBY-)CKYdJ=kVW)hpunMo+FrNS0g145~I}b(18PiIG zu<>0V0cm|XYRY9=RxFR?ze4(Vdiqcr4Lu?FJ}l#R(YIiK&v0ovvPBC&mIbvOzDc0b zxlFT8pSkGSz^x>AoiA;tZ0^iRk?EJ_@LEqBZL$}p?I=-xo0My*7G*XtpYM9r+?}i- zjC|#)?9LqQ;lhM_ZPU0Qs%N7-8R2EF?C)$Z`Q~LfJ}aNsl{=JQhd+7%+OQ zFqK8f@o?e=&(q(jj-hcvr!KSe@q8xloYvP=V1Gx4h$Ce-Je@rB5-ROo5~`a?RuffU z#vr@y7kHIm=>vr(^K_5hbkXP#cTnS985eH(d8g^kmL<(D$#Fc>^g6nd>oY=!649^? z1L`(_H%0h988L<@W5`$?%P>|H_s)Xo<-^(lHn$7k=bK6t zC0Q_pljGGO65Fj9pHmo6kjk+5vL#%C)m9qc@ZFe9e_COpwRGyI(T`B}NN+s(84)4* z&5(z@#>KdYiZA4