From cf10dcd5e31a777ad0bf9e6b2bdd88186d3475a2 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 18 May 2021 18:28:58 +0200 Subject: [PATCH 1/3] fix: Upgrade unit tests to v2 --- tests/unit/__init__.py | 6 + tests/unit/fixtures/config.yaml | 39 +++++ tests/unit/fixtures/dummy_backend.py | 19 +++ tests/unit/fixtures/dummy_config.json | 15 ++ tests/unit/fixtures/dummy_slo_config.json | 19 +++ tests/unit/fixtures/dummy_tests.json | 66 ++++++++ tests/unit/fixtures/exporters.yaml | 10 -- tests/unit/fixtures/fail_exporter.py | 13 ++ tests/unit/fixtures/slo_report.json | 51 ++++--- tests/unit/fixtures/slo_report_v1.json | 30 ++++ tests/unit/test_cli.py | 73 ++++----- tests/unit/test_compute.py | 91 ++++------- tests/unit/test_report.py | 4 +- tests/unit/test_stubs.py | 177 ++++++---------------- tests/unit/test_utils.py | 39 ++--- 15 files changed, 370 insertions(+), 282 deletions(-) create mode 100644 tests/unit/fixtures/config.yaml create mode 100644 tests/unit/fixtures/dummy_backend.py create mode 100644 tests/unit/fixtures/dummy_config.json create mode 100644 tests/unit/fixtures/dummy_slo_config.json create mode 100644 tests/unit/fixtures/dummy_tests.json create mode 100644 tests/unit/fixtures/fail_exporter.py create mode 100644 tests/unit/fixtures/slo_report_v1.json diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index fab6aabf..bb72f5be 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -11,3 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""__init__.py + +Init test environment variables. +""" +import os +os.environ['MIN_VALID_EVENTS'] = '10' diff --git a/tests/unit/fixtures/config.yaml b/tests/unit/fixtures/config.yaml new file mode 100644 index 00000000..a4d63d3e --- /dev/null +++ b/tests/unit/fixtures/config.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: slo-generator-config +data: + receivers: + cloud_monitoring: + project_id: ${PROJECT_ID} + exporters: + cloud_monitoring: + project_id: ${PROJECT_ID} + error_budget_policies: + standard: + steps: + - name: 1 hour + window: 3600 + burn_rate_threshold: 9 + alert: true + message_alert: Page to defend the SLO + message_ok: Last hour on track + - name: 12 hours + window: 43200 + burn_rate_threshold: 3 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 12 hours on track + - name: 7 days + window: 604800 + burn_rate_threshold: 1.5 + alert: false + message_alert: Dev team dedicates 25% of engineers to the reliability backlog + message_ok: Last week on track + - name: 28 days + window: 2419200 + burn_rate_threshold: 1 + alert: false + message_alert: Freeze release, unless related to reliability or security + message_ok: Unfreeze release, per the agreed roll-out policy diff --git a/tests/unit/fixtures/dummy_backend.py b/tests/unit/fixtures/dummy_backend.py new file mode 100644 index 00000000..0022e541 --- /dev/null +++ b/tests/unit/fixtures/dummy_backend.py @@ -0,0 +1,19 @@ +"""dummy_backend.py + +Dummy backend implementation for testing. +""" +# pylint:disable=missing-class-docstring,missing-function-docstring,unused-argument + + +class DummyBackend: + + def __init__(self, client=None, **config): + self.good_events = config.get('good_events', None) + self.bad_events = config.get('bad_events', None) + self.sli_value = config.get('sli', None) + + def good_bad_ratio(self, timestamp, window, slo_config): + return (self.good_events, self.bad_events) + + def sli(self, timestamp, window, slo_config): + return self.sli_value diff --git a/tests/unit/fixtures/dummy_config.json b/tests/unit/fixtures/dummy_config.json new file mode 100644 index 00000000..862f266a --- /dev/null +++ b/tests/unit/fixtures/dummy_config.json @@ -0,0 +1,15 @@ +{ + "backends": { + "dummy": {} + }, + "error_budget_policies": { + "default": [{ + "name": "1 hour", + "window": 3600, + "burn_rate_threshold": 1, + "alert": true, + "message_alert": "Page to defend the SLO", + "message_ok": "Last hour on track" + }] + } +} diff --git a/tests/unit/fixtures/dummy_slo_config.json b/tests/unit/fixtures/dummy_slo_config.json new file mode 100644 index 00000000..b5015816 --- /dev/null +++ b/tests/unit/fixtures/dummy_slo_config.json @@ -0,0 +1,19 @@ +{ + "kind": "ServiceLevelObjective", + "version": "sre.google.com/v2", + "metadata": { + "name": "test-test-test", + "labels": { + "service_name": "test", + "feature_name": "test", + "slo_name": "test" + } + }, + "spec": { + "description": "Test dummy backend", + "goal": 0.99, + "backend": "dummy", + "method": "good_bad_ratio", + "service_level_indicator": {} + } +} diff --git a/tests/unit/fixtures/dummy_tests.json b/tests/unit/fixtures/dummy_tests.json new file mode 100644 index 00000000..a5c7537a --- /dev/null +++ b/tests/unit/fixtures/dummy_tests.json @@ -0,0 +1,66 @@ +{ + "enough_events": { + "method": "good_bad_ratio", + "good_events": 5, + "bad_events": 5, + }, + "no_good_events": { + "method": "good_bad_ratio", + "good_events": -1, + "bad_events": 15, + }, + "no_bad_events": { + "method": "good_bad_ratio", + "good_events": 15, + "bad_events": -1, + }, + "valid_sli_value": { + "method": "sli", + "good_events": -1, + "bad_events": -1, + "sli": 0.991 + }, + "no_events": { + "method": "good_bad_ratio", + "good_events": 0, + "bad_events": 0, + }, + "no_good_bad_events": { + "method": "good_bad_ratio", + "good_events": -1, + "bad_events": -1, + }, + "not_enough_events": { + "method": "good_bad_ratio", + "good_events": 5, + "bad_events": 4, + }, + "no_sli_value": { + "method": "sli", + "good_events": -1, + "bad_events": -1, + }, + "no_backend_response_sli": { + "method": "sli", + "sli": null + }, + "no_backend_response_ratio": { + "method": "good_bad_ratio", + "good_events": null, + "bad_events": null, + }, + "invalid_backend_response_type": { + "method": "good_bad_ratio", + "good_events": { + "data": { + "value": 30 + } + }, + "bad_events": { + "data": { + "value": 400 + } + }, + "sli": null + } +} diff --git a/tests/unit/fixtures/exporters.yaml b/tests/unit/fixtures/exporters.yaml index f4a281b5..d26adf51 100644 --- a/tests/unit/fixtures/exporters.yaml +++ b/tests/unit/fixtures/exporters.yaml @@ -35,16 +35,6 @@ api_url: ${DYNATRACE_API_URL} api_token: ${DYNATRACE_API_TOKEN} - # Old format that will be deprecated in 2.0.0 in favor of the `metrics` block - - class: Stackdriver - project_id: ${STACKDRIVER_HOST_PROJECT_ID} - metric_type: custom.googleapis.com/ebp - metric_description: Test old format - metric_labels: [good_events_count, bad_events_count] - metrics: - - error_budget_burn_rate - - # New format ('metrics' block) - class: Stackdriver project_id: ${STACKDRIVER_HOST_PROJECT_ID} metrics: diff --git a/tests/unit/fixtures/fail_exporter.py b/tests/unit/fixtures/fail_exporter.py new file mode 100644 index 00000000..430c3da9 --- /dev/null +++ b/tests/unit/fixtures/fail_exporter.py @@ -0,0 +1,13 @@ +"""dummy_exporter.py + +Dummy exporter implementation for testing. +""" +# pylint: disable=missing-class-docstring + +from slo_generator.exporters.base import MetricsExporter + + +class FailExporter(MetricsExporter): + + def export_metric(self, data): + raise ValueError("Oops !") diff --git a/tests/unit/fixtures/slo_report.json b/tests/unit/fixtures/slo_report.json index 5c9626e2..c6db547a 100644 --- a/tests/unit/fixtures/slo_report.json +++ b/tests/unit/fixtures/slo_report.json @@ -1,30 +1,37 @@ { - "service_name": "test", - "feature_name": "test", - "slo_name": "slo-pubsub-acked-msg", - "slo_target": 0.9, - "slo_description": "Acked Pub/Sub messages over total number of Pub/Sub messages", - "error_budget_policy_step_name": "b.Last 12 hours", - "error_budget_remaining_minutes": -288.0, + "alert": true, + "backend": "cloud_monitoring", + "bad_events_count": 3556, + "description": "Acked Pub/Sub messages over total number of Pub/Sub messages", + "exporters": [ + "cloud_monitoring" + ], "consequence_message": "Page the SRE team to defend the SLO", + "error_budget_burn_rate": 5.000000000000001, + "error_budget_burn_rate_threshold": 3, + "error_budget_measurement": 0.5, + "error_budget_policy": "default", + "error_budget_policy_step_name": "1h", "error_budget_minutes": 71.99999999999999, - "error_minutes": "360.0", + "error_budget_remaining_minutes": -288, "error_budget_target": 0.09999999999999998, - "timestamp_human": "2019-09-05 11:55:01.004603 UTC", - "timestamp": 1567762279.287761, - "cadence": null, - "window": 43200, + "error_minutes": "360.0", "events_count": 7112, - "bad_events_count": 3556, - "good_events_count": 3556, - "sli_measurement": 0.5, "gap": -0.4, - "error_budget_measurement": 0.5, - "error_budget_burn_rate": 5.000000000000001, - "alerting_burn_rate_threshold": 3.0, - "alert": "true", + "goal": 0.9, + "good_events_count": 3556, "metadata": { - "env": "test", - "team": "test" - } + "name": "test-slo", + "labels": { + "service_name": "test", + "feature_name": "test", + "slo_name": "test", + "env": "test", + "team": "test" + } + }, + "sli_measurement": 0.5, + "timestamp": 1567762279.287761, + "timestamp_human": "2019-09-05 11:55:01.004603 UTC", + "window": 43200 } diff --git a/tests/unit/fixtures/slo_report_v1.json b/tests/unit/fixtures/slo_report_v1.json new file mode 100644 index 00000000..3c8fdbf6 --- /dev/null +++ b/tests/unit/fixtures/slo_report_v1.json @@ -0,0 +1,30 @@ +{ + "service_name": "test", + "feature_name": "test", + "slo_name": "slo-pubsub-acked-msg", + "slo_target": 0.9, + "slo_description": "Acked Pub/Sub messages over total number of Pub/Sub messages", + "error_budget_policy_step_name": "b.Last 12 hours", + "error_budget_remaining_minutes": -288.0, + "consequence_message": "Page the SRE team to defend the SLO", + "error_budget_minutes": 71.99999999999999, + "error_minutes": "360.0", + "error_budget_target": 0.09999999999999998, + "timestamp_human": "2019-09-05 11:55:01.004603 UTC", + "timestamp": 1567762279.287761, + "cadence": null, + "window": 43200, + "events_count": 7112, + "bad_events_count": 3556, + "good_events_count": 3556, + "sli_measurement": 0.5, + "gap": -0.4, + "error_budget_measurement": 0.5, + "error_budget_burn_rate": 5.000000000000001, + "alerting_burn_rate_threshold": 3.0, + "alert": "true", + "metadata": { + "env": "test", + "team": "test" + } +} diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 0424ee20..036731f0 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -17,7 +17,9 @@ from mock import patch -from slo_generator.cli import cli, parse_args +from click.testing import CliRunner +from slo_generator.cli import main +from slo_generator.utils import load_config from .test_stubs import CTX, mock_sd @@ -26,49 +28,50 @@ class TestCLI(unittest.TestCase): + def setUp(self): - for k, v in CTX.items(): - os.environ[k] = v - slo_config = f'{root}/samples/stackdriver/slo_gae_app_availability.yaml' - eb_policy = f'{root}/samples/error_budget_policy.yaml' + for key, value in CTX.items(): + os.environ[key] = value + slo_config = f'{root}/samples/cloud_monitoring/slo_gae_app_availability.yaml' # noqa: E501 + config = f'{root}/samples/config.yaml' self.slo_config = slo_config - self.eb_policy = eb_policy - - def test_parse_args(self): - args = parse_args([ - '--slo-config', self.slo_config, '--error-budget-policy', - self.eb_policy, '--export' - ]) - self.assertEqual(args.slo_config, self.slo_config) - self.assertEqual(args.error_budget_policy, self.eb_policy) - self.assertEqual(args.export, True) + self.slo_metadata_name = load_config(slo_config, + ctx=CTX)['metadata']['name'] + self.config = config + self.cli = CliRunner() @patch('google.api_core.grpc_helpers.create_channel', return_value=mock_sd(8)) - def test_cli(self, mock): - args = parse_args(['-f', self.slo_config, '-b', self.eb_policy]) - all_reports = cli(args) - len_first_report = len(all_reports[self.slo_config]) - self.assertIn(self.slo_config, all_reports.keys()) - self.assertEqual(len_first_report, 4) + def test_cli_compute(self, mock): + args = ['compute', '-f', self.slo_config, '-c', self.config] + result = self.cli.invoke(main, args) + self.assertEqual(result.exit_code, 0) @patch('google.api_core.grpc_helpers.create_channel', return_value=mock_sd(40)) - def test_cli_folder(self, mock): - args = parse_args( - ['-f', f'{root}/samples/stackdriver', '-b', self.eb_policy]) - all_reports = cli(args) - len_first_report = len(all_reports[self.slo_config]) - self.assertIn(self.slo_config, all_reports.keys()) - self.assertEqual(len_first_report, 4) + def test_cli_compute_folder(self, mock): + args = [ + 'compute', '-f', f'{root}/samples/cloud_monitoring', '-c', + self.config + ] + result = self.cli.invoke(main, args) + self.assertEqual(result.exit_code, 0) + + def test_cli_compute_no_config(self): + args = [ + 'compute', '-f', f'{root}/samples', '-c', + f'{root}/samples/config.yaml' + ] + result = self.cli.invoke(main, args) + self.assertEqual(result.exit_code, 1) + + def test_cli_api(self): + # TODO: Write test + pass - def test_cli_no_config(self): - args = parse_args([ - '-f', f'{root}/samples', '-b', - f'{root}/samples/error_budget_policy.yaml' - ]) - all_reports = cli(args) - self.assertEqual(all_reports, {}) + def test_cli_migrate(self): + # TODO: Write test + pass if __name__ == '__main__': diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index 461fe705..acdec4c4 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -23,25 +23,26 @@ from slo_generator.backends.dynatrace import DynatraceClient from slo_generator.compute import compute, export from slo_generator.exporters.bigquery import BigQueryError -from slo_generator.exporters.base import MetricsExporter, DEFAULT_METRIC_LABELS +from slo_generator.exporters.base import MetricsExporter from .test_stubs import (CTX, load_fixture, load_sample, load_slo_samples, mock_dd_metric_query, mock_dd_metric_send, - mock_dd_slo_get, mock_dd_slo_history, - mock_dt, mock_dt_errors, mock_es, mock_prom, mock_sd, + mock_dd_slo_get, mock_dd_slo_history, mock_dt, + mock_dt_errors, mock_es, mock_prom, mock_sd, mock_ssm_client) warnings.filterwarnings("ignore", message=_CLOUD_SDK_CREDENTIALS_WARNING) -ERROR_BUDGET_POLICY = load_sample('error_budget_policy.yaml', **CTX) -STEPS = len(ERROR_BUDGET_POLICY) -SLO_CONFIGS_SD = load_slo_samples('stackdriver', **CTX) -SLO_CONFIGS_SDSM = load_slo_samples('stackdriver_service_monitoring', **CTX) -SLO_CONFIGS_PROM = load_slo_samples('prometheus', **CTX) -SLO_CONFIGS_ES = load_slo_samples('elasticsearch', **CTX) -SLO_CONFIGS_DD = load_slo_samples('datadog', **CTX) -SLO_CONFIGS_DT = load_slo_samples('dynatrace', **CTX) +CONFIG = load_sample('config.yaml', CTX) +STEPS = len(CONFIG['error_budget_policies']['default']['steps']) +SLO_CONFIGS_SD = load_slo_samples('cloud_monitoring', CTX) +SLO_CONFIGS_SDSM = load_slo_samples('cloud_service_monitoring', CTX) +SLO_CONFIGS_PROM = load_slo_samples('prometheus', CTX) +SLO_CONFIGS_ES = load_slo_samples('elasticsearch', CTX) +SLO_CONFIGS_DD = load_slo_samples('datadog', CTX) +SLO_CONFIGS_DT = load_slo_samples('dynatrace', CTX) SLO_REPORT = load_fixture('slo_report.json') -EXPORTERS = load_fixture('exporters.yaml', **CTX) +SLO_REPORT_V1 = load_fixture('slo_report_v1.json') +EXPORTERS = load_fixture('exporters.yaml', CTX) BQ_ERROR = load_fixture('bq_error.json') # Pub/Sub methods to patch @@ -53,8 +54,8 @@ # Service Monitoring method to patch # pylint: ignore=E501 SSM_MOCKS = [ - "slo_generator.backends.stackdriver_service_monitoring.ServiceMonitoringServiceClient", # noqa: E501 - "slo_generator.backends.stackdriver_service_monitoring.SSM.to_json" + "slo_generator.backends.cloud_service_monitoring.ServiceMonitoringServiceClient", # noqa: E501 + "slo_generator.backends.cloud_service_monitoring.SSM.to_json" ] @@ -66,7 +67,7 @@ class TestCompute(unittest.TestCase): def test_compute_stackdriver(self, mock): for config in SLO_CONFIGS_SD: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch(SSM_MOCKS[0], return_value=mock_ssm_client()) @patch(SSM_MOCKS[1], @@ -76,7 +77,7 @@ def test_compute_stackdriver(self, mock): def test_compute_ssm(self, *mocks): for config in SLO_CONFIGS_SDSM: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch(SSM_MOCKS[0], return_value=mock_ssm_client()) @patch(SSM_MOCKS[1], @@ -88,22 +89,19 @@ def test_compute_ssm(self, *mocks): def test_compute_ssm_delete_export(self, *mocks): for config in SLO_CONFIGS_SDSM: with self.subTest(config=config): - compute(config, - ERROR_BUDGET_POLICY, - delete=True, - do_export=True) + compute(config, CONFIG, delete=True, do_export=True) @patch.object(Prometheus, 'query', mock_prom) def test_compute_prometheus(self): for config in SLO_CONFIGS_PROM: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch.object(Elasticsearch, 'search', mock_es) def test_compute_elasticsearch(self): for config in SLO_CONFIGS_ES: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch.object(Metric, 'query', mock_dd_metric_query) @patch.object(ServiceLevelObjective, 'history', mock_dd_slo_history) @@ -111,13 +109,13 @@ def test_compute_elasticsearch(self): def test_compute_datadog(self): for config in SLO_CONFIGS_DD: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch.object(DynatraceClient, 'request', side_effect=mock_dt) def test_compute_dynatrace(self, mock): for config in SLO_CONFIGS_DT: with self.subTest(config=config): - compute(config, ERROR_BUDGET_POLICY) + compute(config, CONFIG) @patch(PUBSUB_MOCKS[0]) @patch(PUBSUB_MOCKS[1]) @@ -163,51 +161,16 @@ def test_export_dynatrace_error(self, mock): codes = [r[0]['response']['error']['code'] for r in responses] self.assertTrue(all(code == 429 for code in codes)) - @patch("google.api_core.grpc_helpers.create_channel", - return_value=mock_sd(STEPS)) - def test_export_deprecated(self, mock): - with self.assertWarns(FutureWarning): - export(SLO_REPORT, EXPORTERS[6]) - - def test_metrics_exporter_build_metrics(self): - exporter = MetricsExporter() - metric = EXPORTERS[7]['metrics'][0] - labels = {} - metric_labels = { - label: str(SLO_REPORT[label]) - for label in DEFAULT_METRIC_LABELS - if label != 'metadata' - } - metadata_labels = SLO_REPORT['metadata'].items() - additional_labels = { - 'good_events_count': str(SLO_REPORT['good_events_count']), - 'bad_events_count': str(SLO_REPORT['bad_events_count']), - } - labels.update(metric_labels) - labels.update(additional_labels) - labels.update(metadata_labels) - metric_expected = { - 'name': 'error_budget_burn_rate', - 'description': "", - 'value': SLO_REPORT['error_budget_burn_rate'], - 'timestamp': SLO_REPORT['timestamp'], - 'labels': labels, - 'additional_labels': metric['additional_labels'] - } - metric = exporter.build_metric(data=SLO_REPORT, metric=metric) - self.assertEqual(labels, metric['labels']) - self.assertEqual(metric, metric_expected) - def test_metrics_exporter_build_data_labels(self): exporter = MetricsExporter() - data = SLO_REPORT + data = SLO_REPORT_V1 labels = ['service_name', 'slo_name', 'metadata'] result = exporter.build_data_labels(data, labels) expected = { - 'service_name': SLO_REPORT['service_name'], - 'slo_name': SLO_REPORT['slo_name'], - 'env': SLO_REPORT['metadata']['env'], - 'team': SLO_REPORT['metadata']['team'] + 'service_name': SLO_REPORT_V1['service_name'], + 'slo_name': SLO_REPORT_V1['slo_name'], + 'env': SLO_REPORT_V1['metadata']['env'], + 'team': SLO_REPORT_V1['metadata']['team'] } self.assertEqual(result, expected) diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index 22f302d2..347ae8b9 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -20,6 +20,7 @@ class TestReport(unittest.TestCase): + def test_report_enough_events(self): report_cfg = mock_slo_report("enough_events") report = SLOReport(**report_cfg) @@ -43,8 +44,7 @@ def test_report_valid_sli_value(self): report_cfg = mock_slo_report("valid_sli_value") report = SLOReport(**report_cfg) self.assertTrue(report.valid) - self.assertEqual(report.sli_measurement, - report_cfg['config']['backend']['sli']) + self.assertEqual(report.sli_measurement, report_cfg['backend']['sli']) self.assertEqual(report.alert, False) def test_report_no_events(self): diff --git a/tests/unit/test_stubs.py b/tests/unit/test_stubs.py index 23bcd28d..5b43d72b 100644 --- a/tests/unit/test_stubs.py +++ b/tests/unit/test_stubs.py @@ -15,7 +15,6 @@ Stubs for mocking backends and exporters. """ -import copy import json import os import sys @@ -23,13 +22,14 @@ from types import ModuleType from google.cloud.monitoring_v3.proto import metric_service_pb2 -from slo_generator.utils import list_slo_configs, parse_config +from slo_generator.utils import load_configs, load_config TEST_DIR = os.path.dirname(os.path.abspath(__file__)) SAMPLE_DIR = os.path.join(os.path.dirname(os.path.dirname(TEST_DIR)), "samples/") CTX = { + 'PROJECT_ID': 'fake', 'PUBSUB_PROJECT_ID': 'fake', 'PUBSUB_TOPIC_NAME': 'fake', 'GAE_PROJECT_ID': 'fake', @@ -57,26 +57,6 @@ 'DYNATRACE_API_TOKEN': 'fake' } -CUSTOM_BACKEND_CODE = """ -class DummyBackend: - def __init__(self, client=None, **config): - self.good_events = config.get('good_events', None) - self.bad_events = config.get('bad_events', None) - self.sli_value = config.get('sli', None) - - def good_bad_ratio(self, timestamp, window, slo_config): - return (self.good_events, self.bad_events) - - def sli(self, timestamp, window, slo_config): - return self.sli_value -""" - -FAIL_EXPORTER_CODE = """ -from slo_generator.exporters.base import MetricsExporter -class FailExporter(MetricsExporter): - def export_metric(self, data): - raise ValueError("Oops !") -""" def add_dynamic(name, code, type): """Dynamically add a backend or exporter to slo-generator. @@ -92,114 +72,28 @@ def add_dynamic(name, code, type): exec(code, mod.__dict__) -# Add backends / exporters for testing purposes -add_dynamic('dummy', CUSTOM_BACKEND_CODE, 'backends') -add_dynamic('fail', FAIL_EXPORTER_CODE, 'exporters') - - -CUSTOM_BASE_CONFIG = { - "service_name": "test", - "feature_name": "test", - "slo_name": "test", - "slo_description": "Test dummy backend", - "slo_target": 0.99, - "backend": { - "class": "Dummy", - } -} - -CUSTOM_STEP = { - "error_budget_policy_step_name": "1 hour", - "measurement_window_seconds": 3600, - "alerting_burn_rate_threshold": 1, - "urgent_notification": True, - "overburned_consequence_message": "Page to defend the SLO", - "achieved_consequence_message": "Last hour on track" -} - -CUSTOM_TESTS = { - "enough_events": { - 'method': 'good_bad_ratio', - 'good_events': 5, - 'bad_events': 5, - }, - "no_good_events": { - 'method': 'good_bad_ratio', - 'good_events': -1, - 'bad_events': 15, - }, - "no_bad_events": { - 'method': 'good_bad_ratio', - 'good_events': 15, - 'bad_events': -1, - }, - "valid_sli_value": { - 'method': 'sli', - 'good_events': -1, - 'bad_events': -1, - 'sli': 0.991 - }, - "no_events": { - 'method': 'good_bad_ratio', - 'good_events': 0, - 'bad_events': 0, - }, - "no_good_bad_events": { - 'method': 'good_bad_ratio', - 'good_events': -1, - 'bad_events': -1, - }, - "not_enough_events": { - 'method': 'good_bad_ratio', - 'good_events': 5, - 'bad_events': 4, - }, - "no_sli_value": { - 'method': 'sli', - 'good_events': -1, - 'bad_events': -1, - }, - "no_backend_response_sli": { - 'method': 'sli', - 'sli': None - }, - "no_backend_response_ratio": { - 'method': 'good_bad_ratio', - 'good_events': None, - 'bad_events': None, - }, - 'invalid_backend_response_type': { - 'method': 'good_bad_ratio', - 'good_events': { - 'data': { - 'value': 30 - } - }, - 'bad_events': { - 'data': { - 'value': 400 - } - }, - 'sli': None - } -} - - def mock_slo_report(key): - """Mock SLO report config with edge cases contained in CUSTOM_TESTS. + """Mock SLO report config with edge cases contained in DUMMY_TESTS. Args: - key (str): Key identifying which config to pick from CUSTOM_TESTS. + key (str): Key identifying which config to pick from DUMMY_TESTS. Returns: dict: Dict configuration for SLOReport class. """ - config = copy.deepcopy(CUSTOM_BASE_CONFIG) - config["backend"].update(CUSTOM_TESTS[key]) + slo_config = load_fixture('dummy_slo_config.json') + ebp_step = load_fixture( + 'dummy_config.json')['error_budget_policies']['default'][0] + dummy_tests = load_fixture('dummy_tests.json') + backend = dummy_tests[key] + slo_config['spec']['method'] = backend['method'] + backend['name'] = 'dummy' + backend['class'] = 'Dummy' timestamp = time.time() return { - "config": config, - "step": CUSTOM_STEP, + "config": slo_config, + "backend": backend, + "step": ebp_step, "timestamp": timestamp, "client": None, "delete": False @@ -209,6 +103,7 @@ def mock_slo_report(key): # pylint: disable=too-few-public-methods class MultiCallableStub: """Stub for the grpc.UnaryUnaryMultiCallable interface.""" + def __init__(self, method, channel_stub): self.method = method self.channel_stub = channel_stub @@ -231,6 +126,7 @@ def __call__(self, request, timeout=None, metadata=None, credentials=None): # pylint: disable=R0903 class ChannelStub: """Stub for the grpc.Channel interface.""" + def __init__(self, responses=[]): self.responses = responses self.requests = [] @@ -344,6 +240,7 @@ def mock_dt(*args, **kwargs): elif args[0] == 'put' and args[1] == 'timeseries': return {} + def mock_dt_errors(*args, **kwargs): """Mock Dynatrace response with errors.""" if args[0] == 'get' and args[1] == 'timeseries': @@ -358,6 +255,7 @@ def mock_dt_errors(*args, **kwargs): elif args[0] == 'put' and args[1] == 'timeseries': return load_fixture('dt_error_rate.json') + class dotdict(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get @@ -383,6 +281,7 @@ def dotize(data): class mock_ssm_client: """Fake Service Monitoring API client.""" + def __init__(self): self.services = [dotize(s) for s in load_fixture('ssm_services.json')] self.service_level_objectives = [ @@ -425,7 +324,19 @@ def to_json(data): return data -def load_fixture(filename, **ctx): +def get_fixture_path(filename): + """Get path for a fixture file. + + Args: + filename (str): Filename of file in fixtures/. + + Returns: + str: Full path of file in fixtures/. + """ + return os.path.join(TEST_DIR, "fixtures/", filename) + + +def load_fixture(filename, ctx=os.environ): """Load a fixture from the test/fixtures/ directory and replace context environmental variables in it. @@ -436,11 +347,11 @@ def load_fixture(filename, **ctx): Returns: dict: Loaded fixture. """ - filename = os.path.join(TEST_DIR, "fixtures/", filename) - return parse_config(filename, ctx) + path = get_fixture_path(filename) + return load_config(path, ctx=ctx) -def load_sample(filename, **ctx): +def load_sample(filename, ctx=os.environ): """Load a sample from the samples/ directory and replace context environmental variables in it. @@ -452,10 +363,10 @@ def load_sample(filename, **ctx): dict: Loaded sample. """ filename = os.path.join(SAMPLE_DIR, filename) - return parse_config(filename, ctx) + return load_config(filename, ctx=ctx) -def load_slo_samples(folder_path, **ctx): +def load_slo_samples(folder_path, ctx=os.environ): """List and load all SLO samples from folder path. Args: @@ -465,7 +376,11 @@ def load_slo_samples(folder_path, **ctx): Returns: list: List of loaded SLO configs. """ - return [ - load_sample(filename, **ctx) - for filename in list_slo_configs(f'{SAMPLE_DIR}/{folder_path}') - ] + return load_configs(f'{SAMPLE_DIR}/{folder_path}', ctx) + + +# Add custom backends / exporters for testing purposes +DUMMY_BACKEND_CODE = open(get_fixture_path('dummy_backend.py')).read() +FAIL_EXPORTER_CODE = open(get_fixture_path('fail_exporter.py')).read() +add_dynamic('dummy', DUMMY_BACKEND_CODE, 'backends') +add_dynamic('fail', FAIL_EXPORTER_CODE, 'exporters') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7acb9346..012a4ab6 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -19,6 +19,7 @@ class TestUtils(unittest.TestCase): + def test_get_human_time(self): # Timezones tz_1 = 'Europe/Paris' @@ -42,54 +43,56 @@ def test_get_human_time(self): self.assertEqual(human_chicago_2, utc_time_2 + "-05:00") def test_get_backend_cls(self): - res1 = get_backend_cls("Stackdriver") + res1 = get_backend_cls("CloudMonitoring") res2 = get_backend_cls("Prometheus") - self.assertEqual(res1.__name__, "StackdriverBackend") - self.assertEqual(res1.__module__, "slo_generator.backends.stackdriver") + self.assertEqual(res1.__name__, "CloudMonitoringBackend") + self.assertEqual(res1.__module__, + "slo_generator.backends.cloud_monitoring") self.assertEqual(res2.__name__, "PrometheusBackend") self.assertEqual(res2.__module__, "slo_generator.backends.prometheus") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_backend_cls("UndefinedBackend") def test_get_backend_dynamic_cls(self): res1 = get_backend_cls("pathlib.Path") self.assertEqual(res1.__name__, "Path") self.assertEqual(res1.__module__, "pathlib") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_exporter_cls("foo.bar.DoesNotExist") def test_get_exporter_cls(self): - res1 = get_exporter_cls("Stackdriver") + res1 = get_exporter_cls("CloudMonitoring") res2 = get_exporter_cls("Pubsub") res3 = get_exporter_cls("Bigquery") - self.assertEqual(res1.__name__, "StackdriverExporter") - self.assertEqual(res1.__module__, "slo_generator.exporters.stackdriver") + self.assertEqual(res1.__name__, "CloudMonitoringExporter") + self.assertEqual(res1.__module__, + "slo_generator.exporters.cloud_monitoring") self.assertEqual(res2.__name__, "PubsubExporter") self.assertEqual(res2.__module__, "slo_generator.exporters.pubsub") self.assertEqual(res3.__name__, "BigqueryExporter") self.assertEqual(res3.__module__, "slo_generator.exporters.bigquery") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_exporter_cls("UndefinedExporter") def test_get_exporter_dynamic_cls(self): res1 = get_exporter_cls("pathlib.Path") self.assertEqual(res1.__name__, "Path") self.assertEqual(res1.__module__, "pathlib") - with self.assertRaises(ModuleNotFoundError): + with self.assertWarns(ImportWarning): get_exporter_cls("foo.bar.DoesNotExist") def test_import_dynamic(self): - res1 = import_dynamic("slo_generator.backends.stackdriver", - "StackdriverBackend", + res1 = import_dynamic("slo_generator.backends.cloud_monitoring", + "CloudMonitoringBackend", prefix="backend") - res2 = import_dynamic("slo_generator.exporters.stackdriver", - "StackdriverExporter", + res2 = import_dynamic("slo_generator.exporters.cloud_monitoring", + "CloudMonitoringExporter", prefix="exporter") - self.assertEqual(res1.__name__, "StackdriverBackend") - self.assertEqual(res2.__name__, "StackdriverExporter") - with self.assertRaises(ModuleNotFoundError): + self.assertEqual(res1.__name__, "CloudMonitoringBackend") + self.assertEqual(res2.__name__, "CloudMonitoringExporter") + with self.assertWarns(ImportWarning): import_dynamic("slo_generator.backends.unknown", - "StackdriverUnknown", + "CloudMonitoringUnknown", prefix="unknown") From 0164921aff8f45cb3c09b2df3ee9e5a405d7cb98 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Sat, 22 May 2021 16:19:39 +0200 Subject: [PATCH 2/3] Add tests for migrator --- tests/unit/fixtures/slo_config_v1.yaml | 35 +++++++++++++++++ tests/unit/fixtures/slo_config_v2.yaml | 38 +++++++++++++++++++ .../{slo_report.json => slo_report_v2.json} | 0 tests/unit/test_compute.py | 2 +- tests/unit/test_migrate.py | 36 ++++++++++++++++++ 5 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 tests/unit/fixtures/slo_config_v1.yaml create mode 100644 tests/unit/fixtures/slo_config_v2.yaml rename tests/unit/fixtures/{slo_report.json => slo_report_v2.json} (100%) create mode 100644 tests/unit/test_migrate.py diff --git a/tests/unit/fixtures/slo_config_v1.yaml b/tests/unit/fixtures/slo_config_v1.yaml new file mode 100644 index 00000000..5b1baf95 --- /dev/null +++ b/tests/unit/fixtures/slo_config_v1.yaml @@ -0,0 +1,35 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +service_name: gae +feature_name: app +slo_description: Availability of App Engine app +slo_name: availability +slo_target: 0.95 +backend: + class: Stackdriver + method: good_bad_ratio + project_id: ${STACKDRIVER_HOST_PROJECT_ID} + measurement: + filter_good: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + metric.labels.response_code = 200 + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" +exporters: +- class: Stackdriver + project_id: ${STACKDRIVER_HOST_PROJECT_ID} diff --git a/tests/unit/fixtures/slo_config_v2.yaml b/tests/unit/fixtures/slo_config_v2.yaml new file mode 100644 index 00000000..0dfa7cde --- /dev/null +++ b/tests/unit/fixtures/slo_config_v2.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: sre.google.com/v2 +kind: ServiceLevelObjective +metadata: + name: gae-app-availability + labels: + service_name: gae + feature_name: app + slo_name: availability +spec: + description: Availability of App Engine app + backend: cloud_monitoring + method: good_bad_ratio + exporters: + - cloud_monitoring + service_level_indicator: + filter_good: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + resource.type="gae_app" + metric.labels.response_code = 200 + filter_valid: > + project=${GAE_PROJECT_ID} + metric.type="appengine.googleapis.com/http/server/response_count" + goal: 0.95 diff --git a/tests/unit/fixtures/slo_report.json b/tests/unit/fixtures/slo_report_v2.json similarity index 100% rename from tests/unit/fixtures/slo_report.json rename to tests/unit/fixtures/slo_report_v2.json diff --git a/tests/unit/test_compute.py b/tests/unit/test_compute.py index acdec4c4..5aa31257 100644 --- a/tests/unit/test_compute.py +++ b/tests/unit/test_compute.py @@ -40,7 +40,7 @@ SLO_CONFIGS_ES = load_slo_samples('elasticsearch', CTX) SLO_CONFIGS_DD = load_slo_samples('datadog', CTX) SLO_CONFIGS_DT = load_slo_samples('dynatrace', CTX) -SLO_REPORT = load_fixture('slo_report.json') +SLO_REPORT = load_fixture('slo_report_v2.json') SLO_REPORT_V1 = load_fixture('slo_report_v1.json') EXPORTERS = load_fixture('exporters.yaml', CTX) BQ_ERROR = load_fixture('bq_error.json') diff --git a/tests/unit/test_migrate.py b/tests/unit/test_migrate.py new file mode 100644 index 00000000..c3ac5498 --- /dev/null +++ b/tests/unit/test_migrate.py @@ -0,0 +1,36 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from slo_generator.migrations.migrator import slo_config_v1tov2 +from .test_stubs import load_fixture + + +class TestMigrator(unittest.TestCase): + + def setUp(self): + self.slo_config_v1 = load_fixture('slo_config_v1.yaml') + self.slo_config_v2 = load_fixture('slo_config_v2.yaml') + self.shared_config = { + 'backends': {}, + 'exporters': {}, + 'error_budget_policies': {} + } + + def test_migrate_v1_to_v2(self): + slo_config_migrated = slo_config_v1tov2(self.slo_config_v1, + self.shared_config, + quiet=True) + self.assertDictEqual(slo_config_migrated, self.slo_config_v2) From 433a69e58ee06fc1750a3fbaccf77a6e063c7f96 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 31 May 2021 09:31:44 +0200 Subject: [PATCH 3/3] Update config --- tests/unit/fixtures/config.yaml | 71 +++++++++++++++------------------ 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/tests/unit/fixtures/config.yaml b/tests/unit/fixtures/config.yaml index a4d63d3e..057c8cf4 100644 --- a/tests/unit/fixtures/config.yaml +++ b/tests/unit/fixtures/config.yaml @@ -1,39 +1,34 @@ --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: slo-generator-config -data: - receivers: - cloud_monitoring: - project_id: ${PROJECT_ID} - exporters: - cloud_monitoring: - project_id: ${PROJECT_ID} - error_budget_policies: - standard: - steps: - - name: 1 hour - window: 3600 - burn_rate_threshold: 9 - alert: true - message_alert: Page to defend the SLO - message_ok: Last hour on track - - name: 12 hours - window: 43200 - burn_rate_threshold: 3 - alert: true - message_alert: Page to defend the SLO - message_ok: Last 12 hours on track - - name: 7 days - window: 604800 - burn_rate_threshold: 1.5 - alert: false - message_alert: Dev team dedicates 25% of engineers to the reliability backlog - message_ok: Last week on track - - name: 28 days - window: 2419200 - burn_rate_threshold: 1 - alert: false - message_alert: Freeze release, unless related to reliability or security - message_ok: Unfreeze release, per the agreed roll-out policy +backends: + cloud_monitoring: + project_id: ${PROJECT_ID} +exporters: + cloud_monitoring: + project_id: ${PROJECT_ID} +error_budget_policies: + standard: + steps: + - name: 1 hour + window: 3600 + burn_rate_threshold: 9 + alert: true + message_alert: Page to defend the SLO + message_ok: Last hour on track + - name: 12 hours + window: 43200 + burn_rate_threshold: 3 + alert: true + message_alert: Page to defend the SLO + message_ok: Last 12 hours on track + - name: 7 days + window: 604800 + burn_rate_threshold: 1.5 + alert: false + message_alert: Dev team dedicates 25% of engineers to the reliability backlog + message_ok: Last week on track + - name: 28 days + window: 2419200 + burn_rate_threshold: 1 + alert: false + message_alert: Freeze release, unless related to reliability or security + message_ok: Unfreeze release, per the agreed roll-out policy