refactor: Upgrade unit tests to v2 (#132)

tests/unit/__init__.py

@@ -11,3 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""__init__.py
+
+Init test environment variables.
+"""
+import os
+os.environ['MIN_VALID_EVENTS'] = '10'

tests/unit/fixtures/config.yaml

@@ -0,0 +1,34 @@
+---
+backends:
+  cloud_monitoring:
+    project_id: ${PROJECT_ID}
+exporters:
+  cloud_monitoring:
+    project_id: ${PROJECT_ID}
+error_budget_policies:
+  standard:
+    steps:
+      - name: 1 hour
+        window: 3600
+        burn_rate_threshold: 9
+        alert: true
+        message_alert: Page to defend the SLO
+        message_ok: Last hour on track
+      - name: 12 hours
+        window: 43200
+        burn_rate_threshold: 3
+        alert: true
+        message_alert: Page to defend the SLO
+        message_ok: Last 12 hours on track
+      - name: 7 days
+        window: 604800
+        burn_rate_threshold: 1.5
+        alert: false
+        message_alert: Dev team dedicates 25% of engineers to the reliability backlog
+        message_ok: Last week on track
+      - name: 28 days
+        window: 2419200
+        burn_rate_threshold: 1
+        alert: false
+        message_alert: Freeze release, unless related to reliability or security
+        message_ok: Unfreeze release, per the agreed roll-out policy

tests/unit/fixtures/dummy_backend.py

@@ -0,0 +1,19 @@
+"""dummy_backend.py
+
+Dummy backend implementation for testing.
+"""
+# pylint:disable=missing-class-docstring,missing-function-docstring,unused-argument
+
+
+class DummyBackend:
+
+    def __init__(self, client=None, **config):
+        self.good_events = config.get('good_events', None)
+        self.bad_events = config.get('bad_events', None)
+        self.sli_value = config.get('sli', None)
+
+    def good_bad_ratio(self, timestamp, window, slo_config):
+        return (self.good_events, self.bad_events)
+
+    def sli(self, timestamp, window, slo_config):
+        return self.sli_value

tests/unit/fixtures/dummy_config.json

@@ -0,0 +1,15 @@
+{
+    "backends": {
+        "dummy": {}
+    },
+    "error_budget_policies": {
+        "default": [{
+            "name": "1 hour",
+            "window": 3600,
+            "burn_rate_threshold": 1,
+            "alert": true,
+            "message_alert": "Page to defend the SLO",
+            "message_ok": "Last hour on track"
+        }]
+    }
+}

tests/unit/fixtures/dummy_slo_config.json

@@ -0,0 +1,19 @@
+{
+    "kind": "ServiceLevelObjective",
+    "version": "sre.google.com/v2",
+    "metadata": {
+        "name": "test-test-test",
+        "labels": {
+            "service_name": "test",
+            "feature_name": "test",
+            "slo_name": "test"
+        }
+    },
+    "spec": {
+        "description": "Test dummy backend",
+        "goal": 0.99,
+        "backend": "dummy",
+        "method": "good_bad_ratio",
+        "service_level_indicator": {}
+    }
+}

tests/unit/fixtures/dummy_tests.json

@@ -0,0 +1,66 @@
+{
+    "enough_events": {
+        "method": "good_bad_ratio",
+        "good_events": 5,
+        "bad_events": 5,
+    },
+    "no_good_events": {
+        "method": "good_bad_ratio",
+        "good_events": -1,
+        "bad_events": 15,
+    },
+    "no_bad_events": {
+        "method": "good_bad_ratio",
+        "good_events": 15,
+        "bad_events": -1,
+    },
+    "valid_sli_value": {
+        "method": "sli",
+        "good_events": -1,
+        "bad_events": -1,
+        "sli": 0.991
+    },
+    "no_events": {
+        "method": "good_bad_ratio",
+        "good_events": 0,
+        "bad_events": 0,
+    },
+    "no_good_bad_events": {
+        "method": "good_bad_ratio",
+        "good_events": -1,
+        "bad_events": -1,
+    },
+    "not_enough_events": {
+        "method": "good_bad_ratio",
+        "good_events": 5,
+        "bad_events": 4,
+    },
+    "no_sli_value": {
+        "method": "sli",
+        "good_events": -1,
+        "bad_events": -1,
+    },
+    "no_backend_response_sli": {
+        "method": "sli",
+        "sli": null
+    },
+    "no_backend_response_ratio": {
+        "method": "good_bad_ratio",
+        "good_events": null,
+        "bad_events": null,
+    },
+    "invalid_backend_response_type": {
+        "method": "good_bad_ratio",
+        "good_events": {
+            "data": {
+                "value": 30
+            }
+        },
+        "bad_events": {
+            "data": {
+                "value": 400
+            }
+        },
+        "sli": null
+    }
+}

tests/unit/fixtures/exporters.yaml

@@ -35,16 +35,6 @@
     api_url:    ${DYNATRACE_API_URL}
     api_token:  ${DYNATRACE_API_TOKEN}
 
-  # Old format that will be deprecated in 2.0.0 in favor of the `metrics` block
-  - class: Stackdriver
-    project_id: ${STACKDRIVER_HOST_PROJECT_ID}
-    metric_type: custom.googleapis.com/ebp
-    metric_description: Test old format
-    metric_labels: [good_events_count, bad_events_count]
-    metrics:
-      - error_budget_burn_rate
-
-  # New format ('metrics' block)
   - class: Stackdriver
     project_id: ${STACKDRIVER_HOST_PROJECT_ID}
     metrics:

tests/unit/fixtures/fail_exporter.py

@@ -0,0 +1,13 @@
+"""dummy_exporter.py
+
+Dummy exporter implementation for testing.
+"""
+# pylint: disable=missing-class-docstring
+
+from slo_generator.exporters.base import MetricsExporter
+
+
+class FailExporter(MetricsExporter):
+
+    def export_metric(self, data):
+        raise ValueError("Oops !")

tests/unit/fixtures/slo_config_v1.yaml

@@ -0,0 +1,35 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#            http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+service_name:     gae
+feature_name:     app
+slo_description:  Availability of App Engine app
+slo_name:         availability
+slo_target:       0.95
+backend:
+  class:          Stackdriver
+  method:         good_bad_ratio
+  project_id:     ${STACKDRIVER_HOST_PROJECT_ID}
+  measurement:
+    filter_good:  >
+      project=${GAE_PROJECT_ID}
+      metric.type="appengine.googleapis.com/http/server/response_count"
+      resource.type="gae_app"
+      metric.labels.response_code = 200
+    filter_valid: >
+      project=${GAE_PROJECT_ID}
+      metric.type="appengine.googleapis.com/http/server/response_count"
+exporters:
+- class:          Stackdriver
+  project_id:     ${STACKDRIVER_HOST_PROJECT_ID}

tests/unit/fixtures/slo_config_v2.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#            http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: sre.google.com/v2
+kind: ServiceLevelObjective
+metadata:
+  name: gae-app-availability
+  labels:
+    service_name: gae
+    feature_name: app
+    slo_name: availability
+spec:
+  description: Availability of App Engine app
+  backend: cloud_monitoring
+  method: good_bad_ratio
+  exporters:
+  - cloud_monitoring
+  service_level_indicator:
+    filter_good: >
+      project=${GAE_PROJECT_ID}
+      metric.type="appengine.googleapis.com/http/server/response_count"
+      resource.type="gae_app"
+      metric.labels.response_code = 200
+    filter_valid: >
+      project=${GAE_PROJECT_ID}
+      metric.type="appengine.googleapis.com/http/server/response_count"
+  goal: 0.95

tests/unit/fixtures/slo_report_v1.json

@@ -0,0 +1,30 @@
+{
+    "service_name": "test",
+    "feature_name": "test",
+    "slo_name": "slo-pubsub-acked-msg",
+    "slo_target": 0.9,
+    "slo_description": "Acked Pub/Sub messages over total number of Pub/Sub messages",
+    "error_budget_policy_step_name": "b.Last 12 hours",
+    "error_budget_remaining_minutes": -288.0,
+    "consequence_message": "Page the SRE team to defend the SLO",
+    "error_budget_minutes": 71.99999999999999,
+    "error_minutes": "360.0",
+    "error_budget_target": 0.09999999999999998,
+    "timestamp_human": "2019-09-05 11:55:01.004603 UTC",
+    "timestamp": 1567762279.287761,
+    "cadence": null,
+    "window": 43200,
+    "events_count": 7112,
+    "bad_events_count": 3556,
+    "good_events_count": 3556,
+    "sli_measurement": 0.5,
+    "gap": -0.4,
+    "error_budget_measurement": 0.5,
+    "error_budget_burn_rate": 5.000000000000001,
+    "alerting_burn_rate_threshold": 3.0,
+    "alert": "true",
+    "metadata": {
+        "env": "test",
+        "team": "test"
+    }
+}

tests/unit/fixtures/slo_report_v2.json

@@ -0,0 +1,37 @@
+{
+  "alert": true,
+  "backend": "cloud_monitoring",
+  "bad_events_count": 3556,
+  "description": "Acked Pub/Sub messages over total number of Pub/Sub messages",
+  "exporters": [
+    "cloud_monitoring"
+  ],
+  "consequence_message": "Page the SRE team to defend the SLO",
+  "error_budget_burn_rate": 5.000000000000001,
+  "error_budget_burn_rate_threshold": 3,
+  "error_budget_measurement": 0.5,
+  "error_budget_policy": "default",
+  "error_budget_policy_step_name": "1h",
+  "error_budget_minutes": 71.99999999999999,
+  "error_budget_remaining_minutes": -288,
+  "error_budget_target": 0.09999999999999998,
+  "error_minutes": "360.0",
+  "events_count": 7112,
+  "gap": -0.4,
+  "goal": 0.9,
+  "good_events_count": 3556,
+  "metadata": {
+    "name": "test-slo",
+    "labels": {
+      "service_name": "test",
+      "feature_name": "test",
+      "slo_name": "test",
+      "env": "test",
+      "team": "test"
+    }
+  },
+  "sli_measurement": 0.5,
+  "timestamp": 1567762279.287761,
+  "timestamp_human": "2019-09-05 11:55:01.004603 UTC",
+  "window": 43200
+}