From b75faed83ca1e9a640f36fc549f26a4e55425548 Mon Sep 17 00:00:00 2001 From: Lindsey Whitley Date: Mon, 7 Aug 2023 16:32:42 -0500 Subject: [PATCH 1/5] feat: backup SLOs (#1) * Add specs for slos * Add SLOs resource --- bin/datadog_backup | 5 +- lib/datadog_backup.rb | 1 + lib/datadog_backup/slos.rb | 53 ++++++++ spec/datadog_backup/slos_spec.rb | 210 +++++++++++++++++++++++++++++++ 4 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 lib/datadog_backup/slos.rb create mode 100644 spec/datadog_backup/slos_spec.rb diff --git a/bin/datadog_backup b/bin/datadog_backup index 1ab5823..d5c2678 100755 --- a/bin/datadog_backup +++ b/bin/datadog_backup @@ -48,6 +48,9 @@ def prereqs(defaults) # rubocop:disable Metrics/AbcSize opts.on('--dashboards-only') do result[:resources] = [DatadogBackup::Dashboards] end + opts.on('--slos-only') do + result[:resources] = [DatadogBackup::SLOs] + end opts.on('--synthetics-only') do result[:resources] = [DatadogBackup::Synthetics] end @@ -83,7 +86,7 @@ defaults = { action: nil, backup_dir: File.join(ENV.fetch('PWD'), 'backup'), diff_format: :color, - resources: [DatadogBackup::Dashboards, DatadogBackup::Monitors, DatadogBackup::Synthetics], + resources: [DatadogBackup::Dashboards, DatadogBackup::Monitors, DatadogBackup::SLOs, DatadogBackup::Synthetics], output_format: :yaml, force_restore: false, disable_array_sort: false diff --git a/lib/datadog_backup.rb b/lib/datadog_backup.rb index b458aea..49e761f 100644 --- a/lib/datadog_backup.rb +++ b/lib/datadog_backup.rb @@ -8,6 +8,7 @@ require_relative 'datadog_backup/resources' require_relative 'datadog_backup/dashboards' require_relative 'datadog_backup/monitors' +require_relative 'datadog_backup/slos' require_relative 'datadog_backup/synthetics' require_relative 'datadog_backup/thread_pool' require_relative 'datadog_backup/version' diff --git a/lib/datadog_backup/slos.rb b/lib/datadog_backup/slos.rb new file mode 100644 index 0000000..9959d68 --- /dev/null +++ b/lib/datadog_backup/slos.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module DatadogBackup + # SLO specific overrides for backup and restore. + class SLOs < Resources + def all + get_all.fetch('data') + end + + def backup + LOGGER.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads") + futures = all.map do |slo| + Concurrent::Promises.future_on(::DatadogBackup::ThreadPool::TPOOL, slo) do |board| + id = board[id_keyname] + get_and_write_file(id) + end + end + + watcher = ::DatadogBackup::ThreadPool.watcher + watcher.join if watcher.status + + Concurrent::Promises.zip(*futures).value! + end + + def get_by_id(id) + begin + slo = except(get(id)) + rescue Faraday::ResourceNotFound => e + slo = {} + end + except(slo) + end + + def initialize(options) + super(options) + @banlist = %w[modified_at url].freeze + end + + private + + def api_version + 'v1' + end + + def api_resource_name + 'slo' + end + + def id_keyname + 'id' + end + end +end diff --git a/spec/datadog_backup/slos_spec.rb b/spec/datadog_backup/slos_spec.rb new file mode 100644 index 0000000..05c763b --- /dev/null +++ b/spec/datadog_backup/slos_spec.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe DatadogBackup::SLOs do + let(:stubs) { Faraday::Adapter::Test::Stubs.new } + let(:api_client_double) { Faraday.new { |f| f.adapter :test, stubs } } + let(:tempdir) { Dir.mktmpdir } + let(:slos) do + slos = described_class.new( + action: 'backup', + backup_dir: tempdir, + output_format: :json, + resources: [] + ) + allow(slos).to receive(:api_service).and_return(api_client_double) + return slos + end + let(:fetched_slos) do + { + "data"=>[ + {"id"=>"abc-123", "name"=>"CI Stability", "tags"=>["kind:availability", "team:my_team"], "monitor_tags"=>[], "thresholds"=>[{"timeframe"=>"7d", "target"=>98.0, "target_display"=>"98."}, {"timeframe"=>"30d", "target"=>98.0, "target_display"=>"98."}, {"timeframe"=>"90d", "target"=>98.0, "target_display"=>"98."}], "type"=>"metric", "type_id"=>1, "description"=>"something helpful", "timeframe"=>"30d", "target_threshold"=>98.0, "query"=>{"denominator"=>"sum:metric.ci_things{*}.as_count()", "numerator"=>"sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count()"}, "creator"=>{"name"=>"Thelma Patterson", "handle"=>"thelma.patterson@example.com", "email"=>"thelma.patterson@example.com"}, "created_at"=>1571335531, "modified_at"=>1687844157}, + {"id"=>"sbc-124", "name"=>"A Latency SLO", "tags"=>["team:my_team", "kind:latency"], "monitor_tags"=>[], "thresholds"=>[{"timeframe"=>"7d", "target"=>95.0, "target_display"=>"95."}, {"timeframe"=>"30d", "target"=>95.0, "target_display"=>"95."}, {"timeframe"=>"90d", "target"=>95.0, "target_display"=>"95."}], "type"=>"monitor", "type_id"=>0, "description"=>"", "timeframe"=>"30d", "target_threshold"=>95.0, "monitor_ids"=>[13158755], "creator"=>{"name"=>"Louise Montague", "handle"=>"louise.montague@example.com", "email"=>"louise.montague@example.com"}, "created_at"=>1573162531, "modified_at"=>1685819875} + ], + "error"=>nil, + "metadata"=>{"page"=>{"total_count"=>359, "total_filtered_count"=>359}} + } + end + let(:slo_abc_123) do + { + "data" => { + "id" => "abc-123", + "name" => "CI Stability", + "tags" => [ + "kind:availability", + "team:my_team", + ], + "monitor_tags" => [], + "thresholds" => [ + { + "timeframe" => "7d", + "target" => 98.0, + "target_display" => "98." + }, + { + "timeframe" => "30d", + "target" => 98.0, + "target_display" => "98." + }, + { + "timeframe" => "90d", + "target" => 98.0, + "target_display" => "98." + } + ], + "type" => "metric", + "type_id" => 1, + "description" => "something helpful", + "timeframe" => "30d", + "target_threshold" => 98.0, + "query" => { + "denominator" => "sum:metric.ci_things{*}.as_count()", + "numerator" => "sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count()" + }, + "creator" => { + "name" => "Thelma Patterson", + "handle" => "thelma.patterson@example.com", + "email" => "thelma.patterson@example.com" + }, + "created_at" => 1571335531, + "modified_at" => 1687844157 + }, + "error" => nil + } + end + let(:slo_sbc_124) do + { + "data" => { + "id" => "sbc-124", + "name" => "A Latency SLO", + "tags" => [ + "kind:latency", + "team:my_team", + ], + "monitor_tags" => [], + "thresholds" => [ + { + "timeframe" => "7d", + "target" => 98.0, + "target_display" => "98." + }, + { + "timeframe" => "30d", + "target" => 98.0, + "target_display" => "98." + }, + { + "timeframe" => "90d", + "target" => 98.0, + "target_display" => "98." + } + ], + "type" => "monitor", + "type_id"=>0, + "description"=>"", + "timeframe"=>"30d", + "target_threshold"=>95.0, + "monitor_ids"=>[ 13158755 ], + "creator"=>{ + "name"=>"Louise Montague", + "handle"=>"louise.montague@example.com", + "email"=>"louise.montague@example.com" + }, + "created_at"=>1573162531, + "modified_at"=>1685819875 + }, + "error" => nil + } + end + let(:all_slos) { respond_with200(fetched_slos) } + let(:example_slo1) { respond_with200(slo_abc_123) } + let(:example_slo2) { respond_with200(slo_sbc_124) } + + before do + stubs.get('/api/v1/slo') { all_slos } + stubs.get('/api/v1/slo/abc-123') { example_slo1 } + stubs.get('/api/v1/slo/sbc-124') { example_slo2 } + end + + describe '#backup' do + subject { slos.backup } + + it 'is expected to create two files' do + file1 = instance_double(File) + allow(File).to receive(:open).with(slos.filename('abc-123'), 'w').and_return(file1) + allow(file1).to receive(:write) + allow(file1).to receive(:close) + + file2 = instance_double(File) + allow(File).to receive(:open).with(slos.filename('sbc-124'), 'w').and_return(file2) + allow(file2).to receive(:write) + allow(file2).to receive(:close) + + slos.backup + expect(file1).to have_received(:write).with(::JSON.pretty_generate(slo_abc_123.deep_sort)) + expect(file2).to have_received(:write).with(::JSON.pretty_generate(slo_sbc_124.deep_sort)) + end + end + + describe '#filename' do + subject { slos.filename('abc-123') } + + it { is_expected.to eq("#{tempdir}/slos/abc-123.json") } + end + + describe '#get_by_id' do + subject { slos.get_by_id('abc-123') } + + it { is_expected.to eq slo_abc_123 } + end + + describe '#diff' do + it 'calls the api only once' do + slos.write_file('{"a":"b"}', slos.filename('abc-123')) + expect(slos.diff('abc-123')).to eq(<<~EODASH + --- + -data: + - created_at: 1571335531 + - creator: + - email: thelma.patterson@example.com + - handle: thelma.patterson@example.com + - name: Thelma Patterson + - description: something helpful + - id: abc-123 + - modified_at: 1687844157 + - monitor_tags: [] + - name: CI Stability + - query: + - denominator: sum:metric.ci_things{*}.as_count() + - numerator: sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count() + - tags: + - - kind:availability + - - team:my_team + - target_threshold: 98.0 + - thresholds: + - - target: 98.0 + - target_display: '98.' + - timeframe: 30d + - - target: 98.0 + - target_display: '98.' + - timeframe: 7d + - - target: 98.0 + - target_display: '98.' + - timeframe: 90d + - timeframe: 30d + - type: metric + - type_id: 1 + -error: + +a: b + EODASH + .chomp) + end + end + + describe '#except' do + subject { slos.except({ :a => :b, 'modified_at' => :c, 'url' => :d }) } + + it { is_expected.to eq({ a: :b }) } + end +end From 7392fd39194f15169fc340b20eaf0894de3fde61 Mon Sep 17 00:00:00 2001 From: Lindsey Whitley Date: Fri, 11 Aug 2023 13:29:28 -0700 Subject: [PATCH 2/5] Update spec/datadog_backup/slos_spec.rb Co-authored-by: Jim Park --- spec/datadog_backup/slos_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/datadog_backup/slos_spec.rb b/spec/datadog_backup/slos_spec.rb index 05c763b..6d5e508 100644 --- a/spec/datadog_backup/slos_spec.rb +++ b/spec/datadog_backup/slos_spec.rb @@ -22,7 +22,7 @@ {"id"=>"abc-123", "name"=>"CI Stability", "tags"=>["kind:availability", "team:my_team"], "monitor_tags"=>[], "thresholds"=>[{"timeframe"=>"7d", "target"=>98.0, "target_display"=>"98."}, {"timeframe"=>"30d", "target"=>98.0, "target_display"=>"98."}, {"timeframe"=>"90d", "target"=>98.0, "target_display"=>"98."}], "type"=>"metric", "type_id"=>1, "description"=>"something helpful", "timeframe"=>"30d", "target_threshold"=>98.0, "query"=>{"denominator"=>"sum:metric.ci_things{*}.as_count()", "numerator"=>"sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count()"}, "creator"=>{"name"=>"Thelma Patterson", "handle"=>"thelma.patterson@example.com", "email"=>"thelma.patterson@example.com"}, "created_at"=>1571335531, "modified_at"=>1687844157}, {"id"=>"sbc-124", "name"=>"A Latency SLO", "tags"=>["team:my_team", "kind:latency"], "monitor_tags"=>[], "thresholds"=>[{"timeframe"=>"7d", "target"=>95.0, "target_display"=>"95."}, {"timeframe"=>"30d", "target"=>95.0, "target_display"=>"95."}, {"timeframe"=>"90d", "target"=>95.0, "target_display"=>"95."}], "type"=>"monitor", "type_id"=>0, "description"=>"", "timeframe"=>"30d", "target_threshold"=>95.0, "monitor_ids"=>[13158755], "creator"=>{"name"=>"Louise Montague", "handle"=>"louise.montague@example.com", "email"=>"louise.montague@example.com"}, "created_at"=>1573162531, "modified_at"=>1685819875} ], - "error"=>nil, + "errors"=>[], "metadata"=>{"page"=>{"total_count"=>359, "total_filtered_count"=>359}} } end From f9d43363ba5db879de8d5b5e8e928094f6f4c9a5 Mon Sep 17 00:00:00 2001 From: Lindsey Whitley Date: Fri, 11 Aug 2023 13:29:36 -0700 Subject: [PATCH 3/5] Update spec/datadog_backup/slos_spec.rb Co-authored-by: Jim Park --- spec/datadog_backup/slos_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/datadog_backup/slos_spec.rb b/spec/datadog_backup/slos_spec.rb index 6d5e508..7ff6d4e 100644 --- a/spec/datadog_backup/slos_spec.rb +++ b/spec/datadog_backup/slos_spec.rb @@ -114,7 +114,7 @@ "created_at"=>1573162531, "modified_at"=>1685819875 }, - "error" => nil + "errors" => [] } end let(:all_slos) { respond_with200(fetched_slos) } From 851a8d262b4dbbdbbdf8e7cce91f6562a778e484 Mon Sep 17 00:00:00 2001 From: Lindsey Whitley Date: Fri, 11 Aug 2023 13:29:49 -0700 Subject: [PATCH 4/5] Update spec/datadog_backup/slos_spec.rb Co-authored-by: Jim Park --- spec/datadog_backup/slos_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/datadog_backup/slos_spec.rb b/spec/datadog_backup/slos_spec.rb index 7ff6d4e..3d2145b 100644 --- a/spec/datadog_backup/slos_spec.rb +++ b/spec/datadog_backup/slos_spec.rb @@ -70,7 +70,7 @@ "created_at" => 1571335531, "modified_at" => 1687844157 }, - "error" => nil + "errors" => [] } end let(:slo_sbc_124) do From 56c07939a10e729ffc3a081f403435703d3878e4 Mon Sep 17 00:00:00 2001 From: Lindsey Whitley Date: Wed, 16 Aug 2023 15:43:50 -0700 Subject: [PATCH 5/5] Go a level deeper for SLOs (do not need data header) --- lib/datadog_backup/slos.rb | 12 +- spec/datadog_backup/slos_spec.rb | 219 +++++++++++++++---------------- 2 files changed, 119 insertions(+), 112 deletions(-) diff --git a/lib/datadog_backup/slos.rb b/lib/datadog_backup/slos.rb index 9959d68..7c91f4d 100644 --- a/lib/datadog_backup/slos.rb +++ b/lib/datadog_backup/slos.rb @@ -4,7 +4,7 @@ module DatadogBackup # SLO specific overrides for backup and restore. class SLOs < Resources def all - get_all.fetch('data') + get_all end def backup @@ -36,6 +36,16 @@ def initialize(options) @banlist = %w[modified_at url].freeze end + # Return the Faraday body from a response with a 2xx status code, otherwise raise an error + def body_with_2xx(response) + unless response.status.to_s =~ /^2/ + raise "#{caller_locations(1, + 1)[0].label} failed with error #{response.status}" + end + + response.body.fetch('data') + end + private def api_version diff --git a/spec/datadog_backup/slos_spec.rb b/spec/datadog_backup/slos_spec.rb index 3d2145b..a952fbc 100644 --- a/spec/datadog_backup/slos_spec.rb +++ b/spec/datadog_backup/slos_spec.rb @@ -28,98 +28,98 @@ end let(:slo_abc_123) do { - "data" => { - "id" => "abc-123", - "name" => "CI Stability", - "tags" => [ - "kind:availability", - "team:my_team", - ], - "monitor_tags" => [], - "thresholds" => [ - { - "timeframe" => "7d", - "target" => 98.0, - "target_display" => "98." - }, - { - "timeframe" => "30d", - "target" => 98.0, - "target_display" => "98." - }, - { - "timeframe" => "90d", - "target" => 98.0, - "target_display" => "98." - } - ], - "type" => "metric", - "type_id" => 1, - "description" => "something helpful", - "timeframe" => "30d", - "target_threshold" => 98.0, - "query" => { - "denominator" => "sum:metric.ci_things{*}.as_count()", - "numerator" => "sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count()" + "id" => "abc-123", + "name" => "CI Stability", + "tags" => [ + "kind:availability", + "team:my_team", + ], + "monitor_tags" => [], + "thresholds" => [ + { + "timeframe" => "7d", + "target" => 98.0, + "target_display" => "98." }, - "creator" => { - "name" => "Thelma Patterson", - "handle" => "thelma.patterson@example.com", - "email" => "thelma.patterson@example.com" + { + "timeframe" => "30d", + "target" => 98.0, + "target_display" => "98." }, - "created_at" => 1571335531, - "modified_at" => 1687844157 + { + "timeframe" => "90d", + "target" => 98.0, + "target_display" => "98." + } + ], + "type" => "metric", + "type_id" => 1, + "description" => "something helpful", + "timeframe" => "30d", + "target_threshold" => 98.0, + "query" => { + "denominator" => "sum:metric.ci_things{*}.as_count()", + "numerator" => "sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count()" }, - "errors" => [] + "creator" => { + "name" => "Thelma Patterson", + "handle" => "thelma.patterson@example.com", + "email" => "thelma.patterson@example.com" + }, + "created_at" => 1571335531, + "modified_at" => 1687844157 } end let(:slo_sbc_124) do { - "data" => { - "id" => "sbc-124", - "name" => "A Latency SLO", - "tags" => [ - "kind:latency", - "team:my_team", - ], - "monitor_tags" => [], - "thresholds" => [ - { - "timeframe" => "7d", - "target" => 98.0, - "target_display" => "98." - }, - { - "timeframe" => "30d", - "target" => 98.0, - "target_display" => "98." - }, - { - "timeframe" => "90d", - "target" => 98.0, - "target_display" => "98." - } - ], - "type" => "monitor", - "type_id"=>0, - "description"=>"", - "timeframe"=>"30d", - "target_threshold"=>95.0, - "monitor_ids"=>[ 13158755 ], - "creator"=>{ - "name"=>"Louise Montague", - "handle"=>"louise.montague@example.com", - "email"=>"louise.montague@example.com" + "id" => "sbc-124", + "name" => "A Latency SLO", + "tags" => [ + "kind:latency", + "team:my_team", + ], + "monitor_tags" => [], + "thresholds" => [ + { + "timeframe" => "7d", + "target" => 98.0, + "target_display" => "98." }, - "created_at"=>1573162531, - "modified_at"=>1685819875 + { + "timeframe" => "30d", + "target" => 98.0, + "target_display" => "98." + }, + { + "timeframe" => "90d", + "target" => 98.0, + "target_display" => "98." + } + ], + "type" => "monitor", + "type_id"=>0, + "description"=>"", + "timeframe"=>"30d", + "target_threshold"=>95.0, + "monitor_ids"=>[ 13158755 ], + "creator"=>{ + "name"=>"Louise Montague", + "handle"=>"louise.montague@example.com", + "email"=>"louise.montague@example.com" }, - "errors" => [] + "created_at"=>1573162531, + "modified_at"=>1685819875 } end + let(:slo_abc_123_response) do + { "data" => slo_abc_123, "errors" => [] } + end + let(:slo_sbc_124_response) do + { "data" => slo_sbc_124, "errors" => [] } + end let(:all_slos) { respond_with200(fetched_slos) } - let(:example_slo1) { respond_with200(slo_abc_123) } - let(:example_slo2) { respond_with200(slo_sbc_124) } + let(:example_slo1) { respond_with200(slo_abc_123_response) } + let(:example_slo2) { respond_with200(slo_sbc_124_response) } before do stubs.get('/api/v1/slo') { all_slos } @@ -164,38 +164,35 @@ slos.write_file('{"a":"b"}', slos.filename('abc-123')) expect(slos.diff('abc-123')).to eq(<<~EODASH --- - -data: - - created_at: 1571335531 - - creator: - - email: thelma.patterson@example.com - - handle: thelma.patterson@example.com - - name: Thelma Patterson - - description: something helpful - - id: abc-123 - - modified_at: 1687844157 - - monitor_tags: [] - - name: CI Stability - - query: - - denominator: sum:metric.ci_things{*}.as_count() - - numerator: sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count() - - tags: - - - kind:availability - - - team:my_team - - target_threshold: 98.0 - - thresholds: - - - target: 98.0 - - target_display: '98.' - - timeframe: 30d - - - target: 98.0 - - target_display: '98.' - - timeframe: 7d - - - target: 98.0 - - target_display: '98.' - - timeframe: 90d + -created_at: 1571335531 + -creator: + - email: thelma.patterson@example.com + - handle: thelma.patterson@example.com + - name: Thelma Patterson + -description: something helpful + -id: abc-123 + -monitor_tags: [] + -name: CI Stability + -query: + - denominator: sum:metric.ci_things{*}.as_count() + - numerator: sum:metric.ci_things{*}.as_count()-sum:metric.ci_things{infra_failure}.as_count() + -tags: + -- kind:availability + -- team:my_team + -target_threshold: 98.0 + -thresholds: + -- target: 98.0 + - target_display: '98.' - timeframe: 30d - - type: metric - - type_id: 1 - -error: + -- target: 98.0 + - target_display: '98.' + - timeframe: 7d + -- target: 98.0 + - target_display: '98.' + - timeframe: 90d + -timeframe: 30d + -type: metric + -type_id: 1 +a: b EODASH .chomp)