diff --git a/config/travis.example.yml b/config/travis.example.yml index 2c6fad868..7499ea241 100644 --- a/config/travis.example.yml +++ b/config/travis.example.yml @@ -45,3 +45,6 @@ development: test: domain: test.travis-ci.local + billing: + url: 'http://localhost:9292' + auth_key: 't0Ps3Cr3t' diff --git a/lib/travis/addons/config/notify.rb b/lib/travis/addons/config/notify.rb index 23a779754..ae8d3b2e7 100644 --- a/lib/travis/addons/config/notify.rb +++ b/lib/travis/addons/config/notify.rb @@ -9,9 +9,9 @@ class Notify DEFAULTS = { start: { email: false, webhooks: false, campfire: false, hipchat: false, irc: false, flowdock: false, sqwiggle: false, slack: false, pushover: false }, - success: { email: :change, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always }, + success: { email: :change, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always, billing: :always }, failure: { email: :always, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always }, - canceled:{ email: :always, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always }, + canceled:{ email: :always, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always, billing: :always }, errored: { email: :always, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always } } diff --git a/lib/travis/addons/handlers.rb b/lib/travis/addons/handlers.rb index d223ea34d..f531f58a4 100644 --- a/lib/travis/addons/handlers.rb +++ b/lib/travis/addons/handlers.rb @@ -16,3 +16,5 @@ require 'travis/addons/handlers/slack' require 'travis/addons/handlers/pushover' require 'travis/addons/handlers/metrics' +require 'travis/addons/handlers/intercom' +require 'travis/addons/handlers/billing' diff --git a/lib/travis/addons/handlers/billing.rb b/lib/travis/addons/handlers/billing.rb new file mode 100644 index 000000000..e4ddc4205 --- /dev/null +++ b/lib/travis/addons/handlers/billing.rb @@ -0,0 +1,161 @@ +require 'travis/addons/handlers/base' +require 'travis/addons/config' +require 'raven' + +module Travis + module Addons + module Handlers + class Billing < Base + EVENTS = ['job:started', 'job:finished', 'job:canceled'].freeze + KEY = :billing + + MSGS = { + failed: 'Failed to push stats to billing-v2: %s' + } + + def handle? + billing_url && billing_auth_key + end + + def handle + publish unless Travis::Hub.context.config.enterprise? + end + + private + + def billing_url + @billing_url ||= Travis::Hub.context.config.billing.url if Travis::Hub.context.config.billing + end + + def billing_auth_key + @billing_auth_key ||= Travis::Hub.context.config.billing.auth_key if Travis::Hub.context.config.billing + end + + def publish + send_usage(data) + rescue => e + logger.error MSGS[:failed] % e.message + end + + def send_usage(data) + logger.info "Hub usage #{data}" + Travis::Sidekiq.billing(data) + end + + def data + @data ||= serialize_data + end + + def serialize_data + { + job: job_data, + repository: repository_data, + owner: owner_data, + build: build_data + } + end + + def job_data + { + id: object.id, + os: config['os'] || 'linux', + instance_size: meta(:vm_size) || vm_size, + arch: config['arch'] || 'amd64', + started_at: object.started_at, + finished_at: object.finished_at, + virt_type: config['virt'], + queue: object.queue, + vm_size: vm_size, + finished: finished? + } + end + + def repository_data + { + id: repository.id, + slug: repository.slug, + private: repository.private + } + end + + def owner_data + { + type: object.owner_type, + id: object.owner_id, + login: object.owner ? object.owner.login : nil + } + end + + def build_data + { + id: object.build.id, + type: object.build.event_type, + number: object.build.number, + branch: object.build.branch, + sender: build_data_sender + } + end + + def build_data_sender + { + id: object.build.sender_id, + type: object.build.sender_type + } + end + + def meta(value) + params[:worker_meta][0][value] if params.has_key?(:worker_meta) && params[:worker_meta].is_a?(Array) && params[:worker_meta].first.respond_to?(:keys) + end + + def vm_size + config.dig('vm', 'size') + end + + def config + @config ||= object.config_id ? JobConfig.find(object.config_id).config : {} + end + + def connection + @connection ||= Faraday.new(url: billing_url, ssl: { ca_path: '/usr/lib/ssl/certs' }) do |conn| + conn.basic_auth '_', billing_auth_key + conn.headers['Content-Type'] = 'application/json' + conn.request :json + conn.response :json + conn.adapter :net_http + end + end + + def handle_usage_executions_response(response) + case response.status + when 404 + raise StandardError, "Not found #{response.body['error'] || response.body}" + when 400 + raise StandardError, "Client error #{response.body['error'] || response.body}" + when 422 + raise StandardError, "Unprocessable entity #{response.body['error'] || response.body}" + else + raise StandardError, "Server error #{response.body['error'] || response.body}" + end + end + + def logger + Addons.logger + end + + def finished? + event != 'job:started' + end + + # EventHandler + class EventHandler < Addons::Instrument + def notify_completed + publish + end + end + EventHandler.attach_to(self) + + class BillingError < StandardError; end + end + end + end +end diff --git a/lib/travis/addons/handlers/intercom.rb b/lib/travis/addons/handlers/intercom.rb new file mode 100644 index 000000000..444361f78 --- /dev/null +++ b/lib/travis/addons/handlers/intercom.rb @@ -0,0 +1,53 @@ +require 'travis/addons/handlers/base' +require 'travis/addons/handlers/task' + +module Travis + module Addons + module Handlers + class Intercom < Base + include Handlers::Task + + EVENTS = /(build):(created|started|restarted)/ + + def handle? + owner_type.downcase == 'user' + end + + def handle + params = { + event: :report_build, + owner_id: owner_id, + last_build_at: last_build_at + } + run_task(:intercom, {}, params) + end + + class Instrument < Addons::Instrument + def notify_completed + publish + end + end + Instrument.attach_to(self) + + private + + def last_build_at + DateTime.now + end + + def owner + object.owner || nil + end + + def owner_id + owner.id.to_s if owner + end + + def owner_type + owner ? owner.class.name : '' + end + + end + end + end +end diff --git a/lib/travis/event.rb b/lib/travis/event.rb index d9e7241ef..cab05a57c 100644 --- a/lib/travis/event.rb +++ b/lib/travis/event.rb @@ -33,7 +33,7 @@ def subscriptions def notify(event, *args) prefix = Underscore.new(self.class.name).string event = PastTense.new(event).string - Event.dispatch("#{prefix}:#{event}", id: id, attrs: attributes) + Event.dispatch("#{prefix}:#{event}", id: id, attrs: attributes, worker_meta: args) end end end diff --git a/lib/travis/hub/config.rb b/lib/travis/hub/config.rb index 8ff8bb3c3..0821d1e2a 100644 --- a/lib/travis/hub/config.rb +++ b/lib/travis/hub/config.rb @@ -38,8 +38,9 @@ def jwt_key(type) repository: { ssl_key: { size: 4096 } }, queue: 'builds', limit: { resets: { max: 50, after: 6 * 60 * 60 } }, - notifications: [], - auth: { jwt_private_key: jwt_key(:private), jwt_public_key: jwt_key(:public), http_basic_auth: http_basic_auth } + notifications: [ 'billing' ], + auth: { jwt_private_key: jwt_key(:private), jwt_public_key: jwt_key(:public), http_basic_auth: http_basic_auth }, + billing: { url: ENV['BILLING_URL'] || 'http://localhost:9292', auth_key: ENV['BILLING_AUTH_KEY'] || 't0Ps3Cr3t' } def metrics # TODO cleanup keychain? diff --git a/lib/travis/hub/service/notify_workers.rb b/lib/travis/hub/service/notify_workers.rb index 7f53bb626..a8dc5057e 100644 --- a/lib/travis/hub/service/notify_workers.rb +++ b/lib/travis/hub/service/notify_workers.rb @@ -11,9 +11,9 @@ class NotifyWorkers < Struct.new(:context) job_board_cancel: 'Canceling via Job Board delete for ' } - def cancel(job) + def cancel(job, reason = '') cancel_via_job_board(job) - cancel_via_amqp(job) + cancel_via_amqp(job, reason) end private @@ -25,11 +25,15 @@ def cancel_via_job_board(job) job_board.cancel(job.id) end - def cancel_via_amqp(job) + def cancel_via_amqp(job, reason) info :amqp_cancel, job.id, job.state + context.amqp.fanout( 'worker.commands', - type: 'cancel_job', job_id: job.id, source: 'hub' + type: 'cancel_job', + job_id: job.id, + source: 'hub', + reason: reason ) end diff --git a/lib/travis/hub/service/update_job.rb b/lib/travis/hub/service/update_job.rb index 41157f142..7d8eead69 100644 --- a/lib/travis/hub/service/update_job.rb +++ b/lib/travis/hub/service/update_job.rb @@ -109,7 +109,7 @@ def error_job end def notify - NotifyWorkers.new(context).cancel(job) if job.reload.state == :canceled + NotifyWorkers.new(context).cancel(job, data[:reason]) if job.reload.state == :canceled NotifyTraceProcessor.new(context).notify(job, data) if event == :finish end diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index df90d13d2..04d20aefc 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -54,6 +54,14 @@ def logsearch(*args) ) end + def billing(*args) + default_client.push( + 'queue' => 'billing', + 'class' => 'Travis::Billing::Worker', + 'args' => [nil, "Travis::Billing::Services::UsageTracker", 'perform', *args] + ) + end + private def default_client diff --git a/spec/support/factories.rb b/spec/support/factories.rb index 0822d52b7..bcfb35f2a 100644 --- a/spec/support/factories.rb +++ b/spec/support/factories.rb @@ -61,6 +61,11 @@ def config=(config) state :created end + factory :job_config do + key 'key' + config { { arch: 'amd64', os: 'linux', virt: 'vm' } } + end + factory :user do end diff --git a/spec/travis/addons/handlers/billing_spec.rb b/spec/travis/addons/handlers/billing_spec.rb new file mode 100644 index 000000000..97f08e308 --- /dev/null +++ b/spec/travis/addons/handlers/billing_spec.rb @@ -0,0 +1,53 @@ +describe Travis::Addons::Handlers::Billing do + let(:build) { FactoryGirl.create(:build) } + let(:job_config) { FactoryGirl.create(:job_config, repository_id: build.repository_id) } + let(:job) { FactoryGirl.create(:job, owner: owner, config_id: job_config.id) } + let(:owner) { FactoryGirl.create(:user) } + let!(:request) do + stub_request(:put, 'http://localhost:9292/usage/executions') + .to_return(status: 200, body: '', headers: {}) + end + + describe 'handle' do + let(:handler) { described_class.new(event_name, id: job.id) } + + context 'job:finished' do + let(:event_name) { 'job:finished' } + + it 'publishes event to billing' do + ::Sidekiq::Client.any_instance.expects(:push).with do |payload| + expect(payload['queue']).to eq('billing') + expect(payload['class']).to eq('Travis::Billing::Worker') + expect(payload['args'][1]).to eq('Travis::Billing::Services::UsageTracker') + end + handler.handle + end + end + + context 'job:canceled' do + let(:event_name) { 'job:canceled' } + + it 'publishes event to billing' do + ::Sidekiq::Client.any_instance.expects(:push).with do |payload| + expect(payload['queue']).to eq('billing') + expect(payload['class']).to eq('Travis::Billing::Worker') + expect(payload['args'][1]).to eq('Travis::Billing::Services::UsageTracker') + end + handler.handle + end + end + + context 'job:started' do + let(:event_name) { 'job:started' } + + it 'publishes event to billing' do + ::Sidekiq::Client.any_instance.expects(:push).with do |payload| + expect(payload['queue']).to eq('billing') + expect(payload['class']).to eq('Travis::Billing::Worker') + expect(payload['args'][1]).to eq('Travis::Billing::Services::UsageTracker') + end + handler.handle + end + end + end +end diff --git a/spec/travis/hub/service/update_build_spec.rb b/spec/travis/hub/service/update_build_spec.rb index 87a5f904b..0206346b2 100644 --- a/spec/travis/hub/service/update_build_spec.rb +++ b/spec/travis/hub/service/update_build_spec.rb @@ -135,7 +135,7 @@ end it 'notifies workers' do - amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub') + amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub', reason: '') subject.run end @@ -168,7 +168,7 @@ end it 'notifies workers' do - amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub') + amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub', reason: '') subject.run end diff --git a/spec/travis/hub/service/update_job_spec.rb b/spec/travis/hub/service/update_job_spec.rb index 59a53994b..8219b5924 100644 --- a/spec/travis/hub/service/update_job_spec.rb +++ b/spec/travis/hub/service/update_job_spec.rb @@ -47,7 +47,7 @@ end it 'broadcasts a cancel message' do - amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub') + amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub', reason: '') subject.run end end @@ -77,7 +77,7 @@ end it 'broadcasts a cancel message' do - amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub') + amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub', reason: '') subject.run end end @@ -102,7 +102,7 @@ describe 'cancel event' do let(:state) { :created } let(:event) { :cancel } - let(:data) { { id: job.id } } + let(:data) { { id: job.id, reason: 'Insufficient funds' } } let(:now) { Time.now } it 'updates the job' do @@ -112,7 +112,7 @@ end it 'notifies workers' do - amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub') + amqp.expects(:fanout).with('worker.commands', type: 'cancel_job', job_id: job.id, source: 'hub', reason: 'Insufficient funds') subject.run end