From 72b737b57d1ac53146bad379684f4f39e2fe22d0 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 25 May 2017 14:13:20 -0400 Subject: [PATCH 1/3] Add the `plugins` command to Jarvis This add a new command to jarvis that allow you to get the status of the build for the default plugins that we ship with Logstash and it will periodically yield in the #logstash channel to tell us to look after them. You can trigger the query manually like this: ``` @jarvis plugins Jarvis Oops, We have currently *14* plugins jobs failing :sadbazpanda: Failures for branch: *master* logstash-codec-edn, logstash-codec-edn_lines, logstash-codec-es_bulk, logstash-codec-graphite, logstash-codec-json_lines, logstash-codec-multiline, logstash-codec-plain, logstash-filter-kv, logstash-filter-mutate, logstash-filter-split, logstash-filter-xml, logstash-output-elasticsearch, logstash-output-null, logstash-output-statsd ``` It will fetch the following file https://github.com/elastic/logstash/blob/master/rakelib/plugins-metadata.json to get the list of plugins. It can check more than one branch for some plugins, you will have to make a PR to this repository and add your plugin and the additional branch to check to the `plugins_config.yml` file. like this: ``` logstash-output-elasticsearch: - "6.x" ``` This was motivated by https://github.com/logstash-plugins/logstash-input-beats/pull/203 and needed for the migration to jruby 9k --- Guardfile | 4 ++ lib/jarvis/commands/plugins.rb | 21 ++++++++++ lib/jarvis/travis/watchdog.rb | 76 ++++++++++++++++++++++++++++++++++ lib/lita/handlers/jarvis.rb | 33 ++++++++++++--- lita-jarvis.gemspec | 2 + plugins_config.yml | 2 + 6 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 lib/jarvis/commands/plugins.rb create mode 100644 lib/jarvis/travis/watchdog.rb create mode 100644 plugins_config.yml diff --git a/Guardfile b/Guardfile index 4bf6626..cc4ad07 100644 --- a/Guardfile +++ b/Guardfile @@ -19,3 +19,7 @@ guard :rspec, :cmd => "rspec" do watch(%r{^(lib|spec)/.*\.rb$}) end + +guard :process, :name => "lita", :command => ['bundle', 'exec', 'lita'] do + watch(%r{^lib/.*\.rb$}) +end diff --git a/lib/jarvis/commands/plugins.rb b/lib/jarvis/commands/plugins.rb new file mode 100644 index 0000000..dd86018 --- /dev/null +++ b/lib/jarvis/commands/plugins.rb @@ -0,0 +1,21 @@ +require "clamp" +require "jarvis/cla" +require "jarvis/patches/i18n" +require "jarvis/github/review_search" +require "jarvis/travis/watchdog" + +module Jarvis module Command class Plugins < Clamp::Command + banner "Get travis status for the default plugins" + + def execute + total, failures = ::Jarvis::Travis::Watchdog.execute + + if total == 0 + puts "Good job, all default plugins are green! :green_heart:" + else + messages = [ "Oops, We have currently *#{total}* plugins jobs failing :sadbazpanda:", ] + messages.concat(::Jarvis::Travis::Watchdog.format_items(failures)) + messages.each { |message| puts message } + end + end +end end end diff --git a/lib/jarvis/travis/watchdog.rb b/lib/jarvis/travis/watchdog.rb new file mode 100644 index 0000000..74c1932 --- /dev/null +++ b/lib/jarvis/travis/watchdog.rb @@ -0,0 +1,76 @@ +require "open-uri" +require "travis" +require "yaml" + +module Jarvis module Travis + class Watchdog + DEFAULT_PLUGINS_URL = "https://raw.githubusercontent.com/elastic/logstash/master/rakelib/plugins-metadata.json" + PLUGINS_CONFIG_FILE = File.join(File.dirname(__FILE__), "../../../plugins_config.yml") + PLUGINS_CONFIG = YAML.load(File.read(PLUGINS_CONFIG_FILE)) + DEFAULT_BRANCHES = ["master"] + + def initialize + end + + def default_plugins + @default_plugins ||= fetch(DEFAULT_PLUGINS_URL).select { |_, v| v["default-plugins"] }.keys + end + + def get_status + status = {} + + default_plugins.each do |plugin_name| + location = "logstash-plugins/#{plugin_name}" + repo = ::Travis::Repository.find(location) + branches = branch_to_monitor(plugin_name) + status[plugin_name] = branches.each_with_object({}) { |branch, hsh| hsh[branch] = repo.branches[branch].passed? } + end + + status + end + + def extract_failures(plugins_status) + failures = {} + + plugins_status.each do |plugin_name, status| + status.each do |branch, state| + if !state + failures[branch] ||= [] + failures[branch] << plugin_name + end + end + end + + total = failures.values.collect(&:size).reduce(&:+) + [total, failures] + end + + def execute + plugins_status = get_status + extract_failures(plugins_status) + end + + def branch_to_monitor(plugin_name) + DEFAULT_BRANCHES.dup.concat(Array(PLUGINS_CONFIG["plugin_name"])).uniq + end + + def fetch(url) + MultiJson.load(open(url) { |f| f.read }) + end + + def self.execute + self.new.execute + end + + def self.format_items(failures) + messages = [] + + failures.each do |branch, plugins| + messages << "Failures for branch: *#{branch}*" + messages << plugins.sort.join(", ") + end + + messages + end + end +end end diff --git a/lib/lita/handlers/jarvis.rb b/lib/lita/handlers/jarvis.rb index c435fc1..f91eddf 100644 --- a/lib/lita/handlers/jarvis.rb +++ b/lib/lita/handlers/jarvis.rb @@ -6,15 +6,18 @@ require "jarvis/commands/cla" require "jarvis/commands/publish" require "jarvis/commands/teamtime" +require "jarvis/commands/plugins" require "jarvis/mixins/fancy_route" require "jarvis/thread_logger" require "jarvis/commands/review" -require 'jarvis/github/review_search' +require "jarvis/github/review_search" +require "travis" module Lita module Handlers class Jarvis < Handler extend ::Jarvis::Mixins::FancyRoute + config :cla_url config :github_token config :organization @@ -24,25 +27,44 @@ class Jarvis < Handler ::Jarvis::Github.client(config.github_token) ::Jarvis::ThreadLogger.setup + # Use github token to get a travis token + Travis.github_auth(config.github_token) + every(60*60) { review_search(robot) } + every(90*60) { travis_watchdog(robot) } end - def review_search(robot) - total, items = ::Jarvis::Github::ReviewSearch.execute + def travis_watchdog(robot) + total, failures = ::Jarvis::Travis::Watchdog.execute return if total == 0 - + + messages = [ "Oops, We have currently *#{total}* plugins jobs failing :sadbazpanda:", ] + messages.concat(::Jarvis::Travis::Watchdog.format_items(failures)) + + send_messages(room_target, messages) + end + + def room_target @target_room ||= Lita::Room.find_by_name("logstash") @target ||= Lita::Source.new(room: @target_room) + end + + def review_search(robot) + total, items = ::Jarvis::Github::ReviewSearch.execute + return if total == 0 messages = [] messages << "There are #{total} items needing PR review. Consider reviewing one of these please :)" messages.concat ::Jarvis::Github::ReviewSearch.format_items(items) + send_messages(room_target, messages) + end + def send_messages(target, messages) messages.each do |message| robot.send_messages(@target, message) # Weirdly, slack displays messages out of order unless you do this. They must have a race # this only happens sometimes - sleep 0.1 + sleep 0.1 end end @@ -64,6 +86,7 @@ def review_search(robot) # and 'review PR_URL' to submit a PR fancy_route("reviews", ::Jarvis::Command::Review, :command => true) fancy_route("review", ::Jarvis::Command::Review, :command => true) + fancy_route("plugins", ::Jarvis::Command::Plugins, :command => true) Lita.register_handler(self) end diff --git a/lita-jarvis.gemspec b/lita-jarvis.gemspec index bdad5ad..bb83dff 100644 --- a/lita-jarvis.gemspec +++ b/lita-jarvis.gemspec @@ -30,6 +30,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "rfc2047" spec.add_runtime_dependency "gems" spec.add_runtime_dependency "tzinfo" + spec.add_runtime_dependency "travis" spec.add_development_dependency "bundler", "~> 1.3" spec.add_development_dependency "pry-byebug" @@ -41,4 +42,5 @@ Gem::Specification.new do |spec| spec.add_development_dependency "guard" spec.add_development_dependency "guard-rspec" spec.add_development_dependency "guard-bundler" + spec.add_development_dependency "guard-process" end diff --git a/plugins_config.yml b/plugins_config.yml new file mode 100644 index 0000000..6e579fd --- /dev/null +++ b/plugins_config.yml @@ -0,0 +1,2 @@ +logstash-output-elasticsearch: + - "6.x" From c8d27c8c313002da1f6126f043bd4931db79b28d Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 25 May 2017 14:54:01 -0400 Subject: [PATCH 2/3] add total plugins check --- lib/jarvis/commands/plugins.rb | 6 +++--- lib/jarvis/travis/watchdog.rb | 4 ++-- lib/lita/handlers/jarvis.rb | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/jarvis/commands/plugins.rb b/lib/jarvis/commands/plugins.rb index dd86018..f042803 100644 --- a/lib/jarvis/commands/plugins.rb +++ b/lib/jarvis/commands/plugins.rb @@ -8,12 +8,12 @@ module Jarvis module Command class Plugins < Clamp::Command banner "Get travis status for the default plugins" def execute - total, failures = ::Jarvis::Travis::Watchdog.execute + total_failures, total_plugins, failures = ::Jarvis::Travis::Watchdog.execute - if total == 0 + if total_failures == 0 puts "Good job, all default plugins are green! :green_heart:" else - messages = [ "Oops, We have currently *#{total}* plugins jobs failing :sadbazpanda:", ] + messages = [ "Oops, We have currently have *#{total_failures}* plugins jobs failing :sadbazpanda: (#{total_plugins} plugins checked)" ] messages.concat(::Jarvis::Travis::Watchdog.format_items(failures)) messages.each { |message| puts message } end diff --git a/lib/jarvis/travis/watchdog.rb b/lib/jarvis/travis/watchdog.rb index 74c1932..c1cf89e 100644 --- a/lib/jarvis/travis/watchdog.rb +++ b/lib/jarvis/travis/watchdog.rb @@ -41,8 +41,8 @@ def extract_failures(plugins_status) end end - total = failures.values.collect(&:size).reduce(&:+) - [total, failures] + total_failures = failures.values.collect(&:size).reduce(&:+) + [total_failures, plugins_status.size, failures] end def execute diff --git a/lib/lita/handlers/jarvis.rb b/lib/lita/handlers/jarvis.rb index f91eddf..c7d91ab 100644 --- a/lib/lita/handlers/jarvis.rb +++ b/lib/lita/handlers/jarvis.rb @@ -35,10 +35,10 @@ class Jarvis < Handler end def travis_watchdog(robot) - total, failures = ::Jarvis::Travis::Watchdog.execute - return if total == 0 + total_failures, total_pl_gins, failures = ::Jarvis::Travis::Watchdog.execute + return if total_failures == 0 - messages = [ "Oops, We have currently *#{total}* plugins jobs failing :sadbazpanda:", ] + messages = [ "Oops, We have currently have *#{total_failures}* plugins jobs failing :sadbazpanda: (#{total_plugins} plugins checked)" ] messages.concat(::Jarvis::Travis::Watchdog.format_items(failures)) send_messages(room_target, messages) From 9eefadfc15b2a0222b18c746c964ca0926549a46 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 25 May 2017 14:54:20 -0400 Subject: [PATCH 3/3] set timer to every 6hr --- lib/lita/handlers/jarvis.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lita/handlers/jarvis.rb b/lib/lita/handlers/jarvis.rb index c7d91ab..7bb9863 100644 --- a/lib/lita/handlers/jarvis.rb +++ b/lib/lita/handlers/jarvis.rb @@ -31,7 +31,7 @@ class Jarvis < Handler Travis.github_auth(config.github_token) every(60*60) { review_search(robot) } - every(90*60) { travis_watchdog(robot) } + every(6*60*60) { travis,watchdog(robot) } end def travis_watchdog(robot)