アクセス可能なリンクがリンク切れと判定されないようにする

Gemfile

@@ -20,6 +20,7 @@ gem 'active_flag'
 gem 'active_storage_validations'
 gem 'acts_as_list'
 gem 'acts-as-taggable-on', '~> 7.0'
+gem 'addressable'
 gem 'any_login'
 gem 'cocoon'
 gem 'coffee-rails', '~> 5.0.0'

Gemfile.lock

@@ -527,6 +527,7 @@ DEPENDENCIES
   active_storage_validations
   acts-as-taggable-on (~> 7.0)
   acts_as_list
+  addressable
   any_login
   bootsnap (>= 1.4.4)
   bullet

app/controllers/scheduler/link_checker_controller.rb

@@ -2,8 +2,12 @@
 
 class Scheduler::LinkCheckerController < SchedulerController
   def show
-    checker = LinkChecker::Checker.new
-    checker.notify_missing_links
-    render plain: checker.errors.join("\n")
+    documents = Page.all + Practice.all
+    links = LinkChecker::Extractor.extract_links_from_multi(documents)
+    summary_of_broken_links = LinkChecker::Checker.check_broken_links(links)
+
+    ChatNotifier.message(summary_of_broken_links, username: 'リンクチェッカー', webhook_url: ENV['DISCORD_BUG_WEBHOOK_URL'])
+
+    render plain: summary_of_broken_links
   end
 end

app/models/link_checker/checker.rb

@@ -3,90 +3,55 @@
 require 'net/http'
 
 module LinkChecker
-  class Checker
-    DENY_LIST = %w[
-      codepen.io
-      www.amazon.co.jp
+  module Checker
+    DENY_HOST = [
+      'codepen.io',
+      'www.amazon.co.jp' # アクセスを繰り返すとリンク切れ判定のレスポンスが返されるようになるため
     ].freeze
-    attr_reader :errors
 
-    def initialize
-      @errors = []
-      @error_links = []
-    end
+    module_function
 
-    def notify_missing_links
-      check
-      return if @error_links.empty?
+    def check_broken_links(links)
+      links_with_valid_url = links.select { |link| valid_url?(link.url) && !denied_host?(link.url) }
+      links_with_response = check_response(links_with_valid_url)
+      broken_links = links_with_response.select { |link| !link.response || link.response > 403 }
 
-      texts = ['リンク切れがありました。']
-      @error_links.map do |link|
-        texts << "- <#{link.url}|#{link.title}> in: <#{link.source_url}|#{link.source_title}>"
-      end
+      summary(broken_links)
+    end
 
-      ChatNotifier.message(texts.join("\n"), username: 'リンクチェッカー')
+    def valid_url?(url)
+      uri = Addressable::URI.parse(url)
+      uri.scheme && uri.host
+    rescue Addressable::URI::InvalidURIError
+      false
     end
 
-    def check
+    def denied_host?(url)
+      uri = Addressable::URI.parse(url)
+      DENY_HOST.include?(uri.host)
+    end
+
+    def check_response(links)
       locks = Queue.new
       5.times { locks.push :lock }
-      all_links.reject! do |link|
-        url = URI.encode_www_form_component(link.url)
-        uri = URI.parse(url)
 
-        !uri || DENY_LIST.include?(uri.host)
-      end
-      all_links.map do |link|
+      links.each do |link|
         Thread.new do
           lock = locks.pop
-          response = Client.request(link.url)
-          link.response = response
-          @error_links << link if !response || response > 403
+          link.response = Client.request(link.url)
           locks.push lock
-        end
-      end.each(&:join)
-
-      @error_links.sort { |a, b| b.source_url <=> a.source_url }
-    end
-
-    def all_links
-      page_links + practice_links
-    end
-
-    private
-
-    def page_links
-      links = []
-      Page.order(:created_at).each do |page|
-        extractor = Extractor.new(
-          page.body,
-          page.title,
-          "https://bootcamp.fjord.jp#{Rails.application.routes.url_helpers.polymorphic_path(page)}"
-        )
-        links += extractor.extract
+        end.join
       end
+
       links
     end
 
-    def practice_links
-      links = []
-      Practice.order(:created_at).each do |practice|
-        practice_url = Rails.application.routes.url_helpers.polymorphic_path(practice)
-        extractor = Extractor.new(
-          practice.description,
-          practice.title,
-          "https://bootcamp.fjord.jp#{practice_url}"
-        )
-        links += extractor.extract
+    def summary(broken_links)
+      return if broken_links.empty?
 
-        extractor = Extractor.new(
-          practice.goal,
-          practice.title,
-          "https://bootcamp.fjord.jp#{practice_url}"
-        )
-        links += extractor.extract
-      end
-      links
+      texts = ['リンク切れがありました。']
+      texts << broken_links.sort.map(&:to_s)
+      texts.join("\n")
     end
   end
 end

app/models/link_checker/client.rb

@@ -2,6 +2,10 @@
 
 module LinkChecker
   class Client
+    SSL_VERIFY_NONE_HOST = [
+      'www.tablesgenerator.com' # 中間証明書を取得できず、SSLサーバー証明書の検証に失敗するため
+    ].freeze
+
     def self.request(url)
       new(url).request
     end
@@ -11,22 +15,15 @@ def initialize(url)
     end
 
     def request
-      @url = encode_ja(@url)
-      uri = URI.parse(@url)
-      response = Net::HTTP.get_response(uri)
-      response.code.to_i
+      uri = Addressable::URI.parse(@url).normalize
+      options = {}
+      options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE if SSL_VERIFY_NONE_HOST.include?(uri.host)
+      response = OpenURI.open_uri(uri, **options)
+      response.status.first.to_i
+    rescue OpenURI::HTTPError => e
+      e.io.status.first.to_i
     rescue StandardError => _e
       false
     end
-
-    def encode_ja(url)
-      url.split(//).map do |c|
-        if c.match?(%r{[-_.!~*'()a-zA-Z0-9;/?:@&=+$,%#]})
-          c
-        else
-          CGI.escape(c)
-        end
-      end.join
-    end
   end
 end

app/models/link_checker/extractor.rb

@@ -1,28 +1,23 @@
 # frozen_string_literal: true
 
 module LinkChecker
-  Link = Struct.new(:title, :url, :source_title, :source_url, :response)
+  module Extractor
+    MARKDOWN_LINK_REGEXP = %r{\[(.*?)\]\((#{URI::DEFAULT_PARSER.make_regexp}|/.*?)\)}.freeze
 
-  class Extractor
-    def initialize(markdown_text, source_title, source_url)
-      @markdown_text = markdown_text
-      @source_title = source_title
-      @source_url = source_url
+    module_function
+
+    def extract_links_from_multi(documents)
+      documents.flat_map { |document| extract_links_from_a(document) }
     end
 
-    def extract
-      links = @markdown_text.scan(/\[(.*?)\]\((.+?)\)/)&.map do |match|
-        title = match[0].strip
-        url = match[1].strip
-        if url.match?(%r{^/})
-          uri = URI(@source_url)
-          uri.path = ''
-          url = uri.to_s + url
-        end
-        Link.new(title, url, @source_title, @source_url)
-      end
+    def extract_links_from_a(document)
+      document.body.scan(MARKDOWN_LINK_REGEXP).map do |title, url_or_path|
+        title = title.strip
+        url_or_path = url_or_path.strip
+        url_or_path = "https://bootcamp.fjord.jp#{url_or_path}" if url_or_path.match?(%r{^/})
 
-      links.select { |link| URI::DEFAULT_PARSER.make_regexp.match(link.url) }
+        Link.new(title, url_or_path, document.title, "https://bootcamp.fjord.jp#{document.path}")
+      end
     end
   end
 end

app/models/link_checker/link.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  Link = Struct.new(:title, :url, :source_title, :source_url, :response) do
+    def to_s
+      "- <#{url} | #{title}> in: <#{source_url} | #{source_title}>"
+    end
+
+    def <=>(other)
+      (source_url <=> other.source_url).nonzero? || url <=> other.url
+    end
+  end
+end

app/models/practice.rb

@@ -99,6 +99,10 @@ def all_text
     [title, description, goal].join("\n")
   end
 
+  def body
+    [description, goal].join("\n")
+  end
+
   def product(user)
     products.find_by(user: user)
   end

cloudbuild.yaml

@@ -77,6 +77,7 @@ steps:
       - '--set-env-vars=DISCORD_NOTICE_WEBHOOK_URL=$_DISCORD_NOTICE_WEBHOOK_URL'
       - '--set-env-vars=DISCORD_ALL_WEBHOOK_URL=$_DISCORD_ALL_WEBHOOK_URL'
       - '--set-env-vars=DISCORD_ADMIN_WEBHOOK_URL=$_DISCORD_ADMIN_WEBHOOK_URL'
+      - '--set-env-vars=DISCORD_BUG_WEBHOOK_URL=$_DISCORD_BUG_WEBHOOK_URL'
       - '--set-env-vars=BASIC_AUTH_USER=$_BASIC_AUTH_USER'
       - '--set-env-vars=BASIC_AUTH_PASSWORD=$_BASIC_AUTH_PASSWORD'
       - '--set-env-vars=ROLLBAR_CLIENT_TOKEN=$_ROLLBAR_CLIENT_TOKEN'

db/fixtures/pages.yml

@@ -35,7 +35,6 @@ page5:
   title: WIPのテスト
   body: WIP
   user: komagata
-  published_at: "2021-04-01 00:00:00"
 
 page6:
   title: ヘルプのページ
@@ -78,3 +77,29 @@ page11:
   user: komagata
   practice: practice1
   published_at: "2021-10-01 00:00:00"
+
+page12:
+  title: apt
+  body: |-
+    aptとはdebianでソフトウェアをネットワークからインストールするコマンドです。
+    [TEST](/test)(/test2)
+    [missing](test)
+    - 参考
+        - [APT - Wikipedia](http://ja.wikipedia.org/wiki/APT)
+    ## Q&A
+    - Q. `$ apt-cache search vim` の検索結果が多すぎる
+    - A. [正規表現](https://ja.wikipedia.org/wiki/%E6%AD%A3%E8%A6%8F%E8%A1%A8%E7%8F%BE) を使う。
+      - 完全一致: `$ apt-cache search ^vim$`
+      - 前方一致: `$ apt-cache search ^vim`
+  user: komagata
+  published_at: "2022-01-01 00:00:00"
+
+page13:
+  title: リンク切れチェッカーのテスト用リンクを載せたページ
+  body: |-
+    [リンク切れ判定対象外の URL へのリンク](https://www.amazon.co.jp)
+    [末尾が閉じ括弧の URL へのリンク](https://ja.wikipedia.org/wiki/マジックナンバー_(プログラム))
+    [SSLサーバー証明書の検証に失敗する host へのリンク](https://www.tablesgenerator.com/markdown_tables)
+    [日本語を含む URL へのリンク](https://ja.wikipedia.org/wiki/あ)
+  user: komagata
+  published_at: "2022-01-01 00:00:00"