shard: update to crystal 0.31.0

Additionally, no longer use the Crystal "markdown" library which has
been removed from the Crystal stdlib in version 0.31.0.
See crystal-lang/crystal#8115.

Also fix some deprecation warnings using the following commands:

    find . \( -type d -name .git -prune \) -o -type f -exec sed -i 's/URI\.escape/URI\.encode_www_form/g' "{}" \;
    find . \( -type d -name .git -prune \) -o -type f -exec sed -i 's/URI\.unescape/URI\.decode_www_form/g' "{}" \;
    sed -i 's/while \%pull\.kind \!\= \:end_object/until \%pull\.kind\.end_object\?/g' src/invidious/helpers/patch_mapping.cr

shard.yml

@@ -19,6 +19,6 @@ dependencies:
     github: kemalcr/kemal
     version: ~> 0.26.0
 
-crystal: 0.30.1
+crystal: 0.31.0
 
 license: AGPLv3

src/invidious.cr

@@ -17,7 +17,6 @@
 require "digest/md5"
 require "file_utils"
 require "kemal"
-require "markdown"
 require "openssl/hmac"
 require "option_parser"
 require "pg"
@@ -296,7 +295,7 @@ before_all do |env|
     current_page += "?#{query}"
   end
 
-  env.set "current_page", URI.escape(current_page)
+  env.set "current_page", URI.encode_www_form(current_page)
 end
 
 get "/" do |env|
@@ -841,7 +840,7 @@ get "/results" do |env|
   page ||= 1
 
   if query
-    env.redirect "/search?q=#{URI.escape(query)}&page=#{page}"
+    env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}"
   else
     env.redirect "/"
   end
@@ -1050,7 +1049,7 @@ post "/login" do |env|
 
       traceback << "done, returned #{response.status_code}.<br/>"
 
-      headers["Cookie"] = URI.unescape(headers["Cookie"])
+      headers["Cookie"] = URI.decode_www_form(headers["Cookie"])
 
       if challenge_results[0][3]?.try &.== 7
         error_message = translate(locale, "Account has temporarily been disabled")
@@ -2423,7 +2422,7 @@ post "/authorize_token" do |env|
   access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB)
 
   if callback_url
-    access_token = URI.escape(access_token)
+    access_token = URI.encode_www_form(access_token)
     url = URI.parse(callback_url)
 
     if url.query
@@ -3327,7 +3326,7 @@ get "/api/v1/captions/:id" do |env|
               json.object do
                 json.field "label", caption.name.simpleText
                 json.field "languageCode", caption.languageCode
-                json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}"
+                json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}"
               end
             end
           end
@@ -3406,7 +3405,7 @@ get "/api/v1/captions/:id" do |env|
 
   if title = env.params.query["title"]?
     # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
-    env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}"
+    env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}"
   end
 
   webvtt
@@ -3594,7 +3593,7 @@ get "/api/v1/annotations/:id" do |env|
         id = id.sub(/^-/, 'A')
       end
 
-      file = URI.escape("#{id[0, 3]}/#{id}.xml")
+      file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
 
       client = make_client(ARCHIVE_URL)
       location = client.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")
@@ -4093,7 +4092,7 @@ get "/api/v1/search/suggestions" do |env|
 
   begin
     client = make_client(URI.parse("https://suggestqueries.google.com"))
-    response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.escape(query)}&callback=suggestCallback").body
+    response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body
 
     body = response[35..-2]
     body = JSON.parse(body).as_a
@@ -4477,7 +4476,7 @@ post "/api/v1/auth/tokens/register" do |env|
     access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB)
 
     if callback_url
-      access_token = URI.escape(access_token)
+      access_token = URI.encode_www_form(access_token)
 
       if query = callback_url.query
         query = HTTP::Params.parse(query.not_nil!)
@@ -4712,7 +4711,7 @@ get "/api/manifest/hls_playlist/*" do |env|
       raw_params = {} of String => Array(String)
       path.each_slice(2) do |pair|
         key, value = pair
-        value = URI.unescape(value)
+        value = URI.decode_www_form(value)
 
         if raw_params[key]?
           raw_params[key] << value
@@ -4837,7 +4836,7 @@ get "/videoplayback/*" do |env|
   raw_params = {} of String => Array(String)
   path.each_slice(2) do |pair|
     key, value = pair
-    value = URI.unescape(value)
+    value = URI.decode_www_form(value)
 
     if raw_params[key]?
       raw_params[key] << value
@@ -5011,7 +5010,7 @@ get "/videoplayback" do |env|
 
             if title = query_params["title"]?
               # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
-              env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}"
+              env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}"
             end
 
             if !response.headers.includes_word?("Transfer-Encoding", "chunked")

src/invidious/channels.cr

@@ -476,7 +476,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "
   end
 
   data = Base64.urlsafe_encode(data)
-  cursor = URI.escape(data)
+  cursor = URI.encode_www_form(data)
 
   data = IO::Memory.new
 
@@ -497,7 +497,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "
   IO.copy data, buffer
 
   continuation = Base64.urlsafe_encode(buffer)
-  continuation = URI.escape(continuation)
+  continuation = URI.encode_www_form(continuation)
 
   url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 
@@ -547,7 +547,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
 
   data.rewind
   data = Base64.urlsafe_encode(data)
-  continuation = URI.escape(data)
+  continuation = URI.encode_www_form(data)
 
   data = IO::Memory.new
 
@@ -568,7 +568,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
   IO.copy data, buffer
 
   continuation = Base64.urlsafe_encode(buffer)
-  continuation = URI.escape(continuation)
+  continuation = URI.encode_www_form(continuation)
 
   url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 
@@ -578,7 +578,7 @@ end
 def extract_channel_playlists_cursor(url, auto_generated)
   continuation = HTTP::Params.parse(URI.parse(url).query.not_nil!)["continuation"]
 
-  continuation = URI.unescape(continuation)
+  continuation = URI.decode_www_form(continuation)
   data = IO::Memory.new(Base64.decode(continuation))
 
   # 0xe2 0xa9 0x85 0xb2 0x02
@@ -597,7 +597,7 @@ def extract_channel_playlists_cursor(url, auto_generated)
   data.read inner_continuation
 
   continuation = String.new(inner_continuation)
-  continuation = URI.unescape(continuation)
+  continuation = URI.decode_www_form(continuation)
   data = IO::Memory.new(Base64.decode(continuation))
 
   # 0x12 0x09 playlists
@@ -614,7 +614,7 @@ def extract_channel_playlists_cursor(url, auto_generated)
   cursor = String.new(cursor)
 
   if !auto_generated
-    cursor = URI.unescape(cursor)
+    cursor = URI.decode_www_form(cursor)
     cursor = Base64.decode_string(cursor)
   end
 
@@ -877,7 +877,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
 end
 
 def produce_channel_community_continuation(ucid, cursor)
-  cursor = URI.escape(cursor)
+  cursor = URI.encode_www_form(cursor)
 
   data = IO::Memory.new
 
@@ -898,13 +898,13 @@ def produce_channel_community_continuation(ucid, cursor)
   IO.copy data, buffer
 
   continuation = Base64.urlsafe_encode(buffer)
-  continuation = URI.escape(continuation)
+  continuation = URI.encode_www_form(continuation)
 
   return continuation
 end
 
 def extract_channel_community_cursor(continuation)
-  continuation = URI.unescape(continuation)
+  continuation = URI.decode_www_form(continuation)
   data = IO::Memory.new(Base64.decode(continuation))
 
   # 0xe2 0xa9 0x85 0xb2 0x02
@@ -923,7 +923,7 @@ def extract_channel_community_cursor(continuation)
     data.read_byte
   end
 
-  return URI.unescape(data.gets_to_end)
+  return URI.decode_www_form(data.gets_to_end)
 end
 
 def get_about_info(ucid, locale)

src/invidious/comments.cr

@@ -573,7 +573,7 @@ def content_to_comment_html(content)
 end
 
 def extract_comment_cursor(continuation)
-  continuation = URI.unescape(continuation)
+  continuation = URI.decode_www_form(continuation)
   data = IO::Memory.new(Base64.decode(continuation))
 
   # 0x12 0x26
@@ -653,7 +653,7 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
   end
 
   continuation = Base64.urlsafe_encode(data)
-  continuation = URI.escape(continuation)
+  continuation = URI.encode_www_form(continuation)
 
   return continuation
 end
@@ -695,7 +695,7 @@ def produce_comment_reply_continuation(video_id, ucid, comment_id)
   data.write(Bytes[0x48, 0x0a])
 
   continuation = Base64.urlsafe_encode(data.to_slice)
-  continuation = URI.escape(continuation)
+  continuation = URI.encode_www_form(continuation)
 
   return continuation
 end

src/invidious/helpers/handlers.cr

@@ -95,8 +95,8 @@ class AuthHandler < Kemal::Handler
 
     begin
       if token = env.request.headers["Authorization"]?
-        token = JSON.parse(URI.unescape(token.lchop("Bearer ")))
-        session = URI.unescape(token["session"].as_s)
+        token = JSON.parse(URI.decode_www_form(token.lchop("Bearer ")))
+        session = URI.decode_www_form(token["session"].as_s)
         scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil)
 
         if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String)

src/invidious/helpers/patch_mapping.cr

@@ -50,7 +50,7 @@ macro patched_json_mapping(_properties_, strict = false)
     rescue exc : ::JSON::ParseException
       raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc)
     end
-    while %pull.kind != :end_object
+    until %pull.kind.end_object?
       %key_location = %pull.location
       key = %pull.read_object_key
       case key

src/invidious/helpers/static_file_handler.cr

@@ -119,7 +119,7 @@ module Kemal
 
       config = Kemal.config.serve_static
       original_path = context.request.path.not_nil!
-      request_path = URI.unescape(original_path)
+      request_path = URI.decode_www_form(original_path)
 
       # File path cannot contains '\0' (NUL) because all filesystem I know
       # don't accept '\0' character as file name.

src/invidious/helpers/tokens.cr

@@ -69,7 +69,7 @@ end
 def validate_request(token, session, request, key, db, locale = nil)
   case token
   when String
-    token = JSON.parse(URI.unescape(token)).as_h
+    token = JSON.parse(URI.decode_www_form(token)).as_h
   when JSON::Any
     token = token.as_h
   when Nil

src/invidious/helpers/utils.cr

@@ -246,7 +246,7 @@ def get_referer(env, fallback = "/", unroll = true)
       if referer.query
         params = HTTP::Params.parse(referer.query.not_nil!)
         if params["referer"]?
-          referer = URI.parse(URI.unescape(params["referer"]))
+          referer = URI.parse(URI.decode_www_form(params["referer"]))
         else
           break
         end

src/invidious/playlists.cr

@@ -172,7 +172,7 @@ def produce_playlist_url(id, index)
   continuation.print data
 
   data = Base64.urlsafe_encode(continuation)
-  cursor = URI.escape(data)
+  cursor = URI.encode_www_form(data)
 
   data = IO::Memory.new
 
@@ -193,7 +193,7 @@ def produce_playlist_url(id, index)
   IO.copy data, buffer
 
   continuation = Base64.urlsafe_encode(buffer)
-  continuation = URI.escape(continuation)
+  continuation = URI.encode_www_form(continuation)
 
   url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 

src/invidious/search.cr

@@ -266,7 +266,7 @@ def search(query, page = 1, search_params = produce_search_params(content_type:
     return {0, [] of SearchItem}
   end
 
-  html = client.get("/results?q=#{URI.escape(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body
+  html = client.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body
   if html.empty?
     return {0, [] of SearchItem}
   end
@@ -371,7 +371,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
   end
 
   token = Base64.urlsafe_encode(token.to_slice)
-  token = URI.escape(token)
+  token = URI.encode_www_form(token)
 
   return token
 end
@@ -396,7 +396,7 @@ def produce_channel_search_url(ucid, query, page)
 
   data.rewind
   data = Base64.urlsafe_encode(data)
-  continuation = URI.escape(data)
+  continuation = URI.encode_www_form(data)
 
   data = IO::Memory.new
 
@@ -421,7 +421,7 @@ def produce_channel_search_url(ucid, query, page)
   IO.copy data, buffer
 
   continuation = Base64.urlsafe_encode(buffer)
-  continuation = URI.escape(continuation)
+  continuation = URI.encode_www_form(continuation)
 
   url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 

src/invidious/trending.cr

@@ -42,7 +42,7 @@ end
 def extract_plid(url)
   wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"]
 
-  wrapper = URI.unescape(wrapper)
+  wrapper = URI.decode_www_form(wrapper)
   wrapper = Base64.decode(wrapper)
 
   # 0xe2 0x02 0x2e

src/invidious/videos.cr

@@ -408,7 +408,7 @@ struct Video
             json.object do
               json.field "label", caption.name.simpleText
               json.field "languageCode", caption.languageCode
-              json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}"
+              json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}"
             end
           end
         end

src/invidious/views/authorize_token.ecr

@@ -72,7 +72,7 @@
                 <input type="hidden" name="expire" value="<%= expire %>">
             <% end %>
 
-            <input type="hidden" name="csrf_token" value="<%= URI.escape(csrf_token) %>">
+            <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>">
         </form>
     </div>
 <% end %>

src/invidious/views/change_password.ecr

@@ -6,7 +6,7 @@
     <div class="pure-u-1 pure-u-lg-1-5"></div>
     <div class="pure-u-1 pure-u-lg-3-5">
         <div class="h-box">
-            <form class="pure-form pure-form-aligned" action="/change_password?referer=<%= URI.escape(referer) %>" method="post">
+            <form class="pure-form pure-form-aligned" action="/change_password?referer=<%= URI.encode_www_form(referer) %>" method="post">
                 <legend><%= translate(locale, "Change password") %></legend>
 
                 <fieldset>
@@ -23,7 +23,7 @@
                         <%= translate(locale, "Change password") %>
                     </button>
 
-                    <input type="hidden" name="csrf_token" value="<%= URI.escape(csrf_token) %>">
+                    <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>">
                 </fieldset>
             </form>
         </div>