From 455db7c9f3a66070a5e885ebd07a1c6566dc8cf7 Mon Sep 17 00:00:00 2001 From: Leon Klingele Date: Mon, 23 Sep 2019 21:52:40 +0200 Subject: [PATCH] shard: update to crystal 0.31.0 Additionally, no longer use the Crystal "markdown" library which has been removed from the Crystal stdlib in version 0.31.0. See https://github.com/crystal-lang/crystal/pull/8115. Also fix some deprecation warnings using the following commands: find . \( -type d -name .git -prune \) -o -type f -exec sed -i 's/URI\.escape/URI\.encode_www_form/g' "{}" \; find . \( -type d -name .git -prune \) -o -type f -exec sed -i 's/URI\.unescape/URI\.decode_www_form/g' "{}" \; sed -i 's/while \%pull\.kind \!\= \:end_object/until \%pull\.kind\.end_object\?/g' src/invidious/helpers/patch_mapping.cr --- shard.yml | 2 +- src/invidious.cr | 25 ++-- src/invidious/channels.cr | 22 ++-- src/invidious/comments.cr | 6 +- src/invidious/helpers/handlers.cr | 4 +- src/invidious/helpers/patch_mapping.cr | 2 +- src/invidious/helpers/static_file_handler.cr | 2 +- src/invidious/helpers/tokens.cr | 2 +- src/invidious/helpers/utils.cr | 2 +- src/invidious/playlists.cr | 4 +- src/invidious/search.cr | 8 +- src/invidious/trending.cr | 2 +- src/invidious/videos.cr | 2 +- src/invidious/views/authorize_token.ecr | 2 +- src/invidious/views/change_password.ecr | 4 +- src/invidious/views/clear_watch_history.ecr | 6 +- src/invidious/views/components/item.ecr | 2 +- .../views/components/subscribe_widget.ecr | 6 +- src/invidious/views/data_control.ecr | 2 +- src/invidious/views/delete_account.ecr | 6 +- src/invidious/views/history.ecr | 4 +- src/invidious/views/login.ecr | 8 +- src/invidious/views/preferences.ecr | 10 +- src/invidious/views/privacy.ecr | 118 ++++++++---------- src/invidious/views/subscription_manager.ecr | 6 +- src/invidious/views/subscriptions.ecr | 2 +- src/invidious/views/template.ecr | 2 +- src/invidious/views/token_manager.ecr | 6 +- src/invidious/views/watch.ecr | 8 +- 29 files changed, 127 insertions(+), 148 deletions(-) diff --git a/shard.yml b/shard.yml index 0f9beaf24..bd5986b8a 100644 --- a/shard.yml +++ b/shard.yml @@ -19,6 +19,6 @@ dependencies: github: kemalcr/kemal version: ~> 0.26.0 -crystal: 0.30.1 +crystal: 0.31.0 license: AGPLv3 diff --git a/src/invidious.cr b/src/invidious.cr index b97aa6b2e..167050fdc 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -17,7 +17,6 @@ require "digest/md5" require "file_utils" require "kemal" -require "markdown" require "openssl/hmac" require "option_parser" require "pg" @@ -296,7 +295,7 @@ before_all do |env| current_page += "?#{query}" end - env.set "current_page", URI.escape(current_page) + env.set "current_page", URI.encode_www_form(current_page) end get "/" do |env| @@ -841,7 +840,7 @@ get "/results" do |env| page ||= 1 if query - env.redirect "/search?q=#{URI.escape(query)}&page=#{page}" + env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" else env.redirect "/" end @@ -1050,7 +1049,7 @@ post "/login" do |env| traceback << "done, returned #{response.status_code}.
" - headers["Cookie"] = URI.unescape(headers["Cookie"]) + headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) if challenge_results[0][3]?.try &.== 7 error_message = translate(locale, "Account has temporarily been disabled") @@ -2423,7 +2422,7 @@ post "/authorize_token" do |env| access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB) if callback_url - access_token = URI.escape(access_token) + access_token = URI.encode_www_form(access_token) url = URI.parse(callback_url) if url.query @@ -3327,7 +3326,7 @@ get "/api/v1/captions/:id" do |env| json.object do json.field "label", caption.name.simpleText json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" end end end @@ -3406,7 +3405,7 @@ get "/api/v1/captions/:id" do |env| if title = env.params.query["title"]? # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" end webvtt @@ -3594,7 +3593,7 @@ get "/api/v1/annotations/:id" do |env| id = id.sub(/^-/, 'A') end - file = URI.escape("#{id[0, 3]}/#{id}.xml") + file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") client = make_client(ARCHIVE_URL) location = client.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}") @@ -4093,7 +4092,7 @@ get "/api/v1/search/suggestions" do |env| begin client = make_client(URI.parse("https://suggestqueries.google.com")) - response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.escape(query)}&callback=suggestCallback").body + response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body body = response[35..-2] body = JSON.parse(body).as_a @@ -4477,7 +4476,7 @@ post "/api/v1/auth/tokens/register" do |env| access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) if callback_url - access_token = URI.escape(access_token) + access_token = URI.encode_www_form(access_token) if query = callback_url.query query = HTTP::Params.parse(query.not_nil!) @@ -4712,7 +4711,7 @@ get "/api/manifest/hls_playlist/*" do |env| raw_params = {} of String => Array(String) path.each_slice(2) do |pair| key, value = pair - value = URI.unescape(value) + value = URI.decode_www_form(value) if raw_params[key]? raw_params[key] << value @@ -4837,7 +4836,7 @@ get "/videoplayback/*" do |env| raw_params = {} of String => Array(String) path.each_slice(2) do |pair| key, value = pair - value = URI.unescape(value) + value = URI.decode_www_form(value) if raw_params[key]? raw_params[key] << value @@ -5011,7 +5010,7 @@ get "/videoplayback" do |env| if title = query_params["title"]? # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" end if !response.headers.includes_word?("Transfer-Encoding", "chunked") diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index d88a8f71a..7b7d37243 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -476,7 +476,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " end data = Base64.urlsafe_encode(data) - cursor = URI.escape(data) + cursor = URI.encode_www_form(data) data = IO::Memory.new @@ -497,7 +497,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" @@ -547,7 +547,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated data.rewind data = Base64.urlsafe_encode(data) - continuation = URI.escape(data) + continuation = URI.encode_www_form(data) data = IO::Memory.new @@ -568,7 +568,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" @@ -578,7 +578,7 @@ end def extract_channel_playlists_cursor(url, auto_generated) continuation = HTTP::Params.parse(URI.parse(url).query.not_nil!)["continuation"] - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0xe2 0xa9 0x85 0xb2 0x02 @@ -597,7 +597,7 @@ def extract_channel_playlists_cursor(url, auto_generated) data.read inner_continuation continuation = String.new(inner_continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0x12 0x09 playlists @@ -614,7 +614,7 @@ def extract_channel_playlists_cursor(url, auto_generated) cursor = String.new(cursor) if !auto_generated - cursor = URI.unescape(cursor) + cursor = URI.decode_www_form(cursor) cursor = Base64.decode_string(cursor) end @@ -877,7 +877,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo end def produce_channel_community_continuation(ucid, cursor) - cursor = URI.escape(cursor) + cursor = URI.encode_www_form(cursor) data = IO::Memory.new @@ -898,13 +898,13 @@ def produce_channel_community_continuation(ucid, cursor) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end def extract_channel_community_cursor(continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0xe2 0xa9 0x85 0xb2 0x02 @@ -923,7 +923,7 @@ def extract_channel_community_cursor(continuation) data.read_byte end - return URI.unescape(data.gets_to_end) + return URI.decode_www_form(data.gets_to_end) end def get_about_info(ucid, locale) diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 04ba6f5d3..740449d3e 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -573,7 +573,7 @@ def content_to_comment_html(content) end def extract_comment_cursor(continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0x12 0x26 @@ -653,7 +653,7 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top") end continuation = Base64.urlsafe_encode(data) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end @@ -695,7 +695,7 @@ def produce_comment_reply_continuation(video_id, ucid, comment_id) data.write(Bytes[0x48, 0x0a]) continuation = Base64.urlsafe_encode(data.to_slice) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 56c1c4886..c8d4da4cd 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -95,8 +95,8 @@ class AuthHandler < Kemal::Handler begin if token = env.request.headers["Authorization"]? - token = JSON.parse(URI.unescape(token.lchop("Bearer "))) - session = URI.unescape(token["session"].as_s) + token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) + session = URI.decode_www_form(token["session"].as_s) scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String) diff --git a/src/invidious/helpers/patch_mapping.cr b/src/invidious/helpers/patch_mapping.cr index e138aa1c9..19bd8ca1b 100644 --- a/src/invidious/helpers/patch_mapping.cr +++ b/src/invidious/helpers/patch_mapping.cr @@ -50,7 +50,7 @@ macro patched_json_mapping(_properties_, strict = false) rescue exc : ::JSON::ParseException raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc) end - while %pull.kind != :end_object + until %pull.kind.end_object? %key_location = %pull.location key = %pull.read_object_key case key diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr index 87edbcd32..20d92b9c8 100644 --- a/src/invidious/helpers/static_file_handler.cr +++ b/src/invidious/helpers/static_file_handler.cr @@ -119,7 +119,7 @@ module Kemal config = Kemal.config.serve_static original_path = context.request.path.not_nil! - request_path = URI.unescape(original_path) + request_path = URI.decode_www_form(original_path) # File path cannot contains '\0' (NUL) because all filesystem I know # don't accept '\0' character as file name. diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index f946fc2c2..30f7d4f49 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -69,7 +69,7 @@ end def validate_request(token, session, request, key, db, locale = nil) case token when String - token = JSON.parse(URI.unescape(token)).as_h + token = JSON.parse(URI.decode_www_form(token)).as_h when JSON::Any token = token.as_h when Nil diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index e17d58e22..ed55dc9c3 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -246,7 +246,7 @@ def get_referer(env, fallback = "/", unroll = true) if referer.query params = HTTP::Params.parse(referer.query.not_nil!) if params["referer"]? - referer = URI.parse(URI.unescape(params["referer"])) + referer = URI.parse(URI.decode_www_form(params["referer"])) else break end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7965d9905..a5383daf8 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -172,7 +172,7 @@ def produce_playlist_url(id, index) continuation.print data data = Base64.urlsafe_encode(continuation) - cursor = URI.escape(data) + cursor = URI.encode_www_form(data) data = IO::Memory.new @@ -193,7 +193,7 @@ def produce_playlist_url(id, index) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 71e1f7379..10475a8f0 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -266,7 +266,7 @@ def search(query, page = 1, search_params = produce_search_params(content_type: return {0, [] of SearchItem} end - html = client.get("/results?q=#{URI.escape(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body + html = client.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body if html.empty? return {0, [] of SearchItem} end @@ -371,7 +371,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte end token = Base64.urlsafe_encode(token.to_slice) - token = URI.escape(token) + token = URI.encode_www_form(token) return token end @@ -396,7 +396,7 @@ def produce_channel_search_url(ucid, query, page) data.rewind data = Base64.urlsafe_encode(data) - continuation = URI.escape(data) + continuation = URI.encode_www_form(data) data = IO::Memory.new @@ -421,7 +421,7 @@ def produce_channel_search_url(ucid, query, page) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 329081578..26db51ea4 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -42,7 +42,7 @@ end def extract_plid(url) wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"] - wrapper = URI.unescape(wrapper) + wrapper = URI.decode_www_form(wrapper) wrapper = Base64.decode(wrapper) # 0xe2 0x02 0x2e diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 4d662e9aa..424d1e50e 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -408,7 +408,7 @@ struct Video json.object do json.field "label", caption.name.simpleText json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" end end end diff --git a/src/invidious/views/authorize_token.ecr b/src/invidious/views/authorize_token.ecr index 53b8f001e..8ea99010b 100644 --- a/src/invidious/views/authorize_token.ecr +++ b/src/invidious/views/authorize_token.ecr @@ -72,7 +72,7 @@ <% end %> - + <% end %> diff --git a/src/invidious/views/change_password.ecr b/src/invidious/views/change_password.ecr index 2e68556b4..fb558f1d5 100644 --- a/src/invidious/views/change_password.ecr +++ b/src/invidious/views/change_password.ecr @@ -6,7 +6,7 @@
-
+ <%= translate(locale, "Change password") %>
@@ -23,7 +23,7 @@ <%= translate(locale, "Change password") %> - +
diff --git a/src/invidious/views/clear_watch_history.ecr b/src/invidious/views/clear_watch_history.ecr index 2bb9884cc..5f9d10326 100644 --- a/src/invidious/views/clear_watch_history.ecr +++ b/src/invidious/views/clear_watch_history.ecr @@ -3,7 +3,7 @@ <% end %>
-
+ <%= translate(locale, "Clear watch history?") %>
@@ -13,12 +13,12 @@
- + <%= translate(locale, "No") %>
- +
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index c722d0bc3..d78d8c4b0 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -91,7 +91,7 @@ <% if env.get? "show_watched" %>
" method="post"> - "> + ">

@@ -11,7 +11,7 @@ <% else %>

" method="post"> - "> + "> @@ -24,7 +24,7 @@ ucid: '<%= ucid %>', author: '<%= HTML.escape(author) %>', sub_count_text: '<%= HTML.escape(sub_count_text) %>', - csrf_token: '<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>', + csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', subscribe_text: '<%= HTML.escape(translate(locale, "Subscribe")) %>', unsubscribe_text: '<%= HTML.escape(translate(locale, "Unsubscribe")) %>' } diff --git a/src/invidious/views/data_control.ecr b/src/invidious/views/data_control.ecr index 463d5fd47..e3edb9ea7 100644 --- a/src/invidious/views/data_control.ecr +++ b/src/invidious/views/data_control.ecr @@ -3,7 +3,7 @@ <% end %>

- +
<%= translate(locale, "Import") %> diff --git a/src/invidious/views/delete_account.ecr b/src/invidious/views/delete_account.ecr index 0fa1e77c4..9103d5b70 100644 --- a/src/invidious/views/delete_account.ecr +++ b/src/invidious/views/delete_account.ecr @@ -3,7 +3,7 @@ <% end %>
- + <%= translate(locale, "Delete account?") %>
@@ -13,12 +13,12 @@
- + <%= translate(locale, "No") %>
- +
diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr index e51545609..7d7ded2c4 100644 --- a/src/invidious/views/history.ecr +++ b/src/invidious/views/history.ecr @@ -20,7 +20,7 @@ @@ -35,7 +35,7 @@ var watched_data = {
" method="post"> - "> + ">