diff --git a/lib/httpoison/base.ex b/lib/httpoison/base.ex index a61faca..fece2ee 100644 --- a/lib/httpoison/base.ex +++ b/lib/httpoison/base.ex @@ -261,7 +261,8 @@ defmodule HTTPoison.Base do * `:follow_redirect` - a boolean that causes redirects to be followed * `:max_redirect` - an integer denoting the maximum number of redirects to follow * `:params` - an enumerable consisting of two-item tuples that will be appended to the url as query string parameters - * `:max_body_length` - a non-negative integer denoting the max response body length. See :hackney.body/2 + * `:max_body_length` - a non-negative integer denoting the max response body length. default: :infinity + * `:partial_response` - a boolean denoting whether exceeding `:max_body_length` returns an error, or a partial response. default: false Timeouts can be an integer or `:infinity` @@ -643,9 +644,12 @@ defmodule HTTPoison.Base do ) {:ok, status_code, headers, client} -> - max_length = Keyword.get(options, :max_body_length, :infinity) + opts = %{ + max_length: Keyword.get(options, :max_body_length, :infinity), + partial_response: Keyword.get(options, :partial_response, false) + } - case :hackney.body(client, max_length) do + case parse_request_body(client, opts, "") do {:ok, body} -> response( process_status_code, @@ -669,6 +673,31 @@ defmodule HTTPoison.Base do end end + defp parse_request_body(client, %{max_length: max} = opts, acc) + when max >= byte_size(acc) do + case :hackney.stream_body(client) do + {:ok, data} -> + parse_request_body(client, opts, acc <> data) + + :done -> + {:ok, acc} + + {:error, reason} = error -> + case reason do + {:closed, bin} when is_binary(bin) -> + {:error, {:closed, acc <> bin}} + + _ -> + error + end + end + end + + defp parse_request_body(_client, %{partial_response: true}, acc), do: {:ok, acc} + + defp parse_request_body(_client, %{partial_response: false}, acc), + do: {:error, {:body_too_large, acc}} + defp do_request(method, request_url, request_headers, {:stream, enumerable}, hn_options) do with {:ok, ref} <- :hackney.request(method, request_url, request_headers, :stream, hn_options) do failures = diff --git a/test/httpoison_base_test.exs b/test/httpoison_base_test.exs index d791ea3..2854303 100644 --- a/test/httpoison_base_test.exs +++ b/test/httpoison_base_test.exs @@ -41,7 +41,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert Example.post!("localhost", "body") == %HTTPoison.Response{ @@ -57,7 +58,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert ExampleParamsOptions.get!("localhost", [], params: %{foo: "bar"}) == %HTTPoison.Response{ @@ -87,7 +89,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body", [], timeout: 12345) == %HTTPoison.Response{ @@ -104,7 +107,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body", [], recv_timeout: 12345) == %HTTPoison.Response{ @@ -120,7 +124,8 @@ defmodule HTTPoisonBaseTest do :post, "http://localhost", [], "body", [proxy: "proxy"] -> {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body", [], proxy: "proxy") == %HTTPoison.Response{ @@ -145,7 +150,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!( "localhost", @@ -173,7 +179,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!( "localhost", @@ -197,7 +204,8 @@ defmodule HTTPoisonBaseTest do :post, "http://localhost", [], "body", [proxy: "proxy"] -> {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body") == %HTTPoison.Response{ @@ -215,7 +223,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body") == %HTTPoison.Response{ @@ -233,7 +242,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("https://localhost", "body") == %HTTPoison.Response{ @@ -251,7 +261,8 @@ defmodule HTTPoisonBaseTest do :post, "http://localhost", [], "body", [] -> {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body") == %HTTPoison.Response{ @@ -271,7 +282,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body", [], ssl: [certfile: "certs/client.crt"]) == %HTTPoison.Response{ @@ -291,7 +303,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body", [], follow_redirect: true) == %HTTPoison.Response{ @@ -307,7 +320,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.post!("localhost", "body", [], max_redirect: 2) == %HTTPoison.Response{ @@ -323,7 +337,8 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, :infinity -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> :done end) assert HTTPoison.get("localhost") == {:ok, @@ -338,14 +353,24 @@ defmodule HTTPoisonBaseTest do {:ok, 200, "headers", :client} end) - expect(:hackney, :body, fn _, _ -> {:ok, "res"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) assert HTTPoison.get("localhost", [], max_body_length: 3) == + {:error, %HTTPoison.Error{id: nil, reason: {:body_too_large, "response"}}} + + expect(:hackney, :request, fn :get, "http://localhost", [], "", [] -> + {:ok, 200, "headers", :client} + end) + + expect(:hackney, :stream_body, fn _ -> {:ok, "response"} end) + expect(:hackney, :stream_body, fn _ -> {:ok, "additionalcontent"} end) + + assert HTTPoison.get("localhost", [], max_body_length: 12, partial_response: true) == {:ok, %HTTPoison.Response{ status_code: 200, headers: "headers", - body: "res", + body: "responseadditionalcontent", request_url: "http://localhost" }} end