From 507ebd1a4d98d4ae5e5dd116a5fa96605ca6ed60 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 1 Aug 2024 17:11:50 +0200 Subject: [PATCH] `encode_body`: Support `%File.Stream{}` in `:form_multipart` Closes #401 --- lib/req/steps.ex | 1 + lib/req/utils.ex | 39 ++++++++++++++++++++++----------------- test/req/steps_test.exs | 1 + test/req/utils_test.exs | 18 ++++++++++++------ 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/lib/req/steps.ex b/lib/req/steps.ex index 140477a..d1671ed 100644 --- a/lib/req/steps.ex +++ b/lib/req/steps.ex @@ -417,6 +417,7 @@ defmodule Req.Steps do %{request | body: multipart.body} |> Req.Request.put_new_header("content-type", multipart.content_type) + |> Req.Request.put_new_header("content-length", Integer.to_string(multipart.size)) data = request.options[:json] -> %{request | body: Jason.encode_to_iodata!(data)} diff --git a/lib/req/utils.ex b/lib/req/utils.ex index f249fa7..f62c921 100644 --- a/lib/req/utils.ex +++ b/lib/req/utils.ex @@ -326,6 +326,8 @@ defmodule Req.Utils do %CollectWithHash{collectable: collectable, type: type} end + @crlf "\r\n" + @doc """ Encodes fields into "multipart/form-data" format. """ @@ -336,40 +338,44 @@ defmodule Req.Utils do options[:boundary] || Base.encode16(:crypto.strong_rand_bytes(16), padding: false, case: :lower) - crlf = "\r\n" + footer = [[@crlf, "--", boundary, "--", @crlf]] - body = + {body, size} = fields - |> Enum.reduce([], &add_form_parts(&2, encode_form_part(&1, boundary))) - |> add_form_parts([[crlf, "--", boundary, "--", crlf]]) + |> Enum.reduce({[], 0}, &add_form_parts(&2, encode_form_part(&1, boundary))) + |> add_form_parts({footer, IO.iodata_length(footer)}) %{ + size: size, content_type: "multipart/form-data; boundary=#{boundary}", body: body } end - defp add_form_parts(parts1, parts2) when is_list(parts1) and is_list(parts2) do - [parts1, parts2] + defp add_form_parts({parts1, size1}, {parts2, size2}) + when is_list(parts1) and is_list(parts2) do + {[parts1, parts2], size1 + size2} end - defp add_form_parts(parts1, parts2) do - Stream.concat(parts1, parts2) + defp add_form_parts({parts1, size1}, {parts2, size2}) do + {Stream.concat(parts1, parts2), size1 + size2} end defp encode_form_part({name, {value, options}}, boundary) do options = Keyword.validate!(options, [:filename, :content_type]) - {parts, options} = + {parts, parts_size, options} = case value do integer when is_integer(integer) -> - {[Integer.to_string(integer)], options} + part = Integer.to_string(integer) + {[part], byte_size(part), options} value when is_binary(value) or is_list(value) -> - {[value], options} + {[value], IO.iodata_length(value), options} stream = %File.Stream{} -> filename = Path.basename(stream.path) + size = File.stat!(stream.path).size options = options @@ -378,7 +384,7 @@ defmodule Req.Utils do MIME.from_path(filename) end) - {stream, options} + {stream, size, options} end params = @@ -388,17 +394,16 @@ defmodule Req.Utils do [] end - crlf = "\r\n" - headers = if content_type = options[:content_type] do - ["content-type: ", content_type, crlf] + ["content-type: ", content_type, @crlf] else [] end - headers = ["content-disposition: form-data; name=\"#{name}\"", params, crlf, headers] - add_form_parts([[crlf, "--", boundary, crlf, headers, crlf]], parts) + headers = ["content-disposition: form-data; name=\"#{name}\"", params, @crlf, headers] + header = [[@crlf, "--", boundary, @crlf, headers, @crlf]] + add_form_parts({header, IO.iodata_length(header)}, {parts, parts_size}) end defp encode_form_part({name, value}, boundary) do diff --git a/test/req/steps_test.exs b/test/req/steps_test.exs index 2e8aadf..7774dd7 100644 --- a/test/req/steps_test.exs +++ b/test/req/steps_test.exs @@ -257,6 +257,7 @@ defmodule Req.StepsTest do plug = fn conn -> conn = Plug.Parsers.call(conn, Plug.Parsers.init(parsers: [:multipart])) + assert Plug.Conn.get_req_header(conn, "content-length") == ["393"] assert %{"a" => "1", "b" => b, "c" => c} = conn.body_params assert b.filename == "b.txt" diff --git a/test/req/utils_test.exs b/test/req/utils_test.exs index 48c786d..0367909 100644 --- a/test/req/utils_test.exs +++ b/test/req/utils_test.exs @@ -72,7 +72,7 @@ defmodule Req.UtilsTest do describe "encode_form_multipart" do test "it works" do - %{content_type: content_type, body: body} = + %{content_type: content_type, body: body, size: size} = Req.Utils.encode_form_multipart( [ field1: 1, @@ -82,9 +82,11 @@ defmodule Req.UtilsTest do boundary: "foo" ) + body = IO.iodata_to_binary(body) + assert size == byte_size(body) assert content_type == "multipart/form-data; boundary=foo" - assert IO.iodata_to_binary(body) == """ + assert body == """ \r\n\ --foo\r\n\ content-disposition: form-data; name=\"field1\"\r\n\ @@ -107,7 +109,7 @@ defmodule Req.UtilsTest do test "can return stream", %{tmp_dir: tmp_dir} do File.write!("#{tmp_dir}/2.txt", "22") - %{body: body} = + %{body: body, size: size} = Req.Utils.encode_form_multipart( [ field1: 1, @@ -117,8 +119,10 @@ defmodule Req.UtilsTest do ) assert is_function(body) + body = body |> Enum.to_list() |> IO.iodata_to_binary() + assert size == byte_size(body) - assert body |> Enum.to_list() |> IO.iodata_to_binary() == """ + assert body == """ \r\n\ --foo\r\n\ content-disposition: form-data; name=\"field1\"\r\n\ @@ -132,7 +136,7 @@ defmodule Req.UtilsTest do --foo--\r\n\ """ - %{body: body} = + %{body: body, size: size} = Req.Utils.encode_form_multipart( [ field2: File.stream!("#{tmp_dir}/2.txt"), @@ -142,8 +146,10 @@ defmodule Req.UtilsTest do ) assert is_function(body) + body = body |> Enum.to_list() |> IO.iodata_to_binary() + assert size == byte_size(body) - assert body |> Enum.to_list() |> IO.iodata_to_binary() == """ + assert body == """ \r\n\ --foo\r\n\ content-disposition: form-data; name=\"field2\"; filename=\"2.txt\"\r\n\