From 4c50256ad8cd6b00272093a662e6e332344a3d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 4 Dec 2023 14:24:34 +0100 Subject: [PATCH 01/32] WIP --- test/Tests/src/Network/Http_Spec.enso | 3 +++ test/Tests/src/Network/URI_Spec.enso | 2 ++ 2 files changed, 5 insertions(+) diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 993d26c5a51c..5735c7acbb21 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -116,6 +116,9 @@ spec = response = Data.fetch url_get response.at "headers" . at "Content-Length" . should_equal "0" + uri_response = url_get.to_uri.fetch + uri_response.at "headers" . at "Content-Length" . should_equal "0" + Test.specify "Can skip auto-parse" <| response = Data.fetch url_get try_auto_parse_response=False response.code.code . should_equal 200 diff --git a/test/Tests/src/Network/URI_Spec.enso b/test/Tests/src/Network/URI_Spec.enso index f1cdc3820ee4..9a501746f600 100644 --- a/test/Tests/src/Network/URI_Spec.enso +++ b/test/Tests/src/Network/URI_Spec.enso @@ -16,6 +16,7 @@ spec = addr.path.should_equal "/foo/bar" addr.query.should_equal "key=val" addr.fragment.should_equal Nothing + Test.specify "should escape URI" <| addr = URI.parse "https://%D0%9B%D0%B8%D0%BD%D1%83%D1%81:pass@ru.wikipedia.org/wiki/%D0%AF%D0%B4%D1%80%D0%BE_Linux?%D0%9A%D0%BE%D0%B4" addr.user_info.should_equal "Линус:pass" @@ -28,6 +29,7 @@ spec = addr.raw_path.should_equal "/wiki/%D0%AF%D0%B4%D1%80%D0%BE_Linux" addr.raw_query.should_equal "%D0%9A%D0%BE%D0%B4" addr.raw_fragment.should_equal Nothing + Test.specify "should return Syntax_Error when parsing invalid URI" <| URI.parse "a b c" . should_fail_with Syntax_Error From a250d3c68249b71d9391440d6855b2c7216a2697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 4 Dec 2023 14:27:14 +0100 Subject: [PATCH 02/32] Fetch tests --- test/Table_Tests/src/IO/Fetch_Spec.enso | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/test/Table_Tests/src/IO/Fetch_Spec.enso b/test/Table_Tests/src/IO/Fetch_Spec.enso index e4183b03591d..0fbddcee140d 100644 --- a/test/Table_Tests/src/IO/Fetch_Spec.enso +++ b/test/Table_Tests/src/IO/Fetch_Spec.enso @@ -29,9 +29,16 @@ spec = r.to Table . should_equal expected_table Test.specify "fetching csv" <| - r = Data.fetch base_url_with_slash+"testfiles/table.csv" + url = base_url_with_slash+"testfiles/table.csv" + r = Data.fetch url expected_table = Table.from_rows ["A", "B"] [[1, "x"], [3, "y"]] - r.to Table . should_equal expected_table + + r.should_be_a Table + r.should_equal expected_table + + r2 = url.to_uri.fetch + r2.should_be_a Table + r2.should_equal expected_table Test.specify "fetching xls" <| url = base_url_with_slash+"testfiles/table.xls" @@ -44,7 +51,7 @@ spec = r2 = Data.fetch url try_auto_parse_response=False . decode (Excel (Excel_Section.Worksheet "MyTestSheet")) r2.should_be_a Table - r2 . should_equal expected_table + r2.should_equal expected_table Test.specify "fetching xlsx" <| url = base_url_with_slash+"testfiles/table.xlsx" @@ -57,4 +64,8 @@ spec = r2 = Data.fetch url try_auto_parse_response=False . decode (Excel (Excel_Section.Worksheet "MyTestSheet")) r2.should_be_a Table - r2 . should_equal expected_table + r2.should_equal expected_table + + r3 = url.fetch try_auto_parse_response=False . decode (Excel (Excel_Section.Worksheet "MyTestSheet")) + r3.should_be_a Table + r3.should_equal expected_table From 381bc293f951282eac7f786bd6ddc3a0cdc36f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 4 Dec 2023 14:37:08 +0100 Subject: [PATCH 03/32] checkpoint --- test/Tests/src/Network/Http_Spec.enso | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 5735c7acbb21..a8870d691310 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -311,6 +311,9 @@ spec = } response . should_equal expected_response + uri_response = url_post.to_uri.post body + uri_response . should_equal expected_response + Test.specify "Can perform a Text POST with explicit content type" <| response = Data.post url_post (Request_Body.Text 'a,b,c\n' content_type="text/csv") expected_response = Json.parse <| ''' @@ -649,3 +652,25 @@ spec = resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "")) . should_contain_the_same_elements_as expected main = Test_Suite.run_main spec + +response_template method data content_type content_length=data.length = + template = ''' + { + "headers": { + "Connection": "Upgrade, HTTP2-Settings", + "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", + "User-Agent": "Java-http-client/21.0.1", + "Upgrade": "h2c", + "Content-Type": "<$CONTENT_TYPE>", + "Content-Length": "<$CONTENT_LENGTH>" + }, + "origin": "127.0.0.1", + "url": "", + "method": "<$METHOD>", + "form": null, + "files": null, + "data": <$DATA>, + "args": {} + } + + template.replace "<$CONTENT_TYPE>" content_type . replace "<$CONTENT_LENGTH>" content_length.to_text . replace "<$METHOD>" method . replace "<$DATA>" data.to_json From 007e9283b26186f27a68a30a76c79e36250c9864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 4 Dec 2023 15:54:04 +0100 Subject: [PATCH 04/32] reduce duplicate code in tests --- test/Tests/src/Network/Http_Spec.enso | 228 ++---------------- .../main/java/org/enso/shttp/TestHandler.java | 56 +++-- 2 files changed, 55 insertions(+), 229 deletions(-) diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index a8870d691310..20cd0d18e48d 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -172,24 +172,7 @@ spec = Test.specify "Can perform a Request_Body.Text POST" <| response = Data.post url_post (Request_Body.Text "hello world") - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "text/plain; charset=UTF-8", - "Content-Length": "11" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "hello world", - "args": {} - } + expected_response = echo_response_template "POST" "hello world" content_type="text/plain; charset=UTF-8" response . should_equal expected_response url_response = url_post.to_uri.post (Request_Body.Text "hello world") @@ -198,117 +181,35 @@ spec = Test.specify "Can perform a Request_Body.Json JSON POST" <| json = Json.parse '{"a": "asdf", "b": 123}' response = Data.post url_post (Request_Body.Json json) - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "application/json", - "Content-Length": "20" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "{\\"a\\":\\"asdf\\",\\"b\\":123}", - "args": {} - } + expected_response = echo_response_template "POST" '{"a":"asdf","b":123}' content_type="application/json" response . should_equal expected_response Test.specify "Can perform a JSON POST" <| json = Json.parse '{"a": "asdf", "b": 123}' response = Data.post url_post json - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "application/json", - "Content-Length": "20" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "{\\"a\\":\\"asdf\\",\\"b\\":123}", - "args": {} - } + expected_response = echo_response_template "POST" '{"a":"asdf","b":123}' content_type="application/json" response . should_equal expected_response Test.specify "Can perform an object Request_Body.Json POST" <| response = Data.post url_post (Request_Body.Json (Test_Type.Aaa "abc")) - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "application/json", - "Content-Length": "50" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "{\\"type\\":\\"Test_Type\\",\\"constructor\\":\\"Aaa\\",\\"s\\":\\"abc\\"}", - "args": {} - } + expected_response = echo_response_template "POST" '{"type":"Test_Type","constructor":"Aaa","s":"abc"}' content_type="application/json" response . should_equal expected_response Test.specify "Can perform an object JSON POST" <| response = Data.post url_post (Test_Type.Bbb 12) - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "application/json", - "Content-Length": "47" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "{\\"type\\":\\"Test_Type\\",\\"constructor\\":\\"Bbb\\",\\"i\\":12}", - "args": {} - } + expected_response = echo_response_template "POST" '{"type":"Test_Type","constructor":"Bbb","i":12}' content_type="application/json" response . should_equal expected_response + uri_response = url_post.to_uri.post (Test_Type.Bbb 12) + uri_response . should_equal expected_response + Test.specify "can handle a bad .to_json" <| Data.post url_post (Bad_To_Json.Aaa "abcd") . should_fail_with Illegal_Argument Test.specify "Can perform a Text POST with explicit encoding" <| body = Request_Body.Text 'Hello World!' encoding=Encoding.utf_16_le response = Data.post url_post body - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "text/plain; charset=UTF-16LE", - "Content-Length": "24" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "Hello World!", - "args": {} - } + expected_response = echo_response_template "POST" "Hello World!" content_type="text/plain; charset=UTF-16LE" content_length=24 response . should_equal expected_response uri_response = url_post.to_uri.post body @@ -316,24 +217,8 @@ spec = Test.specify "Can perform a Text POST with explicit content type" <| response = Data.post url_post (Request_Body.Text 'a,b,c\n' content_type="text/csv") - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "text/csv; charset=UTF-8", - "Content-Length": "6" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "a,b,c\\n", - "args": {} - } + + expected_response = echo_response_template "POST" 'a,b,c\n' content_type="text/csv; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a File POST" <| @@ -347,6 +232,7 @@ spec = test_file = enso_project.data / "sample.png" response = Data.post url_post (Request_Body.Binary test_file) response.at "headers" . at "Content-Type" . should_equal "application/octet-stream" + response.at "headers" . at "Content-Length" . should_equal test_file.size.to_text response.at "data" . should_start_with '\uFFFDPNG' Test.specify "Can perform a url-encoded form POST" <| @@ -373,68 +259,17 @@ spec = Test.specify "Can perform a Text POST with auto-conversion" <| response = Data.post url_post "hello world" - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "text/plain; charset=UTF-8", - "Content-Length": "11" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "hello world", - "args": {} - } + expected_response = echo_response_template "POST" "hello world" content_type="text/plain; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a Request_Body.Text PUT" <| response = Data.post url_put (Request_Body.Text "hello world") method=HTTP_Method.Put - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "text/plain; charset=UTF-8", - "Content-Length": "11" - }, - "origin": "127.0.0.1", - "url": "", - "method": "PUT", - "form": null, - "files": null, - "data": "hello world", - "args": {} - } + expected_response = echo_response_template "PUT" "hello world" content_type="text/plain; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a Request_Body.Text PATCH" <| response = Data.post url_patch (Request_Body.Text "hello world" content_type="application/diff") method=HTTP_Method.Patch - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "application/diff; charset=UTF-8", - "Content-Length": "11" - }, - "origin": "127.0.0.1", - "url": "", - "method": "PATCH", - "form": null, - "files": null, - "data": "hello world", - "args": {} - } + expected_response = echo_response_template "PATCH" "hello world" content_type="application/diff; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a DELETE" <| @@ -460,28 +295,11 @@ spec = Test.specify "Can skip auto-parse" <| response = Data.post url_post (Request_Body.Text "hello world") try_auto_parse_response=False - expected_response = Json.parse <| ''' - { - "headers": { - "Connection": "Upgrade, HTTP2-Settings", - "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", - "User-Agent": "Java-http-client/21.0.1", - "Upgrade": "h2c", - "Content-Type": "text/plain; charset=UTF-8", - "Content-Length": "11" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "hello world", - "args": {} - } + expected_response = echo_response_template "POST" "hello world" content_type="text/plain; charset=UTF-8" response.decode_as_json . should_equal expected_response Test.specify "Can send a custom header" <| - response = Data.post url_post (Request_Body.Text "hello world") headers=[Header.new "Custom" "asdf"] + response = Data.post url_post (Request_Body.Text "hello world") headers=[Header.new "Custom" "asdf", Header.new "Another" 'a:b: c - "ddd"'] expected_response = Json.parse <| ''' { "headers": { @@ -491,7 +309,8 @@ spec = "Upgrade": "h2c", "Content-Type": "text/plain; charset=UTF-8", "Content-Length": "11", - "Custom": "asdf" + "Custom": "asdf", + "Another": "a:b: c - \\"ddd\\"" }, "origin": "127.0.0.1", "url": "", @@ -575,6 +394,8 @@ spec = Test.specify "Multiple content types in the header list are respected" <| response = Data.post url_post (Request_Body.Text '{"a": "asdf", "b": 123}') headers=[Header.content_type "application/json", Header.content_type "text/plain"] + ## Our simple-httpbin server gets 2 Content-Type headers and merges them in the response. + How this is interpreted in practice depends on the server. expected_response = Json.parse <| ''' { "headers": { @@ -582,7 +403,7 @@ spec = "Http2-Settings": "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA", "User-Agent": "Java-http-client/21.0.1", "Upgrade": "h2c", - "Content-Type": "application/json", + "Content-Type": "application/json, text/plain", "Content-Length": "23" }, "origin": "127.0.0.1", @@ -653,7 +474,7 @@ spec = main = Test_Suite.run_main spec -response_template method data content_type content_length=data.length = +echo_response_template method data content_type content_length=data.length = template = ''' { "headers": { @@ -673,4 +494,5 @@ response_template method data content_type content_length=data.length = "args": {} } - template.replace "<$CONTENT_TYPE>" content_type . replace "<$CONTENT_LENGTH>" content_length.to_text . replace "<$METHOD>" method . replace "<$DATA>" data.to_json + replaced = template.replace "<$CONTENT_TYPE>" content_type . replace "<$CONTENT_LENGTH>" content_length.to_text . replace "<$METHOD>" method . replace "<$DATA>" data.to_json + Json.parse replaced diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index 89cc197561a0..32609ac1b69f 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -2,6 +2,8 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; +import org.apache.commons.text.StringEscapeUtils; + import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -13,7 +15,6 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.text.StringEscapeUtils; public class TestHandler implements HttpHandler { private static final Set ignoredHeaders = Set.of("Host"); @@ -44,27 +45,26 @@ public void doHandle(HttpExchange exchange) throws IOException { String textEncoding = "UTF-8"; HttpMethod meth = HttpMethod.valueOf(exchange.getRequestMethod()); - String response; + StringBuilder response; if (meth == HttpMethod.HEAD || meth == HttpMethod.OPTIONS) { - response = ""; + response = new StringBuilder(); exchange.sendResponseHeaders(200, -1); } else { exchange.getResponseHeaders().put("Content-Type", List.of("application/json")); - response = "{\n"; - response += " \"headers\": {\n"; + response = new StringBuilder("{\n"); + response.append(" \"headers\": {\n"); for (Map.Entry> entry : exchange.getRequestHeaders().entrySet()) { if (!ignoredHeaders.contains(entry.getKey())) { if (!first) { - response += ",\n"; + response.append(",\n"); } else { first = false; } - response += - " \"" - + formatHeaderKey(entry.getKey()) - + "\": \"" - + entry.getValue().get(0) - + "\""; + response + .append(" \"") + .append(formatHeaderKey(entry.getKey())) + .append("\": ") + .append(formatHeaderValues(entry.getValue())); } if (entry.getKey().equals("Content-type")) { contentType = entry.getValue().get(0); @@ -74,28 +74,27 @@ public void doHandle(HttpExchange exchange) throws IOException { } } } - response += "\n"; - response += " },\n"; - response += " \"origin\": \"127.0.0.1\",\n"; - response += " \"url\": \"\",\n"; - response += " \"method\": \"" + meth + "\",\n"; + response.append("\n"); + response.append(" },\n"); + response.append(" \"origin\": \"127.0.0.1\",\n"); + response.append(" \"url\": \"\",\n"); + response.append(" \"method\": \"").append(meth).append("\",\n"); if (meth == HttpMethod.POST || meth == HttpMethod.DELETE || meth == HttpMethod.PUT || meth == HttpMethod.PATCH) { - boolean isJson = contentType != null && contentType.equals("application/json"); - response += " \"form\": null,\n"; - response += " \"files\": null,\n"; + response.append(" \"form\": null,\n"); + response.append(" \"files\": null,\n"); String value = readBody(exchange.getRequestBody(), textEncoding); - response += - " \"data\": \"" + (value == null ? "" : StringEscapeUtils.escapeJson(value)) + "\",\n"; + response.append(" \"data\": \"").append(value == null ? "" : StringEscapeUtils.escapeJson(value)).append( + "\",\n"); } - response += " \"args\": {}\n"; - response += "}"; - exchange.sendResponseHeaders(200, response.getBytes().length); + response.append(" \"args\": {}\n"); + response.append("}"); + exchange.sendResponseHeaders(200, response.toString().getBytes().length); } OutputStream os = exchange.getResponseBody(); - os.write(response.getBytes()); + os.write(response.toString().getBytes()); os.close(); } @@ -138,4 +137,9 @@ private String formatHeaderKey(String key) { return key; } } + + private String formatHeaderValues(List key) { + String merged = key.stream().reduce((a, b) -> a + ", " + b).orElse(""); + return "\"" + StringEscapeUtils.escapeJson(merged) + "\""; + } } From 7c693cb5e83081baaf13b3bd40dbbc71f327e18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 5 Dec 2023 12:23:23 +0100 Subject: [PATCH 05/32] checkpoint --- test/Tests/src/Network/Http_Spec.enso | 10 ++++++ test/Tests/src/Network/URI_Spec.enso | 46 ++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 20cd0d18e48d..77c4371c176d 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -472,6 +472,16 @@ spec = expected = [Header.content_type "application/json", Header.content_type "text/plain"] resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "")) . should_contain_the_same_elements_as expected + Test.group "Http Error handling" <| + Test.specify "should be able to handle request errors" <| + err = Data.fetch "http://0.0.0.0:1/" + err.should_fail_with Request_Error + + Test.specify "should be able to handle IO errors" <| + # TODO how to trigger this error??? + err = Data.fetch "TODO" + err.should_fail_with HTTP_Error + main = Test_Suite.run_main spec echo_response_template method data content_type content_length=data.length = diff --git a/test/Tests/src/Network/URI_Spec.enso b/test/Tests/src/Network/URI_Spec.enso index 9a501746f600..34da12c40319 100644 --- a/test/Tests/src/Network/URI_Spec.enso +++ b/test/Tests/src/Network/URI_Spec.enso @@ -17,6 +17,29 @@ spec = addr.query.should_equal "key=val" addr.fragment.should_equal Nothing + Test.specify "should allow to convert a text to URI" <| + addr2 = URI.from "https://example.org:1234/?a=b&c=d+e#line=10,20" + addr2.should_be_a URI + addr2.scheme.should_equal "https" + addr2.user_info.should_equal Nothing + addr2.host.should_equal "example.org" + addr2.authority.should_equal "example.org" + addr2.port.should_equal 1234 + addr2.path.should_equal "/" + addr2.query.should_equal "a=b&c=d+e" + addr2.fragment.should_equal "line=10,20" + + addr3 = "ftp://example.com:21/" . to URI + addr3.should_be_a URI + addr3.scheme.should_equal "ftp" + addr3.user_info.should_equal Nothing + addr3.host.should_equal "example.com" + addr3.authority.should_equal "example.com" + addr3.port.should_equal 21 + addr3.path.should_equal "/" + addr3.query.should_equal Nothing + addr3.fragment.should_equal Nothing + Test.specify "should escape URI" <| addr = URI.parse "https://%D0%9B%D0%B8%D0%BD%D1%83%D1%81:pass@ru.wikipedia.org/wiki/%D0%AF%D0%B4%D1%80%D0%BE_Linux?%D0%9A%D0%BE%D0%B4" addr.user_info.should_equal "Линус:pass" @@ -31,10 +54,31 @@ spec = addr.raw_fragment.should_equal Nothing Test.specify "should return Syntax_Error when parsing invalid URI" <| - URI.parse "a b c" . should_fail_with Syntax_Error + r = URI.parse "a b c" + r.should_fail_with Syntax_Error + r.catch.to_display_text . should_contain "a b c" + URI.from "a b c" . should_fail_with Syntax_Error Test.specify "should compare two URIs for equality" <| (URI.parse "http://google.com").should_equal (URI.parse "http://google.com") (URI.parse "http://google.com").should_not_equal (URI.parse "http://amazon.com") + Test.group "URI_With_Query" <| + Test.specify "should allow to convert URI to URI_With_Query" <| + TODO + + Test.specify "will convert to URI_With_Query if a query argument is added" <| + + Test.specify "should be able to add multiple query arguments" <| + + Test.specify "should be able to convert back to URI" <| + + Test.specify "will not convert back to URI if secrets are present in the query arguments" pending="TODO testing secrets is for later" <| + + Test.specify "should correctly handle various characters within the key and value of arguments" <| + TODO tests both on raw URI_With_Query and on to_uri + + # TODO common tests on URI? + # maybe fetch/post + main = Test_Suite.run_main spec From 3b12d2866f310934affcec81d687729396e90d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 6 Dec 2023 12:58:41 +0100 Subject: [PATCH 06/32] URI_With_Query tests, extending http server to report query params; fix for a bug converting URI_With_Query->URI --- build.sbt | 3 +- .../Base/0.0.0-dev/src/Network/URI.enso | 5 +- .../base/enso_cloud/EnsoSecretHelper.java | 2 +- test/Tests/src/Network/Http_Spec.enso | 64 ++++++----- test/Tests/src/Network/URI_Spec.enso | 100 ++++++++++++++++-- .../main/java/org/enso/shttp/TestHandler.java | 41 ++++++- 6 files changed, 174 insertions(+), 41 deletions(-) diff --git a/build.sbt b/build.sbt index 2e6bae28ff25..55a25fc07dc0 100644 --- a/build.sbt +++ b/build.sbt @@ -2740,7 +2740,8 @@ lazy val `simple-httpbin` = project Compile / run / mainClass := Some("org.enso.shttp.SimpleHTTPBin"), assembly / mainClass := (Compile / run / mainClass).value, libraryDependencies ++= Seq( - "org.apache.commons" % "commons-text" % commonsTextVersion + "org.apache.commons" % "commons-text" % commonsTextVersion, + "org.apache.httpcomponents" % "httpclient" % httpComponentsVersion ), (Compile / run / fork) := true, (Compile / run / connectInput) := true diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso index 08d6aca5a682..a465154c8965 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso @@ -1,5 +1,6 @@ import project.Data.Enso_Cloud.Enso_Secret.Enso_Secret import project.Data.Json.JS_Object +import project.Data.Numbers.Integer import project.Data.Pair.Pair import project.Data.Text.Text import project.Data.Vector.Vector @@ -112,10 +113,10 @@ type URI import Standard.Examples example_port = Examples.uri.port - port : Text | Nothing + port : Integer | Nothing port self = port_number = self.internal_uri.getPort - if port_number == -1 then Nothing else port_number.to_text + if port_number == -1 then Nothing else port_number ## GROUP Metadata Get the path part of this URI. diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index 348751978c7b..1abdbe8ee946 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -71,7 +71,7 @@ public static URI replaceQuery(URI uri, String newQuery) throws URISyntaxExcepti var baseFragment = uri.getFragment(); baseFragment = baseFragment != null && !baseFragment.isBlank() ? "#" + baseFragment : ""; - return URI.create(baseURI + newQuery + baseFragment); + return URI.create(baseURI + "?" + newQuery + baseFragment); } private static String makeQueryAry(EnsoKeyValuePair pair, Function resolver) { diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 77c4371c176d..709db307f5b4 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -3,6 +3,7 @@ from Standard.Base import all import Standard.Base.Errors.Common.Forbidden_Operation import Standard.Base.Errors.Common.Syntax_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Network.HTTP.HTTP_Error.HTTP_Error import Standard.Base.Network.HTTP.Request.Request import Standard.Base.Network.HTTP.Request_Body.Request_Body import Standard.Base.Network.HTTP.Request_Error @@ -14,8 +15,6 @@ import Standard.Test.Extensions from Standard.Test import Test, Test_Suite from Standard.Test.Execution_Context_Helpers import run_with_and_without_output -polyglot java import java.lang.System as Java_System - type Test_Type Aaa (s:Text) Bbb (i:Integer) @@ -28,7 +27,7 @@ spec = ## To run this test locally: $ sbt 'simple-httpbin/run localhost 8080' $ export ENSO_HTTP_TEST_HTTPBIN_URL=http://localhost:8080/ - base_url = Java_System.getenv "ENSO_HTTP_TEST_HTTPBIN_URL" + base_url = Environment.get "ENSO_HTTP_TEST_HTTPBIN_URL" pending_has_url = if base_url != Nothing then Nothing else "The HTTP tests only run when the `ENSO_HTTP_TEST_HTTPBIN_URL` environment variable is set to URL of the httpbin server" @@ -85,7 +84,7 @@ spec = "Content-Length": "0" }, "origin": "127.0.0.1", - "url": "", + "path": "/get", "method": "GET", "args": {} } @@ -132,7 +131,7 @@ spec = "Content-Length": "0" }, "origin": "127.0.0.1", - "url": "", + "path": "/get", "method": "GET", "args": {} } @@ -172,7 +171,7 @@ spec = Test.specify "Can perform a Request_Body.Text POST" <| response = Data.post url_post (Request_Body.Text "hello world") - expected_response = echo_response_template "POST" "hello world" content_type="text/plain; charset=UTF-8" + expected_response = echo_response_template "POST" "/post" "hello world" content_type="text/plain; charset=UTF-8" response . should_equal expected_response url_response = url_post.to_uri.post (Request_Body.Text "hello world") @@ -181,23 +180,23 @@ spec = Test.specify "Can perform a Request_Body.Json JSON POST" <| json = Json.parse '{"a": "asdf", "b": 123}' response = Data.post url_post (Request_Body.Json json) - expected_response = echo_response_template "POST" '{"a":"asdf","b":123}' content_type="application/json" + expected_response = echo_response_template "POST" "/post" '{"a":"asdf","b":123}' content_type="application/json" response . should_equal expected_response Test.specify "Can perform a JSON POST" <| json = Json.parse '{"a": "asdf", "b": 123}' response = Data.post url_post json - expected_response = echo_response_template "POST" '{"a":"asdf","b":123}' content_type="application/json" + expected_response = echo_response_template "POST" "/post" '{"a":"asdf","b":123}' content_type="application/json" response . should_equal expected_response Test.specify "Can perform an object Request_Body.Json POST" <| response = Data.post url_post (Request_Body.Json (Test_Type.Aaa "abc")) - expected_response = echo_response_template "POST" '{"type":"Test_Type","constructor":"Aaa","s":"abc"}' content_type="application/json" + expected_response = echo_response_template "POST" "/post" '{"type":"Test_Type","constructor":"Aaa","s":"abc"}' content_type="application/json" response . should_equal expected_response Test.specify "Can perform an object JSON POST" <| response = Data.post url_post (Test_Type.Bbb 12) - expected_response = echo_response_template "POST" '{"type":"Test_Type","constructor":"Bbb","i":12}' content_type="application/json" + expected_response = echo_response_template "POST" "/post" '{"type":"Test_Type","constructor":"Bbb","i":12}' content_type="application/json" response . should_equal expected_response uri_response = url_post.to_uri.post (Test_Type.Bbb 12) @@ -209,7 +208,7 @@ spec = Test.specify "Can perform a Text POST with explicit encoding" <| body = Request_Body.Text 'Hello World!' encoding=Encoding.utf_16_le response = Data.post url_post body - expected_response = echo_response_template "POST" "Hello World!" content_type="text/plain; charset=UTF-16LE" content_length=24 + expected_response = echo_response_template "POST" "/post" "Hello World!" content_type="text/plain; charset=UTF-16LE" content_length=24 response . should_equal expected_response uri_response = url_post.to_uri.post body @@ -218,7 +217,7 @@ spec = Test.specify "Can perform a Text POST with explicit content type" <| response = Data.post url_post (Request_Body.Text 'a,b,c\n' content_type="text/csv") - expected_response = echo_response_template "POST" 'a,b,c\n' content_type="text/csv; charset=UTF-8" + expected_response = echo_response_template "POST" "/post" 'a,b,c\n' content_type="text/csv; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a File POST" <| @@ -259,17 +258,17 @@ spec = Test.specify "Can perform a Text POST with auto-conversion" <| response = Data.post url_post "hello world" - expected_response = echo_response_template "POST" "hello world" content_type="text/plain; charset=UTF-8" + expected_response = echo_response_template "POST" "/post" "hello world" content_type="text/plain; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a Request_Body.Text PUT" <| response = Data.post url_put (Request_Body.Text "hello world") method=HTTP_Method.Put - expected_response = echo_response_template "PUT" "hello world" content_type="text/plain; charset=UTF-8" + expected_response = echo_response_template "PUT" "/put" "hello world" content_type="text/plain; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a Request_Body.Text PATCH" <| response = Data.post url_patch (Request_Body.Text "hello world" content_type="application/diff") method=HTTP_Method.Patch - expected_response = echo_response_template "PATCH" "hello world" content_type="application/diff; charset=UTF-8" + expected_response = echo_response_template "PATCH" "/patch" "hello world" content_type="application/diff; charset=UTF-8" response . should_equal expected_response Test.specify "Can perform a DELETE" <| @@ -284,7 +283,7 @@ spec = "Content-Length": "0" }, "origin": "127.0.0.1", - "url": "", + "path": "/delete", "method": "DELETE", "form": null, "files": null, @@ -295,7 +294,7 @@ spec = Test.specify "Can skip auto-parse" <| response = Data.post url_post (Request_Body.Text "hello world") try_auto_parse_response=False - expected_response = echo_response_template "POST" "hello world" content_type="text/plain; charset=UTF-8" + expected_response = echo_response_template "POST" "/post" "hello world" content_type="text/plain; charset=UTF-8" response.decode_as_json . should_equal expected_response Test.specify "Can send a custom header" <| @@ -313,7 +312,7 @@ spec = "Another": "a:b: c - \\"ddd\\"" }, "origin": "127.0.0.1", - "url": "", + "path": "/post", "method": "POST", "form": null, "files": null, @@ -322,6 +321,14 @@ spec = } response . should_equal expected_response + Test.specify "can handle HTTP errors" <| + # This should give us bad request + r1 = Data.post url_delete + IO.println r1 + + r2 = Data.post base_url_with_slash + "some/unknown/path" + IO.println r2 + Test.specify "Cannot perform POST when output context is disabled" <| Context.Output.with_disabled <| Data.post url_post (Request_Body.Text "hello world") . should_fail_with Forbidden_Operation @@ -361,7 +368,7 @@ spec = "Content-Length": "23" }, "origin": "127.0.0.1", - "url": "", + "path": "/post", "method": "POST", "form": null, "files": null, @@ -383,7 +390,7 @@ spec = "Content-Length": "23" }, "origin": "127.0.0.1", - "url": "", + "path": "/post", "method": "POST", "form": null, "files": null, @@ -407,7 +414,7 @@ spec = "Content-Length": "23" }, "origin": "127.0.0.1", - "url": "", + "path": "/post", "method": "POST", "form": null, "files": null, @@ -429,7 +436,7 @@ spec = "Content-Length": "23" }, "origin": "127.0.0.1", - "url": "", + "path": "/post", "method": "POST", "form": null, "files": null, @@ -477,14 +484,14 @@ spec = err = Data.fetch "http://0.0.0.0:1/" err.should_fail_with Request_Error - Test.specify "should be able to handle IO errors" <| + Test.specify "should be able to handle IO errors" pending="TODO??" <| # TODO how to trigger this error??? err = Data.fetch "TODO" err.should_fail_with HTTP_Error main = Test_Suite.run_main spec -echo_response_template method data content_type content_length=data.length = +echo_response_template method path data content_type content_length=data.length = template = ''' { "headers": { @@ -496,7 +503,7 @@ echo_response_template method data content_type content_length=data.length = "Content-Length": "<$CONTENT_LENGTH>" }, "origin": "127.0.0.1", - "url": "", + "path": "<$PATH>", "method": "<$METHOD>", "form": null, "files": null, @@ -504,5 +511,10 @@ echo_response_template method data content_type content_length=data.length = "args": {} } - replaced = template.replace "<$CONTENT_TYPE>" content_type . replace "<$CONTENT_LENGTH>" content_length.to_text . replace "<$METHOD>" method . replace "<$DATA>" data.to_json + replaced = template + . replace "<$CONTENT_TYPE>" content_type + . replace "<$CONTENT_LENGTH>" content_length.to_text + . replace "<$METHOD>" method + . replace "<$PATH>" path + . replace "<$DATA>" data.to_json Json.parse replaced diff --git a/test/Tests/src/Network/URI_Spec.enso b/test/Tests/src/Network/URI_Spec.enso index 34da12c40319..273bef79401d 100644 --- a/test/Tests/src/Network/URI_Spec.enso +++ b/test/Tests/src/Network/URI_Spec.enso @@ -1,10 +1,20 @@ from Standard.Base import all import Standard.Base.Errors.Common.Syntax_Error +import Standard.Base.Network.URI_With_Query.URI_With_Query from Standard.Test import Test, Test_Suite import Standard.Test.Extensions spec = + ## To run this test locally: + $ sbt 'simple-httpbin/run localhost 8080' + $ export ENSO_HTTP_TEST_HTTPBIN_URL=http://localhost:8080/ + base_url = case Environment.get "ENSO_HTTP_TEST_HTTPBIN_URL" of + Nothing -> Nothing + str -> if str.ends_with "/" then str else str + "/" + pending_has_url = if base_url != Nothing then Nothing else + "The HTTP tests only run when the `ENSO_HTTP_TEST_HTTPBIN_URL` environment variable is set to URL of the httpbin server" + Test.group "URI" <| Test.specify "should parse URI from string" <| addr = URI.parse "http://user:pass@example.com/foo/bar?key=val" @@ -23,7 +33,7 @@ spec = addr2.scheme.should_equal "https" addr2.user_info.should_equal Nothing addr2.host.should_equal "example.org" - addr2.authority.should_equal "example.org" + addr2.authority.should_equal "example.org:1234" addr2.port.should_equal 1234 addr2.path.should_equal "/" addr2.query.should_equal "a=b&c=d+e" @@ -34,7 +44,7 @@ spec = addr3.scheme.should_equal "ftp" addr3.user_info.should_equal Nothing addr3.host.should_equal "example.com" - addr3.authority.should_equal "example.com" + addr3.authority.should_equal "example.com:21" addr3.port.should_equal 21 addr3.path.should_equal "/" addr3.query.should_equal Nothing @@ -64,21 +74,93 @@ spec = (URI.parse "http://google.com").should_not_equal (URI.parse "http://amazon.com") Test.group "URI_With_Query" <| - Test.specify "should allow to convert URI to URI_With_Query" <| - TODO - Test.specify "will convert to URI_With_Query if a query argument is added" <| + base_uri = URI.parse "http://a_user@example.com" + uri = base_uri . add_query_argument "foo" "bar" + + uri.should_be_a URI_With_Query + # The URI_With_Query should have the same fields as the URI: + uri.scheme.should_equal "http" + uri.user_info.should_equal "a_user" + uri.host.should_equal "example.com" + uri.authority.should_equal "a_user@example.com" + uri.port.should_equal Nothing + uri.path.should_equal "" + uri.query.should_equal "foo=bar" + uri.fragment.should_equal Nothing Test.specify "should be able to add multiple query arguments" <| + base_uri = URI.parse "https://example.com/path?a=b" + uri = base_uri . add_query_argument "c" "d" . add_query_argument "e" "f" + + uri.should_be_a URI_With_Query + uri.query.should_equal "a=b&c=d&e=f" + uri.scheme.should_equal "https" + uri.user_info.should_equal Nothing + uri.host.should_equal "example.com" + uri.authority.should_equal "example.com" + uri.port.should_equal Nothing + uri.path.should_equal "/path" + uri.fragment.should_equal Nothing Test.specify "should be able to convert back to URI" <| + base_uri = URI.parse "https://example.com/path?a=b" + uri1 = base_uri . add_query_argument "c" "d" . add_query_argument "e" "f" + + uri2 = uri1.to_uri + uri2.should_be_a URI + uri2.scheme.should_equal "https" + uri2.host.should_equal "example.com" + uri2.path.should_equal "/path" + uri2.query.should_equal "a=b&c=d&e=f" + uri2.should_equal (URI.parse "https://example.com/path?a=b&c=d&e=f") + + uri3 = uri2 . add_query_argument "g" "h" + uri3.should_be_a URI_With_Query + uri3.query.should_equal "a=b&c=d&e=f&g=h" + uri3.scheme.should_equal "https" + uri3.user_info.should_equal Nothing + uri3.host.should_equal "example.com" + uri3.port.should_equal Nothing + uri3.path.should_equal "/path" + uri3.fragment.should_equal Nothing + uri3.to_uri . should_equal (URI.parse "https://example.com/path?a=b&c=d&e=f&g=h") Test.specify "will not convert back to URI if secrets are present in the query arguments" pending="TODO testing secrets is for later" <| + Error.throw "TODO: secrets tests" + + # We rely on the simple-httpbin server for these tests, to ensure that the encoding is indeed correctly interpreted by a real-life server: + Test.specify "should correctly handle various characters within the key and value of arguments" pending=pending_has_url <| + base_uri = URI.parse base_url+"get" + + uri1 = base_uri . add_query_argument "a" "b" + r1 = uri1.fetch + IO.println r1 + + uri2 = base_uri + . add_query_argument "q1" "b c" + . add_query_argument "q2" "e+f" + . add_query_argument "q3" "e%20f" + r2 = uri2.fetch + IO.println r2 + + uri3 = base_uri + . add_query_argument "q4" "śnieżnobiały" + . add_query_argument "q5" '"f"\'\' ; 🚀🚧a' + . add_query_argument "q6" "[a=b]:[b=c][d=e], ]]] ==>
a" + r3 = uri3.fetch + IO.println r3 - Test.specify "should correctly handle various characters within the key and value of arguments" <| - TODO tests both on raw URI_With_Query and on to_uri + uri4 = base_uri + . add_query_argument "p+r" "b c" + . add_query_argument "p r" "b c" + . add_query_argument "🚀" "🚧" + . add_query_argument "śnieżnobiałą" "łąkę" + . add_query_argument "[a=b]:[b=c][d=e], ]]] ==>
a" "zzz" + r4 = uri4.fetch + IO.println r4 - # TODO common tests on URI? - # maybe fetch/post + Test.specify "should correctly handle various characters within the key and value of arguments" pending=("TODO testing secrets is for later".if_nothing pending_has_url) <| + Error.throw "TODO: test various characters inside of the secret value, like in the raw test above" main = Test_Suite.run_main spec diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index 32609ac1b69f..0f656509179c 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -3,6 +3,8 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import org.apache.commons.text.StringEscapeUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; @@ -10,6 +12,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; @@ -74,10 +78,31 @@ public void doHandle(HttpExchange exchange) throws IOException { } } } + + URI uri = exchange.getRequestURI(); + response.append("\n"); response.append(" },\n"); - response.append(" \"origin\": \"127.0.0.1\",\n"); - response.append(" \"url\": \"\",\n"); + response.append(" \"origin\": \"" + exchange.getRemoteAddress().getAddress().getHostAddress() + "\",\n"); + response.append(" \"path\": \"" + StringEscapeUtils.escapeJson(uri.getPath()) + "\",\n"); + if (uri.getQuery() != null) { + URIBuilder builder = new URIBuilder(uri); + List params = builder.getQueryParams().stream().sorted(nameValuePairComparator).toList(); + response.append(" \"query\": {\n"); + for (NameValuePair param : params) { + String key = StringEscapeUtils.escapeJson(param.getName()); + String value = StringEscapeUtils.escapeJson(param.getValue()); + response.append(" \"").append(key).append("\": \"").append(value).append("\""); + boolean isLast = param == params.get(params.size() - 1); + if (!isLast) { + response.append(",\n"); + } else { + response.append("\n"); + } + } + response.append(" },\n"); + } + response.append(" \"method\": \"").append(meth).append("\",\n"); if (meth == HttpMethod.POST || meth == HttpMethod.DELETE @@ -142,4 +167,16 @@ private String formatHeaderValues(List key) { String merged = key.stream().reduce((a, b) -> a + ", " + b).orElse(""); return "\"" + StringEscapeUtils.escapeJson(merged) + "\""; } + + private final Comparator nameValuePairComparator = new Comparator() { + @Override + public int compare(NameValuePair o1, NameValuePair o2) { + int c = o1.getName().compareTo(o2.getName()); + if (c != 0) { + return c; + } + + return o1.getValue().compareTo(o2.getValue()); + } + }; } From d7492d66bc71675475f897c294dce0b6cc46548d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 6 Dec 2023 13:45:30 +0100 Subject: [PATCH 07/32] another fix, get proper roundtrip test for query argument encoding --- .../lib/Standard/Base/0.0.0-dev/src/Data.enso | 4 +++ .../base/enso_cloud/EnsoSecretHelper.java | 2 +- test/Tests/src/Network/URI_Spec.enso | 33 ++++++++++++++----- .../main/java/org/enso/shttp/TestHandler.java | 26 +++++---------- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index a5a5d1fc36c9..b28b788d8f91 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -180,6 +180,10 @@ fetch : (URI | URI_With_Query | Text) -> HTTP_Method -> Vector (Header | Pair Te fetch (uri:(URI | URI_With_Query | Text)) (method:HTTP_Method=HTTP_Method.Get) (headers:(Vector (Header | Pair Text Text))=[]) (try_auto_parse_response:Boolean=True) = response = HTTP.fetch uri method headers if try_auto_parse_response.not then response.with_materialized_body else + ## We cannot catch decoding errors here and fall-back to the raw response + body, because as soon as decoding is started, at least part of the + input stream may already be consumed, so we cannot easily reconstruct + the whole stream. response.decode if_unsupported=response.with_materialized_body ## ALIAS http post, upload diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index 1abdbe8ee946..dab34ce3acee 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -101,7 +101,7 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U if (queryArguments != null && !queryArguments.isEmpty()) { try { var baseQuery = uri.getQuery(); - baseQuery = baseQuery != null && !baseQuery.isBlank() ? "?" + baseQuery + "&" : "?"; + baseQuery = baseQuery != null && !baseQuery.isBlank() ? baseQuery + "&" : ""; var query = baseQuery + queryArguments.stream().map(p -> makeQueryAry(p, EnsoSecretHelper::resolveValue)).collect(Collectors.joining("&")); resolvedURI = replaceQuery(uri, query); diff --git a/test/Tests/src/Network/URI_Spec.enso b/test/Tests/src/Network/URI_Spec.enso index 273bef79401d..99d21f0a4c3a 100644 --- a/test/Tests/src/Network/URI_Spec.enso +++ b/test/Tests/src/Network/URI_Spec.enso @@ -135,32 +135,49 @@ spec = uri1 = base_uri . add_query_argument "a" "b" r1 = uri1.fetch - IO.println r1 + decode_query_params r1 . should_equal [["a", "b"]] uri2 = base_uri . add_query_argument "q1" "b c" . add_query_argument "q2" "e+f" . add_query_argument "q3" "e%20f" r2 = uri2.fetch - IO.println r2 + # All values should be encoded and decoded correctly so that they retain the original symbols: + decode_query_params r2 . should_equal [["q1", "b c"], ["q2", "e+f"], ["q3", "e%20f"]] + s1 = '"f"\'\' ; 🚀🚧a' + s2 = "[a=b]:[b=c][d=e], ]]] ==>
a" uri3 = base_uri . add_query_argument "q4" "śnieżnobiały" - . add_query_argument "q5" '"f"\'\' ; 🚀🚧a' - . add_query_argument "q6" "[a=b]:[b=c][d=e], ]]] ==>
a" + . add_query_argument "q5" s1 + . add_query_argument "q6" s2 r3 = uri3.fetch - IO.println r3 + decode_query_params r3 . should_equal [["q4", "śnieżnobiały"], ["q5", s1], ["q6", s2]] uri4 = base_uri . add_query_argument "p+r" "b c" . add_query_argument "p r" "b c" - . add_query_argument "🚀" "🚧" + . add_query_argument "🚀" "🚧" . add_query_argument "śnieżnobiałą" "łąkę" - . add_query_argument "[a=b]:[b=c][d=e], ]]] ==>
a" "zzz" + . add_query_argument s2 "zzz" r4 = uri4.fetch - IO.println r4 + decode_query_params r4 . should_equal [["p+r", "b c"], ["p r", "b c"], ["🚀", "🚧"], ["śnieżnobiałą", "łąkę"], [s2, "zzz"]] + + Test.specify "may allow duplicate keys in query parameters" <| + uri = URI.parse base_url+"get" + . add_query_argument "a" "b" + . add_query_argument "a" "c" + . add_query_argument "a" "d" + r = uri.fetch + decode_query_params r . should_equal [["a", "b"], ["a", "c"], ["a", "d"]] Test.specify "should correctly handle various characters within the key and value of arguments" pending=("TODO testing secrets is for later".if_nothing pending_has_url) <| Error.throw "TODO: test various characters inside of the secret value, like in the raw test above" main = Test_Suite.run_main spec + +decode_query_params : Json -> Vector (Pair Text Text) +decode_query_params json_response = + params = json_response.at "queryParameters" + params.map pair-> + [pair.at "name", pair.at "value"] diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index 0f656509179c..0a382ccfadc5 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -80,6 +80,7 @@ public void doHandle(HttpExchange exchange) throws IOException { } URI uri = exchange.getRequestURI(); + System.out.println(exchange.getRequestMethod() + " " + uri); response.append("\n"); response.append(" },\n"); @@ -87,20 +88,21 @@ public void doHandle(HttpExchange exchange) throws IOException { response.append(" \"path\": \"" + StringEscapeUtils.escapeJson(uri.getPath()) + "\",\n"); if (uri.getQuery() != null) { URIBuilder builder = new URIBuilder(uri); - List params = builder.getQueryParams().stream().sorted(nameValuePairComparator).toList(); - response.append(" \"query\": {\n"); - for (NameValuePair param : params) { + List params = builder.getQueryParams(); + response.append(" \"queryParameters\": [\n"); + for (int i = 0; i < params.size(); i++) { + NameValuePair param = params.get(i); String key = StringEscapeUtils.escapeJson(param.getName()); String value = StringEscapeUtils.escapeJson(param.getValue()); - response.append(" \"").append(key).append("\": \"").append(value).append("\""); - boolean isLast = param == params.get(params.size() - 1); + response.append(" {\"name\": \"").append(key).append("\", \"value\": \"").append(value).append("\"}"); + boolean isLast = i == params.size() - 1; if (!isLast) { response.append(",\n"); } else { response.append("\n"); } } - response.append(" },\n"); + response.append(" ],\n"); } response.append(" \"method\": \"").append(meth).append("\",\n"); @@ -167,16 +169,4 @@ private String formatHeaderValues(List key) { String merged = key.stream().reduce((a, b) -> a + ", " + b).orElse(""); return "\"" + StringEscapeUtils.escapeJson(merged) + "\""; } - - private final Comparator nameValuePairComparator = new Comparator() { - @Override - public int compare(NameValuePair o1, NameValuePair o2) { - int c = o1.getName().compareTo(o2.getName()); - if (c != 0) { - return c; - } - - return o1.getValue().compareTo(o2.getValue()); - } - }; } From 44c2a5c133da2a2c359b8e0add75cafd2efdf623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 6 Dec 2023 14:45:29 +0100 Subject: [PATCH 08/32] checkpoint - start relying on httpclient utils for URL encoding of queries instead of rolling our own logic, pt.1 --- build.sbt | 5 +- .../Base/0.0.0-dev/src/Network/URI.enso | 4 +- .../0.0.0-dev/src/Network/URI_With_Query.enso | 51 +++++++++++-------- .../base/enso_cloud/EnsoSecretHelper.java | 39 ++++++++------ .../java/org/enso/base/net/URIHelpers.java | 19 +++++++ .../main/java/org/enso/shttp/TestHandler.java | 1 - 6 files changed, 78 insertions(+), 41 deletions(-) create mode 100644 std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java diff --git a/build.sbt b/build.sbt index 55a25fc07dc0..cf6dfef24542 100644 --- a/build.sbt +++ b/build.sbt @@ -2322,8 +2322,9 @@ lazy val `std-base` = project Compile / packageBin / artifactPath := `base-polyglot-root` / "std-base.jar", libraryDependencies ++= Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided" + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.apache.httpcomponents" % "httpclient" % httpComponentsVersion, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided" ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso index a465154c8965..19604d3a76d8 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso @@ -10,8 +10,8 @@ import project.Network.URI_With_Query.URI_With_Query import project.Nothing.Nothing import project.Panic.Panic -polyglot java import java.lang.Exception polyglot java import java.net.URI as Java_URI +polyglot java import java.net.URISyntaxException type URI ## ALIAS get uri @@ -32,7 +32,7 @@ type URI example_parse = URI.parse "http://example.com" parse : Text -> URI ! Syntax_Error parse uri:Text = - Panic.catch Exception (URI.Value (Java_URI.create uri)) caught_panic-> + Panic.catch URISyntaxException (URI.Value (Java_URI.new uri)) caught_panic-> message = caught_panic.payload.getMessage truncated = if message.is_nothing || message.length > 100 then "Invalid URI '" + uri.to_display_text + "'" else "URI syntax error: " + message diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso index 388999cecb32..3be7d50aab1c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso @@ -14,14 +14,15 @@ import project.Nothing.Nothing import project.Panic.Panic from project.Data.Boolean import Boolean, False, True -polyglot java import java.lang.Exception polyglot java import java.net.URI as Java_URI +polyglot java import java.net.URISyntaxException polyglot java import org.enso.base.enso_cloud.EnsoSecretHelper +polyglot java import org.enso.base.net.URIHelpers ## Represents a URI with a set of query parameters type URI_With_Query ## PRIVATE - Value uri:URI parameters:Vector + Value uri:URI (parameters : Vector (Pair Text (Text | Enso_Secret))) ## PRIVATE Convert this to URI. @@ -29,19 +30,13 @@ type URI_With_Query are used. to_uri : URI ! Enso_Secret_Error to_uri self = - Panic.catch Exception (URI.Value (EnsoSecretHelper.replaceQuery self.uri.internal_uri self.query)) caught_panic-> - message = caught_panic.payload.getMessage - Error.throw (Syntax_Error.Error "Unable to collapse to a URI:"+message) + java_params = make_java_parameters self.get_raw_parameters + URI.Value (build_java_uri_with_parameters self.uri.internal_uri java_params) ## GROUP Metadata Get the query part of this URI, but will error if any secrets are used. query : Text | Nothing ! Enso_Secret_Error - query self = - base_query = self.uri.query.if_nothing "" - query_params = self.parameters.map p-> - if p.second.is_a Enso_Secret then Error.throw Enso_Secret_Error.Access_Denied else - (EnsoSecretHelper.encodeArg p.first True) + "=" + (EnsoSecretHelper.encodeArg p.second False) - (if base_query == "" then "" else base_query + "&") + (query_params.join "&") + query self = self.to_uri.query ## GROUP Calculations Adds a query parameter to the URI @@ -53,19 +48,25 @@ type URI_With_Query add_query_argument self key:Text value:(Text | Enso_Secret) = URI_With_Query.Value self.uri self.parameters+[Pair.new key value] + ## PRIVATE + Returns a list of parameters as name-value pairs. + It will fail if any secrets are used. + get_raw_parameters : Vector (Pair Text Text) ! Enso_Secret_Error + get_raw_parameters self = + self.parameters.map p-> case p.second of + _ : Enso_Secret -> Error.throw Enso_Secret_Error.Access_Denied + _ : Text -> p + ## PRIVATE Convert this URI to text. to_text : Text to_text self = - base_query = self.uri.query.if_nothing "" - query_params = self.parameters.map p-> - if p.second.is_a Enso_Secret then p.first + "=__SECRET__" else - p.first + "=" + p.second - new_query = "?" + (if base_query == "" then "" else base_query + "&") + (query_params.join "&") - - Panic.catch Exception (EnsoSecretHelper.replaceQuery self.uri.internal_uri new_query . toString) caught_panic-> - message = caught_panic.payload.getMessage - Error.throw (Syntax_Error.Error "Unable to render URI_With_Query:"+message) + masked_parameters = self.parameters.map p-> case p.second of + _ : Enso_Secret -> Pair.new p.first "__SECRET__" + _ : Text -> p + java_params = make_java_parameters masked_parameters + java_uri = build_java_uri_with_parameters self.uri.internal_uri java_params + java_uri.to_text ## PRIVATE Convert to a display representation of this URI. @@ -114,3 +115,13 @@ type URI_With_Query Get the fragment part of this URI. fragment : Text | Nothing fragment self = self.uri.fragment + +## PRIVATE +make_java_parameters params = params.map p-> + URIHelpers.NameValuePair.new p.first p.second + +## PRIVATE +build_java_uri_with_parameters java_uri java_params = + Panic.catch URISyntaxException (URIHelpers.addQueryParameters java_uri java_params) caught_panic-> + message = caught_panic.payload.getMessage + Error.throw (Syntax_Error.Error "Unable to collapse to a URI: "+message) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index dab34ce3acee..75998a8b1f0e 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -22,21 +22,21 @@ public class EnsoSecretHelper { /** * Gets the value of an EnsoKeyValuePair resolving secrets. + * * @param pair The pair to resolve. * @return The pair's value. Should not be returned to Enso. */ private static String resolveValue(EnsoKeyValuePair pair) { return switch (pair) { case EnsoKeyStringPair stringPair -> stringPair.value(); - case EnsoKeySecretPair secretPair -> - EnsoSecretReader.readSecret(secretPair.secretId()); - case null -> - throw new IllegalArgumentException("EnsoKeyValuePair should not be NULL."); + case EnsoKeySecretPair secretPair -> EnsoSecretReader.readSecret(secretPair.secretId()); + case null -> throw new IllegalArgumentException("EnsoKeyValuePair should not be NULL."); }; } /** * Converts an EnsoKeyValuePair into a string for display purposes. Does not include secrets. + * * @param pair The pair to render. * @return The rendered string. */ @@ -44,14 +44,13 @@ private static String renderValue(EnsoKeyValuePair pair) { return switch (pair) { case EnsoKeyStringPair stringPair -> stringPair.value(); case EnsoKeySecretPair _ -> "__SECRET__"; - case null -> - throw new IllegalArgumentException("EnsoKeyValuePair should not be NULL."); + case null -> throw new IllegalArgumentException("EnsoKeyValuePair should not be NULL."); }; } /** * Substitutes the minimal parts within the string for the URI parse. - * */ + */ public static String encodeArg(String arg, boolean includeEquals) { var encoded = arg.replace("%", "%25") .replace("&", "%26") @@ -62,9 +61,10 @@ public static String encodeArg(String arg, boolean includeEquals) { return encoded; } - /** - * Replaces the query string in a URI. - * */ + /** + * Replaces the query string in a URI. + */ + @Deprecated public static URI replaceQuery(URI uri, String newQuery) throws URISyntaxException { var baseURI = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), null, null).toString(); @@ -82,7 +82,7 @@ private static String makeQueryAry(EnsoKeyValuePair pair, Function queryArguments, List headerArguments) + public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, URI uri, + List queryArguments, + List headerArguments) throws IOException, InterruptedException { // Build a new URI with the query arguments. @@ -102,7 +104,8 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U try { var baseQuery = uri.getQuery(); baseQuery = baseQuery != null && !baseQuery.isBlank() ? baseQuery + "&" : ""; - var query = baseQuery + queryArguments.stream().map(p -> makeQueryAry(p, EnsoSecretHelper::resolveValue)).collect(Collectors.joining("&")); + var query = + baseQuery + queryArguments.stream().map(p -> makeQueryAry(p, EnsoSecretHelper::resolveValue)).collect(Collectors.joining("&")); resolvedURI = replaceQuery(uri, query); renderedURI = resolvedURI; @@ -112,7 +115,8 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U throw new IllegalArgumentException("Cannot use secrets in query string with non-HTTPS URI."); } - var renderedQuery = baseQuery + queryArguments.stream().map(p -> makeQueryAry(p, EnsoSecretHelper::renderValue)).collect(Collectors.joining("&")); + var renderedQuery = baseQuery + queryArguments.stream().map(p -> makeQueryAry(p, + EnsoSecretHelper::renderValue)).collect(Collectors.joining("&")); renderedURI = replaceQuery(uri, renderedQuery); } } catch (URISyntaxException e) { @@ -134,11 +138,14 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U var javaResponse = client.send(httpRequest, bodyHandler); // Extract parts of the response - return new EnsoHttpResponse(renderedURI, javaResponse.headers().map().keySet().stream().toList(), javaResponse.headers(), javaResponse.body(), javaResponse.statusCode()); + return new EnsoHttpResponse(renderedURI, javaResponse.headers().map().keySet().stream().toList(), + javaResponse.headers(), javaResponse.body(), javaResponse.statusCode()); } /** * A subset of the HttpResponse to avoid leaking the decrypted Enso secrets. */ - public record EnsoHttpResponse(URI uri, List headerNames, HttpHeaders headers, InputStream body, int statusCode) { } + public record EnsoHttpResponse(URI uri, List headerNames, HttpHeaders headers, InputStream body, + int statusCode) { + } } diff --git a/std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java b/std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java new file mode 100644 index 000000000000..21b3cfb3d5cc --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java @@ -0,0 +1,19 @@ +package org.enso.base.net; + +import org.apache.http.client.utils.URIBuilder; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +public class URIHelpers { + public record NameValuePair(String name, String value) {} + + public static URI addQueryParameters(URI uri, List params) throws URISyntaxException { + URIBuilder builder = new URIBuilder(uri); + for (NameValuePair param : params) { + builder.addParameter(param.name(), param.value()); + } + return builder.build(); + } +} diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index 0a382ccfadc5..8dc2e992e821 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -13,7 +13,6 @@ import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URI; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; From 4b8818fd24f7e37a8b604ec371e6e5f8cd107e04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 00:42:11 +0100 Subject: [PATCH 09/32] use new logic in secrets --- .../base/enso_cloud/EnsoSecretHelper.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index 75998a8b1f0e..0ab99f2718d4 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -1,5 +1,7 @@ package org.enso.base.enso_cloud; +import org.enso.base.net.URIHelpers; + import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -101,26 +103,24 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U URI resolvedURI = uri; URI renderedURI = uri; if (queryArguments != null && !queryArguments.isEmpty()) { + boolean hasSecrets = queryArguments.stream().anyMatch(p -> p instanceof EnsoKeySecretPair); + if (hasSecrets && !uri.getScheme().equals("https")) { + // If used a secret then only allow HTTPS + throw new IllegalArgumentException("Cannot use secrets in query string with non-HTTPS URI, but the scheme was: " + uri.getScheme() + "."); + } + try { - var baseQuery = uri.getQuery(); - baseQuery = baseQuery != null && !baseQuery.isBlank() ? baseQuery + "&" : ""; - var query = - baseQuery + queryArguments.stream().map(p -> makeQueryAry(p, EnsoSecretHelper::resolveValue)).collect(Collectors.joining("&")); - - resolvedURI = replaceQuery(uri, query); - renderedURI = resolvedURI; - if (queryArguments.stream().anyMatch(p -> p instanceof EnsoKeySecretPair)) { - if (!resolvedURI.getScheme().equals("https")) { - // If used a secret then only allow HTTPS - throw new IllegalArgumentException("Cannot use secrets in query string with non-HTTPS URI."); - } - - var renderedQuery = baseQuery + queryArguments.stream().map(p -> makeQueryAry(p, - EnsoSecretHelper::renderValue)).collect(Collectors.joining("&")); - renderedURI = replaceQuery(uri, renderedQuery); - } + List resolvedArguments = queryArguments.stream() + .map(p -> new URIHelpers.NameValuePair(p.key(), resolveValue(p))) + .toList(); + List renderedArguments = queryArguments.stream() + .map(p -> new URIHelpers.NameValuePair(p.key(), renderValue(p))) + .toList(); + + resolvedURI = URIHelpers.addQueryParameters(uri, resolvedArguments); + renderedURI = URIHelpers.addQueryParameters(uri, renderedArguments); } catch (URISyntaxException e) { - throw new IllegalArgumentException("Unable to build a valid URI."); + throw new IllegalStateException("Unexpectedly unable to build a valid URI from the base URI: " + uri + " and query arguments: " + queryArguments + "."); } } builder.uri(resolvedURI); From 823c0f8b5d9e9ad512bd773386660077160d7c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 00:43:59 +0100 Subject: [PATCH 10/32] clean --- .../base/enso_cloud/EnsoSecretHelper.java | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index 0ab99f2718d4..3bce0c028556 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -15,8 +15,6 @@ import java.sql.SQLException; import java.util.List; import java.util.Properties; -import java.util.function.Function; -import java.util.stream.Collectors; /** * Makes HTTP requests with secrets in either header or query string. @@ -50,38 +48,6 @@ private static String renderValue(EnsoKeyValuePair pair) { }; } - /** - * Substitutes the minimal parts within the string for the URI parse. - */ - public static String encodeArg(String arg, boolean includeEquals) { - var encoded = arg.replace("%", "%25") - .replace("&", "%26") - .replace(" ", "%20"); - if (includeEquals) { - encoded = encoded.replace("=", "%3D"); - } - return encoded; - } - - /** - * Replaces the query string in a URI. - */ - @Deprecated - public static URI replaceQuery(URI uri, String newQuery) throws URISyntaxException { - var baseURI = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), null, null).toString(); - - var baseFragment = uri.getFragment(); - baseFragment = baseFragment != null && !baseFragment.isBlank() ? "#" + baseFragment : ""; - - return URI.create(baseURI + "?" + newQuery + baseFragment); - } - - private static String makeQueryAry(EnsoKeyValuePair pair, Function resolver) { - String resolvedKey = pair.key() != null && !pair.key().isBlank() ? encodeArg(pair.key(), true) + "=" : ""; - String resolvedValue = encodeArg(resolver.apply(pair), false); - return resolvedKey + resolvedValue; - } - //** Gets a JDBC connection resolving EnsoKeyValuePair into the properties. **// public static Connection getJDBCConnection(String url, EnsoKeyValuePair[] properties) throws SQLException { From 73fe9036307f8cc2a5fb75cd625f71bb6b2464ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 00:49:35 +0100 Subject: [PATCH 11/32] a few more cases --- test/Tests/src/Network/URI_Spec.enso | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/Tests/src/Network/URI_Spec.enso b/test/Tests/src/Network/URI_Spec.enso index 99d21f0a4c3a..2eb1ce02f1db 100644 --- a/test/Tests/src/Network/URI_Spec.enso +++ b/test/Tests/src/Network/URI_Spec.enso @@ -151,8 +151,11 @@ spec = . add_query_argument "q4" "śnieżnobiały" . add_query_argument "q5" s1 . add_query_argument "q6" s2 + . add_query_argument "q7" "" + . add_query_argument "q8" " " + . add_query_argument "q9" "%%%" r3 = uri3.fetch - decode_query_params r3 . should_equal [["q4", "śnieżnobiały"], ["q5", s1], ["q6", s2]] + decode_query_params r3 . should_equal [["q4", "śnieżnobiały"], ["q5", s1], ["q6", s2], ["q7", ""], ["q8", " "], ["q9", "%%%"]] uri4 = base_uri . add_query_argument "p+r" "b c" From 02e5b49ceb0aaf2181eeba1f06a79149a2aab165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 01:23:56 +0100 Subject: [PATCH 12/32] clean --- test/Tests/src/Network/Http_Spec.enso | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 709db307f5b4..0a57b5bd4195 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -30,6 +30,8 @@ spec = base_url = Environment.get "ENSO_HTTP_TEST_HTTPBIN_URL" pending_has_url = if base_url != Nothing then Nothing else "The HTTP tests only run when the `ENSO_HTTP_TEST_HTTPBIN_URL` environment variable is set to URL of the httpbin server" + base_url_with_slash = base_url.if_not_nothing <| + if base_url.ends_with "/" then base_url else base_url + "/" Test.group "HTTP_Method parse" <| Test.specify "should be able to parse a string value into a method" <| @@ -67,7 +69,6 @@ spec = http.version.should_equal version_setting Test.group "fetch" pending=pending_has_url <| - base_url_with_slash = if base_url.ends_with "/" then base_url else base_url + "/" url_get = base_url_with_slash + "get" url_head = base_url_with_slash + "head" url_options = base_url_with_slash + "options" @@ -163,7 +164,6 @@ spec = Data.fetch "" . should_fail_with Request_Error Test.group "post" pending=pending_has_url <| - base_url_with_slash = if base_url.ends_with "/" then base_url else base_url + "/" url_post = base_url_with_slash + "post" url_put = base_url_with_slash + "put" url_patch = base_url_with_slash + "patch" @@ -352,7 +352,6 @@ spec = Data.post url_post (Request_Body.Binary test_file) . should_fail_with Request_Error Test.group "Headers" pending=pending_has_url <| - base_url_with_slash = if base_url.ends_with "/" then base_url else base_url + "/" url_post = base_url_with_slash + "post" Test.specify "Content-type in the body is respected" <| From 750d3c7ba74ee47c6b096f8e275cd7509b2cbf0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 01:26:42 +0100 Subject: [PATCH 13/32] trying to test HTTP_Error but nothing works --- test/Tests/src/Network/Http_Spec.enso | 13 +++++- .../java/org/enso/shttp/SimpleHTTPBin.java | 19 +++++---- .../java/org/enso/shttp/TimeoutHandler.java | 40 +++++++++++++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 0a57b5bd4195..b8a4d423c354 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -7,6 +7,7 @@ import Standard.Base.Network.HTTP.HTTP_Error.HTTP_Error import Standard.Base.Network.HTTP.Request.Request import Standard.Base.Network.HTTP.Request_Body.Request_Body import Standard.Base.Network.HTTP.Request_Error +import Standard.Base.Network.HTTP.Response.Response import Standard.Base.Network.Proxy.Proxy import Standard.Base.Runtime.Context from Standard.Base.Network.HTTP import resolve_headers @@ -483,9 +484,17 @@ spec = err = Data.fetch "http://0.0.0.0:1/" err.should_fail_with Request_Error - Test.specify "should be able to handle IO errors" pending="TODO??" <| + Test.specify "should be able to handle IO errors" <| # TODO how to trigger this error??? - err = Data.fetch "TODO" + # Tried triggering by exceeding timeout but: + # 1. timeout here is just connection timeout + # 2. if the timeout occurs before connecting - we get Request_Error + # 3. if the timeout occurs after connecting - this limit is not applicable and the app just waits patiently... + req = HTTP.new timeout=(Duration.new seconds=1) . request (Request.get (base_url_with_slash + "wait10s")) + req.should_be_a Response + err = req.decode_as_text + IO.println err + IO.println err.length err.should_fail_with HTTP_Error main = Test_Suite.run_main spec diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index 9a6f612e10e3..483ba07d8b3a 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -59,14 +59,9 @@ public static void main(String[] args) { String host = args[0]; SimpleHTTPBin server = null; try { - int port = Integer.valueOf(args[1]); + int port = Integer.parseInt(args[1]); server = new SimpleHTTPBin(host, port); - for (HttpMethod method : HttpMethod.values()) { - String path = "/" + method.toString().toLowerCase(); - server.addHandler(path, new TestHandler()); - } - - setupFileServer(server); + setupServerEndpoints(server); final SimpleHTTPBin server1 = server; SignalHandler stopServerHandler = @@ -103,6 +98,16 @@ boolean isRunning() { } } + private static void setupServerEndpoints(SimpleHTTPBin server) throws URISyntaxException { + for (HttpMethod method : HttpMethod.values()) { + String path = "/" + method.toString().toLowerCase(); + server.addHandler(path, new TestHandler()); + } + + setupFileServer(server); + server.addHandler("/wait10s", new TimeoutHandler(10*1000)); + } + private static void setupFileServer(SimpleHTTPBin server) throws URISyntaxException { Path myRuntimeJar = Path.of(SimpleHTTPBin.class.getProtectionDomain().getCodeSource().getLocation().toURI()) diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java new file mode 100644 index 000000000000..e5023b818dfd --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java @@ -0,0 +1,40 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +public class TimeoutHandler implements HttpHandler { + private final int timeoutMillis; + + public TimeoutHandler(int timeoutMillis) { + this.timeoutMillis = timeoutMillis; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + byte[] responsePart = "OK".getBytes(StandardCharsets.UTF_8); + int responseCount = 2000; + + exchange.sendResponseHeaders(200, (long) responsePart.length * responseCount); + + try (OutputStream os = exchange.getResponseBody()) { + int half = responseCount / 2; + for (int i = 0; i < half; i++) { + os.write(responsePart); + } + os.flush(); + Thread.sleep(timeoutMillis); + + for (int i = half; i < responseCount; i++) { + os.write(responsePart); + } + } catch (InterruptedException e) { + System.out.println("Interrupted"); + e.printStackTrace(); + } + } +} From ce268300e92b686f833a7962fc50ab1757afa8dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 01:26:47 +0100 Subject: [PATCH 14/32] Revert "trying to test HTTP_Error but nothing works" This reverts commit 4a13f3d4986add2f65328a470f456f8abe410b08. --- test/Tests/src/Network/Http_Spec.enso | 13 +----- .../java/org/enso/shttp/SimpleHTTPBin.java | 19 ++++----- .../java/org/enso/shttp/TimeoutHandler.java | 40 ------------------- 3 files changed, 9 insertions(+), 63 deletions(-) delete mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index b8a4d423c354..0a57b5bd4195 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -7,7 +7,6 @@ import Standard.Base.Network.HTTP.HTTP_Error.HTTP_Error import Standard.Base.Network.HTTP.Request.Request import Standard.Base.Network.HTTP.Request_Body.Request_Body import Standard.Base.Network.HTTP.Request_Error -import Standard.Base.Network.HTTP.Response.Response import Standard.Base.Network.Proxy.Proxy import Standard.Base.Runtime.Context from Standard.Base.Network.HTTP import resolve_headers @@ -484,17 +483,9 @@ spec = err = Data.fetch "http://0.0.0.0:1/" err.should_fail_with Request_Error - Test.specify "should be able to handle IO errors" <| + Test.specify "should be able to handle IO errors" pending="TODO??" <| # TODO how to trigger this error??? - # Tried triggering by exceeding timeout but: - # 1. timeout here is just connection timeout - # 2. if the timeout occurs before connecting - we get Request_Error - # 3. if the timeout occurs after connecting - this limit is not applicable and the app just waits patiently... - req = HTTP.new timeout=(Duration.new seconds=1) . request (Request.get (base_url_with_slash + "wait10s")) - req.should_be_a Response - err = req.decode_as_text - IO.println err - IO.println err.length + err = Data.fetch "TODO" err.should_fail_with HTTP_Error main = Test_Suite.run_main spec diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index 483ba07d8b3a..9a6f612e10e3 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -59,9 +59,14 @@ public static void main(String[] args) { String host = args[0]; SimpleHTTPBin server = null; try { - int port = Integer.parseInt(args[1]); + int port = Integer.valueOf(args[1]); server = new SimpleHTTPBin(host, port); - setupServerEndpoints(server); + for (HttpMethod method : HttpMethod.values()) { + String path = "/" + method.toString().toLowerCase(); + server.addHandler(path, new TestHandler()); + } + + setupFileServer(server); final SimpleHTTPBin server1 = server; SignalHandler stopServerHandler = @@ -98,16 +103,6 @@ boolean isRunning() { } } - private static void setupServerEndpoints(SimpleHTTPBin server) throws URISyntaxException { - for (HttpMethod method : HttpMethod.values()) { - String path = "/" + method.toString().toLowerCase(); - server.addHandler(path, new TestHandler()); - } - - setupFileServer(server); - server.addHandler("/wait10s", new TimeoutHandler(10*1000)); - } - private static void setupFileServer(SimpleHTTPBin server) throws URISyntaxException { Path myRuntimeJar = Path.of(SimpleHTTPBin.class.getProtectionDomain().getCodeSource().getLocation().toURI()) diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java deleted file mode 100644 index e5023b818dfd..000000000000 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TimeoutHandler.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.enso.shttp; - -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; - -public class TimeoutHandler implements HttpHandler { - private final int timeoutMillis; - - public TimeoutHandler(int timeoutMillis) { - this.timeoutMillis = timeoutMillis; - } - - @Override - public void handle(HttpExchange exchange) throws IOException { - byte[] responsePart = "OK".getBytes(StandardCharsets.UTF_8); - int responseCount = 2000; - - exchange.sendResponseHeaders(200, (long) responsePart.length * responseCount); - - try (OutputStream os = exchange.getResponseBody()) { - int half = responseCount / 2; - for (int i = 0; i < half; i++) { - os.write(responsePart); - } - os.flush(); - Thread.sleep(timeoutMillis); - - for (int i = half; i < responseCount; i++) { - os.write(responsePart); - } - } catch (InterruptedException e) { - System.out.println("Interrupted"); - e.printStackTrace(); - } - } -} From e018dc175563582863fd0c99d64460ac0a888cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 01:28:47 +0100 Subject: [PATCH 15/32] test also to_uri query --- test/Tests/src/Network/URI_Spec.enso | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/Tests/src/Network/URI_Spec.enso b/test/Tests/src/Network/URI_Spec.enso index 2eb1ce02f1db..204d60e4acaf 100644 --- a/test/Tests/src/Network/URI_Spec.enso +++ b/test/Tests/src/Network/URI_Spec.enso @@ -134,9 +134,13 @@ spec = base_uri = URI.parse base_url+"get" uri1 = base_uri . add_query_argument "a" "b" + uri1.should_be_a URI_With_Query r1 = uri1.fetch decode_query_params r1 . should_equal [["a", "b"]] + # Also check that after converting back to normal URI, we also get the same result: + decode_query_params (uri1.to_uri.fetch) . should_equal (decode_query_params r1) + uri2 = base_uri . add_query_argument "q1" "b c" . add_query_argument "q2" "e+f" @@ -144,6 +148,7 @@ spec = r2 = uri2.fetch # All values should be encoded and decoded correctly so that they retain the original symbols: decode_query_params r2 . should_equal [["q1", "b c"], ["q2", "e+f"], ["q3", "e%20f"]] + decode_query_params (uri2.to_uri.fetch) . should_equal (decode_query_params r2) s1 = '"f"\'\' ; 🚀🚧a' s2 = "[a=b]:[b=c][d=e], ]]] ==>
a" @@ -156,6 +161,7 @@ spec = . add_query_argument "q9" "%%%" r3 = uri3.fetch decode_query_params r3 . should_equal [["q4", "śnieżnobiały"], ["q5", s1], ["q6", s2], ["q7", ""], ["q8", " "], ["q9", "%%%"]] + decode_query_params (uri3.to_uri.fetch) . should_equal (decode_query_params r3) uri4 = base_uri . add_query_argument "p+r" "b c" @@ -165,6 +171,7 @@ spec = . add_query_argument s2 "zzz" r4 = uri4.fetch decode_query_params r4 . should_equal [["p+r", "b c"], ["p r", "b c"], ["🚀", "🚧"], ["śnieżnobiałą", "łąkę"], [s2, "zzz"]] + decode_query_params (uri4.to_uri.fetch) . should_equal (decode_query_params r4) Test.specify "may allow duplicate keys in query parameters" <| uri = URI.parse base_url+"get" @@ -173,6 +180,7 @@ spec = . add_query_argument "a" "d" r = uri.fetch decode_query_params r . should_equal [["a", "b"], ["a", "c"], ["a", "d"]] + decode_query_params (uri.to_uri.fetch) . should_equal (decode_query_params r) Test.specify "should correctly handle various characters within the key and value of arguments" pending=("TODO testing secrets is for later".if_nothing pending_has_url) <| Error.throw "TODO: test various characters inside of the secret value, like in the raw test above" From f8ad5fd43660b43a8de916641c32e1c33ffbca07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 02:17:38 +0100 Subject: [PATCH 16/32] WIP testing http response code errors, some auth, response headers --- .../0.0.0-dev/src/Network/HTTP/Response.enso | 2 +- test/Tests/src/Network/Http_Spec.enso | 37 ++++++++++++++++++- .../java/org/enso/shttp/SimpleHTTPBin.java | 2 +- .../main/java/org/enso/shttp/TestHandler.java | 10 +++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso index e3ff2328ba86..c8e0d1e737c5 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso @@ -53,7 +53,7 @@ type Response import Standard.Examples example_headers = Examples.get_response.headers - headers : Vector + headers : Vector Header headers self = header_keys = self.internal_http_response.headerNames header_keys.flat_map k-> (self.internal_http_response.headers.allValues k).map v-> Header.new k v diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 0a57b5bd4195..743bc7fb747b 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -163,6 +163,17 @@ spec = Data.fetch "zxcv://bad.scheme" . should_fail_with Request_Error Data.fetch "" . should_fail_with Request_Error + Test.specify "can select the version" <| + req = Request.get url_get + r2 = HTTP.new version=HTTP_Version.HTTP_2 . request req . decode_as_json + r2.at "headers" . at "Connection" . should_equal "Upgrade, HTTP2-Settings" + + r1 = HTTP.new version=HTTP_Version.HTTP_1_1 . request req . decode_as_json + r1.at "headers" . keys . should_not_contain "Connection" + r1.at "headers" . keys . should_not_contain "Http2-Settings" + r1.at "headers" . keys . should_not_contain "HTTP2-Settings" + r1.at "headers" . keys . should_not_contain "Upgrade" + Test.group "post" pending=pending_has_url <| url_post = base_url_with_slash + "post" url_put = base_url_with_slash + "put" @@ -322,13 +333,19 @@ spec = response . should_equal expected_response Test.specify "can handle HTTP errors" <| - # This should give us bad request + # This should give us 405 bad method r1 = Data.post url_delete + # TODO upgrade error handling to have better error type for failed requests IO.println r1 - r2 = Data.post base_url_with_slash + "some/unknown/path" + # TODO check for 404 + r2 = Data.post (base_url_with_slash + "some/unknown/path") IO.println r2 + # TODO check for 500? if I can set to custom method? + r3 = Data.post (base_url_with_slash + "get") method=(HTTP_Method.Custom "BREW_COFFEE") + IO.println r3 + Test.specify "Cannot perform POST when output context is disabled" <| Context.Output.with_disabled <| Data.post url_post (Request_Body.Text "hello world") . should_fail_with Forbidden_Operation @@ -450,6 +467,17 @@ spec = Test.specify "Cannot specify content type (implicitly via explicit text encoding) in both body and headers" <| Data.post url_post (Request_Body.Text "hello world" encoding=Encoding.utf_8) headers=[Header.content_type "application/json"] . should_fail_with Illegal_Argument + Test.specify "can also read headers from a response, when returning a raw response" <| + r1 = Data.post url_post (Request_Body.Text "hello world") try_auto_parse_response=False + r1.headers.find (p-> p.name.equals_ignore_case "Content-Type") . value . should_equal "text/plain; charset=UTF-8" + + uri = URI.from (base_url_with_slash + "set_headers") + . add_query_argument "test-header" "test-value" + . add_query_argument "Other-Header" "some other value" + r2 = Data.fetch uri try_auto_parse_response=False + r2.headers.find (p-> p.name.equals_ignore_case "Test-Header") . value . should_equal "test-value" + r2.headers.find (p-> p.name.equals_ignore_case "Other-Header") . value . should_equal "some other value" + Test.group "Header resolution" <| Test.specify "Default content type and encoding" <| expected = [Header.content_type "text/plain; charset=UTF-8"] @@ -488,6 +516,11 @@ spec = err = Data.fetch "TODO" err.should_fail_with HTTP_Error + Test.group "Http Auth" <| + Test.specify "should allow to specify authentication headers" <| + # TODO figure out what is there to check right now? + Error.throw "TODO" + main = Test_Suite.run_main spec echo_response_template method path data content_type content_length=data.length = diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index 9a6f612e10e3..a6c19a7d5d5c 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -63,7 +63,7 @@ public static void main(String[] args) { server = new SimpleHTTPBin(host, port); for (HttpMethod method : HttpMethod.values()) { String path = "/" + method.toString().toLowerCase(); - server.addHandler(path, new TestHandler()); + server.addHandler(path, new TestHandler(method)); } setupFileServer(server); diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index 8dc2e992e821..db172a1e4f0e 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -20,11 +20,16 @@ import java.util.regex.Pattern; public class TestHandler implements HttpHandler { + private final HttpMethod expectedMethod; private static final Set ignoredHeaders = Set.of("Host"); private static final Pattern textEncodingRegex = Pattern.compile(".*; charset=([^;]+).*"); private final boolean logRequests = false; + public TestHandler(HttpMethod expectedMethod) { + this.expectedMethod = expectedMethod; + } + @Override public void handle(HttpExchange exchange) throws IOException { try { @@ -53,6 +58,11 @@ public void doHandle(HttpExchange exchange) throws IOException { response = new StringBuilder(); exchange.sendResponseHeaders(200, -1); } else { + if (meth != expectedMethod) { + exchange.sendResponseHeaders(405, -1); + return; + } + exchange.getResponseHeaders().put("Content-Type", List.of("application/json")); response = new StringBuilder("{\n"); response.append(" \"headers\": {\n"); From 0432ef84ca65ca5df8a98250449415d088547fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 14:51:12 +0100 Subject: [PATCH 17/32] checkpoint: WIP on improving handling of non-200 status codes, fixing other tests --- .../lib/Standard/Base/0.0.0-dev/src/Data.enso | 5 ++- .../Base/0.0.0-dev/src/Network/HTTP.enso | 11 +++-- .../src/Network/HTTP/HTTP_Error.enso | 19 +++++++- .../src/Network/HTTP/HTTP_Status_Code.enso | 11 ++++- test/Tests/src/Network/Http_Spec.enso | 43 ++++++++++++++----- .../java/org/enso/shttp/SimpleHTTPBin.java | 16 ++++--- 6 files changed, 80 insertions(+), 25 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index b28b788d8f91..c983ccdccd3a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -10,6 +10,7 @@ import project.Errors.Problem_Behavior.Problem_Behavior import project.Meta import project.Network.HTTP.Header.Header import project.Network.HTTP.HTTP +import project.Network.HTTP.HTTP_Error.HTTP_Error import project.Network.HTTP.HTTP_Method.HTTP_Method import project.Network.HTTP.Request.Request import project.Network.HTTP.Request_Body.Request_Body @@ -176,7 +177,7 @@ list_directory directory name_filter=Nothing recursive=False = import Standard.Base.Data file = enso_project.data / "spreadsheet.xls" Data.fetch URL . body . to_file file -fetch : (URI | URI_With_Query | Text) -> HTTP_Method -> Vector (Header | Pair Text Text) -> Boolean -> Any +fetch : (URI | URI_With_Query | Text) -> HTTP_Method -> Vector (Header | Pair Text Text) -> Boolean -> Any ! Request_Error | HTTP_Error fetch (uri:(URI | URI_With_Query | Text)) (method:HTTP_Method=HTTP_Method.Get) (headers:(Vector (Header | Pair Text Text))=[]) (try_auto_parse_response:Boolean=True) = response = HTTP.fetch uri method headers if try_auto_parse_response.not then response.with_materialized_body else @@ -304,7 +305,7 @@ fetch (uri:(URI | URI_With_Query | Text)) (method:HTTP_Method=HTTP_Method.Get) ( test_file = enso_project.data / "sample.txt" form_data = Map.from_vector [["key", "val"], ["a_file", test_file]] response = Data.post url_post (Request_Body.Form_Data form_data url_encoded=True) -post : (URI | URI_With_Query | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> Boolean -> Any +post : (URI | URI_With_Query | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> Boolean -> Any ! Request_Error | HTTP_Error post (uri:(URI | URI_With_Query | Text)) (body:Request_Body=Request_Body.Empty) (method:HTTP_Method=HTTP_Method.Post) (headers:(Vector (Header | Pair Text Text))=[]) (try_auto_parse_response:Boolean=True) = response = HTTP.post uri body method headers if try_auto_parse_response.not then response.with_materialized_body else diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso index 5ce967ddf807..85360b2c78d3 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso @@ -12,6 +12,7 @@ import project.Errors.Common.Forbidden_Operation import project.Errors.Illegal_Argument.Illegal_Argument import project.Meta import project.Network.HTTP.Header.Header +import project.Network.HTTP.HTTP_Error.HTTP_Error import project.Network.HTTP.HTTP_Method.HTTP_Method import project.Network.HTTP.HTTP_Version.HTTP_Version import project.Network.HTTP.Request.Request @@ -91,7 +92,7 @@ type HTTP - req: The HTTP request to send using `self` HTTP client. - error_on_failure_code: Whether or not to throw an error if the response code is not a success code. - request : Request -> Boolean -> Response ! Request_Error + request : Request -> Boolean -> Response ! Request_Error | HTTP_Error request self req error_on_failure_code=True = # Prevent request if the method is a write-like method and output context is disabled. check_output_context ~action = @@ -133,11 +134,13 @@ type HTTP response = Response.Value (EnsoSecretHelper.makeRequest self.internal_http_client builder uri_args.first.internal_uri uri_args.second mapped_headers) if error_on_failure_code.not || response.code.is_success then response else - Error.throw (Request_Error.Error "Status Code" ("Request failed with status code: " + response.code.to_text + ". " + response.body.decode_as_text)) + body = response.body.decode_as_text.catch Any _-> "" + message = if body.is_empty then Nothing else body + Error.throw (HTTP_Error.Status_Error response.code message response.uri) ## PRIVATE Static helper for get-like methods - fetch : (URI | URI_With_Query | Text) -> HTTP_Method -> Vector (Header | Pair Text Text) -> Any + fetch : (URI | URI_With_Query | Text) -> HTTP_Method -> Vector (Header | Pair Text Text) -> Response ! Request_Error | HTTP_Error fetch (uri:(URI | URI_With_Query | Text)) (method:HTTP_Method=HTTP_Method.Get) (headers:(Vector (Header | Pair Text Text))=[]) = check_method fetch_methods method <| request = Request.new method uri (parse_headers headers) Request_Body.Empty @@ -145,7 +148,7 @@ type HTTP ## PRIVATE Static helper for post-like methods - post : (URI | URI_With_Query | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> Any + post : (URI | URI_With_Query | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> Response ! Request_Error | HTTP_Error post (uri:(URI | URI_With_Query | Text)) (body:Request_Body=Request_Body.Empty) (method:HTTP_Method=HTTP_Method.Post) (headers:(Vector (Header | Pair Text Text))=[]) = check_method post_methods method <| request = Request.new method uri (parse_headers headers) body diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Error.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Error.enso index 09d75fa1b824..745d4fc0232b 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Error.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Error.enso @@ -2,6 +2,8 @@ import project.Any.Any import project.Data.Text.Text import project.Network.URI.URI import project.Network.URI_With_Query.URI_With_Query +import project.Network.HTTP.HTTP_Status_Code.HTTP_Status_Code +import project.Nothing.Nothing import project.Panic.Panic polyglot java import java.io.IOException @@ -15,10 +17,25 @@ type HTTP_Error - message: The message for the error. IO_Error (uri:URI|URI_With_Query) (message:Text) + ## An error indicating that a non-200 status code was returned. + + Arguments: + - status_code: The status code that was returned. + - message: The message for the error, if it was able to be read. + - uri: The uri that couldn't be read. + Status_Error (status_code:HTTP_Status_Code) (message:Text|Nothing) (uri:URI|URI_With_Query) + ## PRIVATE Convert the HTTP_Error to a human-readable format. to_display_text : Text - to_display_text self = self.message + " (" + self.uri.to_text + ")." + to_display_text self = case self of + HTTP_Error.IO_Error uri message -> + "IO Error: " + message + " (" + uri.to_display_text + ")." + HTTP_Error.Status_Error status_code message uri -> + prefix = "HTTP responded with status " + status_code.to_text + uri_part = " (at URI: " + uri.to_display_text + ")" + suffix = if message.is_nothing then "." else ": " + message.to_text + "." + prefix + uri_part + suffix ## PRIVATE Utility method for running an action with Java exceptions mapping. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Status_Code.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Status_Code.enso index a5fdf6e8bf18..791b63a6b7e4 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Status_Code.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Status_Code.enso @@ -1,5 +1,7 @@ import project.Data.Boolean.Boolean +import project.Data.Numbers.Integer import project.Data.Text.Text +from project.Data.Text.Extensions import all type HTTP_Status_Code ## 100 Continue. @@ -166,12 +168,19 @@ type HTTP_Status_Code Arguments: - code: The numeric representation of the code. - Value code + Value code:Integer ## Does the status code represent a successful response? is_success : Boolean is_success self = self.code >= 200 && self.code < 300 + ## PRIVATE + to_text : Text + to_text self = + text_repr = self.to_display_text + if text_repr.starts_with "HTTP Status Code" then text_repr else + self.code.to_text + " " + text_repr + ## PRIVATE Convert to a display representation of this HTTP_Status_Code. to_display_text : Text diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 743bc7fb747b..c31ff6f33a43 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -49,6 +49,18 @@ spec = Test.specify "should make a custom method" <| "CUSTOM" . to HTTP_Method . should_equal (HTTP_Method.Custom "CUSTOM") + Test.group "HTTP_Status_Code" <| + Test.specify "should have a nice text representation" <| + s1 = HTTP_Status_Code.ok + s1.code . should_equal 200 + s1.to_text . should_equal "200 OK" + s1.to_display_text . should_equal "OK" + + s2 = HTTP_Status_Code.not_found + s2.code . should_equal 404 + s2.to_text . should_equal "404 Not Found" + s2.to_display_text . should_equal "Not Found" + Test.group "HTTP client" pending=pending_has_url <| Test.specify "should create HTTP client with timeout setting" <| http = HTTP.new (timeout = (Duration.new seconds=30)) @@ -169,10 +181,10 @@ spec = r2.at "headers" . at "Connection" . should_equal "Upgrade, HTTP2-Settings" r1 = HTTP.new version=HTTP_Version.HTTP_1_1 . request req . decode_as_json - r1.at "headers" . keys . should_not_contain "Connection" - r1.at "headers" . keys . should_not_contain "Http2-Settings" - r1.at "headers" . keys . should_not_contain "HTTP2-Settings" - r1.at "headers" . keys . should_not_contain "Upgrade" + header_names = r1.at "headers" . field_names . map (s-> s.to_case Case.Lower) + header_names.should_not_contain "connection" + header_names.should_not_contain "http2-settings" + header_names.should_not_contain "upgrade" Test.group "post" pending=pending_has_url <| url_post = base_url_with_slash + "post" @@ -333,18 +345,26 @@ spec = response . should_equal expected_response Test.specify "can handle HTTP errors" <| - # This should give us 405 bad method + # This should give us 405 method not allowed r1 = Data.post url_delete - # TODO upgrade error handling to have better error type for failed requests - IO.println r1 + r1.should_fail_with HTTP_Error + r1.catch.should_be_a HTTP_Error.Status_Error + IO.println r1.catch.to_display_text + r1.catch.status_code.code . should_equal 405 + r1.catch.to_display_text . should_contain "status 405" - # TODO check for 404 r2 = Data.post (base_url_with_slash + "some/unknown/path") - IO.println r2 + r2.should_fail_with HTTP_Error + IO.println r2.catch.to_display_text + r2.catch.should_be_a HTTP_Error.Status_Error + r2.catch.status_code.code . should_equal 404 # TODO check for 500? if I can set to custom method? r3 = Data.post (base_url_with_slash + "get") method=(HTTP_Method.Custom "BREW_COFFEE") - IO.println r3 + r3.should_fail_with HTTP_Error + r3.catch.should_be_a HTTP_Error.Status_Error + IO.println r3.catch.to_display_text + r3.catch.status_code.code . should_equal 405 Test.specify "Cannot perform POST when output context is disabled" <| Context.Output.with_disabled <| @@ -469,7 +489,8 @@ spec = Test.specify "can also read headers from a response, when returning a raw response" <| r1 = Data.post url_post (Request_Body.Text "hello world") try_auto_parse_response=False - r1.headers.find (p-> p.name.equals_ignore_case "Content-Type") . value . should_equal "text/plain; charset=UTF-8" + # The result is JSON data: + r1.headers.find (p-> p.name.equals_ignore_case "Content-Type") . value . should_equal "application/json" uri = URI.from (base_url_with_slash + "set_headers") . add_query_argument "test-header" "test-value" diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index a6c19a7d5d5c..8ff0e9759677 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -61,12 +61,7 @@ public static void main(String[] args) { try { int port = Integer.valueOf(args[1]); server = new SimpleHTTPBin(host, port); - for (HttpMethod method : HttpMethod.values()) { - String path = "/" + method.toString().toLowerCase(); - server.addHandler(path, new TestHandler(method)); - } - - setupFileServer(server); + setupEndpoints(server); final SimpleHTTPBin server1 = server; SignalHandler stopServerHandler = @@ -103,6 +98,15 @@ boolean isRunning() { } } + private static void setupEndpoints(SimpleHTTPBin server) throws URISyntaxException { + for (HttpMethod method : HttpMethod.values()) { + String path = "/" + method.toString().toLowerCase(); + server.addHandler(path, new TestHandler(method)); + } + + setupFileServer(server); + } + private static void setupFileServer(SimpleHTTPBin server) throws URISyntaxException { Path myRuntimeJar = Path.of(SimpleHTTPBin.class.getProtectionDomain().getCodeSource().getLocation().toURI()) From 78f8386bf935aa1e3d85e3068595c889a7259bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 15:02:48 +0100 Subject: [PATCH 18/32] fixing error handling tests --- .../src/Network/HTTP/HTTP_Method.enso | 6 +++++ test/Tests/src/Network/Http_Spec.enso | 14 +++++++----- .../main/java/org/enso/shttp/TestHandler.java | 22 ++++++++++++------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso index e8998b1fc211..2c8b3cadd113 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso @@ -50,6 +50,12 @@ type HTTP_Method HTTP_Method.Connect -> "CONNECT" HTTP_Method.Custom verb -> verb + ## PRIVATE + to_display_text : Text + to_display_text self = case self of + HTTP_Method.Custom verb -> "Custom: "+verb.to_display_text + _ -> self.to_http_method_name + ## PRIVATE Converts from Text to an HTTP_Method. HTTP_Method.from (that:Text) = case that.to_case Case.Upper of diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index c31ff6f33a43..1bec1a942637 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -162,11 +162,11 @@ spec = Test.specify "Unsupported method" <| err = Data.fetch url_get method=HTTP_Method.Post - err.catch.should_equal (Illegal_Argument.Error "Unsupported method Post") + err.catch.should_equal (Illegal_Argument.Error "Unsupported method POST") Test.specify "Cannot DELETE through fetch" <| err = Data.fetch url_get method=HTTP_Method.Delete - err.catch.should_equal (Illegal_Argument.Error "Unsupported method Delete") + err.catch.should_equal (Illegal_Argument.Error "Unsupported method DELETE") Test.specify "unknown host" <| Data.fetch "http://undefined_host.invalid" . should_fail_with Request_Error @@ -359,12 +359,11 @@ spec = r2.catch.should_be_a HTTP_Error.Status_Error r2.catch.status_code.code . should_equal 404 - # TODO check for 500? if I can set to custom method? - r3 = Data.post (base_url_with_slash + "get") method=(HTTP_Method.Custom "BREW_COFFEE") + r3 = HTTP.new.request (Request.new (HTTP_Method.Custom "BREW_COFFEE") (base_url_with_slash + "get")) r3.should_fail_with HTTP_Error r3.catch.should_be_a HTTP_Error.Status_Error IO.println r3.catch.to_display_text - r3.catch.status_code.code . should_equal 405 + r3.catch.status_code.code . should_equal 400 Test.specify "Cannot perform POST when output context is disabled" <| Context.Output.with_disabled <| @@ -379,7 +378,10 @@ spec = Test.specify "Unsupported method" <| err = Data.post url_post (Request_Body.Text "hello world") method=HTTP_Method.Get - err.catch.should_equal (Illegal_Argument.Error "Unsupported method Get") + err.catch.should_equal (Illegal_Argument.Error "Unsupported method GET") + + err2 = Data.post url_post (Request_Body.Text "hello world") method=(HTTP_Method.Custom "BREW_COFFEE") + err2.catch.should_equal (Illegal_Argument.Error "Unsupported method Custom: BREW_COFFEE") Test.specify "unknown host" <| Data.post "http://undefined_host.invalid" (Request_Body.Text "hello world") . should_fail_with Request_Error diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index db172a1e4f0e..29ca768beb82 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -51,14 +51,20 @@ public void doHandle(HttpExchange exchange) throws IOException { boolean first = true; String contentType = null; String textEncoding = "UTF-8"; - HttpMethod meth = HttpMethod.valueOf(exchange.getRequestMethod()); + HttpMethod method; + try { + method = HttpMethod.valueOf(exchange.getRequestMethod()); + } catch (IllegalArgumentException e) { + exchange.sendResponseHeaders(400, -1); + return; + } StringBuilder response; - if (meth == HttpMethod.HEAD || meth == HttpMethod.OPTIONS) { + if (method == HttpMethod.HEAD || method == HttpMethod.OPTIONS) { response = new StringBuilder(); exchange.sendResponseHeaders(200, -1); } else { - if (meth != expectedMethod) { + if (method != expectedMethod) { exchange.sendResponseHeaders(405, -1); return; } @@ -114,11 +120,11 @@ public void doHandle(HttpExchange exchange) throws IOException { response.append(" ],\n"); } - response.append(" \"method\": \"").append(meth).append("\",\n"); - if (meth == HttpMethod.POST - || meth == HttpMethod.DELETE - || meth == HttpMethod.PUT - || meth == HttpMethod.PATCH) { + response.append(" \"method\": \"").append(method).append("\",\n"); + if (method == HttpMethod.POST + || method == HttpMethod.DELETE + || method == HttpMethod.PUT + || method == HttpMethod.PATCH) { response.append(" \"form\": null,\n"); response.append(" \"files\": null,\n"); String value = readBody(exchange.getRequestBody(), textEncoding); From bb4c1762e8d181780b68b3a2537e74171b3434a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 15:19:38 +0100 Subject: [PATCH 19/32] impl endpoint for the header response test --- test/Tests/src/Network/Http_Spec.enso | 2 +- .../org/enso/shttp/HeaderTestHandler.java | 26 +++++++++++++++++++ .../java/org/enso/shttp/SimpleHTTPBin.java | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 1bec1a942637..c59f570ce6cc 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -494,7 +494,7 @@ spec = # The result is JSON data: r1.headers.find (p-> p.name.equals_ignore_case "Content-Type") . value . should_equal "application/json" - uri = URI.from (base_url_with_slash + "set_headers") + uri = URI.from (base_url_with_slash + "test_headers") . add_query_argument "test-header" "test-value" . add_query_argument "Other-Header" "some other value" r2 = Data.fetch uri try_auto_parse_response=False diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java new file mode 100644 index 000000000000..aaab384c4a8d --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java @@ -0,0 +1,26 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.net.URI; + +public class HeaderTestHandler implements HttpHandler { + @Override + public void handle(HttpExchange exchange) throws IOException { + URI uri = exchange.getRequestURI(); + URIBuilder builder = new URIBuilder(uri); + try { + for (var queryPair : builder.getQueryParams()) { + exchange.getResponseHeaders().add(queryPair.getName(), queryPair.getValue()); + } + } catch (Exception e) { + e.printStackTrace(); + exchange.sendResponseHeaders(500, -1); + } + + exchange.sendResponseHeaders(200, -1); + } +} diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index 8ff0e9759677..d697cf4e9353 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -104,6 +104,7 @@ private static void setupEndpoints(SimpleHTTPBin server) throws URISyntaxExcepti server.addHandler(path, new TestHandler(method)); } + server.addHandler("/test_headers", new HeaderTestHandler()); setupFileServer(server); } From 4d93f9d94e96feb17205d69662fdb45d0c463bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 16:54:48 +0100 Subject: [PATCH 20/32] checkpoint --- .../main/java/org/enso/shttp/AuthTestHandler.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java new file mode 100644 index 000000000000..faedf0ba8ac5 --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java @@ -0,0 +1,13 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; + +public class AuthTestHandler implements HttpHandler { + @Override + public void handle(HttpExchange exchange) throws IOException { + + } +} From 7f0d05dc7fdcfe238faafd3f96ae27f103d689a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 18:15:31 +0100 Subject: [PATCH 21/32] WIP: auth tests --- test/Tests/src/Network/Http_Spec.enso | 38 +++++++++++-- .../java/org/enso/shttp/AuthTestHandler.java | 13 ----- .../org/enso/shttp/BasicAuthTestHandler.java | 54 +++++++++++++++++++ .../java/org/enso/shttp/SimpleHTTPBin.java | 2 + .../main/java/org/enso/shttp/TestHandler.java | 1 - .../org/enso/shttp/TokenAuthTestHandler.java | 41 ++++++++++++++ .../src/main/java/org/enso/shttp/Utils.java | 16 ++++++ 7 files changed, 146 insertions(+), 19 deletions(-) delete mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index c59f570ce6cc..d2d2b99cc1af 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -358,6 +358,7 @@ spec = IO.println r2.catch.to_display_text r2.catch.should_be_a HTTP_Error.Status_Error r2.catch.status_code.code . should_equal 404 + r2.catch.message . should_contain "

404 Not Found

" r3 = HTTP.new.request (Request.new (HTTP_Method.Custom "BREW_COFFEE") (base_url_with_slash + "get")) r3.should_fail_with HTTP_Error @@ -534,15 +535,42 @@ spec = err = Data.fetch "http://0.0.0.0:1/" err.should_fail_with Request_Error - Test.specify "should be able to handle IO errors" pending="TODO??" <| - # TODO how to trigger this error??? + Test.specify "should be able to handle IO errors" pending="TODO: Currently I was unable to figure out a way to test such errors" <| + # how to trigger this error??? err = Data.fetch "TODO" err.should_fail_with HTTP_Error Test.group "Http Auth" <| - Test.specify "should allow to specify authentication headers" <| - # TODO figure out what is there to check right now? - Error.throw "TODO" + Test.specify "should support Basic user+password authentication" <| + url = base_url_with_slash + "test_basic_auth" + #r1 = Data.fetch url + #r1.should_fail_with HTTP_Error + #r1.catch.status_code.code . should_equal 401 + + r2 = Data.fetch url headers=[Header.authorization_basic "enso-test-user" "my secret password: 1234@#; ść + 😎"] + r2.should_succeed + r2.should_be_a Text + r2.should_equal "Authorization successful, welcome enso-test-user!" + + r3 = Data.fetch url headers=[Header.authorization_basic "other user" "1234"] + r3.should_fail_with HTTP_Error + r3.catch.status_code.code . should_equal 403 + + Test.specify "should support Bearer token authentication" <| + url = base_url_with_slash + "test_token_auth" + #r1 = Data.fetch url + #r1.should_fail_with HTTP_Error + #r1.catch.status_code.code . should_equal 401 + #IO.println r1.catch.to_display_text + + r2 = Data.fetch url headers=[Header.authorization_bearer "deadbeef-coffee-1234"] + r2.should_succeed + r2.should_be_a Text + r2.should_equal "Authorization successful." + + r3 = Data.fetch url headers=[Header.authorization_bearer "invalid-token"] + r3.should_fail_with HTTP_Error + r3.catch.status_code.code . should_equal 403 main = Test_Suite.run_main spec diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java deleted file mode 100644 index faedf0ba8ac5..000000000000 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/AuthTestHandler.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.enso.shttp; - -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; - -import java.io.IOException; - -public class AuthTestHandler implements HttpHandler { - @Override - public void handle(HttpExchange exchange) throws IOException { - - } -} diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java new file mode 100644 index 000000000000..5b06105f0faa --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java @@ -0,0 +1,54 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; +import java.util.Base64; +import java.util.List; + +import static org.enso.shttp.Utils.sendResponse; + +public class BasicAuthTestHandler implements HttpHandler { + private final String username = "enso-test-user"; + + // ends with 😎 emoji + private final String password = "my secret password: 1234@#; ść + \uD83D\uDE0E"; + + @Override + public void handle(HttpExchange exchange) throws IOException { + List authHeaders = exchange.getRequestHeaders().get("Authorization"); + if (authHeaders.isEmpty()) { + sendResponse(401, "Not authorized.", exchange); + return; + } else if (authHeaders.size() > 1) { + sendResponse(400, "Ambiguous Authorization headers.", exchange); + return; + } + + String authHeader = authHeaders.get(0); + String prefix = "Basic "; + if (!authHeader.startsWith(prefix)) { + sendResponse(400, "Invalid authorization header format.", exchange); + return; + } + + String encodedCredentials = authHeader.substring(prefix.length()); + String decodedCredentials = new String(Base64.getDecoder().decode(encodedCredentials)); + String[] credentials = decodedCredentials.split(":", 2); + if (credentials.length != 2) { + sendResponse(403, "Invalid authorization credential format.", exchange); + return; + } + + String providedUsername = credentials[0]; + String providedPassword = credentials[1]; + boolean authorized = providedUsername.equals(username) && providedPassword.equals(password); + if (!authorized) { + sendResponse(403, "Wrong username or password.", exchange); + return; + } + + sendResponse(200, "Authorization successful, welcome " + username + "!", exchange); + } +} diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index d697cf4e9353..47aeb920fd57 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -105,6 +105,8 @@ private static void setupEndpoints(SimpleHTTPBin server) throws URISyntaxExcepti } server.addHandler("/test_headers", new HeaderTestHandler()); + server.addHandler("/test_token_auth", new TokenAuthTestHandler()); + server.addHandler("/test_basic_auth", new BasicAuthTestHandler()); setupFileServer(server); } diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index 29ca768beb82..de2ca7b6ced3 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -95,7 +95,6 @@ public void doHandle(HttpExchange exchange) throws IOException { } URI uri = exchange.getRequestURI(); - System.out.println(exchange.getRequestMethod() + " " + uri); response.append("\n"); response.append(" },\n"); diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java new file mode 100644 index 000000000000..22e4db27beba --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java @@ -0,0 +1,41 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; +import java.util.List; + +import static org.enso.shttp.Utils.sendResponse; + +public class TokenAuthTestHandler implements HttpHandler { + private final String secretToken = "deadbeef-coffee-1234"; + + @Override + public void handle(HttpExchange exchange) throws IOException { + List authHeaders = exchange.getRequestHeaders().get("Authorization"); + if (authHeaders.isEmpty()) { + sendResponse(401, "Not authorized.", exchange); + return; + } else if (authHeaders.size() > 1) { + sendResponse(400, "Ambiguous Authorization headers.", exchange); + return; + } + + String authHeader = authHeaders.get(0); + String prefix = "Bearer "; + if (!authHeader.startsWith(prefix)) { + sendResponse(400, "Invalid authorization header format.", exchange); + return; + } + + String providedToken = authHeader.substring(prefix.length()); + boolean authorized = providedToken.equals(secretToken); + if (!authorized) { + sendResponse(403, "Invalid token.", exchange); + return; + } + + sendResponse(200, "Authorization successful.", exchange); + } +} diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java new file mode 100644 index 000000000000..30db0b634fde --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java @@ -0,0 +1,16 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class Utils { + static void sendResponse(int code, String message, HttpExchange exchange) throws IOException { + byte[] response = message.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=utf-8"); + exchange.sendResponseHeaders(code, response.length); + exchange.getResponseBody().write(response); + exchange.close(); + } +} From a6956a3e36128634a02ea0beddc5bb144263af03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 18:26:32 +0100 Subject: [PATCH 22/32] make simple-httpbin easier to debug by printing stack trace in all handlers; fixed the auth tests --- test/Tests/src/Network/Http_Spec.enso | 43 ++++++++++++------- .../org/enso/shttp/BasicAuthTestHandler.java | 9 ++-- .../org/enso/shttp/HeaderTestHandler.java | 5 +-- .../java/org/enso/shttp/SimpleHTTPBin.java | 5 ++- .../org/enso/shttp/SimpleHttpHandler.java | 41 ++++++++++++++++++ .../main/java/org/enso/shttp/TestHandler.java | 29 +++---------- .../org/enso/shttp/TokenAuthTestHandler.java | 9 ++-- .../src/main/java/org/enso/shttp/Utils.java | 16 ------- 8 files changed, 85 insertions(+), 72 deletions(-) create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java delete mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index d2d2b99cc1af..8738021bd0ed 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -543,31 +543,44 @@ spec = Test.group "Http Auth" <| Test.specify "should support Basic user+password authentication" <| url = base_url_with_slash + "test_basic_auth" - #r1 = Data.fetch url - #r1.should_fail_with HTTP_Error - #r1.catch.status_code.code . should_equal 401 - r2 = Data.fetch url headers=[Header.authorization_basic "enso-test-user" "my secret password: 1234@#; ść + 😎"] - r2.should_succeed - r2.should_be_a Text - r2.should_equal "Authorization successful, welcome enso-test-user!" + # Correct user and password + r1 = Data.fetch url headers=[Header.authorization_basic "enso-test-user" "my secret password: 1234@#; ść + 😎"] + r1.should_succeed + r1.should_be_a Text + r1.should_equal "Authorization successful, welcome enso-test-user!" + # No auth data + r2 = Data.fetch url + r2.should_fail_with HTTP_Error + r2.catch.status_code.code . should_equal 401 + + # Incorrect credentials r3 = Data.fetch url headers=[Header.authorization_basic "other user" "1234"] r3.should_fail_with HTTP_Error r3.catch.status_code.code . should_equal 403 + # Correct user, incorrect password + r4 = Data.fetch url headers=[Header.authorization_basic "enso-test-user" "1234"] + r4.should_fail_with HTTP_Error + r4.catch.status_code.code . should_equal 403 + Test.specify "should support Bearer token authentication" <| url = base_url_with_slash + "test_token_auth" - #r1 = Data.fetch url - #r1.should_fail_with HTTP_Error - #r1.catch.status_code.code . should_equal 401 - #IO.println r1.catch.to_display_text - r2 = Data.fetch url headers=[Header.authorization_bearer "deadbeef-coffee-1234"] - r2.should_succeed - r2.should_be_a Text - r2.should_equal "Authorization successful." + # Correct token + r1 = Data.fetch url headers=[Header.authorization_bearer "deadbeef-coffee-1234"] + r1.should_succeed + r1.should_be_a Text + r1.should_equal "Authorization successful." + + # No auth data + r2 = Data.fetch url + r2.should_fail_with HTTP_Error + r2.catch.status_code.code . should_equal 401 + IO.println r2.catch.to_display_text + # Invalid token r3 = Data.fetch url headers=[Header.authorization_bearer "invalid-token"] r3.should_fail_with HTTP_Error r3.catch.status_code.code . should_equal 403 diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java index 5b06105f0faa..d88ed548ff6a 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java @@ -1,24 +1,21 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; import java.io.IOException; import java.util.Base64; import java.util.List; -import static org.enso.shttp.Utils.sendResponse; - -public class BasicAuthTestHandler implements HttpHandler { +public class BasicAuthTestHandler extends SimpleHttpHandler { private final String username = "enso-test-user"; // ends with 😎 emoji private final String password = "my secret password: 1234@#; ść + \uD83D\uDE0E"; @Override - public void handle(HttpExchange exchange) throws IOException { + public void doHandle(HttpExchange exchange) throws IOException { List authHeaders = exchange.getRequestHeaders().get("Authorization"); - if (authHeaders.isEmpty()) { + if (authHeaders == null || authHeaders.isEmpty()) { sendResponse(401, "Not authorized.", exchange); return; } else if (authHeaders.size() > 1) { diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java index aaab384c4a8d..b85e16be36b8 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java @@ -1,15 +1,14 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; import org.apache.http.client.utils.URIBuilder; import java.io.IOException; import java.net.URI; -public class HeaderTestHandler implements HttpHandler { +public class HeaderTestHandler extends SimpleHttpHandler { @Override - public void handle(HttpExchange exchange) throws IOException { + public void doHandle(HttpExchange exchange) throws IOException { URI uri = exchange.getRequestURI(); URIBuilder builder = new URIBuilder(uri); try { diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index 47aeb920fd57..e84828610bb0 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -3,6 +3,9 @@ import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.SimpleFileServer; +import sun.misc.Signal; +import sun.misc.SignalHandler; + import java.io.IOException; import java.net.InetSocketAddress; import java.net.URISyntaxException; @@ -10,8 +13,6 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Stream; -import sun.misc.Signal; -import sun.misc.SignalHandler; public class SimpleHTTPBin { diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java new file mode 100644 index 000000000000..0b783f91a503 --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java @@ -0,0 +1,41 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +public abstract class SimpleHttpHandler implements HttpHandler { + private final boolean logRequests = false; + + @Override + public final void handle(HttpExchange exchange) throws IOException { + try { + if (logRequests) { + System.out.println( + "Handling request: " + exchange.getRequestMethod() + " " + exchange.getRequestURI()); + } + + doHandle(exchange); + } catch (IOException e) { + e.printStackTrace(); + throw e; + } catch (Exception e) { + e.printStackTrace(); + } + } + + public abstract void doHandle(HttpExchange exchange) throws IOException; + + protected final void sendResponse(int code, String message, HttpExchange exchange) throws IOException { + byte[] response = message.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=utf-8"); + exchange.sendResponseHeaders(code, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + exchange.close(); + } +} diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index de2ca7b6ced3..b97ff6fcada4 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -1,7 +1,6 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; import org.apache.commons.text.StringEscapeUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URIBuilder; @@ -19,41 +18,23 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -public class TestHandler implements HttpHandler { +public class TestHandler extends SimpleHttpHandler { private final HttpMethod expectedMethod; private static final Set ignoredHeaders = Set.of("Host"); private static final Pattern textEncodingRegex = Pattern.compile(".*; charset=([^;]+).*"); - private final boolean logRequests = false; public TestHandler(HttpMethod expectedMethod) { this.expectedMethod = expectedMethod; } - @Override - public void handle(HttpExchange exchange) throws IOException { - try { - if (logRequests) { - System.out.println( - "Handling request: " + exchange.getRequestMethod() + " " + exchange.getRequestURI()); - } - - doHandle(exchange); - } catch (IOException e) { - e.printStackTrace(); - throw e; - } catch (Exception e) { - e.printStackTrace(); - } - } - public void doHandle(HttpExchange exchange) throws IOException { boolean first = true; String contentType = null; String textEncoding = "UTF-8"; HttpMethod method; try { - method = HttpMethod.valueOf(exchange.getRequestMethod()); + method = HttpMethod.valueOf(exchange.getRequestMethod()); } catch (IllegalArgumentException e) { exchange.sendResponseHeaders(400, -1); return; @@ -133,10 +114,10 @@ public void doHandle(HttpExchange exchange) throws IOException { response.append(" \"args\": {}\n"); response.append("}"); exchange.sendResponseHeaders(200, response.toString().getBytes().length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response.toString().getBytes()); + } } - OutputStream os = exchange.getResponseBody(); - os.write(response.toString().getBytes()); - os.close(); } private String readBody(InputStream inputStream, String encoding) { diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java index 22e4db27beba..cd6995aac323 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java @@ -1,20 +1,17 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; import java.io.IOException; import java.util.List; -import static org.enso.shttp.Utils.sendResponse; - -public class TokenAuthTestHandler implements HttpHandler { +public class TokenAuthTestHandler extends SimpleHttpHandler { private final String secretToken = "deadbeef-coffee-1234"; @Override - public void handle(HttpExchange exchange) throws IOException { + public void doHandle(HttpExchange exchange) throws IOException { List authHeaders = exchange.getRequestHeaders().get("Authorization"); - if (authHeaders.isEmpty()) { + if (authHeaders == null || authHeaders.isEmpty()) { sendResponse(401, "Not authorized.", exchange); return; } else if (authHeaders.size() > 1) { diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java deleted file mode 100644 index 30db0b634fde..000000000000 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/Utils.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.enso.shttp; - -import com.sun.net.httpserver.HttpExchange; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -public class Utils { - static void sendResponse(int code, String message, HttpExchange exchange) throws IOException { - byte[] response = message.getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=utf-8"); - exchange.sendResponseHeaders(code, response.length); - exchange.getResponseBody().write(response); - exchange.close(); - } -} From c5907491c40797cd15ce108bfb369b841083611f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 18:35:18 +0100 Subject: [PATCH 23/32] test crash --- test/Tests/src/Network/Http_Spec.enso | 15 +++++++++++---- .../java/org/enso/shttp/CrashingTestHandler.java | 13 +++++++++++++ .../main/java/org/enso/shttp/SimpleHTTPBin.java | 1 + .../java/org/enso/shttp/SimpleHttpHandler.java | 2 ++ 4 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 8738021bd0ed..87709c9ea9cd 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -349,13 +349,11 @@ spec = r1 = Data.post url_delete r1.should_fail_with HTTP_Error r1.catch.should_be_a HTTP_Error.Status_Error - IO.println r1.catch.to_display_text r1.catch.status_code.code . should_equal 405 r1.catch.to_display_text . should_contain "status 405" r2 = Data.post (base_url_with_slash + "some/unknown/path") r2.should_fail_with HTTP_Error - IO.println r2.catch.to_display_text r2.catch.should_be_a HTTP_Error.Status_Error r2.catch.status_code.code . should_equal 404 r2.catch.message . should_contain "

404 Not Found

" @@ -363,7 +361,6 @@ spec = r3 = HTTP.new.request (Request.new (HTTP_Method.Custom "BREW_COFFEE") (base_url_with_slash + "get")) r3.should_fail_with HTTP_Error r3.catch.should_be_a HTTP_Error.Status_Error - IO.println r3.catch.to_display_text r3.catch.status_code.code . should_equal 400 Test.specify "Cannot perform POST when output context is disabled" <| @@ -535,6 +532,17 @@ spec = err = Data.fetch "http://0.0.0.0:1/" err.should_fail_with Request_Error + ## Checking this error partially as a warning - I spent a lot of time debugging why I'm getting such an error. + Apparently it happens when the httpbin server was crashing without sending any response. + Test.specify "should be able to handle server crash resulting in no response" <| + err = Data.fetch (base_url_with_slash+"crash") + err.should_fail_with Request_Error + err.catch.error_type . should_equal "java.io.IOException" + ## TODO I'm wondering if we should detect this particular error and add some explanation to it - + i.e. "The server did not send back any data." + I think it may be worth adding, because it may be really quite confusing for end users who get that kind of error. + err.catch.message . should_equal "HTTP/1.1 header parser received no bytes" + Test.specify "should be able to handle IO errors" pending="TODO: Currently I was unable to figure out a way to test such errors" <| # how to trigger this error??? err = Data.fetch "TODO" @@ -578,7 +586,6 @@ spec = r2 = Data.fetch url r2.should_fail_with HTTP_Error r2.catch.status_code.code . should_equal 401 - IO.println r2.catch.to_display_text # Invalid token r3 = Data.fetch url headers=[Header.authorization_bearer "invalid-token"] diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java new file mode 100644 index 000000000000..5f556079ec10 --- /dev/null +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java @@ -0,0 +1,13 @@ +package org.enso.shttp; + +import com.sun.net.httpserver.HttpExchange; + +import java.io.IOException; + +public class CrashingTestHandler extends SimpleHttpHandler { + @Override + public void doHandle(HttpExchange exchange) throws IOException { + // This exception will be logged by SimpleHttpHandler, but that's OK - let's know that this crash is happening. + throw new RuntimeException("This handler crashes on purpose."); + } +} diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index e84828610bb0..ce5899d0aa58 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -108,6 +108,7 @@ private static void setupEndpoints(SimpleHTTPBin server) throws URISyntaxExcepti server.addHandler("/test_headers", new HeaderTestHandler()); server.addHandler("/test_token_auth", new TokenAuthTestHandler()); server.addHandler("/test_basic_auth", new BasicAuthTestHandler()); + server.addHandler("/crash", new CrashingTestHandler()); setupFileServer(server); } diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java index 0b783f91a503..0baf318355c6 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java @@ -21,9 +21,11 @@ public final void handle(HttpExchange exchange) throws IOException { doHandle(exchange); } catch (IOException e) { e.printStackTrace(); + exchange.close(); throw e; } catch (Exception e) { e.printStackTrace(); + exchange.close(); } } From e30a46fcb19807c969cf93ede90d23d198e77c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 18:46:18 +0100 Subject: [PATCH 24/32] small test fixes --- test/Tests/src/Network/Http_Spec.enso | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 87709c9ea9cd..b46b9001dc14 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -5,6 +5,7 @@ import Standard.Base.Errors.Common.Syntax_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Network.HTTP.HTTP_Error.HTTP_Error import Standard.Base.Network.HTTP.Request.Request +import Standard.Base.Network.HTTP.Response.Response import Standard.Base.Network.HTTP.Request_Body.Request_Body import Standard.Base.Network.HTTP.Request_Error import Standard.Base.Network.Proxy.Proxy @@ -179,6 +180,7 @@ spec = req = Request.get url_get r2 = HTTP.new version=HTTP_Version.HTTP_2 . request req . decode_as_json r2.at "headers" . at "Connection" . should_equal "Upgrade, HTTP2-Settings" + r2.at "headers" . at "Http2-Settings" . should_contain "AA" r1 = HTTP.new version=HTTP_Version.HTTP_1_1 . request req . decode_as_json header_names = r1.at "headers" . field_names . map (s-> s.to_case Case.Lower) @@ -489,6 +491,7 @@ spec = Test.specify "can also read headers from a response, when returning a raw response" <| r1 = Data.post url_post (Request_Body.Text "hello world") try_auto_parse_response=False + r1.should_be_a Response # The result is JSON data: r1.headers.find (p-> p.name.equals_ignore_case "Content-Type") . value . should_equal "application/json" @@ -496,6 +499,7 @@ spec = . add_query_argument "test-header" "test-value" . add_query_argument "Other-Header" "some other value" r2 = Data.fetch uri try_auto_parse_response=False + r2.should_be_a Response r2.headers.find (p-> p.name.equals_ignore_case "Test-Header") . value . should_equal "test-value" r2.headers.find (p-> p.name.equals_ignore_case "Other-Header") . value . should_equal "some other value" From e97c6a36ebd2ccf8fbc586b1087403e32e747a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 18:50:47 +0100 Subject: [PATCH 25/32] test multi headers --- test/Tests/src/Network/Http_Spec.enso | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index b46b9001dc14..c789c6814ae0 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -503,6 +503,16 @@ spec = r2.headers.find (p-> p.name.equals_ignore_case "Test-Header") . value . should_equal "test-value" r2.headers.find (p-> p.name.equals_ignore_case "Other-Header") . value . should_equal "some other value" + Test.specify "is capable of handling aliasing headers" <| + uri = URI.from (base_url_with_slash + "test_headers") + . add_query_argument "my-header" "value-1" + . add_query_argument "my-header" "value-2" + . add_query_argument "my-header" "value-44" + r1 = Data.fetch uri try_auto_parse_response=False + r1.should_be_a Response + my_headers = r1.headers.filter (p-> p.name.equals_ignore_case "my-header") . map .value + my_headers.sort . should_equal ["value-1", "value-2", "value-44"] + Test.group "Header resolution" <| Test.specify "Default content type and encoding" <| expected = [Header.content_type "text/plain; charset=UTF-8"] From ba4c2f476229d8f4c351fe9e67ceaa5a1ec7f0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 18:59:22 +0100 Subject: [PATCH 26/32] simplify handling of headers in the EnsoHttpResponse --- .../0.0.0-dev/src/Network/HTTP/Response.enso | 19 +++++++++++++++++-- .../base/enso_cloud/EnsoHttpResponse.java | 16 ++++++++++++++++ .../base/enso_cloud/EnsoSecretHelper.java | 19 ++++++------------- 3 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso index c8e0d1e737c5..90caaa111c60 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso @@ -46,6 +46,9 @@ type Response ## GROUP Metadata Get the response headers. + It returns a vector of Header objects and not a mapping, because the + response may contain multiple headers with the same name. + > Example Getting the headers from a response. Note: This example will make a network request. @@ -53,10 +56,22 @@ type Response import Standard.Examples example_headers = Examples.get_response.headers + + > Example + Creating a mapping from the headers, throwing an error if there are duplicates. + Note: This example will make a network request. + + import Standard.Examples + + example_headers = Map.from_vector error_on_duplicates=True (Examples.get_response.headers.map h-> [h.name, h.value]) headers : Vector Header headers self = - header_keys = self.internal_http_response.headerNames - header_keys.flat_map k-> (self.internal_http_response.headers.allValues k).map v-> Header.new k v + # This is a mapping that maps a header name to a list of values (since headers may be duplicated). + multi_map = self.internal_http_response.headers.map + multi_map.to_vector.flat_map p-> + key = p.first + values = p.second + values.map v-> Header.new key v ## Get the response content type. content_type : Text | Nothing diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java new file mode 100644 index 000000000000..ad89b1a88e54 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java @@ -0,0 +1,16 @@ +package org.enso.base.enso_cloud; + +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpHeaders; +import java.util.List; + +/** + * A subset of the HttpResponse to avoid leaking the decrypted Enso secrets. + */ +public record EnsoHttpResponse( + URI uri, + HttpHeaders headers, + InputStream body, + int statusCode) { +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index 3bce0c028556..9f7c834eeb71 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -3,11 +3,9 @@ import org.enso.base.net.URIHelpers; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.net.http.HttpClient; -import java.net.http.HttpHeaders; import java.net.http.HttpRequest.Builder; import java.net.http.HttpResponse; import java.sql.Connection; @@ -72,7 +70,8 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U boolean hasSecrets = queryArguments.stream().anyMatch(p -> p instanceof EnsoKeySecretPair); if (hasSecrets && !uri.getScheme().equals("https")) { // If used a secret then only allow HTTPS - throw new IllegalArgumentException("Cannot use secrets in query string with non-HTTPS URI, but the scheme was: " + uri.getScheme() + "."); + throw new IllegalArgumentException("Cannot use secrets in query string with non-HTTPS URI, but the scheme " + + "was: " + uri.getScheme() + "."); } try { @@ -86,7 +85,9 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U resolvedURI = URIHelpers.addQueryParameters(uri, resolvedArguments); renderedURI = URIHelpers.addQueryParameters(uri, renderedArguments); } catch (URISyntaxException e) { - throw new IllegalStateException("Unexpectedly unable to build a valid URI from the base URI: " + uri + " and query arguments: " + queryArguments + "."); + throw new IllegalStateException( + "Unexpectedly unable to build a valid URI from the base URI: " + uri + " and query arguments: " + queryArguments + "." + ); } } builder.uri(resolvedURI); @@ -104,14 +105,6 @@ public static EnsoHttpResponse makeRequest(HttpClient client, Builder builder, U var javaResponse = client.send(httpRequest, bodyHandler); // Extract parts of the response - return new EnsoHttpResponse(renderedURI, javaResponse.headers().map().keySet().stream().toList(), - javaResponse.headers(), javaResponse.body(), javaResponse.statusCode()); - } - - /** - * A subset of the HttpResponse to avoid leaking the decrypted Enso secrets. - */ - public record EnsoHttpResponse(URI uri, List headerNames, HttpHeaders headers, InputStream body, - int statusCode) { + return new EnsoHttpResponse(renderedURI, javaResponse.headers(), javaResponse.body(), javaResponse.statusCode()); } } From 3f54d64adb54382147d430653bee6835829ccef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 19:04:54 +0100 Subject: [PATCH 27/32] add some simple tests for JS_Object.into Map --- test/Tests/src/Data/Json_Spec.enso | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/Tests/src/Data/Json_Spec.enso b/test/Tests/src/Data/Json_Spec.enso index b39643282604..59eb27606806 100644 --- a/test/Tests/src/Data/Json_Spec.enso +++ b/test/Tests/src/Data/Json_Spec.enso @@ -1,5 +1,6 @@ from Standard.Base import all import Standard.Base.Errors.Common.Index_Out_Of_Bounds +import Standard.Base.Errors.Common.No_Such_Method import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Errors.No_Such_Key.No_Such_Key from Standard.Base.Data.Json import Invalid_JSON @@ -68,6 +69,7 @@ spec = Test.specify "should be able to deserialize using into via conversion" <| Json.parse '{"type":"Time_Zone","constructor":"parse","id":"Europe/Moscow"}' . into Time_Zone . should_equal (Time_Zone.parse "Europe/Moscow") + Json.parse '{}' . into Time_Zone . should_fail_with Illegal_Argument Test.specify "should be able to deserialize using into for single constructor" <| Json.parse '{"first": 1, "second": 2}' . into Pair . should_equal (Pair.Value 1 2) @@ -78,6 +80,14 @@ spec = Json.parse '{"constructor": "Less", "than": 2}' . into Filter_Condition . should_equal (Filter_Condition.Less 2) Json.parse '{"constructor": "NotARealOne", "than": 2}' . into Filter_Condition . should_fail_with Illegal_Argument + Test.specify "should be able to convert a JS_Object into a Map using into" <| + Json.parse '{"a": 15, "b": 20, "c": "X", "d": null}' . into Map . should_equal (Map.from_vector [["a", 15], ["b", 20], ["c", "X"], ["d", Nothing]]) + Json.parse '{}' . into Map . should_equal Map.empty + + # [] parses as a vector/array which does not have the `into` method, that only works for {} objects: + Test.expect_panic No_Such_Method <| + Json.parse '[]' . into Map + Test.specify "should be able to deserialize Date" <| '{"type": "Date", "constructor": "new", "year": 2018, "month": 7, "day": 3}'.should_parse_as (Date.new 2018 7 3) '{"type": "Date", "year": 2025, "month": 5, "day": 12}'.should_parse_as (Date.new 2025 5 12) From addd10bcd58805f433d1ba1b024f6aa627afa8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Thu, 7 Dec 2023 19:11:10 +0100 Subject: [PATCH 28/32] javafmt --- .../java/org/enso/base/net/URIHelpers.java | 6 ++--- .../org/enso/shttp/BasicAuthTestHandler.java | 1 - .../org/enso/shttp/CrashingTestHandler.java | 4 ++-- .../org/enso/shttp/HeaderTestHandler.java | 3 +-- .../java/org/enso/shttp/SimpleHTTPBin.java | 5 ++-- .../org/enso/shttp/SimpleHttpHandler.java | 4 ++-- .../main/java/org/enso/shttp/TestHandler.java | 23 ++++++++++++------- .../org/enso/shttp/TokenAuthTestHandler.java | 1 - 8 files changed, 25 insertions(+), 22 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java b/std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java index 21b3cfb3d5cc..70ebbbf58e96 100644 --- a/std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java +++ b/std-bits/base/src/main/java/org/enso/base/net/URIHelpers.java @@ -1,15 +1,15 @@ package org.enso.base.net; -import org.apache.http.client.utils.URIBuilder; - import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import org.apache.http.client.utils.URIBuilder; public class URIHelpers { public record NameValuePair(String name, String value) {} - public static URI addQueryParameters(URI uri, List params) throws URISyntaxException { + public static URI addQueryParameters(URI uri, List params) + throws URISyntaxException { URIBuilder builder = new URIBuilder(uri); for (NameValuePair param : params) { builder.addParameter(param.name(), param.value()); diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java index d88ed548ff6a..d94a67c99c50 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/BasicAuthTestHandler.java @@ -1,7 +1,6 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; - import java.io.IOException; import java.util.Base64; import java.util.List; diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java index 5f556079ec10..0b18798d57a5 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/CrashingTestHandler.java @@ -1,13 +1,13 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; - import java.io.IOException; public class CrashingTestHandler extends SimpleHttpHandler { @Override public void doHandle(HttpExchange exchange) throws IOException { - // This exception will be logged by SimpleHttpHandler, but that's OK - let's know that this crash is happening. + // This exception will be logged by SimpleHttpHandler, but that's OK - let's know that this + // crash is happening. throw new RuntimeException("This handler crashes on purpose."); } } diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java index b85e16be36b8..6eb10d7923c6 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/HeaderTestHandler.java @@ -1,10 +1,9 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; -import org.apache.http.client.utils.URIBuilder; - import java.io.IOException; import java.net.URI; +import org.apache.http.client.utils.URIBuilder; public class HeaderTestHandler extends SimpleHttpHandler { @Override diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index ce5899d0aa58..fe8a32c3f9d3 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -3,9 +3,6 @@ import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.SimpleFileServer; -import sun.misc.Signal; -import sun.misc.SignalHandler; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.URISyntaxException; @@ -13,6 +10,8 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Stream; +import sun.misc.Signal; +import sun.misc.SignalHandler; public class SimpleHTTPBin { diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java index 0baf318355c6..acb8cbb4b9b2 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHttpHandler.java @@ -2,7 +2,6 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; - import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -31,7 +30,8 @@ public final void handle(HttpExchange exchange) throws IOException { public abstract void doHandle(HttpExchange exchange) throws IOException; - protected final void sendResponse(int code, String message, HttpExchange exchange) throws IOException { + protected final void sendResponse(int code, String message, HttpExchange exchange) + throws IOException { byte[] response = message.getBytes(StandardCharsets.UTF_8); exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=utf-8"); exchange.sendResponseHeaders(code, response.length); diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index b97ff6fcada4..e88b34b8ce3d 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -1,10 +1,6 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; -import org.apache.commons.text.StringEscapeUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URIBuilder; - import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -17,6 +13,9 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.text.StringEscapeUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; public class TestHandler extends SimpleHttpHandler { private final HttpMethod expectedMethod; @@ -79,7 +78,8 @@ public void doHandle(HttpExchange exchange) throws IOException { response.append("\n"); response.append(" },\n"); - response.append(" \"origin\": \"" + exchange.getRemoteAddress().getAddress().getHostAddress() + "\",\n"); + response.append( + " \"origin\": \"" + exchange.getRemoteAddress().getAddress().getHostAddress() + "\",\n"); response.append(" \"path\": \"" + StringEscapeUtils.escapeJson(uri.getPath()) + "\",\n"); if (uri.getQuery() != null) { URIBuilder builder = new URIBuilder(uri); @@ -89,7 +89,12 @@ public void doHandle(HttpExchange exchange) throws IOException { NameValuePair param = params.get(i); String key = StringEscapeUtils.escapeJson(param.getName()); String value = StringEscapeUtils.escapeJson(param.getValue()); - response.append(" {\"name\": \"").append(key).append("\", \"value\": \"").append(value).append("\"}"); + response + .append(" {\"name\": \"") + .append(key) + .append("\", \"value\": \"") + .append(value) + .append("\"}"); boolean isLast = i == params.size() - 1; if (!isLast) { response.append(",\n"); @@ -108,8 +113,10 @@ public void doHandle(HttpExchange exchange) throws IOException { response.append(" \"form\": null,\n"); response.append(" \"files\": null,\n"); String value = readBody(exchange.getRequestBody(), textEncoding); - response.append(" \"data\": \"").append(value == null ? "" : StringEscapeUtils.escapeJson(value)).append( - "\",\n"); + response + .append(" \"data\": \"") + .append(value == null ? "" : StringEscapeUtils.escapeJson(value)) + .append("\",\n"); } response.append(" \"args\": {}\n"); response.append("}"); diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java index cd6995aac323..ef1d5501961c 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TokenAuthTestHandler.java @@ -1,7 +1,6 @@ package org.enso.shttp; import com.sun.net.httpserver.HttpExchange; - import java.io.IOException; import java.util.List; From f95140338bacea9afcc6f23de190b7f786211de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 8 Dec 2023 15:16:29 +0100 Subject: [PATCH 29/32] add a conversion, add one more edge case test --- .../0.0.0-dev/src/Network/URI_With_Query.enso | 3 +++ test/Tests/src/Network/Http_Spec.enso | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso index 3be7d50aab1c..0fa4c1cb7be0 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI_With_Query.enso @@ -125,3 +125,6 @@ build_java_uri_with_parameters java_uri java_params = Panic.catch URISyntaxException (URIHelpers.addQueryParameters java_uri java_params) caught_panic-> message = caught_panic.payload.getMessage Error.throw (Syntax_Error.Error "Unable to collapse to a URI: "+message) + +## PRIVATE +URI.from (that : URI_With_Query) = that.to_uri diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index c789c6814ae0..1a8c4bd5c5e4 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -9,6 +9,7 @@ import Standard.Base.Network.HTTP.Response.Response import Standard.Base.Network.HTTP.Request_Body.Request_Body import Standard.Base.Network.HTTP.Request_Error import Standard.Base.Network.Proxy.Proxy +import Standard.Base.Network.URI_With_Query.URI_With_Query import Standard.Base.Runtime.Context from Standard.Base.Network.HTTP import resolve_headers @@ -104,7 +105,7 @@ spec = } response . should_equal expected_response - uri_response = url_get.to_uri.fetch + uri_response = url_get.to URI . fetch uri_response . should_equal expected_response Test.specify "Can perform a HEAD" <| @@ -354,17 +355,29 @@ spec = r1.catch.status_code.code . should_equal 405 r1.catch.to_display_text . should_contain "status 405" - r2 = Data.post (base_url_with_slash + "some/unknown/path") + uri2 = URI.from (base_url_with_slash + "some/unknown/path") + r2 = Data.post uri2 r2.should_fail_with HTTP_Error r2.catch.should_be_a HTTP_Error.Status_Error r2.catch.status_code.code . should_equal 404 r2.catch.message . should_contain "

404 Not Found

" + r2.catch.uri . should_equal uri2 r3 = HTTP.new.request (Request.new (HTTP_Method.Custom "BREW_COFFEE") (base_url_with_slash + "get")) r3.should_fail_with HTTP_Error r3.catch.should_be_a HTTP_Error.Status_Error r3.catch.status_code.code . should_equal 400 + # Also test the URI_With_Query variant + uri4 = uri2.add_query_argument "a" "b" . add_query_argument "c" "d" + uri4.should_be_a URI_With_Query + r4 = uri4.fetch + r4.should_fail_with HTTP_Error + r4.catch.should_be_a HTTP_Error.Status_Error + r4.catch.status_code.code . should_equal 404 + # The error may not necessarily store the URI_With_Query but a raw URI: + r4.catch.uri . should_equal (URI.from uri4) + Test.specify "Cannot perform POST when output context is disabled" <| Context.Output.with_disabled <| Data.post url_post (Request_Body.Text "hello world") . should_fail_with Forbidden_Operation From 223034526cdaa19e343ad996dbcb39ffb9ee210c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 15 Dec 2023 15:03:53 +0100 Subject: [PATCH 30/32] Adding licenses to the std-base Apache httpclient dependency --- .../Base/0.0.0-dev/THIRD-PARTY/NOTICE | 20 ++ .../LICENSE.txt | 202 +++++++++++++ .../NOTICE.txt | 9 + .../NOTICE.txt | 6 + .../0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 | 201 +++++++++++++ .../NOTICE | 8 + .../LICENSE | 266 ++++++++++++++++++ .../NOTICE | 10 + .../NOTICES | 27 ++ .../copyright-ignore | 1 + .../custom-license | 1 + .../files-keep | 2 + .../copyright-ignore | 1 + .../files-ignore | 1 + .../files-keep | 1 + .../copyright-ignore | 1 + .../files-ignore | 1 + .../files-keep | 1 + .../copyright-add | 26 ++ .../copyright-ignore | 1 + .../custom-license | 1 + .../files-keep | 2 + tools/legal-review/Base/report-state | 4 +- .../Apache_License__Version_2.0 | 1 + 24 files changed, 792 insertions(+), 2 deletions(-) create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/LICENSE.txt create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/NOTICE.txt create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-logging.commons-logging-1.2/NOTICE.txt create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpclient-4.4.1/NOTICE create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/LICENSE create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICE create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICES create mode 100644 tools/legal-review/Base/commons-codec.commons-codec-1.9/copyright-ignore create mode 100644 tools/legal-review/Base/commons-codec.commons-codec-1.9/custom-license create mode 100644 tools/legal-review/Base/commons-codec.commons-codec-1.9/files-keep create mode 100644 tools/legal-review/Base/commons-logging.commons-logging-1.2/copyright-ignore create mode 100644 tools/legal-review/Base/commons-logging.commons-logging-1.2/files-ignore create mode 100644 tools/legal-review/Base/commons-logging.commons-logging-1.2/files-keep create mode 100644 tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/copyright-ignore create mode 100644 tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-ignore create mode 100644 tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-keep create mode 100644 tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-add create mode 100644 tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-ignore create mode 100644 tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/custom-license create mode 100644 tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/files-keep create mode 100644 tools/legal-review/Base/reviewed-licenses/Apache_License__Version_2.0 diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE index ef542c1f288e..1b081cf051f6 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/NOTICE @@ -6,6 +6,26 @@ The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `com.ibm.icu.icu4j-73.1`. +'commons-codec', licensed under the The Apache Software License, Version 2.0, is distributed with the Base. +The license information can be found along with the copyright notices. +Copyright notices related to this dependency can be found in the directory `commons-codec.commons-codec-1.9`. + + +'commons-logging', licensed under the The Apache Software License, Version 2.0, is distributed with the Base. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `commons-logging.commons-logging-1.2`. + + +'httpclient', licensed under the Apache License, Version 2.0, is distributed with the Base. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `org.apache.httpcomponents.httpclient-4.4.1`. + + +'httpcore', licensed under the Apache License, Version 2.0, is distributed with the Base. +The license information can be found along with the copyright notices. +Copyright notices related to this dependency can be found in the directory `org.apache.httpcomponents.httpcore-4.4.1`. + + 'polyglot', licensed under the Universal Permissive License, Version 1.0, is distributed with the Base. The license file can be found at `licenses/Universal_Permissive_License__Version_1.0`. Copyright notices related to this dependency can be found in the directory `org.graalvm.polyglot.polyglot-23.1.0`. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/LICENSE.txt b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/LICENSE.txt new file mode 100644 index 000000000000..75b52484ea47 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/NOTICE.txt b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/NOTICE.txt new file mode 100644 index 000000000000..147f78a298e5 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-codec.commons-codec-1.9/NOTICE.txt @@ -0,0 +1,9 @@ +Apache Commons Codec +Copyright 2002-2013 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-logging.commons-logging-1.2/NOTICE.txt b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-logging.commons-logging-1.2/NOTICE.txt new file mode 100644 index 000000000000..556bd03951d4 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/commons-logging.commons-logging-1.2/NOTICE.txt @@ -0,0 +1,6 @@ +Apache Commons Logging +Copyright 2003-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 new file mode 100644 index 000000000000..261eeb9e9f8b --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpclient-4.4.1/NOTICE b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpclient-4.4.1/NOTICE new file mode 100644 index 000000000000..c05d4e6e933f --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpclient-4.4.1/NOTICE @@ -0,0 +1,8 @@ + +Apache HttpClient +Copyright 1999-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/LICENSE b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/LICENSE new file mode 100644 index 000000000000..54e4285f2d6a --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/LICENSE @@ -0,0 +1,266 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +========================================================================= + +This project contains annotations in the package org.apache.http.annotation +which are derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. +See http://www.jcip.net and the Creative Commons Attribution License +(http://creativecommons.org/licenses/by/2.5) +Full text: http://creativecommons.org/licenses/by/2.5/legalcode + +License + +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + + "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License. + "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License. + "Licensor" means the individual or entity that offers the Work under the terms of this License. + "Original Author" means the individual or entity who created the Work. + "Work" means the copyrightable work of authorship offered under the terms of this License. + "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. + +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: + + to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works; + to create and reproduce Derivative Works; + to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works; + to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works. + + For the avoidance of doubt, where the work is a musical composition: + Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work. + Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions). + Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions). + +The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: + + You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested. + If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit. + +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. Termination + + This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. + Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. + +8. Miscellaneous + + Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. + Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. + If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. + This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICE b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICE new file mode 100644 index 000000000000..976db53ee327 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICE @@ -0,0 +1,10 @@ + +Apache HttpCore +Copyright 2005-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + +This project contains annotations derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICES b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICES new file mode 100644 index 000000000000..76b0b30b05bb --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/THIRD-PARTY/org.apache.httpcomponents.httpcore-4.4.1/NOTICES @@ -0,0 +1,27 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + diff --git a/tools/legal-review/Base/commons-codec.commons-codec-1.9/copyright-ignore b/tools/legal-review/Base/commons-codec.commons-codec-1.9/copyright-ignore new file mode 100644 index 000000000000..8cd8d66408c4 --- /dev/null +++ b/tools/legal-review/Base/commons-codec.commons-codec-1.9/copyright-ignore @@ -0,0 +1 @@ +this work for additional information regarding copyright ownership. diff --git a/tools/legal-review/Base/commons-codec.commons-codec-1.9/custom-license b/tools/legal-review/Base/commons-codec.commons-codec-1.9/custom-license new file mode 100644 index 000000000000..35252fda76e8 --- /dev/null +++ b/tools/legal-review/Base/commons-codec.commons-codec-1.9/custom-license @@ -0,0 +1 @@ +LICENSE.txt diff --git a/tools/legal-review/Base/commons-codec.commons-codec-1.9/files-keep b/tools/legal-review/Base/commons-codec.commons-codec-1.9/files-keep new file mode 100644 index 000000000000..6de1a981f47b --- /dev/null +++ b/tools/legal-review/Base/commons-codec.commons-codec-1.9/files-keep @@ -0,0 +1,2 @@ +META-INF/NOTICE.txt +META-INF/LICENSE.txt diff --git a/tools/legal-review/Base/commons-logging.commons-logging-1.2/copyright-ignore b/tools/legal-review/Base/commons-logging.commons-logging-1.2/copyright-ignore new file mode 100644 index 000000000000..8cd8d66408c4 --- /dev/null +++ b/tools/legal-review/Base/commons-logging.commons-logging-1.2/copyright-ignore @@ -0,0 +1 @@ +this work for additional information regarding copyright ownership. diff --git a/tools/legal-review/Base/commons-logging.commons-logging-1.2/files-ignore b/tools/legal-review/Base/commons-logging.commons-logging-1.2/files-ignore new file mode 100644 index 000000000000..0256724c8d06 --- /dev/null +++ b/tools/legal-review/Base/commons-logging.commons-logging-1.2/files-ignore @@ -0,0 +1 @@ +META-INF/LICENSE.txt diff --git a/tools/legal-review/Base/commons-logging.commons-logging-1.2/files-keep b/tools/legal-review/Base/commons-logging.commons-logging-1.2/files-keep new file mode 100644 index 000000000000..f9a3ec844f02 --- /dev/null +++ b/tools/legal-review/Base/commons-logging.commons-logging-1.2/files-keep @@ -0,0 +1 @@ +META-INF/NOTICE.txt diff --git a/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/copyright-ignore b/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/copyright-ignore new file mode 100644 index 000000000000..dc34027f6f18 --- /dev/null +++ b/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/copyright-ignore @@ -0,0 +1 @@ +regarding copyright ownership. The ASF licenses this file diff --git a/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-ignore b/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-ignore new file mode 100644 index 000000000000..b9005a4d5ae7 --- /dev/null +++ b/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-ignore @@ -0,0 +1 @@ +META-INF/LICENSE diff --git a/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-keep b/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-keep new file mode 100644 index 000000000000..0d1c51375183 --- /dev/null +++ b/tools/legal-review/Base/org.apache.httpcomponents.httpclient-4.4.1/files-keep @@ -0,0 +1 @@ +META-INF/NOTICE diff --git a/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-add b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-add new file mode 100644 index 000000000000..b1eb84d0d452 --- /dev/null +++ b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-add @@ -0,0 +1,26 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ diff --git a/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-ignore b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-ignore new file mode 100644 index 000000000000..dc34027f6f18 --- /dev/null +++ b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/copyright-ignore @@ -0,0 +1 @@ +regarding copyright ownership. The ASF licenses this file diff --git a/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/custom-license b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/custom-license new file mode 100644 index 000000000000..6b1d0bfabc3c --- /dev/null +++ b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/custom-license @@ -0,0 +1 @@ +LICENSE diff --git a/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/files-keep b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/files-keep new file mode 100644 index 000000000000..26e9c87b6125 --- /dev/null +++ b/tools/legal-review/Base/org.apache.httpcomponents.httpcore-4.4.1/files-keep @@ -0,0 +1,2 @@ +META-INF/NOTICE +META-INF/LICENSE diff --git a/tools/legal-review/Base/report-state b/tools/legal-review/Base/report-state index 52e3a3e77a85..39680ade8ab2 100644 --- a/tools/legal-review/Base/report-state +++ b/tools/legal-review/Base/report-state @@ -1,3 +1,3 @@ -58F42EA238F4F16E775412B67F584C74188267FB305705B57A50E10124FE56BC -7C5FEB79459C03EB21D2098EFC33BF7AA26E3C51204A9F32F3DCFC854D5A36A0 +C47CF3A9C954B3EF3C13C65CC3876B735B1EACEB28B3B5187260A86B1FF22C2A +A496F147B6196EB4B1D0744911F51777387851254F053650FA5F79406E121558 0 diff --git a/tools/legal-review/Base/reviewed-licenses/Apache_License__Version_2.0 b/tools/legal-review/Base/reviewed-licenses/Apache_License__Version_2.0 new file mode 100644 index 000000000000..ff46ef6ff419 --- /dev/null +++ b/tools/legal-review/Base/reviewed-licenses/Apache_License__Version_2.0 @@ -0,0 +1 @@ +tools/legal-review/license-texts/APACHE2.0 From bcc9572a51254e8bad3db3c4e586974d787a5bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 15 Dec 2023 17:09:53 +0100 Subject: [PATCH 31/32] fix a test how did I miss it?? :O --- test/Table_Tests/src/IO/Fetch_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/IO/Fetch_Spec.enso b/test/Table_Tests/src/IO/Fetch_Spec.enso index 0fbddcee140d..04b9afb04ae1 100644 --- a/test/Table_Tests/src/IO/Fetch_Spec.enso +++ b/test/Table_Tests/src/IO/Fetch_Spec.enso @@ -66,6 +66,6 @@ spec = r2.should_be_a Table r2.should_equal expected_table - r3 = url.fetch try_auto_parse_response=False . decode (Excel (Excel_Section.Worksheet "MyTestSheet")) + r3 = url.to_uri.fetch try_auto_parse_response=False . decode (Excel (Excel_Section.Worksheet "MyTestSheet")) r3.should_be_a Table r3.should_equal expected_table From 321db41339e56c7ec941a5d11672910bae6c681f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 15 Dec 2023 17:16:48 +0100 Subject: [PATCH 32/32] javafmt --- .../org/enso/base/enso_cloud/EnsoHttpResponse.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java index ad89b1a88e54..93c5a5905ae7 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoHttpResponse.java @@ -3,14 +3,6 @@ import java.io.InputStream; import java.net.URI; import java.net.http.HttpHeaders; -import java.util.List; -/** - * A subset of the HttpResponse to avoid leaking the decrypted Enso secrets. - */ -public record EnsoHttpResponse( - URI uri, - HttpHeaders headers, - InputStream body, - int statusCode) { -} +/** A subset of the HttpResponse to avoid leaking the decrypted Enso secrets. */ +public record EnsoHttpResponse(URI uri, HttpHeaders headers, InputStream body, int statusCode) {}