From 78d9e3484062fc3e83db4ceab308edae981a3f44 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Mon, 28 Oct 2024 20:20:06 +0000 Subject: [PATCH] Excel before 1900 and AWS signed requests. (#11373) --- CHANGELOG.md | 2 + .../lib/Standard/AWS/0.0.0-dev/src/AWS.enso | 114 +++++++ .../Standard/AWS/0.0.0-dev/src/Errors.enso | 9 + .../lib/Standard/AWS/0.0.0-dev/src/Main.enso | 1 + .../lib/Standard/Base/0.0.0-dev/src/Data.enso | 6 +- .../Base/0.0.0-dev/src/Network/HTTP.enso | 199 +++++++------ .../0.0.0-dev/src/Network/HTTP/Header.enso | 20 +- .../src/Extensions/Excel_Extensions.enso | 23 ++ .../Standard/Table/0.0.0-dev/src/Main.enso | 1 + .../main/java/org/enso/aws/ClientBuilder.java | 14 + .../java/org/enso/aws/SignedHttpClient.java | 280 ++++++++++++++++++ .../base/net/http/MultipartBodyBuilder.java | 28 +- .../base/net/http/UrlencodedBodyBuilder.java | 17 +- .../java/org/enso/table/excel/ExcelRow.java | 35 ++- .../java/org/enso/table/excel/ExcelUtils.java | 86 ++++++ .../org/enso/table/write/ExcelWriter.java | 21 +- test/Base_Tests/src/Network/Http_Spec.enso | 24 +- test/Table_Tests/data/OlderDates.xlsx | Bin 0 -> 9123 bytes test/Table_Tests/src/IO/Excel_Spec.enso | 39 +++ 19 files changed, 785 insertions(+), 134 deletions(-) create mode 100644 distribution/lib/Standard/AWS/0.0.0-dev/src/AWS.enso create mode 100644 distribution/lib/Standard/Table/0.0.0-dev/src/Extensions/Excel_Extensions.enso create mode 100644 std-bits/aws/src/main/java/org/enso/aws/SignedHttpClient.java create mode 100644 std-bits/table/src/main/java/org/enso/table/excel/ExcelUtils.java create mode 100644 test/Table_Tests/data/OlderDates.xlsx diff --git a/CHANGELOG.md b/CHANGELOG.md index d18cbd44b81f..570388a6a1b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,10 +28,12 @@ - [The user may set description and labels of an Enso Cloud asset programmatically.][11255] - [DB_Table may be saved as a Data Link.][11371] +- [Support for dates before 1900 in Excel and signed AWS requests.][11373] [11235]: https://github.com/enso-org/enso/pull/11235 [11255]: https://github.com/enso-org/enso/pull/11255 [11371]: https://github.com/enso-org/enso/pull/11371 +[11373]: https://github.com/enso-org/enso/pull/11373 #### Enso Language & Runtime diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/AWS.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/AWS.enso new file mode 100644 index 000000000000..1101b8fa9c9c --- /dev/null +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/AWS.enso @@ -0,0 +1,114 @@ +from Standard.Base import all +import Standard.Base.Errors.Common.Missing_Argument +import Standard.Base.Network.HTTP.HTTP_Error.HTTP_Error +import Standard.Base.Network.HTTP.Request.Request +import Standard.Base.Network.HTTP.Request_Body.Request_Body +import Standard.Base.Network.HTTP.Request_Error +import Standard.Base.Network.HTTP.Response.Response +from Standard.Base.Metadata.Widget import Text_Input +from Standard.Base.Network.HTTP import if_fetch_method, if_post_method, internal_http_client, with_hash_and_client + +import project.AWS_Credential.AWS_Credential +import project.AWS_Region.AWS_Region +import project.Errors.Invalid_AWS_URI + +polyglot java import org.enso.aws.ClientBuilder + +## Methods for interacting with AWS services. +type AWS + ## ALIAS download, http get + ICON data_input + Fetches from an AWS URI signing the request with the necessary headers, + and returns the response, parsing the body if the content-type is + recognised. Returns an error if the status code does not represent a + successful response. + + Arguments: + - method: The HTTP method to use. Must be one of `HTTP_Method.Get`, + `HTTP_Method.Head`, `HTTP_Method.Delete`, `HTTP_Method.Options`. + Defaults to `HTTP_Method.Get`. + - headers: The headers to send with the request. Defaults to an empty + vector. + - format: The format to use for interpreting the response. + Defaults to `Auto_Detect`. If `Raw_Response` is selected or if the + format cannot be determined automatically, a raw HTTP `Response` will + be returned. + - credentials: The credentials to use for signing the request. Defaults + to the default AWS credentials. + - region_service: The region and service to use for signing the request. + Defaults to the region and service parsed from the URI. + @uri (Text_Input display=..Always) + @format File_Format.default_widget + @headers Header.default_widget + @credentials AWS_Credential.default_widget + signed_fetch : URI -> HTTP_Method -> (Vector (Header | Pair Text Text)) -> File_Format -> AWS_Credential -> AWS_Region_Service -> Any + signed_fetch (uri:URI=(Missing_Argument.throw "uri")) (method:HTTP_Method=..Get) (headers:(Vector (Header | Pair Text Text))=[]) (format = Auto_Detect) credentials:AWS_Credential=..Default (region_service:AWS_Region_Service=(AWS.resolve_region_and_service uri)) = if_fetch_method method <| + request = Request.new method uri (Header.unify_vector headers) Request_Body.Empty + http = with_hash_and_client HTTP.new hash_method=AWS.hash_bytes make_client=(_make_client credentials region_service) + raw_response = http.request request + raw_response.decode format=format if_unsupported=raw_response.with_materialized_body + + ## ALIAS http post, upload + ICON data_upload + Writes the provided data to the provided AWS URI signing the request with + the necessary headers. Returns the response, parsing the body if the + content-type is recognised. Returns an error if the status code does not + represent a successful response. + + Arguments: + - uri: The URI to fetch. + - body: The data to write. See `Supported Body Types` below. + - method: The HTTP method to use. Must be one of `HTTP_Method.Post`, + `HTTP_Method.Put`, `HTTP_Method.Patch`. Defaults to `HTTP_Method.Post`. + - headers: The headers to send with the request. Defaults to an empty + vector. + - response_format: The format to use for interpreting the response. + Defaults to `Auto_Detect`. If `Raw_Response` is selected or if the + format cannot be determined automatically, a raw HTTP `Response` will + be returned. + - credentials: The credentials to use for signing the request. Defaults + to the default AWS credentials. + - region_service: The region and service to use for signing the request. + Defaults to the region and service parsed from the URI. + @uri (Text_Input display=..Always) + @format File_Format.default_widget + @headers Header.default_widget + @credentials AWS_Credential.default_widget + signed_post : (URI | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> Response ! Request_Error | HTTP_Error + signed_post (uri:URI=(Missing_Argument.throw "uri")) (body:Request_Body=..Empty) (method:HTTP_Method=..Post) (headers:(Vector (Header | Pair Text Text))=[]) (response_format = Auto_Detect) credentials:AWS_Credential=..Default (region_service:AWS_Region_Service=(AWS.resolve_region_and_service uri)) = if_post_method method <| + request = Request.new method uri (Header.unify_vector headers) body + http = with_hash_and_client HTTP.new hash_method=AWS.hash_bytes make_client=(_make_client credentials region_service) + raw_response = http.request request + raw_response.decode format=response_format if_unsupported=raw_response.with_materialized_body + + ## PRIVATE + Hash a Vector of bytes using SHA256 (as used by AWS). + hash_bytes : Vector Integer -> Text + hash_bytes bytes:Vector = ClientBuilder.getSHA256 bytes + + ## Resolve the region and service from an AWS based URI. + Splits a standard form AWS URI into the region and service. + + The URI must be in the forms: + - `https://(*.)..amazonaws.com`. + - `https://(*.)..amazonaws.com`. + + Arguments: + - uri: The URI to resolve. + resolve_region_and_service : URI -> AWS_Region_Service + resolve_region_and_service (uri:URI=(Missing_Argument.throw "uri")) = + region_regex = regex "^(([a-z]{2}-[^.]+?-\d+)|(global))$" + domain = uri.host.split '.' + if (domain.length < 4 || (domain.at -1) != "com" || (domain.at -2) != "amazonaws") then Error.throw (Invalid_AWS_URI.Error domain.length.to_text+":"+uri.to_text) else + if (domain.at -3).match region_regex then AWS_Region_Service.Region_Service region=(domain.at -3) service=(domain.at -4) else + if (domain.at -4).match region_regex then AWS_Region_Service.Region_Service region=(domain.at -4) service=(domain.at -3) else + Error.throw (Invalid_AWS_URI.Error domain.to_display_text) + +## Holds the region and service of an AWS URI. +type AWS_Region_Service + ## Holds the region and service of an AWS URI. + Region_Service region:Text service:Text + +private _make_client credentials region_service http hash = + builder = ClientBuilder.new credentials.as_java (AWS_Region.Region region_service.region).as_java + builder.createSignedClient region_service.region region_service.service (internal_http_client http "") hash diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Errors.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Errors.enso index 32c90451dccf..b71822295ba2 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Errors.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Errors.enso @@ -2,6 +2,15 @@ from Standard.Base import all polyglot java import software.amazon.awssdk.core.exception.SdkClientException +## An invalid URI was provided. +type Invalid_AWS_URI + ## PRIVATE + Error uri:Text + + ## PRIVATE + to_display_text : Text + to_display_text self = "Invalid AWS URI: " + self.uri + ## An error in the core AWS SDK type AWS_SDK_Error ## PRIVATE diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/Main.enso index 1060a50baf27..beb96125f934 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/Main.enso @@ -1,3 +1,4 @@ +export project.AWS.AWS export project.AWS_Credential.AWS_Credential export project.AWS_Region.AWS_Region export project.Database.Redshift.Redshift_Details.Redshift_Details diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index a862b6f4bd36..b6886fa047df 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -195,7 +195,7 @@ list (directory:(Text | File)=enso_project.root) (name_filter:Text="") recursive import Standard.Base.Data file = enso_project.data / "spreadsheet.xls" Data.fetch URL . body . write file -@uri Text_Input +@uri (Text_Input display=..Always) @format Data_Read_Helpers.format_widget_with_raw_response @headers Header.default_widget fetch : (URI | Text) -> HTTP_Method -> Vector (Header | Pair Text Text) -> File_Format -> Any ! Request_Error | HTTP_Error @@ -322,7 +322,7 @@ fetch (uri:(URI | Text)=(Missing_Argument.throw "uri")) (method:HTTP_Method=..Ge test_file = enso_project.data / "sample.txt" form_data = Dictionary.from_vector [["key", "val"], ["a_file", test_file]] response = Data.post url_post (Request_Body.Form_Data form_data url_encoded=True) -@uri Text_Input +@uri (Text_Input display=..Always) @headers Header.default_widget @response_format Data_Read_Helpers.format_widget_with_raw_response post : (URI | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> File_Format -> Any ! Request_Error | HTTP_Error @@ -342,7 +342,7 @@ post (uri:(URI | Text)=(Missing_Argument.throw "uri")) (body:Request_Body=..Empt `HTTP_Method.Head`, `HTTP_Method.Delete`, `HTTP_Method.Options`. Defaults to `HTTP_Method.Get`. - headers: The headers to send with the request. Defaults to an empty vector. -@uri Text_Input +@uri (Text_Input display=..Always) @headers Header.default_widget download : (URI | Text) -> Writable_File -> HTTP_Method -> Vector (Header | Pair Text Text) -> File ! Request_Error | HTTP_Error download (uri:(URI | Text)=(Missing_Argument.throw "uri")) file:Writable_File (method:HTTP_Method=..Get) (headers:(Vector (Header | Pair Text Text))=[]) = diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso index 5d8575906eed..34511c97e7a9 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso @@ -11,6 +11,7 @@ import project.Enso_Cloud.Enso_Secret.Enso_Secret import project.Error.Error import project.Errors.Common.Forbidden_Operation import project.Errors.Illegal_Argument.Illegal_Argument +import project.Errors.Unimplemented.Unimplemented import project.Function.Function import project.Meta import project.Network.HTTP.Header.Header @@ -28,6 +29,7 @@ import project.Runtime.Context import project.System.File.File from project.Data.Boolean import Boolean, False, True from project.Data.Json.Extensions import all +from project.Data.Text.Extensions import all polyglot java import java.lang.IllegalArgumentException polyglot java import java.io.IOException @@ -48,6 +50,20 @@ polyglot java import org.enso.base.net.http.MultipartBodyBuilder polyglot java import org.enso.base.net.http.UrlencodedBodyBuilder type HTTP + ## PRIVATE + Static helper for get-like methods + fetch : (URI | Text) -> HTTP_Method -> Vector (Header | Pair Text Text) -> Response ! Request_Error | HTTP_Error + fetch (uri:(URI | Text)) (method:HTTP_Method=..Get) (headers:(Vector (Header | Pair Text Text))=[]) = if_fetch_method method <| + request = Request.new method uri (Header.unify_vector headers) Request_Body.Empty + HTTP.new.request request + + ## PRIVATE + Static helper for post-like methods + post : (URI | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> Response ! Request_Error | HTTP_Error + post (uri:(URI | Text)) (body:Request_Body=Request_Body.Empty) (method:HTTP_Method=HTTP_Method.Post) (headers:(Vector (Header | Pair Text Text))=[]) = if_post_method method <| + request = Request.new method uri (Header.unify_vector headers) body + HTTP.new.request request + ## PRIVATE ADVANCED Create a new instance of the HTTP client. @@ -76,7 +92,7 @@ type HTTP example_new = HTTP.new (timeout = (Duration.new seconds=30)) (proxy = Proxy.Address "example.com" 8080) new : Duration -> Boolean -> Proxy -> HTTP_Version -> HTTP - new (timeout:Duration=(Duration.new seconds=10)) (follow_redirects:Boolean=True) (proxy:Proxy=Proxy.System) (version:HTTP_Version=HTTP_Version.HTTP_2) = + new (timeout:Duration=(Duration.new seconds=10)) (follow_redirects:Boolean=True) (proxy:Proxy=..System) (version:HTTP_Version=..HTTP_2) = HTTP.Value timeout follow_redirects proxy version Nothing ## PRIVATE @@ -90,7 +106,9 @@ type HTTP - custom_ssl_context: A custom SSL context to use for requests, or Nothing if the default should be used. For most use cases, it is recommended to use the default. - Value timeout follow_redirects proxy version custom_ssl_context + - hash_method: The hash method to use for body hashing. + - make_client: Creates the Java HTTPClient. + private Value timeout follow_redirects:Boolean proxy:Proxy version:HTTP_Version custom_ssl_context hash_method=Nothing make_client=internal_http_client ## ADVANCED ICON data_download @@ -107,7 +125,7 @@ type HTTP request self req error_on_failure_code=True = # Prevent request if the method is a write-like method and output context is disabled. check_output_context ~action = - if fetch_methods.contains req.method || Context.Output.is_enabled then action else + if (if_fetch_method req.method True if_not=Context.Output.is_enabled) then action else Error.throw (Forbidden_Operation.Error ("As writing is disabled, " + req.method.to_text + " request not sent. Press the Write button ▶ to send it.")) handle_request_error = handler caught_panic = @@ -116,66 +134,25 @@ type HTTP Panic.catch IllegalArgumentException handler=handler <| Panic.catch IOException handler=handler handle_request_error <| Illegal_Argument.handle_java_exception <| check_output_context <| - headers = resolve_headers req + headers = _resolve_headers req headers.if_not_error <| - body_publisher_and_boundary = resolve_body_to_publisher_and_boundary req.body - body_publisher_and_boundary.if_not_error <| + resolved_body = _resolve_body req.body self.hash_method + resolved_body.if_not_error <| # Create builder and set method and body builder = HttpRequest.newBuilder - builder.method req.method.to_http_method_name body_publisher_and_boundary.first + builder.method req.method.to_http_method_name resolved_body.publisher # Create Unified Header list - boundary = body_publisher_and_boundary.second - boundary_header_list = if boundary.is_nothing then [] else [Header.multipart_form_data boundary] + boundary_header_list = if resolved_body.boundary.is_nothing then [] else [Header.multipart_form_data resolved_body.boundary] all_headers = headers + boundary_header_list mapped_headers = all_headers.map on_problems=No_Wrap .to_java_pair - response = Response.Value (EnsoSecretHelper.makeRequest self.internal_http_client builder req.uri.to_java_representation mapped_headers) + response = Response.Value (EnsoSecretHelper.makeRequest (self.make_client self resolved_body.hash) builder req.uri.to_java_representation mapped_headers) if error_on_failure_code.not || response.code.is_success then response else body = response.body.decode_as_text.catch Any _->"" message = if body.is_empty then Nothing else body Error.throw (HTTP_Error.Status_Error response.code message response.uri) - ## PRIVATE - Static helper for get-like methods - fetch : (URI | Text) -> HTTP_Method -> Vector (Header | Pair Text Text) -> Response ! Request_Error | HTTP_Error - fetch (uri:(URI | Text)) (method:HTTP_Method=..Get) (headers:(Vector (Header | Pair Text Text))=[]) = - check_method fetch_methods method <| - request = Request.new method uri (parse_headers headers) Request_Body.Empty - HTTP.new.request request - - ## PRIVATE - Static helper for post-like methods - post : (URI | Text) -> Request_Body -> HTTP_Method -> Vector (Header | Pair Text Text) -> Response ! Request_Error | HTTP_Error - post (uri:(URI | Text)) (body:Request_Body=Request_Body.Empty) (method:HTTP_Method=HTTP_Method.Post) (headers:(Vector (Header | Pair Text Text))=[]) = - check_method post_methods method <| - request = Request.new method uri (parse_headers headers) body - HTTP.new.request request - - ## PRIVATE - - Build an HTTP client. - internal_http_client : HttpClient - internal_http_client self = - builder = HttpClient.newBuilder.connectTimeout self.timeout - - redirect_policy = if self.follow_redirects then HttpClient.Redirect.ALWAYS else HttpClient.Redirect.NEVER - builder.followRedirects redirect_policy - - case self.proxy of - Proxy.Address proxy_host proxy_port -> builder.proxy (ProxySelector.of (InetSocketAddress.new proxy_host proxy_port)) - Proxy.System -> builder.proxy ProxySelector.getDefault - Proxy.None -> Nothing - - case self.version of - HTTP_Version.HTTP_1_1 -> builder.version HttpClient.Version.HTTP_1_1 - HTTP_Version.HTTP_2 -> builder.version HttpClient.Version.HTTP_2 - - if self.custom_ssl_context.is_nothing.not then - builder.sslContext self.custom_ssl_context - - builder.build - ## PRIVATE ADVANCED Create a copy of the HTTP client with a custom SSL context. @@ -183,23 +160,14 @@ type HTTP set_custom_ssl_context self ssl_context = HTTP.Value self.timeout self.follow_redirects self.proxy self.version ssl_context -## PRIVATE -parse_headers : Vector (Header | Pair Text Text) -> Vector Header -parse_headers headers = - headers . map on_problems=No_Wrap h-> case h of - _ : Vector -> Header.new (h.at 0) (h.at 1) - _ : Pair -> Header.new (h.at 0) (h.at 1) - _ : Function -> h:Header - _ : Header -> h - _ -> Error.throw (Illegal_Argument.Error "Invalid header type - all values must be Vector, Pair or Header (got "+(Meta.get_simple_type_name h)+").") - ## PRIVATE If either encoding or content type is specified in the Request_Body, that is used as the content type header. If encoding is specified without content type, "text/plain" is used as the content type. It is an error to specify the content type in both the request body and the header list. If the body is not Request_Body.Empty, and no content type is specified, a default is used. -resolve_headers : Request -> Vector Header -resolve_headers req = + Not explicitly private as allows direct testing. +_resolve_headers : Request -> Vector Header +_resolve_headers req = is_content_type_header h = h.name . equals_ignore_case Header.content_type_header_name # Check for content type and encoding in the Request_Body. @@ -235,66 +203,103 @@ resolve_headers req = all_headers + default_content_type ## PRIVATE - Generate body publisher and optional form content boundary -resolve_body_to_publisher_and_boundary : Request_Body -> Pair BodyPublisher Text -resolve_body_to_publisher_and_boundary body:Request_Body = +type Resolved_Body + private Value publisher:BodyPublisher boundary:Text|Nothing hash:Text|Nothing + +## PRIVATE + Generate body publisher, optional form content boundary and optionally hash from the body +_resolve_body : Request_Body -> Function | Nothing -> Resolved_Body +private _resolve_body body:Request_Body hash_function = body_publishers = HttpRequest.BodyPublishers case body of Request_Body.Text text encoding _ -> body_publisher = case encoding of Nothing -> body_publishers.ofString text _ : Encoding -> body_publishers.ofString text encoding.to_java_charset - Pair.new body_publisher Nothing + hash = if hash_function.is_nothing then "" else hash_function (text.bytes (encoding.if_nothing Encoding.utf_8)) + Resolved_Body.Value body_publisher Nothing hash Request_Body.Json x -> json = x.to_json - json.if_not_error <| - Pair.new (body_publishers.ofString json) Nothing + hash = if hash_function.is_nothing then "" else hash_function json.bytes + json.if_not_error <| Resolved_Body.Value (body_publishers.ofString json) Nothing hash Request_Body.Binary file -> path = File_Utils.toPath file.path - Pair.new (body_publishers.ofFile path) Nothing + ## ToDo: Support hashing a file. + hash = if hash_function.is_nothing then "" else Unimplemented.throw "Hashing a file body is not yet supported." + Resolved_Body.Value (body_publishers.ofFile path) Nothing hash Request_Body.Form_Data form_data url_encoded -> - build_form_body_publisher form_data url_encoded + _resolve_form_body form_data url_encoded hash_function Request_Body.Empty -> - Pair.new (body_publishers.noBody) Nothing + hash = if hash_function.is_nothing then "" else hash_function [] + Resolved_Body.Value body_publishers.noBody Nothing hash _ -> - Error.throw (Illegal_Argument.Error ("Unsupported POST body: " + body.to_display_text + "; this is a bug in the Data library")) - + Error.throw (Illegal_Argument.Error ("Unsupported POST body: " + body.to_display_text + "; this is a bug library.")) ## PRIVATE - Build a BodyPublisher from the given form data. The pair's second value is a content boundary in the case of a `multipart/form-data` form; otherwise, Nothing -build_form_body_publisher : Dictionary Text (Text | File) -> Boolean -> Pair BodyPublisher Text -build_form_body_publisher (form_data:(Dictionary Text (Text | File))) (url_encoded:Boolean=False) = case url_encoded of +_resolve_form_body : Dictionary Text (Text | File) -> Boolean -> Function | Nothing -> Resolved_Body +private _resolve_form_body (form_data:(Dictionary Text (Text | File))) (url_encoded:Boolean=False) hash_function = case url_encoded of True -> body_builder = UrlencodedBodyBuilder.new form_data.map_with_key key-> value-> case value of _ : Text -> body_builder.add_part_text key value _ : File -> body_builder.add_part_file key value.path - Pair.new body_builder.build Nothing + publisher = body_builder.build + hash = if hash_function.is_nothing then "" else hash_function body_builder.getContents.bytes + Resolved_Body.Value publisher Nothing hash False -> body_builder = MultipartBodyBuilder.new form_data.map_with_key key-> value-> case value of _ : Text -> body_builder.add_part_text key value _ : File -> body_builder.add_part_file key value.path - boundary = body_builder.get_boundary - Pair.new body_builder.build boundary + publisher = body_builder.build + hash = if hash_function.is_nothing then "" else hash_function body_builder.getContents + Resolved_Body.Value publisher body_builder.get_boundary hash + +## PRIVATE +if_fetch_method : HTTP_Method -> Function -> Any -> Any ! Illegal_Argument +if_fetch_method method:HTTP_Method ~action ~if_not=(Error.throw (Illegal_Argument.Error ("Unsupported method " + method.to_display_text))) = + if [HTTP_Method.Get, HTTP_Method.Head, HTTP_Method.Options].contains method then action else + if_not ## PRIVATE -fetch_methods : Hashset HTTP_Method -fetch_methods = Hashset.from_vector [HTTP_Method.Get, HTTP_Method.Head, HTTP_Method.Options] +if_post_method : HTTP_Method -> Function -> Any -> Any ! Illegal_Argument +if_post_method method:HTTP_Method ~action ~if_not=(Error.throw (Illegal_Argument.Error ("Unsupported method " + method.to_display_text))) = + if [HTTP_Method.Post, HTTP_Method.Put, HTTP_Method.Patch, HTTP_Method.Delete].contains method then action else + if_not ## PRIVATE -post_methods : Hashset HTTP_Method -post_methods = Hashset.from_vector [HTTP_Method.Post, HTTP_Method.Put, HTTP_Method.Patch, HTTP_Method.Delete] + Build a custom HTTP with hash function and make_client function. +with_hash_and_client : HTTP -> Function -> Function -> HTTP +with_hash_and_client http hash_method make_client = + HTTP.Value http.timeout http.follow_redirects http.proxy http.version http.custom_ssl_context hash_method make_client ## PRIVATE -check_method : Hashset HTTP_Method -> Any -> Any -> Any ! Illegal_Argument -check_method allowed_methods method ~action = - if allowed_methods.contains method then action else - Error.throw (Illegal_Argument.Error ("Unsupported method " + method.to_display_text)) + Build a Java HttpClient with the given settings. +internal_http_client : HTTP -> Text -> HttpClient +internal_http_client http hash = + _ = hash + builder = HttpClient.newBuilder.connectTimeout http.timeout + + redirect_policy = if http.follow_redirects then HttpClient.Redirect.ALWAYS else HttpClient.Redirect.NEVER + builder.followRedirects redirect_policy + + case http.proxy of + Proxy.Address proxy_host proxy_port -> builder.proxy (ProxySelector.of (InetSocketAddress.new proxy_host proxy_port)) + Proxy.System -> builder.proxy ProxySelector.getDefault + Proxy.None -> Nothing + + case http.version of + HTTP_Version.HTTP_1_1 -> builder.version HttpClient.Version.HTTP_1_1 + HTTP_Version.HTTP_2 -> builder.version HttpClient.Version.HTTP_2 + + if http.custom_ssl_context.is_nothing.not then + builder.sslContext http.custom_ssl_context + + builder.build ## PRIVATE An error when sending an HTTP request. @@ -314,3 +319,23 @@ type Request_Error Nothing -> "" _ -> " " + self.message self.error_type + " error when sending request." + description_text + +## PRIVATE + Access the HTTP's timeout (for testing purposes). +get_timeout : HTTP -> Duration +get_timeout http:HTTP = http.timeout + +## PRIVATE + Access the HTTP's follow_redirects (for testing purposes). +get_follow_redirects : HTTP -> Boolean +get_follow_redirects http:HTTP = http.follow_redirects + +## PRIVATE + Access the HTTP's proxy (for testing purposes). +get_proxy : HTTP -> Proxy +get_proxy http:HTTP = http.proxy + +## PRIVATE + Access the HTTP's version (for testing purposes). +get_version : HTTP -> HTTP_Version +get_version http:HTTP = http.version diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso index 0ebcb985d647..c8357c65bfce 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso @@ -1,8 +1,14 @@ import project.Data.Numbers.Integer +import project.Data.Pair.Pair import project.Data.Text.Encoding.Encoding import project.Data.Text.Text +import project.Data.Vector.No_Wrap +import project.Data.Vector.Vector import project.Enso_Cloud.Enso_Secret.Derived_Secret_Value import project.Enso_Cloud.Enso_Secret.Enso_Secret +import project.Error.Error +import project.Errors.Illegal_Argument.Illegal_Argument +import project.Function.Function import project.Meta import project.Metadata.Display import project.Metadata.Widget @@ -20,7 +26,18 @@ polyglot java import org.graalvm.collections.Pair as Java_Pair type Header ## PRIVATE + Normalize a vector of `Header`, `Pair`s or `Vector`s into a vector of + `Header` values. + unify_vector : Vector (Header | Pair Text Text | Vector) -> Vector Header + unify_vector headers:Vector = + headers . map on_problems=No_Wrap h-> case h of + _ : Vector -> Header.new (h.at 0) (h.at 1) + _ : Pair -> Header.new (h.at 0) (h.at 1) + _ : Function -> h:Header + _ : Header -> h + _ -> Error.throw (Illegal_Argument.Error "Invalid header type - all values must be Vector, Pair or Header (got "+(Meta.get_simple_type_name h)+").") + ## PRIVATE A type representing a header. Arguments: @@ -45,7 +62,8 @@ type Header example_new = Header.new "My_Header" "my header's value" @value make_text_secret_selector new : Text -> Text | Enso_Secret | Derived_Secret_Value -> Header - new name:Text value:(Text | Enso_Secret | Derived_Secret_Value) = Header.Value name value + new name:Text value:(Text | Enso_Secret | Derived_Secret_Value) = + Header.Value name value ## ICON text_input Create an "Accept" header. diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Extensions/Excel_Extensions.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Extensions/Excel_Extensions.enso new file mode 100644 index 000000000000..15246e3ab044 --- /dev/null +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Extensions/Excel_Extensions.enso @@ -0,0 +1,23 @@ +from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument + +polyglot java import org.enso.table.excel.ExcelUtils + +## GROUP Standard.Base.Conversions + ICON date_and_time + Converts an Excel date to a `Date`. +Date.from_excel : Integer -> Date +Date.from_excel excel_date:Integer = case excel_date of + 60 -> Error.throw (Illegal_Argument.Error "29th February 1900 does not exist.") + 0 -> Error.throw (Illegal_Argument.Error "0 is not a valid Excel date.") + _ -> ExcelUtils.fromExcelDateTime excel_date + +## GROUP Standard.Base.Conversions + ICON date_and_time + Converts an Excel date time to a `Date_Time`. +Date_Time.from_excel : Number -> Date_Time +Date_Time.from_excel excel_date:Number = + if excel_date >= 60 && excel_date < 61 then Error.throw (Illegal_Argument.Error "29th February 1900 does not exist.") else + if excel_date >= 0 && excel_date < 1 then Error.throw (Illegal_Argument.Error "0 is not a valid Excel date.") else + raw_date = ExcelUtils.fromExcelDateTime excel_date + if raw_date.is_a Date then raw_date.to_date_time else raw_date diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso index 6a93b20fb2f0..ab01be628e6e 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso @@ -25,6 +25,7 @@ export project.Excel.Excel_Workbook.Excel_Workbook export project.Expression.expr export project.Extensions.Column_Vector_Extensions.to_column +export project.Extensions.Excel_Extensions.from_excel export project.Extensions.Table_Conversions.from_objects export project.Extensions.Table_Conversions.parse_to_table export project.Extensions.Table_Conversions.to_table diff --git a/std-bits/aws/src/main/java/org/enso/aws/ClientBuilder.java b/std-bits/aws/src/main/java/org/enso/aws/ClientBuilder.java index cda1edf0d54b..f7cba219cbc6 100644 --- a/std-bits/aws/src/main/java/org/enso/aws/ClientBuilder.java +++ b/std-bits/aws/src/main/java/org/enso/aws/ClientBuilder.java @@ -1,6 +1,7 @@ package org.enso.aws; import java.net.URI; +import java.net.http.HttpClient; import java.util.function.Supplier; import org.enso.base.enso_cloud.ExternalLibrarySecretHelper; import org.enso.base.enso_cloud.HideableValue; @@ -59,6 +60,19 @@ public S3Client buildS3Client() { .build(); } + /** + * Builds an HttpClient that will sign requests and payloads using the AWSv4 Signature algorithm. + */ + public HttpClient createSignedClient( + String regionName, String serviceName, HttpClient baseClient, String bodySHA256) { + return new SignedHttpClient( + regionName, serviceName, unsafeBuildCredentialProvider(), baseClient, bodySHA256); + } + + public static String getSHA256(byte[] rawData) { + return SignedHttpClient.getSHA256(rawData); + } + /** * Instantiates an S3Client configured in such a way that it can query buckets regardless of their * region. diff --git a/std-bits/aws/src/main/java/org/enso/aws/SignedHttpClient.java b/std-bits/aws/src/main/java/org/enso/aws/SignedHttpClient.java new file mode 100644 index 000000000000..df3c59a759a4 --- /dev/null +++ b/std-bits/aws/src/main/java/org/enso/aws/SignedHttpClient.java @@ -0,0 +1,280 @@ +package org.enso.aws; + +import java.io.IOException; +import java.net.Authenticator; +import java.net.CookieHandler; +import java.net.ProxySelector; +import java.net.URL; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; +import javax.crypto.Mac; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +/** + * Wraps an HttpClient to sign requests with AWS signature v4. Designed to be called by + * EnsoSecretHelper.makeRequest. + */ +class SignedHttpClient extends HttpClient { + private static final String SCHEME = "AWS4"; + private static final String ALGORITHM = "HMAC-SHA256"; + private static final String TERMINATOR = "aws4_request"; + + private final String regionName; + private final String serviceName; + private final AwsCredentialsProvider credentialsProvider; + private final HttpClient parent; + private final String bodyHash; + + SignedHttpClient( + String regionName, + String serviceName, + AwsCredentialsProvider credentialsProvider, + HttpClient parent, + String bodyHash) { + this.regionName = regionName; + this.serviceName = serviceName; + this.credentialsProvider = credentialsProvider; + this.parent = parent; + this.bodyHash = bodyHash; + } + + @Override + public Optional cookieHandler() { + return parent.cookieHandler(); + } + + @Override + public Optional connectTimeout() { + return parent.connectTimeout(); + } + + @Override + public Redirect followRedirects() { + return parent.followRedirects(); + } + + @Override + public Optional proxy() { + return parent.proxy(); + } + + @Override + public SSLContext sslContext() { + return parent.sslContext(); + } + + @Override + public SSLParameters sslParameters() { + return parent.sslParameters(); + } + + @Override + public Optional authenticator() { + return parent.authenticator(); + } + + @Override + public Version version() { + return parent.version(); + } + + @Override + public Optional executor() { + return parent.executor(); + } + + @Override + public CompletableFuture> sendAsync( + HttpRequest request, HttpResponse.BodyHandler responseBodyHandler) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public CompletableFuture> sendAsync( + HttpRequest request, + HttpResponse.BodyHandler responseBodyHandler, + HttpResponse.PushPromiseHandler pushPromiseHandler) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public HttpResponse send( + HttpRequest request, HttpResponse.BodyHandler responseBodyHandler) + throws IOException, InterruptedException { + URL url = request.uri().toURL(); + + var headerMap = request.headers().map(); + var output = new HashMap(); + + var bodyPublisher = request.bodyPublisher().orElse(HttpRequest.BodyPublishers.noBody()); + long bodyLength = bodyPublisher.contentLength(); + output.put("content-length", bodyLength == 0 ? "" : Long.toString(bodyLength)); + + output.put("x-amz-content-sha256", bodyHash); + + output.put( + "x-amz-date", + ZonedDateTime.now(ZoneId.of("UTC")) + .format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'"))); + + int port = url.getPort(); + String hostHeader = url.getHost() + (port == -1 ? "" : ":" + port); + output.put("Host", hostHeader); + + // Create canonical headers + var sortedHeaders = new ArrayList(); + sortedHeaders.addAll(headerMap.keySet()); + sortedHeaders.addAll(output.keySet()); + sortedHeaders.sort(String.CASE_INSENSITIVE_ORDER); + var canonicalHeaderNames = + sortedHeaders.stream().map(String::toLowerCase).collect(Collectors.joining(";")); + var canonicalHeaders = + sortedHeaders.stream() + .map( + k -> + k.toLowerCase().replaceAll("\\s+", " ") + + ":" + + output.getOrDefault( + k, headerMap.containsKey(k) ? headerMap.get(k).get(0) : null)) + .collect(Collectors.joining("\n")); + + // Create canonical query string. + var queryParameters = ""; + if (url.getQuery() != null) { + var parameters = Arrays.stream(url.getQuery().split("&")).map(p -> p.split("=", 2)); + queryParameters = + parameters + .sorted(Comparator.comparing(l -> l[0])) + .map(p -> urlEncode(p[0]) + "=" + urlEncode(p[1])) + .collect(Collectors.joining("&")); + } + + // Create canonical request + var canonicalPath = url.getPath(); + if (!canonicalPath.startsWith("/")) { + canonicalPath = "/" + canonicalPath; + } + canonicalPath = urlEncode(canonicalPath).replace("%2F", "/"); + var canonicalRequest = + String.join( + "\n", + request.method(), + canonicalPath, + queryParameters, + canonicalHeaders, + "", + canonicalHeaderNames, + bodyHash); + var canonicalRequestHash = getSHA256(canonicalRequest.getBytes(StandardCharsets.UTF_8)); + + // Need the credentials + var credentials = credentialsProvider.resolveCredentials(); + + // Create signing string + String dateStamp = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE); + String scope = dateStamp + "/" + regionName + "/" + serviceName + "/" + TERMINATOR; + String toSign = + String.join( + "\n", SCHEME + "-" + ALGORITHM, output.get("x-amz-date"), scope, canonicalRequestHash); + var signature = + sign( + SCHEME + credentials.secretAccessKey(), + dateStamp, + regionName, + serviceName, + TERMINATOR, + toSign); + + // Build the authorization header + var authorizationHeader = + SCHEME + + "-" + + ALGORITHM + + " " + + "Credential=" + + credentials.accessKeyId() + + "/" + + scope + + ", " + + "SignedHeaders=" + + canonicalHeaderNames + + ", " + + "Signature=" + + signature; + output.put("Authorization", authorizationHeader); + + // Build a new request with the additional headers + output.remove("Host"); + output.remove("content-length"); + var newBuilder = HttpRequest.newBuilder(request, (n, v) -> !output.containsKey(n)); + output.keySet().forEach(n -> newBuilder.header(n, output.get(n))); + + // Send the request + return parent.send(newBuilder.build(), responseBodyHandler); + } + + private static String sign(String init, String... values) { + try { + var mac = Mac.getInstance("HmacSHA256"); + byte[] key = init.getBytes(StandardCharsets.UTF_8); + try { + for (String value : values) { + mac.init(new javax.crypto.spec.SecretKeySpec(key, ALGORITHM)); + key = mac.doFinal(value.getBytes(StandardCharsets.UTF_8)); + } + return bytesToHex(key); + } catch (java.security.InvalidKeyException e) { + throw new RuntimeException("Failed to sign the request.", e); + } + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Failed to get HMAC-SHA-256 algorithm.", e); + } + } + + /** + * Returns the SHA-256 hash of the given data. + * + * @param rawData the data to hash + * @return the SHA-256 hash of the data + */ + static String getSHA256(byte[] rawData) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(rawData); + return bytesToHex(hash); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Failed to get SHA-256 algorithm.", e); + } + } + + private static String bytesToHex(byte[] hash) { + StringBuilder hexString = new StringBuilder(2 * hash.length); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } + + private static String urlEncode(String input) { + return URLEncoder.encode(input, StandardCharsets.UTF_8); + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/net/http/MultipartBodyBuilder.java b/std-bits/base/src/main/java/org/enso/base/net/http/MultipartBodyBuilder.java index e7e54e74ebd5..4ac1074e9e8f 100644 --- a/std-bits/base/src/main/java/org/enso/base/net/http/MultipartBodyBuilder.java +++ b/std-bits/base/src/main/java/org/enso/base/net/http/MultipartBodyBuilder.java @@ -23,13 +23,39 @@ public class MultipartBodyBuilder { * @return the body publisher. */ public HttpRequest.BodyPublisher build() { - if (partsSpecificationList.size() == 0) { + if (partsSpecificationList.isEmpty()) { throw new IllegalStateException("Must have at least one part to build multipart message."); } addFinalBoundaryPart(); return HttpRequest.BodyPublishers.ofByteArrays(PartsIterator::new); } + /** + * Get the content of the multipart form. The form needs to be built before this. + * + * @return the content of the multipart form. + */ + public byte[] getContents() { + if (partsSpecificationList.isEmpty() + || partsSpecificationList.get(partsSpecificationList.size() - 1).type + != PartsSpecification.TYPE.FINAL_BOUNDARY) { + throw new IllegalStateException( + "Must have built the MultipartBodyBuilder, before calling getContents."); + } + + var iterator = new PartsIterator(); + byte[] output = new byte[0]; + while (iterator.hasNext()) { + byte[] part = iterator.next(); + byte[] newOutput = new byte[output.length + part.length]; + System.arraycopy(output, 0, newOutput, 0, output.length); + System.arraycopy(part, 0, newOutput, output.length, part.length); + output = newOutput; + } + + return output; + } + /** * Get the multipart boundary separator. * diff --git a/std-bits/base/src/main/java/org/enso/base/net/http/UrlencodedBodyBuilder.java b/std-bits/base/src/main/java/org/enso/base/net/http/UrlencodedBodyBuilder.java index 4865471c1d10..4244ecb2723a 100644 --- a/std-bits/base/src/main/java/org/enso/base/net/http/UrlencodedBodyBuilder.java +++ b/std-bits/base/src/main/java/org/enso/base/net/http/UrlencodedBodyBuilder.java @@ -12,6 +12,7 @@ public final class UrlencodedBodyBuilder { private final ArrayList parts = new ArrayList<>(); + private String contents = null; /** * Create HTTP body publisher for an url-encoded form data. @@ -19,10 +20,24 @@ public final class UrlencodedBodyBuilder { * @return the body publisher. */ public HttpRequest.BodyPublisher build() { - String contents = String.join("&", parts); + contents = String.join("&", parts); return HttpRequest.BodyPublishers.ofString(contents); } + /** + * Get the contents of the form data. + * + * @return the contents. + */ + public String getContents() { + if (contents == null) { + throw new IllegalStateException( + "Must have built the UrlencodedBodyBuilder, before calling getContents."); + } + + return contents; + } + /** * Add text field to the form. * diff --git a/std-bits/table/src/main/java/org/enso/table/excel/ExcelRow.java b/std-bits/table/src/main/java/org/enso/table/excel/ExcelRow.java index e99d62596b8d..5e2b25fe3203 100644 --- a/std-bits/table/src/main/java/org/enso/table/excel/ExcelRow.java +++ b/std-bits/table/src/main/java/org/enso/table/excel/ExcelRow.java @@ -1,8 +1,14 @@ package org.enso.table.excel; -import java.time.LocalDateTime; +import java.time.LocalDate; import java.time.ZoneId; -import org.apache.poi.ss.usermodel.*; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.usermodel.ExcelNumberFormat; +import org.apache.poi.ss.usermodel.FormulaError; +import org.apache.poi.ss.usermodel.Row; import org.graalvm.polyglot.Context; /** Wrapper class to handle Excel rows. */ @@ -37,20 +43,21 @@ public Object getCellValue(int column) { switch (cellType) { case NUMERIC: double dblValue = cell.getNumericCellValue(); - if (DateUtil.isCellDateFormatted(cell) && DateUtil.isValidExcelDate(dblValue)) { - var dateTime = DateUtil.getLocalDateTime(dblValue); - if (dateTime.isBefore(LocalDateTime.of(1900, 1, 2, 0, 0))) { - // Excel stores times as if they are on the 1st January 1900. - // Due to the 1900 leap year bug might be 31st December 1899. - return dateTime.toLocalTime(); + var nf = ExcelNumberFormat.from(cell, null); + if (nf != null && DateUtil.isADateFormat(nf.getIdx(), nf.getFormat())) { + var temporal = ExcelUtils.fromExcelDateTime(dblValue); + if (temporal == null) { + return null; } - if (dateTime.getHour() == 0 && dateTime.getMinute() == 0 && dateTime.getSecond() == 0) { - var dateFormat = cell.getCellStyle().getDataFormatString(); - if (!dateFormat.contains("h") && !dateFormat.contains("H")) { - return dateTime.toLocalDate(); + return switch (temporal) { + case LocalDate date -> { + var dateFormat = cell.getCellStyle().getDataFormatString(); + yield (dateFormat.contains("h") || dateFormat.contains("H")) + ? date.atStartOfDay(ZoneId.systemDefault()) + : date; } - } - return dateTime.atZone(ZoneId.systemDefault()); + default -> temporal; + }; } else { if (dblValue == (long) dblValue) { return (long) dblValue; diff --git a/std-bits/table/src/main/java/org/enso/table/excel/ExcelUtils.java b/std-bits/table/src/main/java/org/enso/table/excel/ExcelUtils.java new file mode 100644 index 000000000000..6b1a432982f2 --- /dev/null +++ b/std-bits/table/src/main/java/org/enso/table/excel/ExcelUtils.java @@ -0,0 +1,86 @@ +package org.enso.table.excel; + +import java.time.*; +import java.time.temporal.ChronoUnit; +import java.time.temporal.Temporal; + +public class ExcelUtils { + // The epoch for Excel date-time values. Due to 1900-02-29 being a valid date + // in Excel, it is actually 1899-12-30. Excel dates are counted from 1 being + // 1900-01-01. + private static final LocalDate EPOCH_1900 = LocalDate.of(1899, 12, 30); + private static final long MILLIS_PER_DAY = 24 * 60 * 60 * 1000L; + + /** Converts an Excel date-time value to a {@link Temporal}. */ + public static Temporal fromExcelDateTime(double value) { + // Excel treats 1900-02-29 as a valid date, which it is not a valid date. + if (value >= 60 && value < 61) { + return null; + } + + // For days before 1900-01-01, Stored as milliseconds before 1900-01-01. + long days = (long) value; + + // Extract the milliseconds part of the value. + long millis = (long) ((value - days) * MILLIS_PER_DAY + (value < 0 ? -0.5 : 0.5)); + if (millis < 0) { + millis += MILLIS_PER_DAY; + } + if (millis != 0 && value < 0) { + days--; + } + + // Excel stores times as 0 to 1. + if (days == 0) { + return LocalTime.ofNanoOfDay(millis * 1000000); + } + + int shift = 0; + if (days > 0 && days < 60) { + // Due to a bug in Excel, 1900-02-29 is treated as a valid date. + // So within the first two months of 1900, the epoch needs to be 1 day later. + shift = 1; + } else if (days < 0) { + // For days before 1900-01-01, Excel has no representation. + // 0 is 1900-01-00 in Excel. + // We make -1 as 1899-12-31, -2 as 1899-12-30, etc. + // This needs the shift to be 2 days later. + shift = 2; + } + LocalDate date = EPOCH_1900.plusDays(days + shift); + + return millis < 1000 + ? date + : date.atTime(LocalTime.ofNanoOfDay(millis * 1000000)).atZone(ZoneId.systemDefault()); + } + + /** Converts a {@link Temporal} to an Excel date-time value. */ + public static double toExcelDateTime(Temporal temporal) { + return switch (temporal) { + case ZonedDateTime zonedDateTime -> toExcelDateTime(zonedDateTime.toLocalDateTime()); + case LocalDateTime dateTime -> toExcelDateTime(dateTime.toLocalDate()) + + toExcelDateTime(dateTime.toLocalTime()); + case LocalDate date -> { + long days = ChronoUnit.DAYS.between(EPOCH_1900, date); + + if (date.getYear() == 1900 && date.getMonthValue() < 3) { + // Due to a bug in Excel, 1900-02-29 is treated as a valid date. + // So within the first two months of 1900, the epoch needs to be 1 day later. + days--; + } + if (date.getYear() < 1900) { + // For days before 1900-01-01, Excel has no representation. + // 0 is 1900-01-00 in Excel. + // We make -1 as 1899-12-31, -2 as 1899-12-30, etc. + // This means the epoch needs to be 2 days later. + days -= 2; + } + + yield days; + } + case LocalTime time -> time.toNanoOfDay() / 1000000.0 / MILLIS_PER_DAY; + default -> throw new IllegalArgumentException( + "Unsupported Temporal type: " + temporal.getClass()); + }; + } +} diff --git a/std-bits/table/src/main/java/org/enso/table/write/ExcelWriter.java b/std-bits/table/src/main/java/org/enso/table/write/ExcelWriter.java index 29e7dea7faab..93787612165a 100644 --- a/std-bits/table/src/main/java/org/enso/table/write/ExcelWriter.java +++ b/std-bits/table/src/main/java/org/enso/table/write/ExcelWriter.java @@ -1,8 +1,8 @@ package org.enso.table.write; import java.time.LocalDate; -import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.function.Function; import org.apache.poi.ss.usermodel.Cell; @@ -14,7 +14,6 @@ import org.apache.poi.ss.usermodel.Workbook; import org.enso.table.data.column.storage.BoolStorage; import org.enso.table.data.column.storage.Storage; -import org.enso.table.data.column.storage.datetime.DateTimeStorage; import org.enso.table.data.column.storage.numeric.AbstractLongStorage; import org.enso.table.data.column.storage.numeric.DoubleStorage; import org.enso.table.data.table.Column; @@ -24,16 +23,11 @@ import org.enso.table.error.ExistingDataException; import org.enso.table.error.InvalidLocationException; import org.enso.table.error.RangeExceededException; -import org.enso.table.excel.ExcelHeaders; -import org.enso.table.excel.ExcelRange; -import org.enso.table.excel.ExcelRow; -import org.enso.table.excel.ExcelSheet; +import org.enso.table.excel.*; import org.enso.table.util.ColumnMapper; import org.enso.table.util.NameDeduplicator; public class ExcelWriter { - private static final double SECONDS_IN_A_DAY = 86400.0; - private static Function ensoToTextCallback; public static Function getEnsoToTextCallback() { @@ -499,9 +493,6 @@ private static void writeValueToCell(Cell cell, int j, Storage storage, Workb cell.setCellValue(longStorage.getItem(j)); } else if (storage instanceof BoolStorage boolStorage) { cell.setCellValue(boolStorage.getItem(j)); - } else if (storage instanceof DateTimeStorage dateTimeStorage) { - cell.setCellValue(dateTimeStorage.getItem(j).toLocalDateTime()); - cell.setCellStyle(getDateTimeStyle(workbook, "yyyy-MM-dd HH:mm:ss")); } else { Object value = storage.getItemBoxed(j); switch (value) { @@ -509,16 +500,16 @@ private static void writeValueToCell(Cell cell, int j, Storage storage, Workb case Boolean b -> cell.setCellValue(b); case Double d -> cell.setCellValue(d); case Long l -> cell.setCellValue(l); - case LocalDateTime ldt -> { - cell.setCellValue(ldt); + case ZonedDateTime zdt -> { + cell.setCellValue(ExcelUtils.toExcelDateTime(zdt)); cell.setCellStyle(getDateTimeStyle(workbook, "yyyy-MM-dd HH:mm:ss")); } case LocalDate ld -> { - cell.setCellValue(ld); + cell.setCellValue(ExcelUtils.toExcelDateTime(ld)); cell.setCellStyle(getDateTimeStyle(workbook, "yyyy-MM-dd")); } case LocalTime lt -> { - cell.setCellValue(lt.toSecondOfDay() / SECONDS_IN_A_DAY); + cell.setCellValue(ExcelUtils.toExcelDateTime(lt)); cell.setCellStyle(getDateTimeStyle(workbook, "HH:mm:ss")); } default -> { diff --git a/test/Base_Tests/src/Network/Http_Spec.enso b/test/Base_Tests/src/Network/Http_Spec.enso index c3a03d5a8c15..bcf256f71fd8 100644 --- a/test/Base_Tests/src/Network/Http_Spec.enso +++ b/test/Base_Tests/src/Network/Http_Spec.enso @@ -10,7 +10,7 @@ import Standard.Base.Network.HTTP.Request_Body.Request_Body import Standard.Base.Network.HTTP.Request_Error import Standard.Base.Network.Proxy.Proxy import Standard.Base.Runtime.Context -from Standard.Base.Network.HTTP import resolve_headers +from Standard.Base.Network.HTTP import _resolve_headers, get_follow_redirects, get_proxy, get_timeout, get_version from Standard.Test import all from Standard.Test.Execution_Context_Helpers import run_with_and_without_output @@ -65,11 +65,11 @@ add_specs suite_builder = suite_builder.group "HTTP client" pending=pending_has_url group_builder-> group_builder.specify "should create HTTP client with timeout setting" <| http = HTTP.new (timeout = (Duration.new seconds=30)) - http.timeout.should_equal (Duration.new seconds=30) + (get_timeout http).should_equal (Duration.new seconds=30) group_builder.specify "should create HTTP client with follow_redirects setting" <| http = HTTP.new (follow_redirects = False) - http.follow_redirects.should_equal False + (get_follow_redirects http).should_equal False Test.with_retries <| r = http.request (Request.new HTTP_Method.Get base_url_with_slash+"test_redirect") @@ -80,12 +80,12 @@ add_specs suite_builder = group_builder.specify "should create HTTP client with proxy setting" <| proxy_setting = Proxy.Address "example.com" 80 http = HTTP.new (proxy = proxy_setting) - http.proxy.should_equal proxy_setting + (get_proxy http).should_equal proxy_setting group_builder.specify "should create HTTP client with version setting" <| version_setting = HTTP_Version.HTTP_2 http = HTTP.new (version = version_setting) - http.version.should_equal version_setting + (get_version http).should_equal version_setting url_get = base_url_with_slash.if_not_nothing <| base_url_with_slash + "get" suite_builder.group "fetch" pending=pending_has_url group_builder-> @@ -559,30 +559,30 @@ add_specs suite_builder = suite_builder.group "Header resolution" group_builder-> group_builder.specify "Default content type and encoding" <| expected = [Header.content_type "text/plain; charset=UTF-8"] - resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "")) . should_equal_ignoring_order expected + _resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "")) . should_equal_ignoring_order expected group_builder.specify "Content type specified in body" <| expected = [Header.content_type "application/json; charset=UTF-8"] - resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" content_type="application/json")) . should_equal_ignoring_order expected + _resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" content_type="application/json")) . should_equal_ignoring_order expected group_builder.specify "Content type specified in header list" <| expected = [Header.content_type "application/json"] - resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "")) . should_equal_ignoring_order expected + _resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "")) . should_equal_ignoring_order expected group_builder.specify "Text encoding specified in body" <| expected = [Header.content_type "text/plain; charset=UTF-16LE"] - resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" encoding=Encoding.utf_16_le)) . should_equal_ignoring_order expected + _resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" encoding=Encoding.utf_16_le)) . should_equal_ignoring_order expected group_builder.specify "Can't specify content type in both places" <| - resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "" content_type="text/plain")) . should_fail_with Illegal_Argument + _resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "" content_type="text/plain")) . should_fail_with Illegal_Argument group_builder.specify "Custom header" <| expected = [Header.new "some" "header", Header.content_type "application/json; charset=UTF-8"] - resolve_headers (Request.new HTTP_Method.Get "" [Header.new "some" "header"] (Request_Body.Text "" content_type="application/json")) . should_equal_ignoring_order expected + _resolve_headers (Request.new HTTP_Method.Get "" [Header.new "some" "header"] (Request_Body.Text "" content_type="application/json")) . should_equal_ignoring_order expected group_builder.specify "Multiple content types in header list are ok" <| expected = [Header.content_type "application/json", Header.content_type "text/plain"] - resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "")) . should_equal_ignoring_order expected + _resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "")) . should_equal_ignoring_order expected suite_builder.group "Http Error handling" group_builder-> group_builder.specify "should be able to handle request errors" <| diff --git a/test/Table_Tests/data/OlderDates.xlsx b/test/Table_Tests/data/OlderDates.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..c73b322331a3f6b03191160b97a493e7b8cf856a GIT binary patch literal 9123 zcmeHtgEiprPHv=QxAwzddOCupI-HJ4*l(e*zbV@oPsnmD$zW3+7 z-23|r-gkfI>^T|02P1^003wKx$KTm2Lu2B9|Zs)0-z&8 zq@hmkR!;6FIzG-;ZbqEmPaJ6qkP%t(0f_MH|JVMDpFq{~PPJ}s>(&-hWmQsp#hT=ox9!cIOm#*UZ#a-Yjifm~js>IY zi4-4HSi*+BT;*Cl2-ocr!eX39dNX(O!~V^JUq|d4jO&oArThtS%nox_^Ej)-vmy7t$cnHS z>Dplo4ya1sV;Muk$8-pu&7iB6rLVu<5?jv;O)9klax3_~XISVa*a<>X9~~k{R!N}k zeTP8Ny02J2+CqZx34+Z}`V{Hv)7RC+u6`d?4qj&hwbn0J<QeXx5tE6Y0+$0xCAVut(thnM8k9@GMVm=3xO&~*A^|l1#?b~{Zu%4Wj0&8BSa6P- zxLP^7adH0a|HsY$VnY7q)~k|L)&Jn;Rb>D9)M6sOl(LtkLL044V6fr>UPD|V1NmY* zGX=g5*;6F>pm%{+gY%2PxZQr*vo-#zXN1I}^o?HC5$Qi%JkVH~T+`%Ss#ZS`cut*8 zoxPA(@?r6Om%v)tT=r6VV3}TS@<^r@f0R?741=hcB7y|;BE+y))nLi|q6T3~O8cNX z;&s!b{N0q%te}~+k{ulJC}EZTi5K_!-OO#~YW({hX-~hC>1x{w+r2i=brqrWF|l;) zIF`w3C%X3MP|NPuqT?sL;2o9kW6VDdGOXu68p`qc@Cc*3vj1=(^ci*865L(=lO)IH zTb%SraDu@78Xo+~csp`=L0uipp-_jPPOM1x2{elv|3=L27U8>S50eCu4(}#TwU(%^ z-L#Wd(I84G80DkmWKrhXvR6bR=PPn1df+261Umsk3|b$k z%(2(PV}2myD0{JfqQO=-;AMEZPJC51D@eJ;+Ad7ZjAhypDm6XTyd?}?6mvoAC;htN z&u9i{px(?JNVXRwC`K)RQr@nwhjYe2L}TtS68PycuaGDWEjpTEX4r|bk!FrT!$*34 zjC40WjE3w~C3c^|G%`8^&R7;0$zes>!zuUgkt91W4{xR8_<07LX%ILD&aORpgDbH` zdyMStIxoMUAifDy&C5sC*m3uh8KoU4^3rGGW$+TeoPEY~_Sla&)?&Up7kd7#v;4r< zn>!v>~y03tbM?jLx>Y?uieMdl;~6Gh`>(Nd&9YYU8I2XU?09ie+DpIyZI-o7 zn5^FH0q}1{sHZMX}5l`@8mkA<|w^P$`&#Wtl+sZp}Bfb4uzP)PLhsGTQTf zWH{5r;hZ7^pd-Nf^oQU1E35wWK?v~p5Pt1{cdJSsvj4!1Eqfk*^K|^Z3vYl%gp2jK zR>qU8z+mU+LMrBHkbn)CgORo9dfyopf(8u3ZuNUad@=S?&*FB3*?9oj1F`9Y%B<4K!XB)o@Xy(lsv@0s)t zA#I782-6;UPMkhkx{9MUDT*7(ZpOP)1SeO$>4Lb z93H9zjwMpDN$t$ez*I#HC-@H}B&o;`&uog1p$%TVa5uVD&2X3FaFm_xz+%13DB>5d zdxPcpLlW!~5x)7xifcFKN;Q54dD-qk@{n`pr98>;OLgM|b0=S}@CbeK7UR_j%J-rC zZ84^mR5(!c#UG~m&xApJQ-}}-c4f<#<^AdPk4t2oNr8bFvn^iBNj8-*nh-yU)Q8bh zPhZ8(A@|+qXOx^=!B33!s|{yPBpeWC{m#cUz(pptqBwf#INF`m;c9OS=DJQvYL`8L ze9Y|W;crvtL#AS0{`i$VcC*;WKa%LJ*HT#>fxck9wE=b7icl_w4jDaQZ$r*JTu4qS ztUC@gd}i^);=KWIs8qU>$qd68y#I=J5x^SiWr8!j@ zCJ6=H#69hpJA2ksY2xuRiuu6(8E)K~V{Sfjm&HYBJ4X}EyMEdipP1=Ek!SLL!ZQJW zcA1cXx0KJWPmk5zLFf+)&Ss7Z1(v_b$OKur0ykgX?v{WAsGFY7R+($=t9#no^?70* z`#A)@yPtLa!+3v&`B6aufs#x&`cA@_Hr{4JnVk&qTT~>01K*z6?v~(sxyeV6RqN*x zI+1H9XdX;Fvng*eK|Q^A!pvxTn`U1G)^i@l5zWc8wblZe+Vp;6EbIPc1WmL2ZSo7v zc2bQ3mSL@tQ5F6dv!ZIM^@AQzI$6zfD6cUaAI_T?eZ?Py31Sb|WtZl(VTPL?uxgo? zkr;^j3AH}hW)C3r=0HY`UZ01L!p4E&6ZjAGMsrc;RB};4!5)1#wUAb$N=05XD5F-Q z3?G%?i0MvYH{>EQx^I-EQnTgOceq|IV0a^cn+&l>I0?}uxspF= zZo7Mk#(+ktqsyWmr>CE671XN+k;RmW&r{c;akY=8Frabm=%PO-jL8=0+5o4%QV8ik zI!Ann-{Sq}Toab;2$`8Ha3v8j$pTXqQ-;%Q!&{La4cTd_bA7ref(E794GtYB&qjV} zL3qFrWz^a+VnE}4OdK@mI z%2PAuR1Jy|Z#5UPdaf2ZQ5@v{r@fXYoJ~Y?ZzJ)z!J@R1;^RmgE()9RKyJ* z3q;OvUevjNe}aeE2Zk&LL)~>iq%=nn)EAjx96{h(_3JQvfe7-ft|siq6u-(_p6-@c3nBF-A-T5bjwW}CEcNM z1vyiuZ~0X$k+;cO#4??g3i7oy=POO7h22Nqf~*Q70SPIH3r>7WsVC}WP3hSFZj24I z=H`_-6JmnzG5DooPV9%nyyRyGsnX=ce{8ljtoCNxR3@! z(?u1upPx8WIhC9)k@N%40!(sMnN!9(AvesRk*wq6)7jGmn)RU1-RdPa$IfdP@`;_1 z0k>ImJwnch=)4yOc;Y!ZCRH}LRFb4a-&OG_k=lMSPB%va;8@CRE@&s*Q&m8I=fmXM#ZUgG$a0oyfL4^-nc}{1P^pnf-)o!uED=V{ znzstczyj}y0wh5q|tVZ zIcLwm>vw~irk~Xm;ev8vw)oxo^GTdG7rHz zW1`8W@8!Xm=g|rnZYw4Q?5T;nq5U5jC}jJ}B$L=!fob>IOd^*(k=Dy-`#FS=B`;oY zkkGJE$59Y3a`37Wl09tOPeUoCMU+^L)5E@bHe`Z3%{ByKITw5ed&lO<7D#AYd&n|| z5$jfN`YFS)x8-$M)n0o}r_RvVjyO%2EI2jEw_w)XX@#YLgVLW*a{c*JWRZu?%X^aE zsk;c5yXBWBWz*x}g1KQ8dEJvw6eN@90 z7XgnEX>C_LPs_Fm81D`k#Q5du{TSg=sc2>rrZ z%G%Ln@UQKfaO@cyKfJ!T{&Am@M6MvW-AK83Am(XB0FYK-cU^_KYzB?i=?qIabjr}9 zLLfbuyrR)c;96_ZKnP1QKniiAl(xt20i#Xh`HyIY6pcwV>z!^ZPOk#O66uD|irOoc z>%`yKpyVq;S*9BXEt)tNy^J_yUy?zH%EJNWenW@|_lz@~3xuoNK9XS=w~K1^lT={;E%{FM-tq7}xJPbS_vEnc*wQp5Q+`XNbAG zI+RTjVOz+5iEaMmOelh)Dd4i1D><*hx(uBA8Ybu0PQ+Hh=5-CG4w$!a*{Yc1N?Wy` za~pEj#B>qHyr~-#N9B2gQr!WoRu)J0ig)SW(C?z+&boQ7Py;6OYilJIRSIFS zw^jCBSfIX-<-dC?N#01*vG95i$xowKlKA3Nkj*h@#1S~Y^mZ>_Mf#a%=Dmx&`JT$T zZ;eW!cJ&#kDQuajb?&v;69dEMEmhWK+^$fmg62|@{%6+dO8{CCrvYriX~Hf2Y9pCohDf+cXGv!81chh_LqtE zzPmeR7zyT)=3azx0}g$o6-T_oe%!uwC^Ts$1O8_s^Kg=h9AWLVWX{?tCgj; zyQ`g(%`Z|q#t&h3apTLLhu-2_zbeO*VTi`+DQ5Ga{ydJFsd%6wd%F}BMZ-DTLeXp; zkPz2u)<>HZKF|yZ*C}Bg>*+drP0k@%>s{fWg3bXSkwc`aoiz9fhJaDf*N_;CGJr?)FKmb$$+ zB;fPhc}isqxLUZbCP(d^=*-pi!KqW<3zJyU7iO-FTg%hO>#Jwh%b72ya&{=OP`@}V zaSUx1SNPA*jbBW;*K+!Q&)6DgU-b~vzu|?reamSW64J>!^@YSqcHmv64GW~3cNSpezwL6LwyGNwCHVqjl3>`Id{mA*~WST{(+R96I8h1JpB>z05!E z$poe)@K@&V+(QvTd_(EOE-O~r%32gM7BG@^Z@7k@X73n`xnh}UzQPX(2dP%Sq-N@A zmef+q?`oqhdRg~FdsmU z!D8AD|M?cH)Uk!5p?yP!u0Jw+s5$1-G3CNieVz@ty`xq%8VsIV$C)Xe1 zo%RO6XKvtiC;Y!^n3=Ql{}c?aS^sI-$wTnMDBPSYTuE<$JTTE;4)RC=KOo8!lYq%> zCcw>}8>9tkuWXW|k`dWFkDFV?-Z!?+Rs~E;N!^#GElGK7iSNj61irUs=i}$j$;&{b zv~CeB<%rM|E(X*%CMH+d0Pr=UgQGIbUMNO4Ml^I{yh4IjWK2gitFn<+4jMGap9SEA zY`Em+DbC`D5e^nw+pwVa87SWha1t~)DmfYwj@(ri_^{t1jTEetk5C++>Tjh4dM~#fMyL<-8k2{<+CPi5*UUm@?0Yxl`AEZy z0`e_qK=(-(<{;NFO!ja>E0)LK0@F4pywb#4AyA`P4&;fXc{Z2jVF8J^1_TNwdfD|6q|NoQk zcjeqox&D@9g#G{i#6J?Scct7-5&V{NOz^J>gS!Im?n!NixA z^bhFWaO$pvyD`dd3EX7AB>XL6xr_e0U;T{*09wfbfPZ+|yYRmo&A-AasQv=~j|tUK VMuF?b&$2!i;62>Tj?(-*`#(+&niBv3 literal 0 HcmV?d00001 diff --git a/test/Table_Tests/src/IO/Excel_Spec.enso b/test/Table_Tests/src/IO/Excel_Spec.enso index f8821df65900..63f66f552b89 100644 --- a/test/Table_Tests/src/IO/Excel_Spec.enso +++ b/test/Table_Tests/src/IO/Excel_Spec.enso @@ -9,6 +9,7 @@ import Standard.Base.Runtime.Managed_Resource.Managed_Resource import Standard.Base.Runtime.Ref.Ref from Standard.Table import Table, Match_Columns, Excel_Format, Excel_Range, Data_Formatter, Delimited_Format, Excel_Workbook, Value_Type +from Standard.Table.Extensions.Excel_Extensions import all from Standard.Table.Errors import Invalid_Column_Names, Duplicate_Output_Column_Names, Invalid_Location, Range_Exceeded, Existing_Data, Column_Count_Mismatch, Column_Name_Mismatch, Empty_Sheet @@ -1025,6 +1026,44 @@ add_specs suite_builder = table.at "InvMixDates" . to_vector . should_equal [(Date_Time.new 1997 7 25), (Date_Time.new 1993 5 3 10 58 45 millisecond=980), (Date_Time.new 2010 8 1 17 9 29 millisecond=923), (Date_Time.new 1988 7 12 12 39 20 millisecond=185), (Date_Time.new 2009 10 22)] table.at "Mixed" . to_vector . should_equal [(Date.new 1997 7 25), (Date_Time.new 1993 5 3 10 58 45 millisecond=980), (Date_Time.new 2010 8 1 17 9 29 millisecond=923), (Time_Of_Day.new 18 39 24 millisecond=572), 85] + group_builder.specify "should be able to read dates before 1900-01-01 (treating 1899-12-31 as -1)" <| + table = (enso_project.data / "OlderDates.xlsx") . read ..Sheet + table.row_count . should_equal 15 + table.column_names . should_equal ["Num","Date","Date13","NumVal"] + table.at "Date" . to_vector . should_equal [(Date.new 1899 12 28),(Date.new 1899 12 29),(Date.new 1899 12 30),(Date.new 1899 12 31),(Date.new 1900 1 1),(Date.new 1900 1 2),(Date.new 1900 1 3),(Date.new 1900 1 4),(Date.new 1900 1 5),(Date.new 1900 2 28),Nothing,(Date.new 1900 3 1),(Date.new 1900 3 2),(Date.new 1900 3 3),(Date.new 1900 3 4)] + table.at "Date13" . to_vector . should_equal [(Date_Time.new 1899 12 28 hour=13),(Date_Time.new 1899 12 29 hour=13),(Date_Time.new 1899 12 30 hour=13),(Date_Time.new 1899 12 31 hour=13),(Date_Time.new 1900 1 1 hour=13),(Date_Time.new 1900 1 2 hour=13),(Date_Time.new 1900 1 3 hour=13),(Date_Time.new 1900 1 4 hour=13),(Date_Time.new 1900 1 5 hour=13),(Date_Time.new 1900 2 28 hour=13),Nothing,(Date_Time.new 1900 3 1 hour=13),(Date_Time.new 1900 3 2 hour=13),(Date_Time.new 1900 3 3 hour=13),(Date_Time.new 1900 3 4 hour=13)] + + group_builder.specify "should be able to write and then read dates before 1900-01-01 (treating 1899-12-31 as -1)" <| + numbers = [-100,-1,1,2,50,61,100] + dates = numbers.map n-> (Date.new 1899 12 31).date_add n ..Day + date_times = dates.map d-> d.to_date_time (Time_Of_Day.new 12 34 56) + table = Table.new [["Num", numbers], ["Date", dates], ["DateTimes", date_times]] + file = enso_project.data / "transient" / "TestOlderDates.xlsx" + file.delete_if_exists . should_succeed + table.write file . should_succeed + read_table = file.read ..Sheet + read_table.should_equal table + + group_builder.specify "should be able to convert Excel dates from Integer to Date" <| + numbers = [-100,-1,1,2,50,61,100] + dates = [Date.new 1899 09 23, Date.new 1899 12 31, Date.new 1900 01 01, Date.new 1900 01 02, Date.new 1900 02 19, Date.new 1900 03 01, Date.new 1900 04 09] + parsed = numbers.map n-> Date.from_excel n + parsed . should_equal dates + + Date.from_excel 0 . should_fail_with Illegal_Argument + Date.from_excel 60 . should_fail_with Illegal_Argument + + group_builder.specify "should be able to convert Excel dates from Integer to Date" <| + numbers = [-100,-1,1,2,50,61,100] + dates = [Date.new 1899 09 23, Date.new 1899 12 31, Date.new 1900 01 01, Date.new 1900 01 02, Date.new 1900 02 19, Date.new 1900 03 01, Date.new 1900 04 09] + numbers_2 = numbers.map_with_index i->n-> n+(1+i*2)/24 + date_times = dates.map_with_index i->d-> (d.to_date_time (Time_Of_Day.new hour=1+i*2)) + parsed = numbers_2.map n-> Date_Time.from_excel n + parsed . should_equal date_times + + Date_Time.from_excel 0.25 . should_fail_with Illegal_Argument + Date_Time.from_excel 60.45 . should_fail_with Illegal_Argument + ci_pending = if Environment.get "CI" != Nothing then "This test takes a lot of time so it is disabled on CI." group_builder.specify "should be able to write and read a big XLSX file (>110MB)" pending=ci_pending <| n = 10^6