From a26952420db07cf693505626089e9ebf3a39d71a Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Fri, 14 Jun 2024 12:44:14 +0200 Subject: [PATCH 01/52] Fix: use `SOCK_CLOEXEC` with `FD_CLOEXEC` fallback (#14672) Harmonizes the implementations between Darwin and other POSIX platforms for the "close on exec" behavior. When `SOCK_CLOEXEC` is available, we always use it in the `socket`, `socketpair` and `accept4` syscalls. When `SOCK_CLOEXEC` isn't available, we don't delay to `Socket#initialize_handle` anymore to set `FD_CLOEXEC` for Darwin only, but immediately call `fcntl` to set it after the above syscalls. The `accept4` syscall is non-standard but widely available: Linux, all supported BSD, except for Darwin (obviously). The patch also fixes an issue where TCP and UNIX client sockets didn't have `FD_CLOEXEC` on POSIX platforms... except for Darwin. closes #14650 --- spec/std/socket/socket_spec.cr | 3 +++ spec/std/socket/tcp_server_spec.cr | 14 ++++++++++++++ spec/std/socket/unix_server_spec.cr | 14 ++++++++++++++ spec/std/socket/unix_socket_spec.cr | 3 +++ src/crystal/system/unix/event_loop_libevent.cr | 11 ++++++++++- src/crystal/system/unix/socket.cr | 17 +++++++++-------- src/lib_c/aarch64-android/c/sys/socket.cr | 1 + src/lib_c/aarch64-linux-gnu/c/sys/socket.cr | 1 + src/lib_c/aarch64-linux-musl/c/sys/socket.cr | 1 + src/lib_c/arm-linux-gnueabihf/c/sys/socket.cr | 1 + src/lib_c/i386-linux-gnu/c/sys/socket.cr | 1 + src/lib_c/i386-linux-musl/c/sys/socket.cr | 1 + src/lib_c/x86_64-dragonfly/c/sys/socket.cr | 1 + src/lib_c/x86_64-freebsd/c/sys/socket.cr | 1 + src/lib_c/x86_64-linux-gnu/c/sys/socket.cr | 1 + src/lib_c/x86_64-linux-musl/c/sys/socket.cr | 1 + src/lib_c/x86_64-netbsd/c/sys/socket.cr | 1 + src/lib_c/x86_64-openbsd/c/sys/socket.cr | 1 + src/lib_c/x86_64-solaris/c/sys/socket.cr | 1 + src/socket/unix_socket.cr | 9 +++++++-- 20 files changed, 73 insertions(+), 11 deletions(-) diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index 56fb07f41db5..d4e7051d12bd 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -57,6 +57,9 @@ describe Socket, tags: "network" do client.family.should eq(Socket::Family::INET) client.type.should eq(Socket::Type::STREAM) client.protocol.should eq(Socket::Protocol::TCP) + {% unless flag?(:win32) %} + client.close_on_exec?.should be_true + {% end %} ensure client.close end diff --git a/spec/std/socket/tcp_server_spec.cr b/spec/std/socket/tcp_server_spec.cr index bb8fd03dad5f..0c6113a4a7ff 100644 --- a/spec/std/socket/tcp_server_spec.cr +++ b/spec/std/socket/tcp_server_spec.cr @@ -136,4 +136,18 @@ describe TCPServer, tags: "network" do end end {% end %} + + describe "accept" do + {% unless flag?(:win32) %} + it "sets close on exec flag" do + TCPServer.open("localhost", 0) do |server| + TCPSocket.open("localhost", server.local_address.port) do |client| + server.accept? do |sock| + sock.close_on_exec?.should be_true + end + end + end + end + {% end %} + end end diff --git a/spec/std/socket/unix_server_spec.cr b/spec/std/socket/unix_server_spec.cr index 098bdb3e7d53..ca364f08667c 100644 --- a/spec/std/socket/unix_server_spec.cr +++ b/spec/std/socket/unix_server_spec.cr @@ -147,6 +147,20 @@ describe UNIXServer do ret.should be_nil end end + + {% unless flag?(:win32) %} + it "sets close on exec flag" do + with_tempfile("unix_socket-accept.sock") do |path| + UNIXServer.open(path) do |server| + UNIXSocket.open(path) do |client| + server.accept? do |sock| + sock.close_on_exec?.should be_true + end + end + end + end + end + {% end %} end # Datagram socket type is not supported on Windows yet: diff --git a/spec/std/socket/unix_socket_spec.cr b/spec/std/socket/unix_socket_spec.cr index 5968ffe381aa..24777bada67f 100644 --- a/spec/std/socket/unix_socket_spec.cr +++ b/spec/std/socket/unix_socket_spec.cr @@ -101,6 +101,9 @@ describe UNIXSocket do (left.recv_buffer_size = size).should eq(size) sizes.should contain(left.recv_buffer_size) + + left.close_on_exec?.should be_true + right.close_on_exec?.should be_true end end {% end %} diff --git a/src/crystal/system/unix/event_loop_libevent.cr b/src/crystal/system/unix/event_loop_libevent.cr index 0ae968c7a15f..3d8cecf694f2 100644 --- a/src/crystal/system/unix/event_loop_libevent.cr +++ b/src/crystal/system/unix/event_loop_libevent.cr @@ -148,7 +148,13 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop def accept(socket : ::Socket) : ::Socket::Handle? loop do - client_fd = LibC.accept(socket.fd, nil, nil) + client_fd = + {% if LibC.has_method?(:accept4) %} + LibC.accept4(socket.fd, nil, nil, LibC::SOCK_CLOEXEC) + {% else %} + LibC.accept(socket.fd, nil, nil) + {% end %} + if client_fd == -1 if socket.closed? return @@ -161,6 +167,9 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop raise ::Socket::Error.from_errno("accept") end else + {% unless LibC.has_method?(:accept4) %} + Crystal::System::Socket.fcntl(client_fd, LibC::F_SETFD, LibC::FD_CLOEXEC) + {% end %} return client_fd end end diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index a263e7742301..ad065bf3ba23 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -9,21 +9,22 @@ module Crystal::System::Socket alias Handle = Int32 private def create_handle(family, type, protocol, blocking) : Handle + socktype = type.value {% if LibC.has_constant?(:SOCK_CLOEXEC) %} - # Forces opened sockets to be closed on `exec(2)`. - type = type.to_i | LibC::SOCK_CLOEXEC + socktype |= LibC::SOCK_CLOEXEC {% end %} - fd = LibC.socket(family, type, protocol) + + fd = LibC.socket(family, socktype, protocol) raise ::Socket::Error.from_errno("Failed to create socket") if fd == -1 + + {% unless LibC.has_constant?(:SOCK_CLOEXEC) %} + Socket.fcntl(fd, LibC::F_SETFD, LibC::FD_CLOEXEC) + {% end %} + fd end private def initialize_handle(fd) - {% unless LibC.has_constant?(:SOCK_CLOEXEC) %} - # Forces opened sockets to be closed on `exec(2)`. Only for platforms that don't - # support `SOCK_CLOEXEC` (e.g., Darwin). - LibC.fcntl(fd, LibC::F_SETFD, LibC::FD_CLOEXEC) - {% end %} end # Tries to bind the socket to a local address. diff --git a/src/lib_c/aarch64-android/c/sys/socket.cr b/src/lib_c/aarch64-android/c/sys/socket.cr index 241be043248b..d52a5c1110ab 100644 --- a/src/lib_c/aarch64-android/c/sys/socket.cr +++ b/src/lib_c/aarch64-android/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(__fd : Int, __addr : Sockaddr*, __addr_length : SocklenT*) : Int + fun accept4(__fd : Int, __addr : Sockaddr*, __addr_length : SocklenT*, __flags : Int) : Int fun bind(__fd : Int, __addr : Sockaddr*, __addr_length : SocklenT) : Int fun connect(__fd : Int, __addr : Sockaddr*, __addr_length : SocklenT) : Int fun getpeername(__fd : Int, __addr : Sockaddr*, __addr_length : SocklenT*) : Int diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/socket.cr b/src/lib_c/aarch64-linux-gnu/c/sys/socket.cr index 82e48d78c9f2..7935dd8b3550 100644 --- a/src/lib_c/aarch64-linux-gnu/c/sys/socket.cr +++ b/src/lib_c/aarch64-linux-gnu/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(fd : Int, addr : Sockaddr*, addr_len : SocklenT*) : Int + fun accept4(fd : Int, addr : Sockaddr*, addr_len : SocklenT*, flags : Int) : Int fun bind(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun connect(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun getpeername(fd : Int, addr : Sockaddr*, len : SocklenT*) : Int diff --git a/src/lib_c/aarch64-linux-musl/c/sys/socket.cr b/src/lib_c/aarch64-linux-musl/c/sys/socket.cr index 361e5b8040d6..51211386e8bd 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/socket.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/socket.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/socket.cr index 3a61519d9baa..4a2641d3ecd3 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/sys/socket.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(fd : Int, addr : Sockaddr*, addr_len : SocklenT*) : Int + fun accept4(fd : Int, addr : Sockaddr*, addr_len : SocklenT*, flags : Int) : Int fun bind(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun connect(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun getpeername(fd : Int, addr : Sockaddr*, len : SocklenT*) : Int diff --git a/src/lib_c/i386-linux-gnu/c/sys/socket.cr b/src/lib_c/i386-linux-gnu/c/sys/socket.cr index 6651736a41c0..6473b6bad757 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/socket.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(fd : Int, addr : Sockaddr*, addr_len : SocklenT*) : Int + fun accept4(fd : Int, addr : Sockaddr*, addr_len : SocklenT*, flags : Int) : Int fun bind(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun connect(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun getpeername(fd : Int, addr : Sockaddr*, len : SocklenT*) : Int diff --git a/src/lib_c/i386-linux-musl/c/sys/socket.cr b/src/lib_c/i386-linux-musl/c/sys/socket.cr index 361e5b8040d6..51211386e8bd 100644 --- a/src/lib_c/i386-linux-musl/c/sys/socket.cr +++ b/src/lib_c/i386-linux-musl/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/lib_c/x86_64-dragonfly/c/sys/socket.cr b/src/lib_c/x86_64-dragonfly/c/sys/socket.cr index ff439b8524d1..0d30f295ed04 100644 --- a/src/lib_c/x86_64-dragonfly/c/sys/socket.cr +++ b/src/lib_c/x86_64-dragonfly/c/sys/socket.cr @@ -51,6 +51,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/lib_c/x86_64-freebsd/c/sys/socket.cr b/src/lib_c/x86_64-freebsd/c/sys/socket.cr index 8a731c8ee82c..052b897af1a7 100644 --- a/src/lib_c/x86_64-freebsd/c/sys/socket.cr +++ b/src/lib_c/x86_64-freebsd/c/sys/socket.cr @@ -51,6 +51,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/socket.cr b/src/lib_c/x86_64-linux-gnu/c/sys/socket.cr index 82e48d78c9f2..7935dd8b3550 100644 --- a/src/lib_c/x86_64-linux-gnu/c/sys/socket.cr +++ b/src/lib_c/x86_64-linux-gnu/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(fd : Int, addr : Sockaddr*, addr_len : SocklenT*) : Int + fun accept4(fd : Int, addr : Sockaddr*, addr_len : SocklenT*, flags : Int) : Int fun bind(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun connect(fd : Int, addr : Sockaddr*, len : SocklenT) : Int fun getpeername(fd : Int, addr : Sockaddr*, len : SocklenT*) : Int diff --git a/src/lib_c/x86_64-linux-musl/c/sys/socket.cr b/src/lib_c/x86_64-linux-musl/c/sys/socket.cr index 361e5b8040d6..51211386e8bd 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/socket.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/socket.cr @@ -48,6 +48,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/lib_c/x86_64-netbsd/c/sys/socket.cr b/src/lib_c/x86_64-netbsd/c/sys/socket.cr index d96f245bc42a..3d196098492f 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/socket.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/socket.cr @@ -51,6 +51,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/lib_c/x86_64-openbsd/c/sys/socket.cr b/src/lib_c/x86_64-openbsd/c/sys/socket.cr index cb2f1fa8123b..e812ddca2236 100644 --- a/src/lib_c/x86_64-openbsd/c/sys/socket.cr +++ b/src/lib_c/x86_64-openbsd/c/sys/socket.cr @@ -51,6 +51,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/lib_c/x86_64-solaris/c/sys/socket.cr b/src/lib_c/x86_64-solaris/c/sys/socket.cr index 4c2572c288ec..0031c66d0da0 100644 --- a/src/lib_c/x86_64-solaris/c/sys/socket.cr +++ b/src/lib_c/x86_64-solaris/c/sys/socket.cr @@ -50,6 +50,7 @@ lib LibC end fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun accept4(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*, x3 : Int) : Int fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int diff --git a/src/socket/unix_socket.cr b/src/socket/unix_socket.cr index 0639dce97ca9..e672d812f631 100644 --- a/src/socket/unix_socket.cr +++ b/src/socket/unix_socket.cr @@ -75,13 +75,18 @@ class UNIXSocket < Socket socktype = type.value {% if LibC.has_constant?(:SOCK_CLOEXEC) %} - socktype |= LibC::SOCK_CLOEXEC + socktype |= LibC::SOCK_CLOEXEC {% end %} if LibC.socketpair(Family::UNIX, socktype, 0, fds) != 0 - raise Socket::Error.new("socketpair:") + raise Socket::Error.new("socketpair() failed") end + {% unless LibC.has_constant?(:SOCK_CLOEXEC) %} + Crystal::System::Socket.fcntl(fds[0], LibC::F_SETFD, LibC::FD_CLOEXEC) + Crystal::System::Socket.fcntl(fds[1], LibC::F_SETFD, LibC::FD_CLOEXEC) + {% end %} + {UNIXSocket.new(fd: fds[0], type: type), UNIXSocket.new(fd: fds[1], type: type)} {% end %} end From 47e7b16baedb30c81bc4e2e4617af40c470b91df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 14 Jun 2024 16:37:59 +0200 Subject: [PATCH 02/52] Add type restriction `host : String` in `TCPSocket` and `Addrinfo` (#14703) --- src/socket/addrinfo.cr | 12 ++++++------ src/socket/tcp_socket.cr | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/socket/addrinfo.cr b/src/socket/addrinfo.cr index 01415d9b642c..83ef561c88ac 100644 --- a/src/socket/addrinfo.cr +++ b/src/socket/addrinfo.cr @@ -30,7 +30,7 @@ class Socket # # addrinfos = Socket::Addrinfo.resolve("example.org", "http", type: Socket::Type::STREAM, protocol: Socket::Protocol::TCP) # ``` - def self.resolve(domain, service, family : Family? = nil, type : Type = nil, protocol : Protocol = Protocol::IP, timeout = nil) : Array(Addrinfo) + def self.resolve(domain : String, service, family : Family? = nil, type : Type = nil, protocol : Protocol = Protocol::IP, timeout = nil) : Array(Addrinfo) addrinfos = [] of Addrinfo getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| @@ -56,7 +56,7 @@ class Socket # # The iteration will be stopped once the block returns something that isn't # an `Exception` (e.g. a `Socket` or `nil`). - def self.resolve(domain, service, family : Family? = nil, type : Type = nil, protocol : Protocol = Protocol::IP, timeout = nil, &) + def self.resolve(domain : String, service, family : Family? = nil, type : Type = nil, protocol : Protocol = Protocol::IP, timeout = nil, &) getaddrinfo(domain, service, family, type, protocol, timeout) do |addrinfo| loop do value = yield addrinfo.not_nil! @@ -196,13 +196,13 @@ class Socket # # addrinfos = Socket::Addrinfo.tcp("example.org", 80) # ``` - def self.tcp(domain, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) + def self.tcp(domain : String, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) resolve(domain, service, family, Type::STREAM, Protocol::TCP) end # Resolves a domain for the TCP protocol with STREAM type, and yields each # possible `Addrinfo`. See `#resolve` for details. - def self.tcp(domain, service, family = Family::UNSPEC, timeout = nil, &) + def self.tcp(domain : String, service, family = Family::UNSPEC, timeout = nil, &) resolve(domain, service, family, Type::STREAM, Protocol::TCP) { |addrinfo| yield addrinfo } end @@ -215,13 +215,13 @@ class Socket # # addrinfos = Socket::Addrinfo.udp("example.org", 53) # ``` - def self.udp(domain, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) + def self.udp(domain : String, service, family = Family::UNSPEC, timeout = nil) : Array(Addrinfo) resolve(domain, service, family, Type::DGRAM, Protocol::UDP) end # Resolves a domain for the UDP protocol with DGRAM type, and yields each # possible `Addrinfo`. See `#resolve` for details. - def self.udp(domain, service, family = Family::UNSPEC, timeout = nil, &) + def self.udp(domain : String, service, family = Family::UNSPEC, timeout = nil, &) resolve(domain, service, family, Type::DGRAM, Protocol::UDP) { |addrinfo| yield addrinfo } end diff --git a/src/socket/tcp_socket.cr b/src/socket/tcp_socket.cr index 06e3d6f9b138..387417211a1a 100644 --- a/src/socket/tcp_socket.cr +++ b/src/socket/tcp_socket.cr @@ -26,7 +26,7 @@ class TCPSocket < IPSocket # must be in seconds (integers or floats). # # Note that `dns_timeout` is currently ignored. - def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false) + def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false) Addrinfo.tcp(host, port, timeout: dns_timeout) do |addrinfo| super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) connect(addrinfo, timeout: connect_timeout) do |error| @@ -53,7 +53,7 @@ class TCPSocket < IPSocket # eventually closes the socket when the block returns. # # Returns the value of the block. - def self.open(host, port, &) + def self.open(host : String, port, &) sock = new(host, port) begin yield sock From c2dd54806ee8bd0489206e87020104d46dfe8ef5 Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Fri, 14 Jun 2024 11:39:28 -0300 Subject: [PATCH 03/52] Improve compile time error for `#sort(&block : T, T -> U)` (#14693) --- src/array.cr | 8 ++++---- src/slice.cr | 8 ++++---- src/static_array.cr | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/array.cr b/src/array.cr index cef70c4704c7..30425c6869e3 100644 --- a/src/array.cr +++ b/src/array.cr @@ -1658,7 +1658,7 @@ class Array(T) # Raises `ArgumentError` if for any two elements the block returns `nil`. def sort(&block : T, T -> U) : Array(T) forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#sort_by`?" %} {% end %} dup.sort! &block @@ -1680,7 +1680,7 @@ class Array(T) # Raises `ArgumentError` if for any two elements the block returns `nil`. def unstable_sort(&block : T, T -> U) : Array(T) forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#unstable_sort_by`?" %} {% end %} dup.unstable_sort!(&block) @@ -1701,7 +1701,7 @@ class Array(T) # :inherit: def sort!(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#sort_by!`?" %} {% end %} to_unsafe_slice.sort!(&block) @@ -1711,7 +1711,7 @@ class Array(T) # :inherit: def unstable_sort!(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#unstable_sort_by!`?" %} {% end %} to_unsafe_slice.unstable_sort!(&block) diff --git a/src/slice.cr b/src/slice.cr index 7a27218221a2..196a29a768dd 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -932,7 +932,7 @@ struct Slice(T) # Raises `ArgumentError` if for any two elements the block returns `nil`. def sort(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#sort_by`?" %} {% end %} dup.sort! &block @@ -954,7 +954,7 @@ struct Slice(T) # Raises `ArgumentError` if for any two elements the block returns `nil`. def unstable_sort(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#unstable_sort_by`?" %} {% end %} dup.unstable_sort!(&block) @@ -1055,7 +1055,7 @@ struct Slice(T) # Raises `ArgumentError` if for any two elements the block returns `nil`. def sort!(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#sort_by!`?" %} {% end %} Slice.merge_sort!(self, block) @@ -1098,7 +1098,7 @@ struct Slice(T) # Raises `ArgumentError` if for any two elements the block returns `nil`. def unstable_sort!(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#unstable_sort_by!`?" %} {% end %} Slice.intro_sort!(to_unsafe, size, block) diff --git a/src/static_array.cr b/src/static_array.cr index 4cb2b186f200..2c09e21df166 100644 --- a/src/static_array.cr +++ b/src/static_array.cr @@ -228,7 +228,7 @@ struct StaticArray(T, N) # Raises `ArgumentError` if for any two elements the block returns `nil`.= def sort(&block : T, T -> U) : StaticArray(T, N) forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#sort_by`?" %} {% end %} ary = dup @@ -251,7 +251,7 @@ struct StaticArray(T, N) # Raises `ArgumentError` if for any two elements the block returns `nil`. def unstable_sort(&block : T, T -> U) : StaticArray(T, N) forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#unstable_sort_by`?" %} {% end %} ary = dup @@ -273,7 +273,7 @@ struct StaticArray(T, N) # :inherit: def sort!(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#sort_by!`?" %} {% end %} to_slice.sort!(&block) @@ -283,7 +283,7 @@ struct StaticArray(T, N) # :inherit: def unstable_sort!(&block : T, T -> U) : self forall U {% unless U <= Int32? %} - {% raise "Expected block to return Int32 or Nil, not #{U}" %} + {% raise "Expected block to return Int32 or Nil, not #{U}.\nThe block is supposed to be a custom comparison operation, compatible with `Comparable#<=>`.\nDid you mean to use `#unstable_sort_by!`?" %} {% end %} to_slice.unstable_sort!(&block) From 0de7abfdfa21a0e8a4fd6f528bf62dbe558d60eb Mon Sep 17 00:00:00 2001 From: Linus Sellberg Date: Sat, 15 Jun 2024 10:11:56 +0200 Subject: [PATCH 04/52] Drop obsolete workaround in `Range#reverse_each` (#14709) The workaround is no longer needed since Crystal 0.36 --- src/range.cr | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/range.cr b/src/range.cr index db1b3f32902a..39d8119dff6e 100644 --- a/src/range.cr +++ b/src/range.cr @@ -163,17 +163,15 @@ struct Range(B, E) yield end_value if !@exclusive && (begin_value.nil? || !(end_value < begin_value)) current = end_value - # TODO: The macro interpolations are a workaround until #9324 is fixed. - {% if B == Nil %} while true current = current.pred - {{ "yield current".id }} + yield current end {% else %} while begin_value.nil? || begin_value < current current = current.pred - {{ "yield current".id }} + yield current end {% end %} end From 42ed5c46f2b8cbffd47d654cefff2de4abea43f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Hovs=C3=A4ter?= Date: Sat, 15 Jun 2024 16:42:16 +0200 Subject: [PATCH 05/52] Do not strip the macOS target triple (#14466) Starting with Xcode 15, the minimum deployment target is required in the target triple. With the release of Xcode 15, a [new linker was introduced](https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes#Linking) and with it came warning messages like this: ld: warning: no platform load command found in '/Users/pascal/.cache/crystal/Users-pascal-Documents-tutorials-crystal-hello_world.cr/F-loat32.o', assuming: macOS It appears that the new linker is stricter in which target triples it considers valid. Specifically, it looks like the minimum deployment target is required. E.g., `aarch64-apple-darwin23.3.0` is valid, while `aarch64-apple-darwin` is not. See https://github.com/crystal-lang/crystal/issues/13846#issuecomment-2040146898 for details. This patch removes code which strips the minimum deployment target in `LLVM.default_target_triple` as an effort for standardization. See also: https://github.com/crystal-lang/distribution-scripts/pull/296 --- spec/std/llvm/llvm_spec.cr | 2 +- src/llvm.cr | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/spec/std/llvm/llvm_spec.cr b/spec/std/llvm/llvm_spec.cr index d232931db848..17ea96d5e261 100644 --- a/spec/std/llvm/llvm_spec.cr +++ b/spec/std/llvm/llvm_spec.cr @@ -22,7 +22,7 @@ describe LLVM do it ".default_target_triple" do triple = LLVM.default_target_triple {% if flag?(:darwin) %} - triple.should match(/-apple-macosx$/) + triple.should match(/-apple-(darwin|macosx)/) {% elsif flag?(:android) %} triple.should match(/-android$/) {% elsif flag?(:linux) %} diff --git a/src/llvm.cr b/src/llvm.cr index 6ad1bf6c796d..6fb8767cad54 100644 --- a/src/llvm.cr +++ b/src/llvm.cr @@ -107,12 +107,6 @@ module LLVM def self.default_target_triple : String chars = LibLLVM.get_default_target_triple case triple = string_and_dispose(chars) - when .starts_with?("x86_64-apple-macosx"), .starts_with?("x86_64-apple-darwin") - # normalize on `macosx` and remove minimum deployment target version - "x86_64-apple-macosx" - when .starts_with?("aarch64-apple-macosx"), .starts_with?("aarch64-apple-darwin") - # normalize on `macosx` and remove minimum deployment target version - "aarch64-apple-macosx" when .starts_with?("aarch64-unknown-linux-android") # remove API version "aarch64-unknown-linux-android" From ff014992554c19cf33485f9e6f37abc5d192fc4d Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sat, 15 Jun 2024 17:08:12 +0200 Subject: [PATCH 06/52] Add `Crystal::Tracing` for runtime tracing (#14659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements tracing of the garbage collector and the scheduler as per #14618 Tracing is enabled by compiling with `-Dtracing` then individual tracing must be enabled at runtime with the `CRYSTAL_TRACE` environment variable that is a comma separated list of sections to enable, for example: - ` ` (empty value) or `none` to disable any tracing (default) - `gc` - `sched` - `gc,sched` - `all` to enable everything The traces are printed to the standard error by default, but this can be changed at runtime with the `CRYSTAL_TRACE_FILE` environment variable. For example `trace.log`. You can also redirect the standard error to a file (e.g. `2> trace.log` on UNIX shell). Example tracing calls: Crystal.trace :sched, "spawn", fiber: fiber Crystal.trace :gc, "malloc", size: size, atomic: 1 **Technical note:** tracing happens before the stdlib is initialized, so the implementation must rely on some `LibC` methods directly (i.e. read environment variable, write to file descriptor) and can't use the core/stdlib abstractions. Co-authored-by: Johannes Müller --- src/concurrent.cr | 2 + src/crystal/main.cr | 1 + src/crystal/scheduler.cr | 39 +++-- src/crystal/system/print_error.cr | 33 +++- src/crystal/tracing.cr | 272 ++++++++++++++++++++++++++++++ src/gc/boehm.cr | 99 ++++++++++- src/gc/none.cr | 5 + 7 files changed, 430 insertions(+), 21 deletions(-) create mode 100644 src/crystal/tracing.cr diff --git a/src/concurrent.cr b/src/concurrent.cr index af2f0aecf736..6f3a58291a22 100644 --- a/src/concurrent.cr +++ b/src/concurrent.cr @@ -1,6 +1,7 @@ require "fiber" require "channel" require "crystal/scheduler" +require "crystal/tracing" # Blocks the current fiber for the specified number of seconds. # @@ -59,6 +60,7 @@ end # ``` def spawn(*, name : String? = nil, same_thread = false, &block) fiber = Fiber.new(name, &block) + Crystal.trace :sched, "spawn", fiber: fiber {% if flag?(:preview_mt) %} fiber.set_current_thread if same_thread {% end %} fiber.enqueue fiber diff --git a/src/crystal/main.cr b/src/crystal/main.cr index 059a822c5ff4..625238229c58 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -34,6 +34,7 @@ module Crystal # same can be accomplished with `at_exit`. But in some cases # redefinition of C's main is needed. def self.main(&block) + {% if flag?(:tracing) %} Crystal::Tracing.init {% end %} GC.init status = diff --git a/src/crystal/scheduler.cr b/src/crystal/scheduler.cr index 4796226ce8e9..d3634e9aea6a 100644 --- a/src/crystal/scheduler.cr +++ b/src/crystal/scheduler.cr @@ -25,21 +25,23 @@ class Crystal::Scheduler end def self.enqueue(fiber : Fiber) : Nil - thread = Thread.current - scheduler = thread.scheduler + Crystal.trace :sched, "enqueue", fiber: fiber do + thread = Thread.current + scheduler = thread.scheduler - {% if flag?(:preview_mt) %} - th = fiber.get_current_thread - th ||= fiber.set_current_thread(scheduler.find_target_thread) + {% if flag?(:preview_mt) %} + th = fiber.get_current_thread + th ||= fiber.set_current_thread(scheduler.find_target_thread) - if th == thread + if th == thread + scheduler.enqueue(fiber) + else + th.scheduler.send_fiber(fiber) + end + {% else %} scheduler.enqueue(fiber) - else - th.scheduler.send_fiber(fiber) - end - {% else %} - scheduler.enqueue(fiber) - {% end %} + {% end %} + end end def self.enqueue(fibers : Enumerable(Fiber)) : Nil @@ -49,6 +51,7 @@ class Crystal::Scheduler end def self.reschedule : Nil + Crystal.trace :sched, "reschedule" Thread.current.scheduler.reschedule end @@ -58,10 +61,13 @@ class Crystal::Scheduler end def self.sleep(time : Time::Span) : Nil + Crystal.trace :sched, "sleep", for: time.total_nanoseconds.to_i64! Thread.current.scheduler.sleep(time) end def self.yield : Nil + Crystal.trace :sched, "yield" + # TODO: Fiber switching and libevent for wasm32 {% unless flag?(:wasm32) %} Thread.current.scheduler.sleep(0.seconds) @@ -109,6 +115,7 @@ class Crystal::Scheduler end protected def resume(fiber : Fiber) : Nil + Crystal.trace :sched, "resume", fiber: fiber validate_resumable(fiber) {% if flag?(:preview_mt) %} @@ -149,7 +156,9 @@ class Crystal::Scheduler resume(runnable) unless runnable == @thread.current_fiber break else - @event_loop.run(blocking: true) + Crystal.trace :sched, "event_loop" do + @event_loop.run(blocking: true) + end end end end @@ -190,7 +199,9 @@ class Crystal::Scheduler else @sleeping = true @lock.unlock - fiber = fiber_channel.receive + + Crystal.trace :sched, "mt:sleeping" + fiber = Crystal.trace(:sched, "mt:slept") { fiber_channel.receive } @lock.lock @sleeping = false diff --git a/src/crystal/system/print_error.cr b/src/crystal/system/print_error.cr index f58bef1c4ff6..796579bf256a 100644 --- a/src/crystal/system/print_error.cr +++ b/src/crystal/system/print_error.cr @@ -14,6 +14,37 @@ module Crystal::System {% end %} end + # Print a UTF-16 slice as UTF-8 directly to stderr. Useful on Windows to print + # strings returned from the unicode variant of the Win32 API. + def self.print_error(bytes : Slice(UInt16)) : Nil + utf8 = uninitialized UInt8[512] + appender = utf8.to_unsafe.appender + + String.each_utf16_char(bytes) do |char| + if appender.size > utf8.size - char.bytesize + # buffer is full (char won't fit) + print_error utf8.to_slice[0...appender.size] + appender = utf8.to_unsafe.appender + end + + char.each_byte do |byte| + appender << byte + end + end + + if appender.size > 0 + print_error utf8.to_slice[0...appender.size] + end + end + + def self.print(handle : FileDescriptor::Handle, bytes : Bytes) : Nil + {% if flag?(:unix) || flag?(:wasm32) %} + LibC.write handle, bytes, bytes.size + {% elsif flag?(:win32) %} + LibC.WriteFile(Pointer(FileDescriptor::Handle).new(handle), bytes, bytes.size, out _, nil) + {% end %} + end + # Minimal drop-in replacement for C `printf` function. Yields successive # non-empty `Bytes` to the block, which should do the actual printing. # @@ -109,7 +140,7 @@ module Crystal::System end # simplified version of `Int#internal_to_s` - private def self.to_int_slice(num, base, signed, width, &) + protected def self.to_int_slice(num, base, signed, width, &) if num == 0 yield "0".to_slice return diff --git a/src/crystal/tracing.cr b/src/crystal/tracing.cr new file mode 100644 index 000000000000..708956ad8feb --- /dev/null +++ b/src/crystal/tracing.cr @@ -0,0 +1,272 @@ +module Crystal + # :nodoc: + module Tracing + @[Flags] + enum Section + GC + Sched + + def self.from_id(slice) : self + {% begin %} + case slice + {% for name in @type.constants %} + when {{name.underscore.stringify}}.to_slice + {{name}} + {% end %} + else + None + end + {% end %} + end + + def to_id : String + {% begin %} + case self + {% for name in @type.constants %} + when {{name}} + {{name.underscore.stringify}} + {% end %} + else + "???" + end + {% end %} + end + end + end + + {% if flag?(:tracing) %} + # :nodoc: + module Tracing + # IO-like object with a fixed capacity but dynamic size within the + # buffer's capacity (i.e. `0 <= size <= N`). Stops writing to the internal + # buffer when capacity is reached; further writes are skipped. + struct BufferIO(N) + getter size : Int32 + + def initialize + @buf = uninitialized UInt8[N] + @size = 0 + end + + def write(bytes : Bytes) : Nil + pos = @size + remaining = N - pos + return if remaining == 0 + + n = bytes.size.clamp(..remaining) + bytes.to_unsafe.copy_to(@buf.to_unsafe + pos, n) + @size = pos + n + end + + def write(string : String) : Nil + write string.to_slice + end + + def write(fiber : Fiber) : Nil + write fiber.as(Void*) + write ":" + write fiber.name || "?" + end + + def write(ptr : Pointer) : Nil + write "0x" + System.to_int_slice(ptr.address, 16, true, 2) { |bytes| write(bytes) } + end + + def write(int : Int::Signed) : Nil + System.to_int_slice(int, 10, true, 2) { |bytes| write(bytes) } + end + + def write(uint : Int::Unsigned) : Nil + System.to_int_slice(uint, 10, false, 2) { |bytes| write(bytes) } + end + + def to_slice : Bytes + Bytes.new(@buf.to_unsafe, @size) + end + end + + @@sections = Section::None + @@handle = uninitialized System::FileDescriptor::Handle + + @[AlwaysInline] + def self.enabled?(section : Section) : Bool + @@sections.includes?(section) + end + + # Setup tracing. + # + # Parses the `CRYSTAL_TRACE` environment variable to enable the sections + # to trace. See `Section`. By default no sections are enabled. + # + # Parses the `CRYSTAL_TRACE_FILE` environment variable to open the trace + # file to write to. Exits with an error message when the file can't be + # opened, created or truncated. Uses the standard error when unspecified. + # + # This should be the first thing called in main, maybe even before the GC + # itself is initialized. The function assumes neither the GC nor ENV nor + # anything is available and musn't allocate into the GC HEAP. + def self.init : Nil + @@sections = Section::None + + {% if flag?(:win32) %} + buf = uninitialized UInt16[256] + + name = UInt16.static_array({% for chr in "CRYSTAL_TRACE".chars %}{{chr.ord}}, {% end %} 0) + len = LibC.GetEnvironmentVariableW(name, buf, buf.size) + parse_sections(buf.to_slice[0...len]) if len > 0 + + name = UInt16.static_array({% for chr in "CRYSTAL_TRACE_FILE".chars %}{{chr.ord}}, {% end %} 0) + len = LibC.GetEnvironmentVariableW(name, buf, buf.size) + if len > 0 + @@handle = open_trace_file(buf.to_slice[0...len]) + else + @@handle = LibC.GetStdHandle(LibC::STD_ERROR_HANDLE).address + end + {% else %} + if ptr = LibC.getenv("CRYSTAL_TRACE") + len = LibC.strlen(ptr) + parse_sections(Slice.new(ptr, len)) if len > 0 + end + + if (ptr = LibC.getenv("CRYSTAL_TRACE_FILE")) && (LibC.strlen(ptr) > 0) + @@handle = open_trace_file(ptr) + else + @@handle = 2 + end + {% end %} + end + + private def self.open_trace_file(filename) + {% if flag?(:win32) %} + handle = LibC.CreateFileW(filename, LibC::FILE_GENERIC_WRITE, LibC::DEFAULT_SHARE_MODE, nil, LibC::CREATE_ALWAYS, LibC::FILE_ATTRIBUTE_NORMAL, LibC::HANDLE.null) + # not using LibC::INVALID_HANDLE_VALUE because it doesn't exist (yet) + return handle.address unless handle == LibC::HANDLE.new(-1) + + error = uninitialized UInt16[256] + len = LibC.FormatMessageW(LibC::FORMAT_MESSAGE_FROM_SYSTEM, nil, WinError.value, 0, error, error.size, nil) + + # not using printf because filename and error are UTF-16 slices: + System.print_error "ERROR: failed to open " + System.print_error filename + System.print_error " for writing: " + System.print_error error.to_slice[0...len] + System.print_error "\n" + {% else %} + fd = LibC.open(filename, LibC::O_CREAT | LibC::O_WRONLY | LibC::O_TRUNC | LibC::O_CLOEXEC, 0o644) + return fd unless fd < 0 + + System.print_error "ERROR: failed to open %s for writing: %s\n", filename, LibC.strerror(Errno.value) + {% end %} + + LibC._exit(1) + end + + private def self.parse_sections(slice) + each_token(slice) do |token| + @@sections |= Section.from_id(token) + end + end + + private def self.each_token(slice, delim = ',', &) + while e = slice.index(delim.ord) + yield slice[0, e] + slice = slice[(e + 1)..] + end + yield slice[0..] unless slice.size == 0 + end + + # :nodoc: + # + # Formats and prints a log message to stderr. The generated message is + # limited to 512 bytes (PIPE_BUF) after which it will be truncated. Being + # below PIPE_BUF the message shall be written atomically to stderr, + # avoiding interleaved or smashed traces from multiple threads. + # + # Windows may not have the same guarantees but the buffering should limit + # these from happening. + def self.log(section : String, operation : String, time : UInt64, **metadata) : Nil + buf = BufferIO(512).new + buf.write section + buf.write "." + buf.write operation + buf.write " " + buf.write time + + {% unless flag?(:wasm32) %} + # WASM doesn't have threads (and fibers aren't implemented either) + # + # We also start to trace *before* Thread.current and other objects have + # been allocated, they're lazily allocated and since we trace GC.malloc we + # must skip the objects until they're allocated (otherwise we hit infinite + # recursion): malloc -> trace -> malloc -> trace -> ... + thread = ::Thread.current? + + buf.write " thread=" + {% if flag?(:linux) %} + buf.write Pointer(Void).new(thread ? thread.@system_handle : System::Thread.current_handle) + {% else %} + buf.write thread ? thread.@system_handle : System::Thread.current_handle + {% end %} + buf.write ":" + buf.write thread.try(&.name) || "?" + + if thread && (fiber = thread.current_fiber?) + buf.write " fiber=" + buf.write fiber + end + {% end %} + + metadata.each do |key, value| + buf.write " " + buf.write key.to_s + buf.write "=" + buf.write value + end + + buf.write "\n" + System.print(@@handle, buf.to_slice) + end + end + + def self.trace(section : Tracing::Section, operation : String, time : UInt64? = nil, **metadata, &) + if Tracing.enabled?(section) + time ||= System::Time.ticks + begin + yield + ensure + duration = System::Time.ticks - time + Tracing.log(section.to_id, operation, time, **metadata, duration: duration) + end + else + yield + end + end + + def self.trace(section : Tracing::Section, operation : String, time : UInt64? = nil, **metadata) : Nil + if Tracing.enabled?(section) + Tracing.log(section.to_id, operation, time || System::Time.ticks, **metadata) + end + end + {% else %} + # :nodoc: + module Tracing + def self.init + end + + def self.enabled?(section) + false + end + + def self.log(section : String, operation : String, time : UInt64, **metadata) + end + end + + def self.trace(section : Tracing::Section, operation : String, time : UInt64? = nil, **metadata, &) + yield + end + + def self.trace(section : Tracing::Section, operation : String, time : UInt64? = nil, **metadata) + end + {% end %} +end diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index 29ae825adab1..8ccc1bb7b6e8 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -1,6 +1,7 @@ {% if flag?(:preview_mt) %} require "crystal/rw_lock" {% end %} +require "crystal/tracing" # MUSL: On musl systems, libpthread is empty. The entire library is already included in libc. # The empty library is only available for POSIX compatibility. We don't need to link it. @@ -113,7 +114,32 @@ lib LibGC $stackbottom = GC_stackbottom : Void* {% end %} - fun set_on_collection_event = GC_set_on_collection_event(cb : ->) + alias OnHeapResizeProc = Word -> + fun set_on_heap_resize = GC_set_on_heap_resize(OnHeapResizeProc) + fun get_on_heap_resize = GC_get_on_heap_resize : OnHeapResizeProc + + enum EventType + START # COLLECTION + MARK_START + MARK_END + RECLAIM_START + RECLAIM_END + END # COLLECTION + PRE_STOP_WORLD # STOPWORLD_BEGIN + POST_STOP_WORLD # STOPWORLD_END + PRE_START_WORLD # STARTWORLD_BEGIN + POST_START_WORLD # STARTWORLD_END + THREAD_SUSPENDED + THREAD_UNSUSPENDED + end + + alias OnCollectionEventProc = EventType -> + fun set_on_collection_event = GC_set_on_collection_event(cb : OnCollectionEventProc) + fun get_on_collection_event = GC_get_on_collection_event : OnCollectionEventProc + + alias OnThreadEventProc = EventType, Void* -> + fun set_on_thread_event = GC_set_on_thread_event(cb : OnThreadEventProc) + fun get_on_thread_event = GC_get_on_thread_event : OnThreadEventProc $gc_no = GC_gc_no : Word $bytes_found = GC_bytes_found : SignedWord @@ -144,17 +170,23 @@ module GC # :nodoc: def self.malloc(size : LibC::SizeT) : Void* - LibGC.malloc(size) + Crystal.trace :gc, "malloc", size: size do + LibGC.malloc(size) + end end # :nodoc: def self.malloc_atomic(size : LibC::SizeT) : Void* - LibGC.malloc_atomic(size) + Crystal.trace :gc, "malloc", size: size, atomic: 1 do + LibGC.malloc_atomic(size) + end end # :nodoc: def self.realloc(ptr : Void*, size : LibC::SizeT) : Void* - LibGC.realloc(ptr, size) + Crystal.trace :gc, "realloc", size: size do + LibGC.realloc(ptr, size) + end end def self.init : Nil @@ -166,6 +198,14 @@ module GC LibGC.set_start_callback ->do GC.lock_write end + + {% if flag?(:tracing) %} + if ::Crystal::Tracing.enabled?(:gc) + set_on_heap_resize_proc + set_on_collection_events_proc + end + {% end %} + # By default the GC warns on big allocations/reallocations. This # is of limited use and pollutes program output with warnings. LibGC.set_warn_proc ->(msg, v) do @@ -178,8 +218,53 @@ module GC end end + {% if flag?(:tracing) %} + @@on_heap_resize : LibGC::OnHeapResizeProc? + @@on_collection_event : LibGC::OnCollectionEventProc? + + @@collect_start = 0_u64 + @@mark_start = 0_u64 + @@sweep_start = 0_u64 + + private def self.set_on_heap_resize_proc : Nil + @@on_heap_resize = LibGC.get_on_heap_resize + + LibGC.set_on_heap_resize(->(new_size : LibGC::Word) { + Crystal.trace :gc, "heap_resize", size: new_size + @@on_heap_resize.try(&.call(new_size)) + }) + end + + private def self.set_on_collection_events_proc : Nil + @@on_collection_event = LibGC.get_on_collection_event + + LibGC.set_on_collection_event(->(event_type : LibGC::EventType) { + case event_type + when .start? + @@collect_start = Crystal::System::Time.ticks + when .mark_start? + @@mark_start = Crystal::System::Time.ticks + when .reclaim_start? + @@sweep_start = Crystal::System::Time.ticks + when .end? + duration = Crystal::System::Time.ticks - @@collect_start + Crystal.trace :gc, "collect", @@collect_start, duration: duration + when .mark_end? + duration = Crystal::System::Time.ticks - @@mark_start + Crystal.trace :gc, "collect:mark", @@mark_start, duration: duration + when .reclaim_end? + duration = Crystal::System::Time.ticks - @@sweep_start + Crystal.trace :gc, "collect:sweep", @@sweep_start, duration: duration + end + @@on_collection_event.try(&.call(event_type)) + }) + end + {% end %} + def self.collect - LibGC.collect + Crystal.trace :gc, "collect" do + LibGC.collect + end end def self.enable @@ -195,7 +280,9 @@ module GC end def self.free(pointer : Void*) : Nil - LibGC.free(pointer) + Crystal.trace :gc, "free" do + LibGC.free(pointer) + end end def self.add_finalizer(object : Reference) : Nil diff --git a/src/gc/none.cr b/src/gc/none.cr index c71ab05ccd8d..1121caef1bf4 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -1,6 +1,7 @@ {% if flag?(:win32) %} require "c/process" {% end %} +require "crystal/tracing" module GC def self.init @@ -8,16 +9,19 @@ module GC # :nodoc: def self.malloc(size : LibC::SizeT) : Void* + Crystal.trace :gc, "malloc", size: size LibC.malloc(size) end # :nodoc: def self.malloc_atomic(size : LibC::SizeT) : Void* + Crystal.trace :gc, "malloc", size: size, atomic: 1 LibC.malloc(size) end # :nodoc: def self.realloc(pointer : Void*, size : LibC::SizeT) : Void* + Crystal.trace :gc, "realloc", size: size LibC.realloc(pointer, size) end @@ -31,6 +35,7 @@ module GC end def self.free(pointer : Void*) : Nil + Crystal.trace :gc, "free" LibC.free(pointer) end From 6a2097d4478590b2c3a5fb0a54363ce39b82c14a Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Sun, 16 Jun 2024 04:59:12 -0400 Subject: [PATCH 07/52] Fix formatter to skip trailing comma for single-line parameters (#14713) --- spec/compiler/formatter/formatter_spec.cr | 35 +++++++++++++++++++++++ src/compiler/crystal/tools/formatter.cr | 4 +-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index e4e76279f4d5..7bb7a1034e72 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -1120,6 +1120,41 @@ describe Crystal::Formatter do ) end CRYSTAL + + assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + def foo(a) + end + CRYSTAL + + assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + def foo(a, b) + end + CRYSTAL + + assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + def foo(a, *args) + end + CRYSTAL + + assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + def foo(a, *args, &block) + end + CRYSTAL + + assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + def foo(a, **kwargs) + end + CRYSTAL + + assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + def foo(a, **kwargs, &block) + end + CRYSTAL + + assert_format <<-CRYSTAL, flags: %w[def_trailing_comma] + def foo(a, &block) + end + CRYSTAL end assert_format "1 + 2", "1 + 2" diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 614ecc836637..dc14c70a90ad 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -1562,7 +1562,7 @@ module Crystal args.each_with_index do |arg, i| has_more = !last?(i, args) || double_splat || block_arg || yields || variadic - wrote_newline = format_def_arg(wrote_newline, has_more, true) do + wrote_newline = format_def_arg(wrote_newline, has_more, found_first_newline && !has_more) do if i == splat_index write_token :OP_STAR skip_space_or_newline @@ -1577,7 +1577,7 @@ module Crystal end if double_splat - wrote_newline = format_def_arg(wrote_newline, block_arg || yields, true) do + wrote_newline = format_def_arg(wrote_newline, block_arg || yields, found_first_newline) do write_token :OP_STAR_STAR skip_space_or_newline From bcb5aeb5d2c432ccb1e5e2385189ed15599e8ba8 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sun, 16 Jun 2024 10:59:34 +0200 Subject: [PATCH 08/52] Add `Thread.sleep(Time::Span)` (#14715) Blocks the current thread for the given duration. --- src/crystal/system/thread.cr | 8 ++++++++ src/crystal/system/unix/pthread.cr | 12 ++++++++++++ src/crystal/system/wasi/thread.cr | 12 ++++++++++++ src/crystal/system/win32/thread.cr | 4 ++++ src/lib_c/aarch64-android/c/time.cr | 1 + src/lib_c/aarch64-darwin/c/time.cr | 1 + src/lib_c/aarch64-linux-gnu/c/time.cr | 1 + src/lib_c/aarch64-linux-musl/c/time.cr | 1 + src/lib_c/arm-linux-gnueabihf/c/time.cr | 1 + src/lib_c/i386-linux-gnu/c/time.cr | 1 + src/lib_c/i386-linux-musl/c/time.cr | 1 + src/lib_c/wasm32-wasi/c/time.cr | 1 + src/lib_c/x86_64-darwin/c/time.cr | 1 + src/lib_c/x86_64-dragonfly/c/time.cr | 1 + src/lib_c/x86_64-freebsd/c/time.cr | 1 + src/lib_c/x86_64-linux-gnu/c/time.cr | 1 + src/lib_c/x86_64-linux-musl/c/time.cr | 1 + src/lib_c/x86_64-netbsd/c/time.cr | 1 + src/lib_c/x86_64-openbsd/c/time.cr | 1 + src/lib_c/x86_64-solaris/c/time.cr | 1 + 20 files changed, 52 insertions(+) diff --git a/src/crystal/system/thread.cr b/src/crystal/system/thread.cr index 03b00f2779f4..d9dc6acf17dc 100644 --- a/src/crystal/system/thread.cr +++ b/src/crystal/system/thread.cr @@ -14,6 +14,8 @@ module Crystal::System::Thread # def self.current_thread=(thread : ::Thread) + # def self.sleep(time : ::Time::Span) : Nil + # private def system_join : Exception? # private def system_close @@ -99,6 +101,12 @@ class Thread end end + # Blocks the current thread for the duration of *time*. Clock precision is + # dependent on the operating system and hardware. + def self.sleep(time : Time::Span) : Nil + Crystal::System::Thread.sleep(time) + end + # Returns the Thread object associated to the running system thread. def self.current : Thread Crystal::System::Thread.current_thread diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 4b357b04281c..d38e52ee012a 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -75,6 +75,18 @@ module Crystal::System::Thread end {% end %} + def self.sleep(time : ::Time::Span) : Nil + req = uninitialized LibC::Timespec + req.tv_sec = typeof(req.tv_sec).new(time.seconds) + req.tv_nsec = typeof(req.tv_nsec).new(time.nanoseconds) + + loop do + return if LibC.nanosleep(pointerof(req), out rem) == 0 + raise RuntimeError.from_errno("nanosleep() failed") unless Errno.value == Errno::EINTR + req = rem + end + end + private def system_join : Exception? ret = GC.pthread_join(@system_handle) RuntimeError.from_os_error("pthread_join", Errno.new(ret)) unless ret == 0 diff --git a/src/crystal/system/wasi/thread.cr b/src/crystal/system/wasi/thread.cr index 0e641faba785..6f0c0cbe8260 100644 --- a/src/crystal/system/wasi/thread.cr +++ b/src/crystal/system/wasi/thread.cr @@ -15,6 +15,18 @@ module Crystal::System::Thread class_property current_thread : ::Thread { ::Thread.new } + def self.sleep(time : ::Time::Span) : Nil + req = uninitialized LibC::Timespec + req.tv_sec = typeof(req.tv_sec).new(time.seconds) + req.tv_nsec = typeof(req.tv_nsec).new(time.nanoseconds) + + loop do + return if LibC.nanosleep(pointerof(req), out rem) == 0 + raise RuntimeError.from_errno("nanosleep() failed") unless Errno.value == Errno::EINTR + req = rem + end + end + private def system_join : Exception? NotImplementedError.new("Crystal::System::Thread#system_join") end diff --git a/src/crystal/system/win32/thread.cr b/src/crystal/system/win32/thread.cr index 9507e332b422..ddfe3298b20a 100644 --- a/src/crystal/system/win32/thread.cr +++ b/src/crystal/system/win32/thread.cr @@ -51,6 +51,10 @@ module Crystal::System::Thread @@current_thread end + def self.sleep(time : ::Time::Span) : Nil + LibC.Sleep(time.total_milliseconds.to_i.clamp(1..)) + end + private def system_join : Exception? if LibC.WaitForSingleObject(@system_handle, LibC::INFINITE) != LibC::WAIT_OBJECT_0 return RuntimeError.from_winerror("WaitForSingleObject") diff --git a/src/lib_c/aarch64-android/c/time.cr b/src/lib_c/aarch64-android/c/time.cr index 3108f2e94bff..8f8b81291f0d 100644 --- a/src/lib_c/aarch64-android/c/time.cr +++ b/src/lib_c/aarch64-android/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(__t : TimeT*, __tm : Tm*) : Tm* fun localtime_r(__t : TimeT*, __tm : Tm*) : Tm* fun mktime(__tm : Tm*) : TimeT + fun nanosleep(__req : Timespec*, __rem : Timespec*) : Int fun tzset : Void fun timegm(__tm : Tm*) : TimeT diff --git a/src/lib_c/aarch64-darwin/c/time.cr b/src/lib_c/aarch64-darwin/c/time.cr index e20477a6a004..7e76fb969fbe 100644 --- a/src/lib_c/aarch64-darwin/c/time.cr +++ b/src/lib_c/aarch64-darwin/c/time.cr @@ -23,6 +23,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/aarch64-linux-gnu/c/time.cr b/src/lib_c/aarch64-linux-gnu/c/time.cr index b93b8e698dd9..710d477e269b 100644 --- a/src/lib_c/aarch64-linux-gnu/c/time.cr +++ b/src/lib_c/aarch64-linux-gnu/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* fun localtime_r(timer : TimeT*, tp : Tm*) : Tm* fun mktime(tp : Tm*) : TimeT + fun nanosleep(req : Timespec*, rem : Timespec*) : Int fun tzset : Void fun timegm(tp : Tm*) : TimeT diff --git a/src/lib_c/aarch64-linux-musl/c/time.cr b/src/lib_c/aarch64-linux-musl/c/time.cr index 22fdf7a86ebf..f687c8b35db4 100644 --- a/src/lib_c/aarch64-linux-musl/c/time.cr +++ b/src/lib_c/aarch64-linux-musl/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/arm-linux-gnueabihf/c/time.cr b/src/lib_c/arm-linux-gnueabihf/c/time.cr index b93b8e698dd9..710d477e269b 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/time.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* fun localtime_r(timer : TimeT*, tp : Tm*) : Tm* fun mktime(tp : Tm*) : TimeT + fun nanosleep(req : Timespec*, rem : Timespec*) : Int fun tzset : Void fun timegm(tp : Tm*) : TimeT diff --git a/src/lib_c/i386-linux-gnu/c/time.cr b/src/lib_c/i386-linux-gnu/c/time.cr index b93b8e698dd9..710d477e269b 100644 --- a/src/lib_c/i386-linux-gnu/c/time.cr +++ b/src/lib_c/i386-linux-gnu/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* fun localtime_r(timer : TimeT*, tp : Tm*) : Tm* fun mktime(tp : Tm*) : TimeT + fun nanosleep(req : Timespec*, rem : Timespec*) : Int fun tzset : Void fun timegm(tp : Tm*) : TimeT diff --git a/src/lib_c/i386-linux-musl/c/time.cr b/src/lib_c/i386-linux-musl/c/time.cr index 22fdf7a86ebf..f687c8b35db4 100644 --- a/src/lib_c/i386-linux-musl/c/time.cr +++ b/src/lib_c/i386-linux-musl/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/wasm32-wasi/c/time.cr b/src/lib_c/wasm32-wasi/c/time.cr index 9d77b0f53fec..9965c3a7d324 100644 --- a/src/lib_c/wasm32-wasi/c/time.cr +++ b/src/lib_c/wasm32-wasi/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun timegm(x0 : Tm*) : TimeT fun futimes(fd : Int, times : Timeval[2]) : Int end diff --git a/src/lib_c/x86_64-darwin/c/time.cr b/src/lib_c/x86_64-darwin/c/time.cr index e20477a6a004..7e76fb969fbe 100644 --- a/src/lib_c/x86_64-darwin/c/time.cr +++ b/src/lib_c/x86_64-darwin/c/time.cr @@ -23,6 +23,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/x86_64-dragonfly/c/time.cr b/src/lib_c/x86_64-dragonfly/c/time.cr index 7b7c5a3b54b7..d4f0d2111e28 100644 --- a/src/lib_c/x86_64-dragonfly/c/time.cr +++ b/src/lib_c/x86_64-dragonfly/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/x86_64-freebsd/c/time.cr b/src/lib_c/x86_64-freebsd/c/time.cr index e0a72c914d82..6b84331c8361 100644 --- a/src/lib_c/x86_64-freebsd/c/time.cr +++ b/src/lib_c/x86_64-freebsd/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/x86_64-linux-gnu/c/time.cr b/src/lib_c/x86_64-linux-gnu/c/time.cr index b93b8e698dd9..710d477e269b 100644 --- a/src/lib_c/x86_64-linux-gnu/c/time.cr +++ b/src/lib_c/x86_64-linux-gnu/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(timer : TimeT*, tp : Tm*) : Tm* fun localtime_r(timer : TimeT*, tp : Tm*) : Tm* fun mktime(tp : Tm*) : TimeT + fun nanosleep(req : Timespec*, rem : Timespec*) : Int fun tzset : Void fun timegm(tp : Tm*) : TimeT diff --git a/src/lib_c/x86_64-linux-musl/c/time.cr b/src/lib_c/x86_64-linux-musl/c/time.cr index 22fdf7a86ebf..f687c8b35db4 100644 --- a/src/lib_c/x86_64-linux-musl/c/time.cr +++ b/src/lib_c/x86_64-linux-musl/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/x86_64-netbsd/c/time.cr b/src/lib_c/x86_64-netbsd/c/time.cr index 17fb6b2dcaa6..a0f11bb50283 100644 --- a/src/lib_c/x86_64-netbsd/c/time.cr +++ b/src/lib_c/x86_64-netbsd/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r = __gmtime_r50(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r = __localtime_r50(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime = __mktime50(x0 : Tm*) : TimeT + fun nanosleep = __nanosleep50(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm = __timegm50(x0 : Tm*) : TimeT diff --git a/src/lib_c/x86_64-openbsd/c/time.cr b/src/lib_c/x86_64-openbsd/c/time.cr index 704a722c2a7e..e7979bfba679 100644 --- a/src/lib_c/x86_64-openbsd/c/time.cr +++ b/src/lib_c/x86_64-openbsd/c/time.cr @@ -28,6 +28,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT diff --git a/src/lib_c/x86_64-solaris/c/time.cr b/src/lib_c/x86_64-solaris/c/time.cr index c8fc7ea9231f..531f8e373f4b 100644 --- a/src/lib_c/x86_64-solaris/c/time.cr +++ b/src/lib_c/x86_64-solaris/c/time.cr @@ -26,6 +26,7 @@ lib LibC fun gmtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun localtime_r(x0 : TimeT*, x1 : Tm*) : Tm* fun mktime(x0 : Tm*) : TimeT + fun nanosleep(x0 : Timespec*, x1 : Timespec*) : Int fun tzset : Void fun timegm(x0 : Tm*) : TimeT From a82dd3ae23a14bf1be9255142d7364e192388074 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Sun, 16 Jun 2024 22:31:44 +0200 Subject: [PATCH 09/52] Extract system implementation of `UNIXSocket.pair` as `Crystal::System::Socket.socketpair` (#14675) Co-authored-by: Sijawusz Pur Rahnama --- src/crystal/system/socket.cr | 2 ++ src/crystal/system/unix/socket.cr | 20 ++++++++++++++++++++ src/crystal/system/wasi/socket.cr | 4 ++++ src/crystal/system/win32/socket.cr | 4 ++++ src/socket/unix_socket.cr | 24 +++--------------------- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/crystal/system/socket.cr b/src/crystal/system/socket.cr index 7e7b939fbeae..2669b4c57bca 100644 --- a/src/crystal/system/socket.cr +++ b/src/crystal/system/socket.cr @@ -79,6 +79,8 @@ module Crystal::System::Socket # def self.fcntl(fd, cmd, arg = 0) + # def self.socketpair(type : ::Socket::Type, protocol : ::Socket::Protocol) : {Handle, Handle} + private def system_read(slice : Bytes) : Int32 event_loop.read(self, slice) end diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index ad065bf3ba23..4f010d7d29f6 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -179,6 +179,26 @@ module Crystal::System::Socket r end + def self.socketpair(type : ::Socket::Type, protocol : ::Socket::Protocol) : {Handle, Handle} + fds = uninitialized Handle[2] + socktype = type.value + + {% if LibC.has_constant?(:SOCK_CLOEXEC) %} + socktype |= LibC::SOCK_CLOEXEC + {% end %} + + if LibC.socketpair(::Socket::Family::UNIX, socktype, protocol, fds) != 0 + raise ::Socket::Error.new("socketpair() failed") + end + + {% unless LibC.has_constant?(:SOCK_CLOEXEC) %} + fcntl(fds[0], LibC::F_SETFD, LibC::FD_CLOEXEC) + fcntl(fds[1], LibC::F_SETFD, LibC::FD_CLOEXEC) + {% end %} + + {fds[0], fds[1]} + end + private def system_tty? LibC.isatty(fd) == 1 end diff --git a/src/crystal/system/wasi/socket.cr b/src/crystal/system/wasi/socket.cr index 901e8a4db1cb..712f3538d249 100644 --- a/src/crystal/system/wasi/socket.cr +++ b/src/crystal/system/wasi/socket.cr @@ -135,6 +135,10 @@ module Crystal::System::Socket r end + def self.socketpair(type : ::Socket::Type, protocol : ::Socket::Protocol) : {Handle, Handle} + raise NotImplementedError.new("Crystal::System::Socket.socketpair") + end + private def system_tty? LibC.isatty(fd) == 1 end diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index a39382c252d6..623ec0ae8954 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -363,6 +363,10 @@ module Crystal::System::Socket raise NotImplementedError.new "Crystal::System::Socket.fcntl" end + def self.socketpair(type : ::Socket::Type, protocol : ::Socket::Protocol) : {Handle, Handle} + raise NotImplementedError.new("Crystal::System::Socket.socketpair") + end + private def system_tty? LibC.GetConsoleMode(LibC::HANDLE.new(fd), out _) != 0 end diff --git a/src/socket/unix_socket.cr b/src/socket/unix_socket.cr index e672d812f631..201fd8410bf7 100644 --- a/src/socket/unix_socket.cr +++ b/src/socket/unix_socket.cr @@ -68,27 +68,9 @@ class UNIXSocket < Socket # left.gets # => "message" # ``` def self.pair(type : Type = Type::STREAM) : {UNIXSocket, UNIXSocket} - {% if flag?(:wasm32) || flag?(:win32) %} - raise NotImplementedError.new "UNIXSocket.pair" - {% else %} - fds = uninitialized Int32[2] - - socktype = type.value - {% if LibC.has_constant?(:SOCK_CLOEXEC) %} - socktype |= LibC::SOCK_CLOEXEC - {% end %} - - if LibC.socketpair(Family::UNIX, socktype, 0, fds) != 0 - raise Socket::Error.new("socketpair() failed") - end - - {% unless LibC.has_constant?(:SOCK_CLOEXEC) %} - Crystal::System::Socket.fcntl(fds[0], LibC::F_SETFD, LibC::FD_CLOEXEC) - Crystal::System::Socket.fcntl(fds[1], LibC::F_SETFD, LibC::FD_CLOEXEC) - {% end %} - - {UNIXSocket.new(fd: fds[0], type: type), UNIXSocket.new(fd: fds[1], type: type)} - {% end %} + Crystal::System::Socket + .socketpair(type, Protocol::IP) + .map { |fd| UNIXSocket.new(fd: fd, type: type) } end # Creates a pair of unnamed UNIX sockets (see `pair`) and yields them to the From 09e8a01273ab7d1a3a3b0cd6457979dd7b31b645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sun, 16 Jun 2024 23:24:41 +0200 Subject: [PATCH 10/52] Update distribution-scripts (#14714) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/fe82a34ad7855ddb432a26ef7e48c46e7b440e49 This includes the following changes: * crystal-lang/distribution-scripts#312 * crystal-lang/distribution-scripts#315 * crystal-lang/distribution-scripts#314 * crystal-lang/distribution-scripts#284 * crystal-lang/distribution-scripts#283 * crystal-lang/distribution-scripts#282 * crystal-lang/distribution-scripts#313 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0ad974b4f20..cf6d612d61b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "7a013f14ed64e7e569b5e453eab02af63cf62b61" + default: "fe82a34ad7855ddb432a26ef7e48c46e7b440e49" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string From e5032d0e55e076fbf08548f4cad6ea7651e53d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 18 Jun 2024 09:17:35 +0200 Subject: [PATCH 11/52] Fix `IO::FileDescriptor.new` for closed fd (#14697) Skips any operations for detecting/setting blocking mode in `IO::FileDescriptor.new` when the file descriptor is closed. --- spec/std/io/file_descriptor_spec.cr | 11 +++++++++++ src/crystal/system/win32/file_descriptor.cr | 15 +++++++++++++-- src/io/file_descriptor.cr | 5 +++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/spec/std/io/file_descriptor_spec.cr b/spec/std/io/file_descriptor_spec.cr index f64d5fd2d8b9..e497ac1061a3 100644 --- a/spec/std/io/file_descriptor_spec.cr +++ b/spec/std/io/file_descriptor_spec.cr @@ -14,6 +14,17 @@ private def shell_command(command) end describe IO::FileDescriptor do + describe "#initialize" do + it "handles closed file descriptor gracefully" do + a, b = IO.pipe + a.close + b.close + + fd = IO::FileDescriptor.new(a.fd) + fd.closed?.should be_true + end + end + it "reopen STDIN with the right mode", tags: %w[slow] do code = %q(puts "#{STDIN.blocking} #{STDIN.info.type}") compile_source(code) do |binpath| diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index c836391e49ef..dc8d479532be 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -90,8 +90,19 @@ module Crystal::System::FileDescriptor raise NotImplementedError.new("Crystal::System::FileDescriptor#system_close_on_exec=") if close_on_exec end - private def system_closed? - false + private def system_closed? : Bool + file_type = LibC.GetFileType(windows_handle) + + if file_type == LibC::FILE_TYPE_UNKNOWN + case error = WinError.value + when .error_invalid_handle? + return true + else + raise IO::Error.from_os_error("Unable to get info", error, target: self) + end + else + false + end end def self.fcntl(fd, cmd, arg = 0) diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index bdcc6cafde91..d4459e9bbe0c 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -41,8 +41,13 @@ class IO::FileDescriptor < IO def initialize(fd : Handle, blocking = nil, *, @close_on_finalize = true) @volatile_fd = Atomic.new(fd) + @closed = true # This is necessary so we can reference `self` in `system_closed?` (in case of an exception) + + # TODO: Refactor to avoid calling `GetFileType` twice on Windows (once in `system_closed?` and once in `system_info`) @closed = system_closed? + return if @closed + if blocking.nil? blocking = case system_info.type From 245edd15476d50cb6fc28d82cc5d422819ee9496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 18 Jun 2024 09:17:50 +0200 Subject: [PATCH 12/52] Add specs for `Pointer::Appender` (#14719) --- spec/std/pointer/appender_spec.cr | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 spec/std/pointer/appender_spec.cr diff --git a/spec/std/pointer/appender_spec.cr b/spec/std/pointer/appender_spec.cr new file mode 100644 index 000000000000..02ca18e0188e --- /dev/null +++ b/spec/std/pointer/appender_spec.cr @@ -0,0 +1,28 @@ +require "spec" + +describe Pointer::Appender do + it ".new" do + Pointer::Appender.new(Pointer(Void).null) + end + + it "#<<" do + data = Slice(Int32).new(5) + appender = data.to_unsafe.appender + 4.times do |i| + appender << (i + 1) * 2 + end + + data.should eq Slice[2, 4, 6, 8, 0] + end + + it "#size" do + data = Slice(Int32).new(5) + appender = data.to_unsafe.appender + appender.size.should eq 0 + 4.times do |i| + appender << 0 + appender.size.should eq i + 1 + end + appender.size.should eq 4 + end +end From 1f3a4fa54c138eafec9b42eaac37454678260cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 18 Jun 2024 22:52:15 +0200 Subject: [PATCH 13/52] Remove calls to `Pointer.new(Int)` (#14683) --- src/crystal/system/win32/thread_mutex.cr | 2 +- src/crystal/tracing.cr | 2 +- src/lib_c/aarch64-android/c/sys/mman.cr | 2 +- src/lib_c/aarch64-darwin/c/dlfcn.cr | 2 +- src/lib_c/aarch64-darwin/c/sys/mman.cr | 2 +- src/lib_c/aarch64-linux-gnu/c/sys/mman.cr | 2 +- src/lib_c/aarch64-linux-musl/c/dlfcn.cr | 2 +- src/lib_c/aarch64-linux-musl/c/sys/mman.cr | 2 +- src/lib_c/arm-linux-gnueabihf/c/sys/mman.cr | 2 +- src/lib_c/i386-linux-gnu/c/sys/mman.cr | 2 +- src/lib_c/i386-linux-musl/c/dlfcn.cr | 2 +- src/lib_c/i386-linux-musl/c/sys/mman.cr | 2 +- src/lib_c/x86_64-darwin/c/dlfcn.cr | 2 +- src/lib_c/x86_64-darwin/c/sys/mman.cr | 2 +- src/lib_c/x86_64-dragonfly/c/dlfcn.cr | 2 +- src/lib_c/x86_64-dragonfly/c/sys/mman.cr | 2 +- src/lib_c/x86_64-freebsd/c/dlfcn.cr | 2 +- src/lib_c/x86_64-freebsd/c/sys/mman.cr | 2 +- src/lib_c/x86_64-linux-gnu/c/sys/mman.cr | 2 +- src/lib_c/x86_64-linux-musl/c/dlfcn.cr | 2 +- src/lib_c/x86_64-linux-musl/c/sys/mman.cr | 2 +- src/lib_c/x86_64-netbsd/c/dlfcn.cr | 2 +- src/lib_c/x86_64-netbsd/c/sys/mman.cr | 2 +- src/lib_c/x86_64-openbsd/c/dlfcn.cr | 2 +- src/lib_c/x86_64-openbsd/c/sys/mman.cr | 2 +- src/lib_c/x86_64-solaris/c/dlfcn.cr | 2 +- src/lib_c/x86_64-solaris/c/sys/mman.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/handleapi.cr | 2 +- src/lib_c/x86_64-windows-msvc/c/winreg.cr | 14 +++++++------- 29 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/crystal/system/win32/thread_mutex.cr b/src/crystal/system/win32/thread_mutex.cr index 559af6acb4f0..44c1ab8a9679 100644 --- a/src/crystal/system/win32/thread_mutex.cr +++ b/src/crystal/system/win32/thread_mutex.cr @@ -37,7 +37,7 @@ class Thread def unlock : Nil # `owningThread` is declared as `LibC::HANDLE` for historical reasons, so # the following comparison is correct - unless @cs.owningThread == LibC::HANDLE.new(LibC.GetCurrentThreadId) + unless @cs.owningThread == LibC::HANDLE.new(LibC.GetCurrentThreadId.to_u64!) raise RuntimeError.new "Attempt to unlock a mutex locked by another thread" end LibC.LeaveCriticalSection(self) diff --git a/src/crystal/tracing.cr b/src/crystal/tracing.cr index 708956ad8feb..ad3ae184a54a 100644 --- a/src/crystal/tracing.cr +++ b/src/crystal/tracing.cr @@ -141,7 +141,7 @@ module Crystal {% if flag?(:win32) %} handle = LibC.CreateFileW(filename, LibC::FILE_GENERIC_WRITE, LibC::DEFAULT_SHARE_MODE, nil, LibC::CREATE_ALWAYS, LibC::FILE_ATTRIBUTE_NORMAL, LibC::HANDLE.null) # not using LibC::INVALID_HANDLE_VALUE because it doesn't exist (yet) - return handle.address unless handle == LibC::HANDLE.new(-1) + return handle.address unless handle == LibC::HANDLE.new(-1.to_u64!) error = uninitialized UInt16[256] len = LibC.FormatMessageW(LibC::FORMAT_MESSAGE_FROM_SYSTEM, nil, WinError.value, 0, error, error.size, nil) diff --git a/src/lib_c/aarch64-android/c/sys/mman.cr b/src/lib_c/aarch64-android/c/sys/mman.cr index b38ec92b9f0e..cf8525cbf3a9 100644 --- a/src/lib_c/aarch64-android/c/sys/mman.cr +++ b/src/lib_c/aarch64-android/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = LibC::MAP_ANONYMOUS MAP_ANONYMOUS = 0x20 - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/aarch64-darwin/c/dlfcn.cr b/src/lib_c/aarch64-darwin/c/dlfcn.cr index e4f1dffc933e..25c53eaeba2a 100644 --- a/src/lib_c/aarch64-darwin/c/dlfcn.cr +++ b/src/lib_c/aarch64-darwin/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 0x8 RTLD_LOCAL = 0x4 RTLD_DEFAULT = Pointer(Void).new(-2) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/aarch64-darwin/c/sys/mman.cr b/src/lib_c/aarch64-darwin/c/sys/mman.cr index e9e65125c3eb..dc5f1e79d6c7 100644 --- a/src/lib_c/aarch64-darwin/c/sys/mman.cr +++ b/src/lib_c/aarch64-darwin/c/sys/mman.cr @@ -9,7 +9,7 @@ lib LibC MAP_PRIVATE = 0x0002 MAP_SHARED = 0x0001 MAP_ANON = 0x1000 - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/aarch64-linux-gnu/c/sys/mman.cr b/src/lib_c/aarch64-linux-gnu/c/sys/mman.cr index 8c44b210a24e..0b6e318b8d2d 100644 --- a/src/lib_c/aarch64-linux-gnu/c/sys/mman.cr +++ b/src/lib_c/aarch64-linux-gnu/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = LibC::MAP_ANONYMOUS MAP_ANONYMOUS = 0x20 - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/aarch64-linux-musl/c/dlfcn.cr b/src/lib_c/aarch64-linux-musl/c/dlfcn.cr index 2f48a19e8092..b0dc20d93f1a 100644 --- a/src/lib_c/aarch64-linux-musl/c/dlfcn.cr +++ b/src/lib_c/aarch64-linux-musl/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 256 RTLD_LOCAL = 0 RTLD_DEFAULT = Pointer(Void).new(0) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/aarch64-linux-musl/c/sys/mman.cr b/src/lib_c/aarch64-linux-musl/c/sys/mman.cr index 4dd11dad0918..b0ce2d629a81 100644 --- a/src/lib_c/aarch64-linux-musl/c/sys/mman.cr +++ b/src/lib_c/aarch64-linux-musl/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = 0x20 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 0 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/arm-linux-gnueabihf/c/sys/mman.cr b/src/lib_c/arm-linux-gnueabihf/c/sys/mman.cr index 8c44b210a24e..0b6e318b8d2d 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/sys/mman.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = LibC::MAP_ANONYMOUS MAP_ANONYMOUS = 0x20 - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/i386-linux-gnu/c/sys/mman.cr b/src/lib_c/i386-linux-gnu/c/sys/mman.cr index 158228f6946d..2c2678495b1a 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/mman.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = LibC::MAP_ANONYMOUS MAP_ANONYMOUS = 0x20 - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/i386-linux-musl/c/dlfcn.cr b/src/lib_c/i386-linux-musl/c/dlfcn.cr index 2f48a19e8092..b0dc20d93f1a 100644 --- a/src/lib_c/i386-linux-musl/c/dlfcn.cr +++ b/src/lib_c/i386-linux-musl/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 256 RTLD_LOCAL = 0 RTLD_DEFAULT = Pointer(Void).new(0) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/i386-linux-musl/c/sys/mman.cr b/src/lib_c/i386-linux-musl/c/sys/mman.cr index 4dd11dad0918..b0ce2d629a81 100644 --- a/src/lib_c/i386-linux-musl/c/sys/mman.cr +++ b/src/lib_c/i386-linux-musl/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = 0x20 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 0 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/x86_64-darwin/c/dlfcn.cr b/src/lib_c/x86_64-darwin/c/dlfcn.cr index e4f1dffc933e..25c53eaeba2a 100644 --- a/src/lib_c/x86_64-darwin/c/dlfcn.cr +++ b/src/lib_c/x86_64-darwin/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 0x8 RTLD_LOCAL = 0x4 RTLD_DEFAULT = Pointer(Void).new(-2) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/x86_64-darwin/c/sys/mman.cr b/src/lib_c/x86_64-darwin/c/sys/mman.cr index 934bd88ff5ad..1d2717b0061d 100644 --- a/src/lib_c/x86_64-darwin/c/sys/mman.cr +++ b/src/lib_c/x86_64-darwin/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x0001 MAP_ANON = 0x1000 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/x86_64-dragonfly/c/dlfcn.cr b/src/lib_c/x86_64-dragonfly/c/dlfcn.cr index 035ccf873319..fe95d81f85a1 100644 --- a/src/lib_c/x86_64-dragonfly/c/dlfcn.cr +++ b/src/lib_c/x86_64-dragonfly/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 0x100 RTLD_LOCAL = 0 RTLD_DEFAULT = Pointer(Void).new(-2) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/x86_64-dragonfly/c/sys/mman.cr b/src/lib_c/x86_64-dragonfly/c/sys/mman.cr index eafa58cc00d3..06f2643d4788 100644 --- a/src/lib_c/x86_64-dragonfly/c/sys/mman.cr +++ b/src/lib_c/x86_64-dragonfly/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x0001 MAP_ANON = 0x1000 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_NORMAL = LibC::MADV_NORMAL POSIX_MADV_RANDOM = LibC::MADV_RANDOM POSIX_MADV_SEQUENTIAL = LibC::MADV_SEQUENTIAL diff --git a/src/lib_c/x86_64-freebsd/c/dlfcn.cr b/src/lib_c/x86_64-freebsd/c/dlfcn.cr index 035ccf873319..fe95d81f85a1 100644 --- a/src/lib_c/x86_64-freebsd/c/dlfcn.cr +++ b/src/lib_c/x86_64-freebsd/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 0x100 RTLD_LOCAL = 0 RTLD_DEFAULT = Pointer(Void).new(-2) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/x86_64-freebsd/c/sys/mman.cr b/src/lib_c/x86_64-freebsd/c/sys/mman.cr index 4990727db9c5..dfada5da1552 100644 --- a/src/lib_c/x86_64-freebsd/c/sys/mman.cr +++ b/src/lib_c/x86_64-freebsd/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x0001 MAP_ANON = 0x1000 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/x86_64-linux-gnu/c/sys/mman.cr b/src/lib_c/x86_64-linux-gnu/c/sys/mman.cr index 8c44b210a24e..0b6e318b8d2d 100644 --- a/src/lib_c/x86_64-linux-gnu/c/sys/mman.cr +++ b/src/lib_c/x86_64-linux-gnu/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = LibC::MAP_ANONYMOUS MAP_ANONYMOUS = 0x20 - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/x86_64-linux-musl/c/dlfcn.cr b/src/lib_c/x86_64-linux-musl/c/dlfcn.cr index 2f48a19e8092..b0dc20d93f1a 100644 --- a/src/lib_c/x86_64-linux-musl/c/dlfcn.cr +++ b/src/lib_c/x86_64-linux-musl/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 256 RTLD_LOCAL = 0 RTLD_DEFAULT = Pointer(Void).new(0) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/x86_64-linux-musl/c/sys/mman.cr b/src/lib_c/x86_64-linux-musl/c/sys/mman.cr index 4dd11dad0918..b0ce2d629a81 100644 --- a/src/lib_c/x86_64-linux-musl/c/sys/mman.cr +++ b/src/lib_c/x86_64-linux-musl/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x01 MAP_ANON = 0x20 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 0 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/x86_64-netbsd/c/dlfcn.cr b/src/lib_c/x86_64-netbsd/c/dlfcn.cr index cbdf854f1912..abb0c0fcc951 100644 --- a/src/lib_c/x86_64-netbsd/c/dlfcn.cr +++ b/src/lib_c/x86_64-netbsd/c/dlfcn.cr @@ -3,7 +3,7 @@ lib LibC RTLD_NOW = 2 RTLD_GLOBAL = 0x100 RTLD_LOCAL = 0x200 - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) RTLD_DEFAULT = Pointer(Void).new(-2) struct DlInfo diff --git a/src/lib_c/x86_64-netbsd/c/sys/mman.cr b/src/lib_c/x86_64-netbsd/c/sys/mman.cr index 2c6675659c2f..3557a00ab788 100644 --- a/src/lib_c/x86_64-netbsd/c/sys/mman.cr +++ b/src/lib_c/x86_64-netbsd/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_FIXED = 0x0010 MAP_ANON = 0x1000 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) MAP_STACK = 0x2000 POSIX_MADV_NORMAL = 0 POSIX_MADV_RANDOM = 1 diff --git a/src/lib_c/x86_64-openbsd/c/dlfcn.cr b/src/lib_c/x86_64-openbsd/c/dlfcn.cr index 595a6e059563..8c6bbe3fc7e6 100644 --- a/src/lib_c/x86_64-openbsd/c/dlfcn.cr +++ b/src/lib_c/x86_64-openbsd/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 0x100 RTLD_LOCAL = 0x000 RTLD_DEFAULT = Pointer(Void).new(-2) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/x86_64-openbsd/c/sys/mman.cr b/src/lib_c/x86_64-openbsd/c/sys/mman.cr index 4b6714e7efa1..7c857527adbf 100644 --- a/src/lib_c/x86_64-openbsd/c/sys/mman.cr +++ b/src/lib_c/x86_64-openbsd/c/sys/mman.cr @@ -10,7 +10,7 @@ lib LibC MAP_SHARED = 0x0001 MAP_ANON = 0x1000 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) MAP_STACK = 0x4000 POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 diff --git a/src/lib_c/x86_64-solaris/c/dlfcn.cr b/src/lib_c/x86_64-solaris/c/dlfcn.cr index 3afc6fd37cbf..792f4d4fcb33 100644 --- a/src/lib_c/x86_64-solaris/c/dlfcn.cr +++ b/src/lib_c/x86_64-solaris/c/dlfcn.cr @@ -4,7 +4,7 @@ lib LibC RTLD_GLOBAL = 0x00100 RTLD_LOCAL = 0x00000 RTLD_DEFAULT = Pointer(Void).new(-2) - RTLD_NEXT = Pointer(Void).new(-1) + RTLD_NEXT = Pointer(Void).new(-1.to_u64!) struct DlInfo dli_fname : Char* diff --git a/src/lib_c/x86_64-solaris/c/sys/mman.cr b/src/lib_c/x86_64-solaris/c/sys/mman.cr index 55f912792fb8..c2319455c16f 100644 --- a/src/lib_c/x86_64-solaris/c/sys/mman.cr +++ b/src/lib_c/x86_64-solaris/c/sys/mman.cr @@ -12,7 +12,7 @@ lib LibC MAP_ANON = 0x100 MAP_ANONYMOUS = LibC::MAP_ANON - MAP_FAILED = Pointer(Void).new(-1) + MAP_FAILED = Pointer(Void).new(-1.to_u64!) POSIX_MADV_DONTNEED = 4 POSIX_MADV_NORMAL = 0 diff --git a/src/lib_c/x86_64-windows-msvc/c/handleapi.cr b/src/lib_c/x86_64-windows-msvc/c/handleapi.cr index c2d02e741c27..527a5ba94a58 100644 --- a/src/lib_c/x86_64-windows-msvc/c/handleapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/handleapi.cr @@ -1,7 +1,7 @@ require "c/winnt" lib LibC - INVALID_HANDLE_VALUE = HANDLE.new(-1) + INVALID_HANDLE_VALUE = HANDLE.new(-1.to_u64!) fun CloseHandle(hObject : HANDLE) : BOOL diff --git a/src/lib_c/x86_64-windows-msvc/c/winreg.cr b/src/lib_c/x86_64-windows-msvc/c/winreg.cr index cdcdd6f1a64a..0be83b90b707 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winreg.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winreg.cr @@ -18,13 +18,13 @@ lib LibC QWORD_LITTLE_ENDIAN = QWORD end - HKEY_CLASSES_ROOT = Pointer(Void).new(0x80000000).as(HKEY) - HKEY_CURRENT_USER = Pointer(Void).new(0x80000001).as(HKEY) - HKEY_LOCAL_MACHINE = Pointer(Void).new(0x80000002).as(HKEY) - HKEY_USERS = Pointer(Void).new(0x80000003).as(HKEY) - HKEY_PERFORMANCE_DATA = Pointer(Void).new(0x80000004).as(HKEY) - HKEY_CURRENT_CONFIG = Pointer(Void).new(0x80000005).as(HKEY) - HKEY_DYN_DATA = Pointer(Void).new(0x8000006).as(HKEY) + HKEY_CLASSES_ROOT = Pointer(Void).new(0x80000000_u64).as(HKEY) + HKEY_CURRENT_USER = Pointer(Void).new(0x80000001_u64).as(HKEY) + HKEY_LOCAL_MACHINE = Pointer(Void).new(0x80000002_u64).as(HKEY) + HKEY_USERS = Pointer(Void).new(0x80000003_u64).as(HKEY) + HKEY_PERFORMANCE_DATA = Pointer(Void).new(0x80000004_u64).as(HKEY) + HKEY_CURRENT_CONFIG = Pointer(Void).new(0x80000005_u64).as(HKEY) + HKEY_DYN_DATA = Pointer(Void).new(0x8000006_u64).as(HKEY) fun RegOpenKeyExW(hKey : HKEY, lpSubKey : LPWSTR, ulOptions : DWORD, samDesired : REGSAM, phkResult : HKEY*) : LSTATUS fun RegCloseKey(hKey : HKEY) : LSTATUS From ded40c46353233a37c6086ff8a2d699c3c7ab2fb Mon Sep 17 00:00:00 2001 From: meatball <69751659+meatball133@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:54:05 +0200 Subject: [PATCH 14/52] Fix result formatting in code example for `Indexable#[]?` (#14721) --- src/indexable.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/indexable.cr b/src/indexable.cr index d39ddaaef197..4a3990e83870 100644 --- a/src/indexable.cr +++ b/src/indexable.cr @@ -101,8 +101,8 @@ module Indexable(T) # ary[-1]? # => 'c' # ary[-2]? # => 'b' # - # ary[3]? # nil - # ary[-4]? # nil + # ary[3]? # => nil + # ary[-4]? # => nil # ``` @[AlwaysInline] def []?(index : Int) From e0754ca20a5ad2dbf202106383c7fbf677701776 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Thu, 20 Jun 2024 08:54:56 +0200 Subject: [PATCH 15/52] Add process rwlock to wrap fork/exec on Darwin (#14674) In addition to the standard `O_CLOEXEC` flag to `open` (POSIX.1-2008), most modern POSIX systems implement non-standard syscalls (`accept4`, `dup3` and `pipe2`) along with the `SOCK_CLOEXEC` flag to atomically create file descriptors with the `FD_CLOEXEC` flag. A notable exception is Darwin that only implements `O_CLOEXEC`. We thus have to support falling back to `accept`, `dup2` and `pipe` that won't set `FD_CLOEXEC` or `SOCK_CLOEXEC` atomically, which creates a time window during which another thread may fork the process before `FD_CLOEXEC` is set, which will leak the file descriptor to a child process. This patch introduces a RWLock to prevent fork/exec in such situations. **Prior art**: Go does exactly that. --- .../system/unix/event_loop_libevent.cr | 15 ++++-- src/crystal/system/unix/file_descriptor.cr | 18 ++++--- src/crystal/system/unix/process.cr | 49 ++++++++++++++++++- src/crystal/system/unix/socket.cr | 43 ++++++++-------- 4 files changed, 90 insertions(+), 35 deletions(-) diff --git a/src/crystal/system/unix/event_loop_libevent.cr b/src/crystal/system/unix/event_loop_libevent.cr index 3d8cecf694f2..32c9c8409b17 100644 --- a/src/crystal/system/unix/event_loop_libevent.cr +++ b/src/crystal/system/unix/event_loop_libevent.cr @@ -152,7 +152,17 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop {% if LibC.has_method?(:accept4) %} LibC.accept4(socket.fd, nil, nil, LibC::SOCK_CLOEXEC) {% else %} - LibC.accept(socket.fd, nil, nil) + # we may fail to set FD_CLOEXEC between `accept` and `fcntl` but we + # can't call `Crystal::System::Socket.lock_read` because the socket + # might be in blocking mode and accept would block until the socket + # receives a connection. + # + # we could lock when `socket.blocking?` is false, but another thread + # could change the socket back to blocking mode between the condition + # check and the `accept` call. + fd = LibC.accept(socket.fd, nil, nil) + Crystal::System::Socket.fcntl(fd, LibC::F_SETFD, LibC::FD_CLOEXEC) unless fd == -1 + fd {% end %} if client_fd == -1 @@ -167,9 +177,6 @@ class Crystal::LibEvent::EventLoop < Crystal::EventLoop raise ::Socket::Error.from_errno("accept") end else - {% unless LibC.has_method?(:accept4) %} - Crystal::System::Socket.fcntl(client_fd, LibC::F_SETFD, LibC::FD_CLOEXEC) - {% end %} return client_fd end end diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 765f7a989f3d..0c3ece9cfff8 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -97,10 +97,12 @@ module Crystal::System::FileDescriptor raise IO::Error.from_errno("Could not reopen file descriptor") end {% else %} - if LibC.dup2(other.fd, fd) == -1 - raise IO::Error.from_errno("Could not reopen file descriptor") + Process.lock_read do + if LibC.dup2(other.fd, fd) == -1 + raise IO::Error.from_errno("Could not reopen file descriptor") + end + self.close_on_exec = other.close_on_exec? end - self.close_on_exec = other.close_on_exec? {% end %} # Mark the handle open, since we had to have dup'd a live handle. @@ -195,11 +197,13 @@ module Crystal::System::FileDescriptor raise IO::Error.from_errno("Could not create pipe") end {% else %} - if LibC.pipe(pipe_fds) != 0 - raise IO::Error.from_errno("Could not create pipe") + Process.lock_read do + if LibC.pipe(pipe_fds) != 0 + raise IO::Error.from_errno("Could not create pipe") + end + fcntl(pipe_fds[0], LibC::F_SETFD, LibC::FD_CLOEXEC) + fcntl(pipe_fds[1], LibC::F_SETFD, LibC::FD_CLOEXEC) end - fcntl(pipe_fds[0], LibC::F_SETFD, LibC::FD_CLOEXEC) - fcntl(pipe_fds[1], LibC::F_SETFD, LibC::FD_CLOEXEC) {% end %} r = IO::FileDescriptor.new(pipe_fds[0], read_blocking) diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 4fd2b658cd59..f3d5dbf3eddb 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -2,6 +2,7 @@ require "c/signal" require "c/stdlib" require "c/sys/resource" require "c/unistd" +require "crystal/rw_lock" require "file/error" struct Crystal::System::Process @@ -127,6 +128,50 @@ struct Crystal::System::Process ) end + # The RWLock is trying to protect against file descriptors leaking to + # sub-processes. + # + # There is a race condition in the POSIX standard between the creation of a + # file descriptor (`accept`, `dup`, `open`, `pipe`, `socket`) and setting the + # `FD_CLOEXEC` flag with `fcntl`. During the time window between those two + # syscalls, another thread may fork the process and exec another process, + # which will leak the file descriptor to that process. + # + # Most systems have long implemented non standard syscalls that prevent the + # race condition, except for Darwin that implements `O_CLOEXEC` but doesn't + # implement `SOCK_CLOEXEC` nor `accept4`, `dup3` or `pipe2`. + # + # NOTE: there may still be some potential leaks (e.g. calling `accept` on a + # blocking socket). + {% if LibC.has_constant?(:SOCK_CLOEXEC) && LibC.has_method?(:accept4) && LibC.has_method?(:dup3) && LibC.has_method?(:pipe2) %} + # we don't implement .lock_read so compilation will fail if we need to + # support another case, instead of silently skipping the rwlock! + + def self.lock_write(&) + yield + end + {% else %} + @@rwlock = Crystal::RWLock.new + + def self.lock_read(&) + @@rwlock.read_lock + begin + yield + ensure + @@rwlock.read_unlock + end + end + + def self.lock_write(&) + @@rwlock.write_lock + begin + yield + ensure + @@rwlock.write_unlock + end + end + {% end %} + def self.fork(*, will_exec = false) newmask = uninitialized LibC::SigsetT oldmask = uninitialized LibC::SigsetT @@ -135,7 +180,7 @@ struct Crystal::System::Process ret = LibC.pthread_sigmask(LibC::SIG_SETMASK, pointerof(newmask), pointerof(oldmask)) raise RuntimeError.from_errno("Failed to disable signals") unless ret == 0 - case pid = LibC.fork + case pid = lock_write { LibC.fork } when 0 # child: pid = nil @@ -274,7 +319,7 @@ struct Crystal::System::Process argv = command_args.map &.check_no_null_byte.to_unsafe argv << Pointer(UInt8).null - LibC.execvp(command, argv) + lock_write { LibC.execvp(command, argv) } end def self.replace(command_args, env, clear_env, input, output, error, chdir) diff --git a/src/crystal/system/unix/socket.cr b/src/crystal/system/unix/socket.cr index 4f010d7d29f6..33ac70659b9f 100644 --- a/src/crystal/system/unix/socket.cr +++ b/src/crystal/system/unix/socket.cr @@ -9,19 +9,18 @@ module Crystal::System::Socket alias Handle = Int32 private def create_handle(family, type, protocol, blocking) : Handle - socktype = type.value {% if LibC.has_constant?(:SOCK_CLOEXEC) %} - socktype |= LibC::SOCK_CLOEXEC - {% end %} - - fd = LibC.socket(family, socktype, protocol) - raise ::Socket::Error.from_errno("Failed to create socket") if fd == -1 - - {% unless LibC.has_constant?(:SOCK_CLOEXEC) %} - Socket.fcntl(fd, LibC::F_SETFD, LibC::FD_CLOEXEC) + fd = LibC.socket(family, type.value | LibC::SOCK_CLOEXEC, protocol) + raise ::Socket::Error.from_errno("Failed to create socket") if fd == -1 + fd + {% else %} + Process.lock_read do + fd = LibC.socket(family, type, protocol) + raise ::Socket::Error.from_errno("Failed to create socket") if fd == -1 + Socket.fcntl(fd, LibC::F_SETFD, LibC::FD_CLOEXEC) + fd + end {% end %} - - fd end private def initialize_handle(fd) @@ -181,19 +180,19 @@ module Crystal::System::Socket def self.socketpair(type : ::Socket::Type, protocol : ::Socket::Protocol) : {Handle, Handle} fds = uninitialized Handle[2] - socktype = type.value {% if LibC.has_constant?(:SOCK_CLOEXEC) %} - socktype |= LibC::SOCK_CLOEXEC - {% end %} - - if LibC.socketpair(::Socket::Family::UNIX, socktype, protocol, fds) != 0 - raise ::Socket::Error.new("socketpair() failed") - end - - {% unless LibC.has_constant?(:SOCK_CLOEXEC) %} - fcntl(fds[0], LibC::F_SETFD, LibC::FD_CLOEXEC) - fcntl(fds[1], LibC::F_SETFD, LibC::FD_CLOEXEC) + if LibC.socketpair(::Socket::Family::UNIX, type.value | LibC::SOCK_CLOEXEC, protocol, fds) == -1 + raise ::Socket::Error.new("socketpair() failed") + end + {% else %} + Process.lock_read do + if LibC.socketpair(::Socket::Family::UNIX, type, protocol, fds) == -1 + raise ::Socket::Error.new("socketpair() failed") + end + fcntl(fds[0], LibC::F_SETFD, LibC::FD_CLOEXEC) + fcntl(fds[1], LibC::F_SETFD, LibC::FD_CLOEXEC) + end {% end %} {fds[0], fds[1]} From b14be1ebc25513f2ef8ab591288fe91e3b8a87fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 20 Jun 2024 08:56:17 +0200 Subject: [PATCH 16/52] Cleanup `IOCP::OverlappedOperation` (#14723) Light refactor of `IOCP::OverlappedOperation` to simplify the implementation. * Add `OverlappedOperation#to_unsafe` as standard format for passing to C functions * Add `OverlappedOperation.unbox` for the reverse * Drop unnecessary `OverlappedOperation#start` to simplify the logic --- src/crystal/system/win32/event_loop_iocp.cr | 4 +-- src/crystal/system/win32/iocp.cr | 30 +++++++++------------ src/crystal/system/win32/socket.cr | 4 +-- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/crystal/system/win32/event_loop_iocp.cr b/src/crystal/system/win32/event_loop_iocp.cr index d05e15162171..25c8db41d9ff 100644 --- a/src/crystal/system/win32/event_loop_iocp.cr +++ b/src/crystal/system/win32/event_loop_iocp.cr @@ -233,7 +233,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop def connect(socket : ::Socket, address : ::Socket::Addrinfo | ::Socket::Address, timeout : ::Time::Span?) : IO::Error? socket.overlapped_connect(socket.fd, "ConnectEx") do |overlapped| # This is: LibC.ConnectEx(fd, address, address.size, nil, 0, nil, overlapped) - Crystal::System::Socket.connect_ex.call(socket.fd, address.to_unsafe, address.size, Pointer(Void).null, 0_u32, Pointer(UInt32).null, overlapped) + Crystal::System::Socket.connect_ex.call(socket.fd, address.to_unsafe, address.size, Pointer(Void).null, 0_u32, Pointer(UInt32).null, overlapped.to_unsafe) end end @@ -256,7 +256,7 @@ class Crystal::IOCP::EventLoop < Crystal::EventLoop received_bytes = uninitialized UInt32 Crystal::System::Socket.accept_ex.call(socket.fd, client_handle, output_buffer.to_unsafe.as(Void*), buffer_size.to_u32!, - address_size.to_u32!, address_size.to_u32!, pointerof(received_bytes), overlapped) + address_size.to_u32!, address_size.to_u32!, pointerof(received_bytes), overlapped.to_unsafe) end if success diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index 53cf704ad760..780b6f1ac6f0 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -40,7 +40,8 @@ module Crystal::IOCP # I/O operations, including socket ones, do not set this field case completion_key = Pointer(Void).new(entry.lpCompletionKey).as(CompletionKey?) when Nil - OverlappedOperation.schedule(entry.lpOverlapped) { |fiber| yield fiber } + operation = OverlappedOperation.unbox(entry.lpOverlapped) + operation.schedule { |fiber| yield fiber } else case entry.dwNumberOfBytesTransferred when LibC::JOB_OBJECT_MSG_EXIT_PROCESS, LibC::JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS @@ -62,15 +63,14 @@ module Crystal::IOCP class OverlappedOperation enum State - INITIALIZED STARTED DONE CANCELLED end @overlapped = LibC::OVERLAPPED.new - @fiber : Fiber? = nil - @state : State = :initialized + @fiber = Fiber.current + @state : State = :started property next : OverlappedOperation? property previous : OverlappedOperation? @@canceled = Thread::LinkedList(OverlappedOperation).new @@ -84,22 +84,18 @@ module Crystal::IOCP end end - def self.schedule(overlapped : LibC::OVERLAPPED*, &) + def self.unbox(overlapped : LibC::OVERLAPPED*) start = overlapped.as(Pointer(UInt8)) - offsetof(OverlappedOperation, @overlapped) - operation = Box(OverlappedOperation).unbox(start.as(Pointer(Void))) - operation.schedule { |fiber| yield fiber } + Box(OverlappedOperation).unbox(start.as(Pointer(Void))) end - def start - raise Exception.new("Invalid state #{@state}") unless @state.initialized? - @fiber = Fiber.current - @state = State::STARTED + def to_unsafe pointerof(@overlapped) end def result(handle, &) raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? - result = LibC.GetOverlappedResult(handle, pointerof(@overlapped), out bytes, 0) + result = LibC.GetOverlappedResult(handle, self, out bytes, 0) if result.zero? error = WinError.value yield error @@ -113,7 +109,7 @@ module Crystal::IOCP def wsa_result(socket, &) raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? flags = 0_u32 - result = LibC.WSAGetOverlappedResult(socket, pointerof(@overlapped), out bytes, false, pointerof(flags)) + result = LibC.WSAGetOverlappedResult(socket, self, out bytes, false, pointerof(flags)) if result.zero? error = WinError.wsa_value yield error @@ -127,7 +123,7 @@ module Crystal::IOCP protected def schedule(&) case @state when .started? - yield @fiber.not_nil! + yield @fiber done! when .cancelled? @@canceled.delete(self) @@ -144,7 +140,7 @@ module Crystal::IOCP # https://learn.microsoft.com/en-us/windows/win32/api/ioapiset/nf-ioapiset-cancelioex # > The application must not free or reuse the OVERLAPPED structure # associated with the canceled I/O operations until they have completed - if LibC.CancelIoEx(handle, pointerof(@overlapped)) != 0 + if LibC.CancelIoEx(handle, self) != 0 @state = :cancelled @@canceled.push(self) # to increase lifetime end @@ -176,7 +172,7 @@ module Crystal::IOCP def self.overlapped_operation(target, handle, method, timeout, *, writing = false, &) OverlappedOperation.run(handle) do |operation| - result, value = yield operation.start + result, value = yield operation if result == 0 case error = WinError.value @@ -214,7 +210,7 @@ module Crystal::IOCP def self.wsa_overlapped_operation(target, socket, method, timeout, connreset_is_error = true, &) OverlappedOperation.run(socket) do |operation| - result, value = yield operation.start + result, value = yield operation if result == LibC::SOCKET_ERROR case error = WinError.wsa_value diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 623ec0ae8954..c04d3a9ad868 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -130,7 +130,7 @@ module Crystal::System::Socket # :nodoc: def overlapped_connect(socket, method, &) IOCP::OverlappedOperation.run(socket) do |operation| - result = yield operation.start + result = yield operation if result == 0 case error = WinError.wsa_value @@ -196,7 +196,7 @@ module Crystal::System::Socket def overlapped_accept(socket, method, &) IOCP::OverlappedOperation.run(socket) do |operation| - result = yield operation.start + result = yield operation if result == 0 case error = WinError.wsa_value From 2c62b19469247bbaea32a0c365122e243edb0354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 20 Jun 2024 08:56:55 +0200 Subject: [PATCH 17/52] Fix relative file paths in spec output on Windows (#14725) --- src/spec/context.cr | 8 +++++--- src/spec/source.cr | 10 ---------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/spec/context.cr b/src/spec/context.cr index 12501adf8360..1cc473819580 100644 --- a/src/spec/context.cr +++ b/src/spec/context.cr @@ -191,6 +191,8 @@ module Spec failures = results_for(:fail) errors = results_for(:error) + cwd = Dir.current + failures_and_errors = failures + errors unless failures_and_errors.empty? puts @@ -216,7 +218,7 @@ module Spec if ex.is_a?(SpecError) puts - puts Spec.color(" # #{Spec.relative_file(ex.file)}:#{ex.line}", :comment) + puts Spec.color(" # #{Path[ex.file].relative_to(cwd)}:#{ex.line}", :comment) end end end @@ -232,7 +234,7 @@ module Spec top_n.each do |res| puts " #{res.description}" res_elapsed = res.elapsed.not_nil!.total_seconds.humanize - puts " #{res_elapsed.colorize.bold} seconds #{Spec.relative_file(res.file)}:#{res.line}" + puts " #{res_elapsed.colorize.bold} seconds #{Path[res.file].relative_to(cwd)}:#{res.line}" end end @@ -258,7 +260,7 @@ module Spec puts "Failed examples:" puts failures_and_errors.each do |fail| - print Spec.color("crystal spec #{Spec.relative_file(fail.file)}:#{fail.line}", :error) + print Spec.color("crystal spec #{Path[fail.file].relative_to(cwd)}:#{fail.line}", :error) puts Spec.color(" # #{fail.description}", :comment) end end diff --git a/src/spec/source.cr b/src/spec/source.cr index 6db12c936ae4..cf057240abe5 100644 --- a/src/spec/source.cr +++ b/src/spec/source.cr @@ -11,14 +11,4 @@ module Spec lines = lines_cache.put_if_absent(file) { File.read_lines(file) } lines[line - 1]? end - - # :nodoc: - def self.relative_file(file) - cwd = Dir.current - if basename = file.lchop? cwd - basename.lchop '/' - else - file - end - end end From 884f3827814fe000ca80e0d88de6cf6668fb2d89 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Sat, 22 Jun 2024 15:26:46 -0400 Subject: [PATCH 18/52] Allow new formatter styles for trailing comma and whitespace around proc literal (#14726) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In preparation of changing the formatter style, this change accepts the new style without enforcing it. Existing code is not affected (this will happen in a follow-up). Co-authored-by: Johannes Müller --- spec/compiler/formatter/formatter_spec.cr | 50 ++++++++++++++++++++--- src/compiler/crystal/tools/formatter.cr | 7 ++-- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 7bb7a1034e72..7c332aac3b0a 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -1,8 +1,8 @@ require "spec" require "../../../src/compiler/crystal/formatter" -private def assert_format(input, output = input, strict = false, flags = nil, file = __FILE__, line = __LINE__) - it "formats #{input.inspect}", file, line do +private def assert_format(input, output = input, strict = false, flags = nil, file = __FILE__, line = __LINE__, focus = false) + it "formats #{input.inspect}", file, line, focus: focus do output = "#{output}\n" unless strict result = Crystal.format(input, flags: flags) unless result == output @@ -812,7 +812,7 @@ describe Crystal::Formatter do end CRYSTAL def foo(x, - y) + y,) yield end CRYSTAL @@ -888,7 +888,7 @@ describe Crystal::Formatter do end CRYSTAL def foo( - x + x, ) yield end @@ -901,6 +901,39 @@ describe Crystal::Formatter do CRYSTAL end + # Allows trailing commas, but doesn't enforce them + assert_format <<-CRYSTAL + def foo( + a, + b + ) + end + CRYSTAL + + assert_format <<-CRYSTAL + def foo( + a, + b, + ) + end + CRYSTAL + + assert_format <<-CRYSTAL + macro foo( + a, + *b, + ) + end + CRYSTAL + + assert_format <<-CRYSTAL + macro foo( + a, + **b, + ) + end + CRYSTAL + context "adds trailing comma to def multi-line normal, splat, and double splat parameters" do assert_format <<-CRYSTAL, <<-CRYSTAL, flags: %w[def_trailing_comma] macro foo( @@ -1693,6 +1726,13 @@ describe Crystal::Formatter do assert_format "-> : Int32 {}", "-> : Int32 { }", flags: %w[proc_literal_whitespace] assert_format "->do\nend", "-> do\nend", flags: %w[proc_literal_whitespace] + # Allows whitespace around proc literal, but doesn't enforce them + assert_format "-> { }" + assert_format "-> { 1 }" + assert_format "->(x : Int32) { }" + assert_format "-> : Int32 { }" + assert_format "-> do\nend" + assert_format "-> : Int32 {}" assert_format "-> : Int32 | String { 1 }" assert_format "-> : Array(Int32) {}" @@ -1703,7 +1743,7 @@ describe Crystal::Formatter do assert_format "-> : {Int32} { String }" assert_format "-> : {x: Int32, y: String} {}" assert_format "->\n:\nInt32\n{\n}", "-> : Int32 {\n}" - assert_format "->( x )\n:\nInt32 { }", "->(x) : Int32 {}" + assert_format "->( x )\n:\nInt32 { }", "->(x) : Int32 { }" assert_format "->: Int32 do\nx\nend", "-> : Int32 do\n x\nend" {:+, :-, :*, :/, :^, :>>, :<<, :|, :&, :&+, :&-, :&*, :&**}.each do |sym| diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index dc14c70a90ad..796afe0730de 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -1651,7 +1651,7 @@ module Crystal yield # Write "," before skipping spaces to prevent inserting comment between argument and comma. - write "," if has_more || (write_trailing_comma && flag?("def_trailing_comma")) + write "," if has_more || (wrote_newline && @token.type.op_comma?) || (write_trailing_comma && flag?("def_trailing_comma")) just_wrote_newline = skip_space if @token.type.newline? @@ -4242,6 +4242,7 @@ module Crystal def visit(node : ProcLiteral) write_token :OP_MINUS_GT + whitespace_after_op_minus_gt = @token.type.space? skip_space_or_newline a_def = node.def @@ -4272,7 +4273,7 @@ module Crystal skip_space_or_newline end - write " " if a_def.args.present? || return_type || flag?("proc_literal_whitespace") + write " " if a_def.args.present? || return_type || flag?("proc_literal_whitespace") || whitespace_after_op_minus_gt is_do = false if @token.keyword?(:do) @@ -4280,7 +4281,7 @@ module Crystal is_do = true else write_token :OP_LCURLY - write " " if a_def.body.is_a?(Nop) && flag?("proc_literal_whitespace") + write " " if a_def.body.is_a?(Nop) && (flag?("proc_literal_whitespace") || @token.type.space?) end skip_space From a63e874948a0e26643e7c3e48ac1d7575f87dc93 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 23 Jun 2024 03:27:12 +0800 Subject: [PATCH 19/52] Fix ECR escape sequences containing `-` (#14739) --- spec/std/data/test_template7.ecr | 5 ++ spec/std/ecr/ecr_lexer_spec.cr | 81 ++++++++++++++++++++++++++++++++ spec/std/ecr/ecr_spec.cr | 12 +++++ src/ecr.cr | 3 +- src/ecr/lexer.cr | 36 +++++++------- 5 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 spec/std/data/test_template7.ecr diff --git a/spec/std/data/test_template7.ecr b/spec/std/data/test_template7.ecr new file mode 100644 index 000000000000..0c1a9eff0806 --- /dev/null +++ b/spec/std/data/test_template7.ecr @@ -0,0 +1,5 @@ +<%% if @name %> +Greetings, <%%= @name %>! + <%-% else -%> +Greetings! +<%-% end -%> \ No newline at end of file diff --git a/spec/std/ecr/ecr_lexer_spec.cr b/spec/std/ecr/ecr_lexer_spec.cr index 05e3f5436b93..4a1968b8458f 100644 --- a/spec/std/ecr/ecr_lexer_spec.cr +++ b/spec/std/ecr/ecr_lexer_spec.cr @@ -210,6 +210,87 @@ describe "ECR::Lexer" do token.type.should eq(t :eof) end + it "lexes with <%-% %> (#14734)" do + lexer = ECR::Lexer.new("hello <%-% foo %> bar") + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq("hello ") + token.column_number.should eq(1) + token.line_number.should eq(1) + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq("<%- foo %>") + token.line_number.should eq(1) + token.column_number.should eq(11) + token.suppress_leading?.should be_false + token.suppress_trailing?.should be_false + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq(" bar") + token.line_number.should eq(1) + token.column_number.should eq(18) + + token = lexer.next_token + token.type.should eq(t :eof) + end + + it "lexes with <%-%= %> (#14734)" do + lexer = ECR::Lexer.new("hello <%-%= foo %> bar") + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq("hello ") + token.column_number.should eq(1) + token.line_number.should eq(1) + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq("<%-= foo %>") + token.line_number.should eq(1) + token.column_number.should eq(11) + token.suppress_leading?.should be_false + token.suppress_trailing?.should be_false + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq(" bar") + token.line_number.should eq(1) + token.column_number.should eq(19) + + token = lexer.next_token + token.type.should eq(t :eof) + end + + it "lexes with <%% -%> (#14734)" do + lexer = ECR::Lexer.new("hello <%% foo -%> bar") + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq("hello ") + token.column_number.should eq(1) + token.line_number.should eq(1) + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq("<% foo -%>") + token.line_number.should eq(1) + token.column_number.should eq(10) + token.suppress_leading?.should be_false + token.suppress_trailing?.should be_false + + token = lexer.next_token + token.type.should eq(t :string) + token.value.should eq(" bar") + token.line_number.should eq(1) + token.column_number.should eq(18) + + token = lexer.next_token + token.type.should eq(t :eof) + end + it "lexes with <% %> and correct location info" do lexer = ECR::Lexer.new("hi\nthere <% foo\nbar %> baz") diff --git a/spec/std/ecr/ecr_spec.cr b/spec/std/ecr/ecr_spec.cr index ce424785d805..0e35ea1dd1f1 100644 --- a/spec/std/ecr/ecr_spec.cr +++ b/spec/std/ecr/ecr_spec.cr @@ -65,6 +65,18 @@ describe "ECR" do io.to_s.should eq("string with -%") end + it "does with <%% %>" do + io = IO::Memory.new + ECR.embed "#{__DIR__}/../data/test_template7.ecr", io + io.to_s.should eq(<<-ECR) + <% if @name %> + Greetings, <%= @name %>! + <%- else -%> + Greetings! + <%- end -%> + ECR + end + it ".render" do ECR.render("#{__DIR__}/../data/test_template2.ecr").should eq("123") end diff --git a/src/ecr.cr b/src/ecr.cr index 42bee548b22c..785e47c5a762 100644 --- a/src/ecr.cr +++ b/src/ecr.cr @@ -14,7 +14,8 @@ # # A comment can be created the same as normal code: `<% # hello %>` or by the special # tag: `<%# hello %>`. An ECR tag can be inserted directly (i.e. the tag itself may be -# escaped) by using a second `%` like so: `<%% a = b %>` or `<%%= foo %>`. +# escaped) by using a second `%` like so: `<%% a = b %>` or `<%%= foo %>`. Dashes may +# also be present in those escaped tags and have no effect on the surrounding text. # # NOTE: To use `ECR`, you must explicitly import it with `require "ecr"` # diff --git a/src/ecr/lexer.cr b/src/ecr/lexer.cr index b5a30cae8e84..e32de726040f 100644 --- a/src/ecr/lexer.cr +++ b/src/ecr/lexer.cr @@ -44,12 +44,8 @@ class ECR::Lexer next_char next_char - if current_char == '-' - @token.suppress_leading = true - next_char - else - @token.suppress_leading = false - end + suppress_leading = current_char == '-' + next_char if suppress_leading case current_char when '=' @@ -64,7 +60,7 @@ class ECR::Lexer copy_location_info_to_token end - return consume_control(is_output, is_escape) + return consume_control(is_output, is_escape, suppress_leading) end else # consume string @@ -97,7 +93,7 @@ class ECR::Lexer @token end - private def consume_control(is_output, is_escape) + private def consume_control(is_output, is_escape, suppress_leading) start_pos = current_pos while true case current_char @@ -126,8 +122,7 @@ class ECR::Lexer @column_number = column_number if is_end - @token.suppress_trailing = true - setup_control_token(start_pos, is_escape) + setup_control_token(start_pos, is_escape, suppress_leading, true) raise "Expecting '>' after '-%'" if current_char != '>' next_char break @@ -135,8 +130,7 @@ class ECR::Lexer end when '%' if peek_next_char == '>' - @token.suppress_trailing = false - setup_control_token(start_pos, is_escape) + setup_control_token(start_pos, is_escape, suppress_leading, false) break end else @@ -155,12 +149,18 @@ class ECR::Lexer @token end - private def setup_control_token(start_pos, is_escape) - @token.value = if is_escape - "<%#{string_range(start_pos, current_pos + 2)}" - else - string_range(start_pos) - end + private def setup_control_token(start_pos, is_escape, suppress_leading, suppress_trailing) + @token.suppress_leading = !is_escape && suppress_leading + @token.suppress_trailing = !is_escape && suppress_trailing + @token.value = + if is_escape + head = suppress_leading ? "<%-" : "<%" + tail = string_range(start_pos, current_pos + (suppress_trailing ? 3 : 2)) + head + tail + else + string_range(start_pos) + end + next_char next_char end From 4f31615c50aa694ab46c13608080b9f551704cf7 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 23 Jun 2024 03:27:46 +0800 Subject: [PATCH 20/52] Make `ReferenceStorage(T)` non-atomic if `T` is non-atomic (#14730) --- spec/primitives/pointer_spec.cr | 25 +++++++++++++++++++++++++ src/compiler/crystal/codegen/types.cr | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 spec/primitives/pointer_spec.cr diff --git a/spec/primitives/pointer_spec.cr b/spec/primitives/pointer_spec.cr new file mode 100644 index 000000000000..1b62ec54a8d4 --- /dev/null +++ b/spec/primitives/pointer_spec.cr @@ -0,0 +1,25 @@ +require "spec" +require "../support/finalize" +require "../support/interpreted" + +private class Inner + include FinalizeCounter + + def initialize(@key : String) + end +end + +private class Outer + @inner = Inner.new("reference-storage") +end + +describe "Primitives: pointer" do + describe ".malloc" do + pending_interpreted "is non-atomic for ReferenceStorage(T) if T is non-atomic (#14692)" do + FinalizeState.reset + outer = Outer.unsafe_construct(Pointer(ReferenceStorage(Outer)).malloc(1)) + GC.collect + FinalizeState.count("reference-storage").should eq(0) + end + end +end diff --git a/src/compiler/crystal/codegen/types.cr b/src/compiler/crystal/codegen/types.cr index 654e7a281421..470fe7424dcd 100644 --- a/src/compiler/crystal/codegen/types.cr +++ b/src/compiler/crystal/codegen/types.cr @@ -69,6 +69,8 @@ module Crystal self.tuple_types.any? &.has_inner_pointers? when NamedTupleInstanceType self.entries.any? &.type.has_inner_pointers? + when ReferenceStorageType + self.reference_type.has_inner_pointers? when PrimitiveType false when EnumType From 7d8e2434edde9fde1777612500be81c2e1a3ee12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Cla=C3=9Fen?= Date: Mon, 24 Jun 2024 10:49:50 +0200 Subject: [PATCH 21/52] Fix docs for `CSV::Builder#row(&)` (#14736) --- src/csv/builder.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csv/builder.cr b/src/csv/builder.cr index d0fae5cf6dfe..de53018795ad 100644 --- a/src/csv/builder.cr +++ b/src/csv/builder.cr @@ -51,7 +51,7 @@ class CSV::Builder @first_cell_in_row = true end - # Yields a `CSV::Row` to append a row. A newline is appended + # Yields a `CSV::Builder::Row` to append a row. A newline is appended # to `IO` after the block exits. def row(&) yield Row.new(self, @separator, @quote_char, @quoting) From feb612b6425ed216919c9e7ce796e6319dc5bd4a Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Mon, 24 Jun 2024 04:50:24 -0400 Subject: [PATCH 22/52] Add `Time::Error` (#14743) --- src/time.cr | 6 +++++- src/time/location.cr | 4 ++-- src/time/location/loader.cr | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/time.cr b/src/time.cr index fe5a21d7c77c..4b29114dd190 100644 --- a/src/time.cr +++ b/src/time.cr @@ -212,7 +212,11 @@ require "crystal/system/time" # elapsed_time # => 20.milliseconds (approximately) # ``` struct Time - class FloatingTimeConversionError < Exception + # Raised when an error occurs while performing a `Time` based operation. + class Error < Exception + end + + class FloatingTimeConversionError < Error end include Comparable(Time) diff --git a/src/time/location.cr b/src/time/location.cr index 7e0e8f160cb9..21d1e7a6e56d 100644 --- a/src/time/location.cr +++ b/src/time/location.cr @@ -51,7 +51,7 @@ class Time::Location # the time zone database. # # See `Time::Location.load` for details. - class InvalidLocationNameError < Exception + class InvalidLocationNameError < Time::Error getter name, source def initialize(@name : String, @source : String? = nil) @@ -63,7 +63,7 @@ class Time::Location # `InvalidTimezoneOffsetError` is raised if `Time::Location::Zone.new` # receives an invalid time zone offset. - class InvalidTimezoneOffsetError < Exception + class InvalidTimezoneOffsetError < Time::Error def initialize(offset : Int) super "Invalid time zone offset: #{offset}" end diff --git a/src/time/location/loader.cr b/src/time/location/loader.cr index 6555125e6ff7..6a104101405c 100644 --- a/src/time/location/loader.cr +++ b/src/time/location/loader.cr @@ -5,7 +5,7 @@ class Time::Location # time zone data. # # Details on the exact cause can be found in the error message. - class InvalidTZDataError < Exception + class InvalidTZDataError < Time::Error def self.initialize(message : String? = "Malformed time zone information", cause : Exception? = nil) super(message, cause) end From a1db18c3b8ca68dca0481b3622c04107321eb5be Mon Sep 17 00:00:00 2001 From: David Keller Date: Mon, 24 Jun 2024 20:57:21 +0200 Subject: [PATCH 23/52] Fix calls to `retry_with_buffer` when big buffer is necessary (#14622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller --- src/crystal/system/unix/group.cr | 8 ++++---- src/crystal/system/unix/path.cr | 8 ++++---- src/crystal/system/unix/user.cr | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/crystal/system/unix/group.cr b/src/crystal/system/unix/group.cr index 020c76dab51b..d7d408f77608 100644 --- a/src/crystal/system/unix/group.cr +++ b/src/crystal/system/unix/group.cr @@ -12,9 +12,9 @@ module Crystal::System::Group groupname.check_no_null_byte grp = uninitialized LibC::Group - grp_pointer = pointerof(grp) + grp_pointer = Pointer(LibC::Group).null System.retry_with_buffer("getgrnam_r", GETGR_R_SIZE_MAX) do |buf| - LibC.getgrnam_r(groupname, grp_pointer, buf, buf.size, pointerof(grp_pointer)).tap do + LibC.getgrnam_r(groupname, pointerof(grp), buf, buf.size, pointerof(grp_pointer)).tap do # It's not necessary to check success with `ret == 0` because `grp_pointer` will be NULL on failure return from_struct(grp) if grp_pointer end @@ -26,9 +26,9 @@ module Crystal::System::Group return unless groupid grp = uninitialized LibC::Group - grp_pointer = pointerof(grp) + grp_pointer = Pointer(LibC::Group).null System.retry_with_buffer("getgrgid_r", GETGR_R_SIZE_MAX) do |buf| - LibC.getgrgid_r(groupid, grp_pointer, buf, buf.size, pointerof(grp_pointer)).tap do + LibC.getgrgid_r(groupid, pointerof(grp), buf, buf.size, pointerof(grp_pointer)).tap do # It's not necessary to check success with `ret == 0` because `grp_pointer` will be NULL on failure return from_struct(grp) if grp_pointer end diff --git a/src/crystal/system/unix/path.cr b/src/crystal/system/unix/path.cr index 4392486cbf6d..09588d688bc1 100644 --- a/src/crystal/system/unix/path.cr +++ b/src/crystal/system/unix/path.cr @@ -8,16 +8,16 @@ module Crystal::System::Path id = LibC.getuid pwd = uninitialized LibC::Passwd - pwd_pointer = pointerof(pwd) - ret = nil + pwd_pointer = Pointer(LibC::Passwd).null + ret = LibC::Int.new(0) System.retry_with_buffer("getpwuid_r", User::GETPW_R_SIZE_MAX) do |buf| - ret = LibC.getpwuid_r(id, pwd_pointer, buf, buf.size, pointerof(pwd_pointer)).tap do + ret = LibC.getpwuid_r(id, pointerof(pwd), buf, buf.size, pointerof(pwd_pointer)).tap do # It's not necessary to check success with `ret == 0` because `pwd_pointer` will be NULL on failure return String.new(pwd.pw_dir) if pwd_pointer end end - raise RuntimeError.from_os_error("getpwuid_r", Errno.new(ret.not_nil!)) + raise RuntimeError.from_os_error("getpwuid_r", Errno.new(ret)) end end end diff --git a/src/crystal/system/unix/user.cr b/src/crystal/system/unix/user.cr index 9695f349957c..8e4f16e8c1c4 100644 --- a/src/crystal/system/unix/user.cr +++ b/src/crystal/system/unix/user.cr @@ -15,9 +15,9 @@ module Crystal::System::User username.check_no_null_byte pwd = uninitialized LibC::Passwd - pwd_pointer = pointerof(pwd) + pwd_pointer = Pointer(LibC::Passwd).null System.retry_with_buffer("getpwnam_r", GETPW_R_SIZE_MAX) do |buf| - LibC.getpwnam_r(username, pwd_pointer, buf, buf.size, pointerof(pwd_pointer)).tap do + LibC.getpwnam_r(username, pointerof(pwd), buf, buf.size, pointerof(pwd_pointer)).tap do # It's not necessary to check success with `ret == 0` because `pwd_pointer` will be NULL on failure return from_struct(pwd) if pwd_pointer end @@ -29,9 +29,9 @@ module Crystal::System::User return unless id pwd = uninitialized LibC::Passwd - pwd_pointer = pointerof(pwd) + pwd_pointer = Pointer(LibC::Passwd).null System.retry_with_buffer("getpwuid_r", GETPW_R_SIZE_MAX) do |buf| - LibC.getpwuid_r(id, pwd_pointer, buf, buf.size, pointerof(pwd_pointer)).tap do + LibC.getpwuid_r(id, pointerof(pwd), buf, buf.size, pointerof(pwd_pointer)).tap do # It's not necessary to check success with `ret == 0` because `pwd_pointer` will be NULL on failure return from_struct(pwd) if pwd_pointer end From 7e33ecc01ec62564d2cd68fde0e757fbbf509b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 25 Jun 2024 16:14:54 +0200 Subject: [PATCH 24/52] Refactor `IOCP::OverlappedOperation` internalize `handle` and `wait_for_completion` (#14724) * Add ivar `@handle` * Internalize `schedule_overlapped` as `wait_for_completion` * Add temporary special case for `overlapped_accept` --- src/crystal/system/win32/iocp.cr | 70 +++++++++++++++++------------- src/crystal/system/win32/socket.cr | 8 ++-- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/crystal/system/win32/iocp.cr b/src/crystal/system/win32/iocp.cr index 780b6f1ac6f0..ba0f11eb2af5 100644 --- a/src/crystal/system/win32/iocp.cr +++ b/src/crystal/system/win32/iocp.cr @@ -75,12 +75,19 @@ module Crystal::IOCP property previous : OverlappedOperation? @@canceled = Thread::LinkedList(OverlappedOperation).new + def initialize(@handle : LibC::HANDLE) + end + + def initialize(handle : LibC::SOCKET) + @handle = LibC::HANDLE.new(handle) + end + def self.run(handle, &) - operation = OverlappedOperation.new + operation = OverlappedOperation.new(handle) begin yield operation ensure - operation.done(handle) + operation.done end end @@ -93,9 +100,12 @@ module Crystal::IOCP pointerof(@overlapped) end - def result(handle, &) + def wait_for_result(timeout, &) + wait_for_completion(timeout) + raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? - result = LibC.GetOverlappedResult(handle, self, out bytes, 0) + + result = LibC.GetOverlappedResult(@handle, self, out bytes, 0) if result.zero? error = WinError.value yield error @@ -106,10 +116,15 @@ module Crystal::IOCP bytes end - def wsa_result(socket, &) + def wait_for_wsa_result(timeout, &) + wait_for_completion(timeout) + wsa_result { |error| yield error } + end + + def wsa_result(&) raise Exception.new("Invalid state #{@state}") unless @state.done? || @state.started? flags = 0_u32 - result = LibC.WSAGetOverlappedResult(socket, self, out bytes, false, pointerof(flags)) + result = LibC.WSAGetOverlappedResult(LibC::SOCKET.new(@handle.address), self, out bytes, false, pointerof(flags)) if result.zero? error = WinError.wsa_value yield error @@ -132,15 +147,13 @@ module Crystal::IOCP end end - protected def done(handle) + protected def done case @state when .started? - handle = LibC::HANDLE.new(handle) if handle.is_a?(LibC::SOCKET) - # https://learn.microsoft.com/en-us/windows/win32/api/ioapiset/nf-ioapiset-cancelioex # > The application must not free or reuse the OVERLAPPED structure # associated with the canceled I/O operations until they have completed - if LibC.CancelIoEx(handle, self) != 0 + if LibC.CancelIoEx(@handle, self) != 0 @state = :cancelled @@canceled.push(self) # to increase lifetime end @@ -150,24 +163,23 @@ module Crystal::IOCP def done! @state = :done end - end - # Returns `false` if the operation timed out. - def self.schedule_overlapped(timeout : Time::Span?, line = __LINE__) : Bool - if timeout - timeout_event = Crystal::IOCP::Event.new(Fiber.current) - timeout_event.add(timeout) - else - timeout_event = Crystal::IOCP::Event.new(Fiber.current, Time::Span::MAX) - end - # memoize event loop to make sure that we still target the same instance - # after wakeup (guaranteed by current MT model but let's be future proof) - event_loop = Crystal::EventLoop.current - event_loop.enqueue(timeout_event) + def wait_for_completion(timeout) + if timeout + timeout_event = Crystal::IOCP::Event.new(Fiber.current) + timeout_event.add(timeout) + else + timeout_event = Crystal::IOCP::Event.new(Fiber.current, Time::Span::MAX) + end + # memoize event loop to make sure that we still target the same instance + # after wakeup (guaranteed by current MT model but let's be future proof) + event_loop = Crystal::EventLoop.current + event_loop.enqueue(timeout_event) - Fiber.suspend + Fiber.suspend - event_loop.dequeue(timeout_event) + event_loop.dequeue(timeout_event) + end end def self.overlapped_operation(target, handle, method, timeout, *, writing = false, &) @@ -192,9 +204,7 @@ module Crystal::IOCP return value end - schedule_overlapped(timeout) - - operation.result(handle) do |error| + operation.wait_for_result(timeout) do |error| case error when .error_io_incomplete? raise IO::TimeoutError.new("#{method} timed out") @@ -224,9 +234,7 @@ module Crystal::IOCP return value end - schedule_overlapped(timeout) - - operation.wsa_result(socket) do |error| + operation.wait_for_wsa_result(timeout) do |error| case error when .wsa_io_incomplete? raise IO::TimeoutError.new("#{method} timed out") diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index c04d3a9ad868..2a540f4df88d 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -146,9 +146,7 @@ module Crystal::System::Socket return nil end - IOCP.schedule_overlapped(read_timeout || 1.seconds) - - operation.wsa_result(socket) do |error| + operation.wait_for_wsa_result(read_timeout || 1.seconds) do |error| case error when .wsa_io_incomplete?, .wsaeconnrefused? return ::Socket::ConnectError.from_os_error(method, error) @@ -210,11 +208,11 @@ module Crystal::System::Socket return true end - unless IOCP.schedule_overlapped(read_timeout) + unless operation.wait_for_completion(read_timeout) raise IO::TimeoutError.new("#{method} timed out") end - operation.wsa_result(socket) do |error| + operation.wsa_result do |error| case error when .wsa_io_incomplete?, .wsaenotsock? return false From da8a9bda835cae3473d2977938be8e712236143c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 26 Jun 2024 00:01:34 +0200 Subject: [PATCH 25/52] Fix `Process.run` with closed IO (#14698) IOs passed to `Process.run` experience some indirection which blew up if an IO is closed. This patch ensures that closed IOs are handled correctly. * A closed IO that is not a `IO::FileDescriptor` is replaced with a closed file descriptor * A closed file descriptor won't be reopened in the new process --- spec/std/process_spec.cr | 8 ++++++++ src/crystal/system/unix/process.cr | 5 +++++ src/process.cr | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index d656e9353589..f067d2f5c775 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -181,6 +181,14 @@ pending_interpreted describe: Process do $?.exit_code.should eq(0) end + it "forwards closed io" do + closed_io = IO::Memory.new + closed_io.close + Process.run(*stdin_to_stdout_command, input: closed_io) + Process.run(*stdin_to_stdout_command, output: closed_io) + Process.run(*stdin_to_stdout_command, error: closed_io) + end + it "sets working directory with string" do parent = File.dirname(Dir.current) command = {% if flag?(:win32) %} diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index f3d5dbf3eddb..83f95cc8648c 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -337,6 +337,11 @@ struct Crystal::System::Process end private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) + if src_io.closed? + dst_io.close + return + end + src_io = to_real_fd(src_io) dst_io.reopen(src_io) diff --git a/src/process.cr b/src/process.cr index a1b827d73754..045615c814a7 100644 --- a/src/process.cr +++ b/src/process.cr @@ -294,6 +294,14 @@ class Process when IO::FileDescriptor stdio when IO + if stdio.closed? + if dst_io == STDIN + return File.open(File::NULL, "r").tap(&.close) + else + return File.open(File::NULL, "w").tap(&.close) + end + end + if dst_io == STDIN fork_io, process_io = IO.pipe(read_blocking: true) From 28342af2066f9c2ae83bcdbc4f1e62f47dd31584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 26 Jun 2024 00:02:40 +0200 Subject: [PATCH 26/52] Remove unnecessary calls to `#unsafe_as(UInt64)` etc. (#14686) Rewrites calls of `unsafe_as` which are casting between integer types. These conversions are not unsafe and don't need `unsafe_as`. They can be expressed with appropriate `to_uX!` calls. --- src/compiler/crystal/interpreter/compiler.cr | 2 +- src/compiler/crystal/interpreter/instructions.cr | 2 +- src/crystal/hasher.cr | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/compiler/crystal/interpreter/compiler.cr b/src/compiler/crystal/interpreter/compiler.cr index b25b6e7e25f9..50024d8b65e3 100644 --- a/src/compiler/crystal/interpreter/compiler.cr +++ b/src/compiler/crystal/interpreter/compiler.cr @@ -3350,7 +3350,7 @@ class Crystal::Repl::Compiler < Crystal::Visitor end private def append(value : Int8) - append value.unsafe_as(UInt8) + append value.to_u8! end private def append(value : Symbol) diff --git a/src/compiler/crystal/interpreter/instructions.cr b/src/compiler/crystal/interpreter/instructions.cr index af1af55301e2..8fae94f5ee62 100644 --- a/src/compiler/crystal/interpreter/instructions.cr +++ b/src/compiler/crystal/interpreter/instructions.cr @@ -1472,7 +1472,7 @@ require "./repl" symbol_to_s: { pop_values: [index : Int32], push: true, - code: @context.index_to_symbol(index).object_id.unsafe_as(UInt64), + code: @context.index_to_symbol(index).object_id.to_u64!, }, # >>> Symbol (1) diff --git a/src/crystal/hasher.cr b/src/crystal/hasher.cr index 0c80fe5a0c50..6d5c90853af8 100644 --- a/src/crystal/hasher.cr +++ b/src/crystal/hasher.cr @@ -77,7 +77,7 @@ struct Crystal::Hasher HASH_NAN = 0_u64 HASH_INF_PLUS = 314159_u64 - HASH_INF_MINUS = (-314159_i64).unsafe_as(UInt64) + HASH_INF_MINUS = (-314159_i64).to_u64! @@seed = uninitialized UInt64[2] Crystal::System::Random.random_bytes(@@seed.to_slice.to_unsafe_bytes) @@ -106,7 +106,7 @@ struct Crystal::Hasher end def self.reduce_num(value : Int8 | Int16 | Int32) - value.to_i64.unsafe_as(UInt64) + value.to_u64! end def self.reduce_num(value : UInt8 | UInt16 | UInt32) @@ -118,7 +118,9 @@ struct Crystal::Hasher end def self.reduce_num(value : Int) - value.remainder(HASH_MODULUS).to_i64.unsafe_as(UInt64) + # The result of `remainder(HASH_MODULUS)` is a 64-bit integer, + # and thus guaranteed to fit into `UInt64` + value.remainder(HASH_MODULUS).to_u64! end # This function is for reference implementation, and it is used for `BigFloat`. @@ -157,7 +159,7 @@ struct Crystal::Hasher exp = exp >= 0 ? exp % HASH_BITS : HASH_BITS - 1 - ((-1 - exp) % HASH_BITS) x = ((x << exp) & HASH_MODULUS) | x >> (HASH_BITS - exp) - (x * (value < 0 ? -1 : 1)).to_i64.unsafe_as(UInt64) + (x * (value < 0 ? -1 : 1)).to_u64! end def self.reduce_num(value : Float32) From b08f4a252e9dca8bd8c05c09b82dda7c0e8e38e6 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 26 Jun 2024 00:03:58 +0200 Subject: [PATCH 27/52] Add `StringLiteral#to_utf16` (#14676) Implements `{{ "hello".to_utf16 }}` by exposing `String#to_utf16` in the macro language. Co-authored-by: Quinton Miller --- spec/compiler/macro/macro_methods_spec.cr | 5 +++++ src/compiler/crystal/macros/methods.cr | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 4f5ebf299677..29de1a51c2be 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -591,6 +591,11 @@ module Crystal assert_macro %({{"hello world".titleize}}), %("Hello World") end + it "executes to_utf16" do + assert_macro %({{"hello".to_utf16}}), "(::Slice(::UInt16).literal(104_u16, 101_u16, 108_u16, 108_u16, 111_u16, 0_u16))[0, 5]" + assert_macro %({{"TEST 😐🐙 ±∀ の".to_utf16}}), "(::Slice(::UInt16).literal(84_u16, 69_u16, 83_u16, 84_u16, 32_u16, 55357_u16, 56848_u16, 55357_u16, 56345_u16, 32_u16, 177_u16, 8704_u16, 32_u16, 12398_u16, 0_u16))[0, 14]" + end + it "executes to_i" do assert_macro %({{"1234".to_i}}), %(1234) end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 10e091e3a456..a44bba1b76f9 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -820,6 +820,21 @@ module Crystal else raise "StringLiteral#to_i: #{@value} is not an integer" end + when "to_utf16" + interpret_check_args do + slice = @value.to_utf16 + + # include the trailing zero that isn't counted in the slice but was + # generated by String#to_utf16 so the literal can be passed to C + # functions that expect a null terminated UInt16* + args = Slice(UInt16).new(slice.to_unsafe, slice.size + 1).to_a do |codepoint| + NumberLiteral.new(codepoint).as(ASTNode) + end + literal_node = Call.new(Generic.new(Path.global("Slice"), [Path.global("UInt16")] of ASTNode), "literal", args) + + # but keep the trailing zero hidden in the exposed slice + Call.new(literal_node, "[]", [NumberLiteral.new("0", :i32), NumberLiteral.new(slice.size)] of ASTNode) + end when "tr" interpret_check_args do |first, second| raise "first argument to StringLiteral#tr must be a string, not #{first.class_desc}" unless first.is_a?(StringLiteral) From da33258d547df54220775c90039d50773439c220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 26 Jun 2024 10:49:30 +0200 Subject: [PATCH 28/52] Restore leading zero in exponent for `printf("%e")` and `printf("%g")` (#14695) --- spec/std/sprintf_spec.cr | 204 +++++++++++++++++++-------------------- src/string/formatter.cr | 12 ++- 2 files changed, 112 insertions(+), 104 deletions(-) diff --git a/spec/std/sprintf_spec.cr b/spec/std/sprintf_spec.cr index 67cc1ac1604c..a91ce8030915 100644 --- a/spec/std/sprintf_spec.cr +++ b/spec/std/sprintf_spec.cr @@ -411,14 +411,14 @@ describe "::sprintf" do context "scientific format" do it "works" do - assert_sprintf "%e", 123.45, "1.234500e+2" - assert_sprintf "%E", 123.45, "1.234500E+2" + assert_sprintf "%e", 123.45, "1.234500e+02" + assert_sprintf "%E", 123.45, "1.234500E+02" assert_sprintf "%e", Float64::MAX, "1.797693e+308" assert_sprintf "%e", Float64::MIN_POSITIVE, "2.225074e-308" assert_sprintf "%e", Float64::MIN_SUBNORMAL, "4.940656e-324" - assert_sprintf "%e", 0.0, "0.000000e+0" - assert_sprintf "%e", -0.0, "-0.000000e+0" + assert_sprintf "%e", 0.0, "0.000000e+00" + assert_sprintf "%e", -0.0, "-0.000000e+00" assert_sprintf "%e", -Float64::MIN_SUBNORMAL, "-4.940656e-324" assert_sprintf "%e", -Float64::MIN_POSITIVE, "-2.225074e-308" assert_sprintf "%e", Float64::MIN, "-1.797693e+308" @@ -426,45 +426,45 @@ describe "::sprintf" do context "width specifier" do it "sets the minimum length of the string" do - assert_sprintf "%20e", 123.45, " 1.234500e+2" - assert_sprintf "%20e", -123.45, " -1.234500e+2" - assert_sprintf "%+20e", 123.45, " +1.234500e+2" + assert_sprintf "%20e", 123.45, " 1.234500e+02" + assert_sprintf "%20e", -123.45, " -1.234500e+02" + assert_sprintf "%+20e", 123.45, " +1.234500e+02" - assert_sprintf "%12e", 123.45, " 1.234500e+2" - assert_sprintf "%12e", -123.45, "-1.234500e+2" - assert_sprintf "%+12e", 123.45, "+1.234500e+2" + assert_sprintf "%13e", 123.45, " 1.234500e+02" + assert_sprintf "%13e", -123.45, "-1.234500e+02" + assert_sprintf "%+13e", 123.45, "+1.234500e+02" - assert_sprintf "%11e", 123.45, "1.234500e+2" - assert_sprintf "%11e", -123.45, "-1.234500e+2" - assert_sprintf "%+11e", 123.45, "+1.234500e+2" + assert_sprintf "%12e", 123.45, "1.234500e+02" + assert_sprintf "%12e", -123.45, "-1.234500e+02" + assert_sprintf "%+12e", 123.45, "+1.234500e+02" - assert_sprintf "%2e", 123.45, "1.234500e+2" - assert_sprintf "%2e", -123.45, "-1.234500e+2" - assert_sprintf "%+2e", 123.45, "+1.234500e+2" + assert_sprintf "%2e", 123.45, "1.234500e+02" + assert_sprintf "%2e", -123.45, "-1.234500e+02" + assert_sprintf "%+2e", 123.45, "+1.234500e+02" end it "left-justifies on negative width" do - assert_sprintf "%*e", [-20, 123.45], "1.234500e+2 " + assert_sprintf "%*e", [-20, 123.45], "1.234500e+02 " end end context "precision specifier" do it "sets the minimum length of the fractional part" do - assert_sprintf "%.0e", 2.0, "2e+0" - assert_sprintf "%.0e", 2.5.prev_float, "2e+0" - assert_sprintf "%.0e", 2.5, "2e+0" - assert_sprintf "%.0e", 2.5.next_float, "3e+0" - assert_sprintf "%.0e", 3.0, "3e+0" - assert_sprintf "%.0e", 3.5.prev_float, "3e+0" - assert_sprintf "%.0e", 3.5, "4e+0" - assert_sprintf "%.0e", 3.5.next_float, "4e+0" - assert_sprintf "%.0e", 4.0, "4e+0" + assert_sprintf "%.0e", 2.0, "2e+00" + assert_sprintf "%.0e", 2.5.prev_float, "2e+00" + assert_sprintf "%.0e", 2.5, "2e+00" + assert_sprintf "%.0e", 2.5.next_float, "3e+00" + assert_sprintf "%.0e", 3.0, "3e+00" + assert_sprintf "%.0e", 3.5.prev_float, "3e+00" + assert_sprintf "%.0e", 3.5, "4e+00" + assert_sprintf "%.0e", 3.5.next_float, "4e+00" + assert_sprintf "%.0e", 4.0, "4e+00" - assert_sprintf "%.0e", 9.5, "1e+1" + assert_sprintf "%.0e", 9.5, "1e+01" - assert_sprintf "%.100e", 1.1, "1.1000000000000000888178419700125232338905334472656250000000000000000000000000000000000000000000000000e+0" + assert_sprintf "%.100e", 1.1, "1.1000000000000000888178419700125232338905334472656250000000000000000000000000000000000000000000000000e+00" - assert_sprintf "%.10000e", 1.0, "1.#{"0" * 10000}e+0" + assert_sprintf "%.10000e", 1.0, "1.#{"0" * 10000}e+00" assert_sprintf "%.1000e", Float64::MIN_POSITIVE.prev_float, "2.2250738585072008890245868760858598876504231122409594654935248025624400092282356951" \ @@ -482,103 +482,103 @@ describe "::sprintf" do end it "can be used with width" do - assert_sprintf "%20.12e", 123.45, " 1.234500000000e+2" - assert_sprintf "%20.12e", -123.45, " -1.234500000000e+2" - assert_sprintf "%20.12e", 0.0, " 0.000000000000e+0" + assert_sprintf "%20.13e", 123.45, " 1.2345000000000e+02" + assert_sprintf "%20.13e", -123.45, "-1.2345000000000e+02" + assert_sprintf "%20.13e", 0.0, " 0.0000000000000e+00" - assert_sprintf "%-20.12e", 123.45, "1.234500000000e+2 " - assert_sprintf "%-20.12e", -123.45, "-1.234500000000e+2 " - assert_sprintf "%-20.12e", 0.0, "0.000000000000e+0 " + assert_sprintf "%-20.13e", 123.45, "1.2345000000000e+02 " + assert_sprintf "%-20.13e", -123.45, "-1.2345000000000e+02" + assert_sprintf "%-20.13e", 0.0, "0.0000000000000e+00 " - assert_sprintf "%8.12e", 123.45, "1.234500000000e+2" - assert_sprintf "%8.12e", -123.45, "-1.234500000000e+2" - assert_sprintf "%8.12e", 0.0, "0.000000000000e+0" + assert_sprintf "%8.13e", 123.45, "1.2345000000000e+02" + assert_sprintf "%8.13e", -123.45, "-1.2345000000000e+02" + assert_sprintf "%8.13e", 0.0, "0.0000000000000e+00" end it "is ignored if precision argument is negative" do - assert_sprintf "%.*e", [-2, 123.45], "1.234500e+2" + assert_sprintf "%.*e", [-2, 123.45], "1.234500e+02" end end context "sharp flag" do it "prints a decimal point even if no digits follow" do - assert_sprintf "%#.0e", 1.0, "1.e+0" - assert_sprintf "%#.0e", 10000.0, "1.e+4" + assert_sprintf "%#.0e", 1.0, "1.e+00" + assert_sprintf "%#.0e", 10000.0, "1.e+04" assert_sprintf "%#.0e", 1.0e+23, "1.e+23" assert_sprintf "%#.0e", 1.0e-100, "1.e-100" - assert_sprintf "%#.0e", 0.0, "0.e+0" - assert_sprintf "%#.0e", -0.0, "-0.e+0" + assert_sprintf "%#.0e", 0.0, "0.e+00" + assert_sprintf "%#.0e", -0.0, "-0.e+00" end end context "plus flag" do it "writes a plus sign for positive values" do - assert_sprintf "%+e", 123.45, "+1.234500e+2" - assert_sprintf "%+e", -123.45, "-1.234500e+2" - assert_sprintf "%+e", 0.0, "+0.000000e+0" + assert_sprintf "%+e", 123.45, "+1.234500e+02" + assert_sprintf "%+e", -123.45, "-1.234500e+02" + assert_sprintf "%+e", 0.0, "+0.000000e+00" end it "writes plus sign after left space-padding" do - assert_sprintf "%+20e", 123.45, " +1.234500e+2" - assert_sprintf "%+20e", -123.45, " -1.234500e+2" - assert_sprintf "%+20e", 0.0, " +0.000000e+0" + assert_sprintf "%+20e", 123.45, " +1.234500e+02" + assert_sprintf "%+20e", -123.45, " -1.234500e+02" + assert_sprintf "%+20e", 0.0, " +0.000000e+00" end it "writes plus sign before left zero-padding" do - assert_sprintf "%+020e", 123.45, "+000000001.234500e+2" - assert_sprintf "%+020e", -123.45, "-000000001.234500e+2" - assert_sprintf "%+020e", 0.0, "+000000000.000000e+0" + assert_sprintf "%+020e", 123.45, "+00000001.234500e+02" + assert_sprintf "%+020e", -123.45, "-00000001.234500e+02" + assert_sprintf "%+020e", 0.0, "+00000000.000000e+00" end end context "space flag" do it "writes a space for positive values" do - assert_sprintf "% e", 123.45, " 1.234500e+2" - assert_sprintf "% e", -123.45, "-1.234500e+2" - assert_sprintf "% e", 0.0, " 0.000000e+0" + assert_sprintf "% e", 123.45, " 1.234500e+02" + assert_sprintf "% e", -123.45, "-1.234500e+02" + assert_sprintf "% e", 0.0, " 0.000000e+00" end it "writes space before left space-padding" do - assert_sprintf "% 20e", 123.45, " 1.234500e+2" - assert_sprintf "% 20e", -123.45, " -1.234500e+2" - assert_sprintf "% 20e", 0.0, " 0.000000e+0" + assert_sprintf "% 20e", 123.45, " 1.234500e+02" + assert_sprintf "% 20e", -123.45, " -1.234500e+02" + assert_sprintf "% 20e", 0.0, " 0.000000e+00" - assert_sprintf "% 020e", 123.45, " 000000001.234500e+2" - assert_sprintf "% 020e", -123.45, "-000000001.234500e+2" - assert_sprintf "% 020e", 0.0, " 000000000.000000e+0" + assert_sprintf "% 020e", 123.45, " 00000001.234500e+02" + assert_sprintf "% 020e", -123.45, "-00000001.234500e+02" + assert_sprintf "% 020e", 0.0, " 00000000.000000e+00" end it "is ignored if plus flag is also specified" do - assert_sprintf "% +e", 123.45, "+1.234500e+2" - assert_sprintf "%+ e", -123.45, "-1.234500e+2" + assert_sprintf "% +e", 123.45, "+1.234500e+02" + assert_sprintf "%+ e", -123.45, "-1.234500e+02" end end context "zero flag" do it "left-pads the result with zeros" do - assert_sprintf "%020e", 123.45, "0000000001.234500e+2" - assert_sprintf "%020e", -123.45, "-000000001.234500e+2" - assert_sprintf "%020e", 0.0, "0000000000.000000e+0" + assert_sprintf "%020e", 123.45, "000000001.234500e+02" + assert_sprintf "%020e", -123.45, "-00000001.234500e+02" + assert_sprintf "%020e", 0.0, "000000000.000000e+00" end it "is ignored if string is left-justified" do - assert_sprintf "%-020e", 123.45, "1.234500e+2 " - assert_sprintf "%-020e", -123.45, "-1.234500e+2 " - assert_sprintf "%-020e", 0.0, "0.000000e+0 " + assert_sprintf "%-020e", 123.45, "1.234500e+02 " + assert_sprintf "%-020e", -123.45, "-1.234500e+02 " + assert_sprintf "%-020e", 0.0, "0.000000e+00 " end it "can be used with precision" do - assert_sprintf "%020.12e", 123.45, "0001.234500000000e+2" - assert_sprintf "%020.12e", -123.45, "-001.234500000000e+2" - assert_sprintf "%020.12e", 0.0, "0000.000000000000e+0" + assert_sprintf "%020.12e", 123.45, "001.234500000000e+02" + assert_sprintf "%020.12e", -123.45, "-01.234500000000e+02" + assert_sprintf "%020.12e", 0.0, "000.000000000000e+00" end end context "minus flag" do it "left-justifies the string" do - assert_sprintf "%-20e", 123.45, "1.234500e+2 " - assert_sprintf "%-20e", -123.45, "-1.234500e+2 " - assert_sprintf "%-20e", 0.0, "0.000000e+0 " + assert_sprintf "%-20e", 123.45, "1.234500e+02 " + assert_sprintf "%-20e", -123.45, "-1.234500e+02 " + assert_sprintf "%-20e", 0.0, "0.000000e+00 " end end end @@ -588,8 +588,8 @@ describe "::sprintf" do assert_sprintf "%g", 123.45, "123.45" assert_sprintf "%G", 123.45, "123.45" - assert_sprintf "%g", 1.2345e-5, "1.2345e-5" - assert_sprintf "%G", 1.2345e-5, "1.2345E-5" + assert_sprintf "%g", 1.2345e-5, "1.2345e-05" + assert_sprintf "%G", 1.2345e-5, "1.2345E-05" assert_sprintf "%g", 1.2345e+25, "1.2345e+25" assert_sprintf "%G", 1.2345e+25, "1.2345E+25" @@ -630,9 +630,9 @@ describe "::sprintf" do context "precision specifier" do it "sets the precision of the value" do - assert_sprintf "%.0g", 123.45, "1e+2" - assert_sprintf "%.1g", 123.45, "1e+2" - assert_sprintf "%.2g", 123.45, "1.2e+2" + assert_sprintf "%.0g", 123.45, "1e+02" + assert_sprintf "%.1g", 123.45, "1e+02" + assert_sprintf "%.2g", 123.45, "1.2e+02" assert_sprintf "%.3g", 123.45, "123" assert_sprintf "%.4g", 123.45, "123.5" assert_sprintf "%.5g", 123.45, "123.45" @@ -650,41 +650,41 @@ describe "::sprintf" do assert_sprintf "%.5g", 1.23e-45, "1.23e-45" assert_sprintf "%.6g", 1.23e-45, "1.23e-45" - assert_sprintf "%.1000g", 1e-5, "1.0000000000000000818030539140313095458623138256371021270751953125e-5" + assert_sprintf "%.1000g", 1e-5, "1.0000000000000000818030539140313095458623138256371021270751953125e-05" end it "can be used with width" do - assert_sprintf "%10.1g", 123.45, " 1e+2" - assert_sprintf "%10.2g", 123.45, " 1.2e+2" + assert_sprintf "%10.1g", 123.45, " 1e+02" + assert_sprintf "%10.2g", 123.45, " 1.2e+02" assert_sprintf "%10.3g", 123.45, " 123" assert_sprintf "%10.4g", 123.45, " 123.5" assert_sprintf "%10.5g", 123.45, " 123.45" - assert_sprintf "%10.1g", -123.45, " -1e+2" - assert_sprintf "%10.2g", -123.45, " -1.2e+2" + assert_sprintf "%10.1g", -123.45, " -1e+02" + assert_sprintf "%10.2g", -123.45, " -1.2e+02" assert_sprintf "%10.3g", -123.45, " -123" assert_sprintf "%10.4g", -123.45, " -123.5" assert_sprintf "%10.5g", -123.45, " -123.45" assert_sprintf "%10.5g", 0, " 0" - assert_sprintf "%-10.1g", 123.45, "1e+2 " - assert_sprintf "%-10.2g", 123.45, "1.2e+2 " + assert_sprintf "%-10.1g", 123.45, "1e+02 " + assert_sprintf "%-10.2g", 123.45, "1.2e+02 " assert_sprintf "%-10.3g", 123.45, "123 " assert_sprintf "%-10.4g", 123.45, "123.5 " assert_sprintf "%-10.5g", 123.45, "123.45 " - assert_sprintf "%-10.1g", -123.45, "-1e+2 " - assert_sprintf "%-10.2g", -123.45, "-1.2e+2 " + assert_sprintf "%-10.1g", -123.45, "-1e+02 " + assert_sprintf "%-10.2g", -123.45, "-1.2e+02 " assert_sprintf "%-10.3g", -123.45, "-123 " assert_sprintf "%-10.4g", -123.45, "-123.5 " assert_sprintf "%-10.5g", -123.45, "-123.45 " assert_sprintf "%-10.5g", 0, "0 " - assert_sprintf "%3.1g", 123.45, "1e+2" - assert_sprintf "%3.2g", 123.45, "1.2e+2" + assert_sprintf "%3.1g", 123.45, "1e+02" + assert_sprintf "%3.2g", 123.45, "1.2e+02" assert_sprintf "%3.3g", 123.45, "123" assert_sprintf "%3.4g", 123.45, "123.5" assert_sprintf "%3.5g", 123.45, "123.45" - assert_sprintf "%3.1g", -123.45, "-1e+2" - assert_sprintf "%3.2g", -123.45, "-1.2e+2" + assert_sprintf "%3.1g", -123.45, "-1e+02" + assert_sprintf "%3.2g", -123.45, "-1.2e+02" assert_sprintf "%3.3g", -123.45, "-123" assert_sprintf "%3.4g", -123.45, "-123.5" assert_sprintf "%3.5g", -123.45, "-123.45" @@ -699,19 +699,19 @@ describe "::sprintf" do context "sharp flag" do it "prints decimal point and trailing zeros" do - assert_sprintf "%#.0g", 12345, "1.e+4" + assert_sprintf "%#.0g", 12345, "1.e+04" assert_sprintf "%#.6g", 12345, "12345.0" assert_sprintf "%#.10g", 12345, "12345.00000" assert_sprintf "%#.100g", 12345, "12345.#{"0" * 95}" assert_sprintf "%#.1000g", 12345, "12345.#{"0" * 995}" - assert_sprintf "%#.0g", 1e-5, "1.e-5" - assert_sprintf "%#.6g", 1e-5, "1.00000e-5" - assert_sprintf "%#.10g", 1e-5, "1.000000000e-5" - assert_sprintf "%#.100g", 1e-5, "1.0000000000000000818030539140313095458623138256371021270751953125#{"0" * 35}e-5" - assert_sprintf "%#.1000g", 1e-5, "1.0000000000000000818030539140313095458623138256371021270751953125#{"0" * 935}e-5" + assert_sprintf "%#.0g", 1e-5, "1.e-05" + assert_sprintf "%#.6g", 1e-5, "1.00000e-05" + assert_sprintf "%#.10g", 1e-5, "1.000000000e-05" + assert_sprintf "%#.100g", 1e-5, "1.0000000000000000818030539140313095458623138256371021270751953125#{"0" * 35}e-05" + assert_sprintf "%#.1000g", 1e-5, "1.0000000000000000818030539140313095458623138256371021270751953125#{"0" * 935}e-05" - assert_sprintf "%#15.0g", 12345, " 1.e+4" + assert_sprintf "%#15.0g", 12345, " 1.e+04" assert_sprintf "%#15.6g", 12345, " 12345.0" assert_sprintf "%#15.10g", 12345, " 12345.00000" end @@ -774,8 +774,8 @@ describe "::sprintf" do end it "can be used with precision" do - assert_sprintf "%010.2g", 123.45, "00001.2e+2" - assert_sprintf "%010.2g", -123.45, "-0001.2e+2" + assert_sprintf "%010.2g", 123.45, "0001.2e+02" + assert_sprintf "%010.2g", -123.45, "-001.2e+02" assert_sprintf "%010.2g", 0.0, "0000000000" end end diff --git a/src/string/formatter.cr b/src/string/formatter.cr index 0d4956dc0c05..60da55a2601f 100644 --- a/src/string/formatter.cr +++ b/src/string/formatter.cr @@ -430,6 +430,7 @@ struct String::Formatter(A) str_size = printf_size + trailing_zeros str_size += 1 if sign < 0 || flags.plus || flags.space str_size += 1 if flags.sharp && dot_index.nil? + str_size += 1 if printf_slice.size - e_index < 4 pad(str_size, flags) if flags.left_padding? && flags.padding_char != '0' @@ -441,7 +442,9 @@ struct String::Formatter(A) @io.write_string(printf_slice[0, e_index]) trailing_zeros.times { @io << '0' } @io << '.' if flags.sharp && dot_index.nil? - @io.write_string(printf_slice[e_index..]) + @io.write_string(printf_slice[e_index, 2]) + @io << '0' if printf_slice.size - e_index < 4 + @io.write_string(printf_slice[(e_index + 2)..]) pad(str_size, flags) if flags.right_padding? end @@ -465,6 +468,7 @@ struct String::Formatter(A) str_size = printf_size str_size += 1 if sign < 0 || flags.plus || flags.space str_size += (dot_index.nil? ? 1 : 0) + trailing_zeros if flags.sharp + str_size += 1 if printf_slice.size - e_index < 4 if e_index pad(str_size, flags) if flags.left_padding? && flags.padding_char != '0' @@ -476,7 +480,11 @@ struct String::Formatter(A) @io.write_string(printf_slice[0...e_index]) trailing_zeros.times { @io << '0' } if flags.sharp @io << '.' if flags.sharp && dot_index.nil? - @io.write_string(printf_slice[e_index..]) if e_index + if e_index + @io.write_string(printf_slice[e_index, 2]) + @io << '0' if printf_slice.size - e_index < 4 + @io.write_string(printf_slice[(e_index + 2)..]) + end pad(str_size, flags) if flags.right_padding? end From 6c344943792be2e581c54e21b317d0dae12bb0e4 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Wed, 26 Jun 2024 10:50:58 +0200 Subject: [PATCH 29/52] Add `Crystal::System.panic` (#14733) Prints a system error message on the standard error then exits with an error status. Raising an exception should always be preferred but there are a few cases where we can't allocate any memory (e.g. stop the world) and still need to fail when reaching a system error. --- src/crystal/system/panic.cr | 16 ++++++++++++++++ src/crystal/tracing.cr | 22 +++++++++++----------- src/errno.cr | 10 ++++++++-- src/wasi_error.cr | 5 +++++ src/winerror.cr | 12 ++++++++---- 5 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 src/crystal/system/panic.cr diff --git a/src/crystal/system/panic.cr b/src/crystal/system/panic.cr new file mode 100644 index 000000000000..192b735e4d0f --- /dev/null +++ b/src/crystal/system/panic.cr @@ -0,0 +1,16 @@ +module Crystal::System + # Prints a system error message on the standard error then exits with an error + # status. + # + # You should always prefer raising an exception, built with + # `RuntimeError.from_os_error` for example, but there are a few cases where we + # can't allocate any memory (e.g. stop the world) and still need to fail when + # reaching a system error. + def self.panic(syscall_name : String, error : Errno | WinError | WasiError) : NoReturn + System.print_error("%s failed with ", syscall_name) + error.unsafe_message { |slice| System.print_error(slice) } + System.print_error(" (%s)\n", error.to_s) + + LibC._exit(1) + end +end diff --git a/src/crystal/tracing.cr b/src/crystal/tracing.cr index ad3ae184a54a..a680bfea717f 100644 --- a/src/crystal/tracing.cr +++ b/src/crystal/tracing.cr @@ -1,3 +1,5 @@ +require "./system/panic" + module Crystal # :nodoc: module Tracing @@ -143,23 +145,21 @@ module Crystal # not using LibC::INVALID_HANDLE_VALUE because it doesn't exist (yet) return handle.address unless handle == LibC::HANDLE.new(-1.to_u64!) - error = uninitialized UInt16[256] - len = LibC.FormatMessageW(LibC::FORMAT_MESSAGE_FROM_SYSTEM, nil, WinError.value, 0, error, error.size, nil) - - # not using printf because filename and error are UTF-16 slices: - System.print_error "ERROR: failed to open " - System.print_error filename - System.print_error " for writing: " - System.print_error error.to_slice[0...len] - System.print_error "\n" + syscall_name = "CreateFileW" + error = WinError.value {% else %} fd = LibC.open(filename, LibC::O_CREAT | LibC::O_WRONLY | LibC::O_TRUNC | LibC::O_CLOEXEC, 0o644) return fd unless fd < 0 - System.print_error "ERROR: failed to open %s for writing: %s\n", filename, LibC.strerror(Errno.value) + syscall_name = "open" + error = Errno.value {% end %} - LibC._exit(1) + System.print_error "ERROR: failed to open " + System.print_error filename + System.print_error " for writing\n" + + System.panic(syscall_name, Errno.value) end private def self.parse_sections(slice) diff --git a/src/errno.cr b/src/errno.cr index 03ca6085eb8a..2a68371f4a19 100644 --- a/src/errno.cr +++ b/src/errno.cr @@ -40,10 +40,16 @@ enum Errno # Convert an Errno to an error message def message : String - String.new(LibC.strerror(value)) + unsafe_message { |slice| String.new(slice) } end - # Returns the value of libc's errno. + # :nodoc: + def unsafe_message(&) + pointer = LibC.strerror(value) + yield Bytes.new(pointer, LibC.strlen(pointer)) + end + + # returns the value of libc's errno. def self.value : self {% if flag?(:netbsd) || flag?(:openbsd) || flag?(:android) %} Errno.new LibC.__errno.value diff --git a/src/wasi_error.cr b/src/wasi_error.cr index 9a04ed315463..a026de8c7ee2 100644 --- a/src/wasi_error.cr +++ b/src/wasi_error.cr @@ -83,6 +83,11 @@ enum WasiError : UInt16 end end + # :nodoc: + def unsafe_message(&) + yield message.to_slice + end + # Transforms this `WasiError` value to the equivalent `Errno` value. # # This is only defined for some values. If no transformation is defined for diff --git a/src/winerror.cr b/src/winerror.cr index 8da56d7a905e..ab978769d553 100644 --- a/src/winerror.cr +++ b/src/winerror.cr @@ -61,17 +61,21 @@ enum WinError : UInt32 # # On non-win32 platforms the result is always an empty string. def message : String - formatted_message + {% if flag?(:win32) %} + unsafe_message { |slice| String.from_utf16(slice).strip } + {% else %} + "" + {% end %} end # :nodoc: - def formatted_message : String + def unsafe_message(&) {% if flag?(:win32) %} buffer = uninitialized UInt16[256] size = LibC.FormatMessageW(LibC::FORMAT_MESSAGE_FROM_SYSTEM, nil, value, 0, buffer, buffer.size, nil) - String.from_utf16(buffer.to_slice[0, size]).strip + yield buffer.to_slice[0, size] {% else %} - "" + yield "".to_slice {% end %} end From 7d8a6a0704c1dc1bed8d66737982dab294a5effd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 26 Jun 2024 16:30:39 +0200 Subject: [PATCH 30/52] Undocument `IO::Evented` (#14749) --- src/io/evented.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/io/evented.cr b/src/io/evented.cr index 57d71254f24b..ccc040932285 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -2,6 +2,7 @@ require "crystal/thread_local_value" +# :nodoc: module IO::Evented @read_timed_out = false @write_timed_out = false From 6c8542aa608a0cd207b136893faddbd3d5ff7b4e Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Wed, 26 Jun 2024 10:31:51 -0400 Subject: [PATCH 31/52] Add UUID v7 (#14732) Implementation of [RFC 9562](https://www.rfc-editor.org/rfc/rfc9562#name-uuid-version-7) Co-authored-by: Julien Portalier --- spec/std/uuid_spec.cr | 17 +++++++++++++++++ src/uuid.cr | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/spec/std/uuid_spec.cr b/spec/std/uuid_spec.cr index 12e497829b16..48cc3351a3c6 100644 --- a/spec/std/uuid_spec.cr +++ b/spec/std/uuid_spec.cr @@ -269,4 +269,21 @@ describe "UUID" do UUID.v5_x500(data).v5?.should eq(true) end end + + describe "v7" do + it "generates a v7 UUID" do + uuid = UUID.v7 + uuid.v7?.should eq true + uuid.variant.rfc9562?.should eq true + end + + pending_wasm32 "generates UUIDs that are sortable with 1ms precision" do + uuids = Array.new(10) do + sleep 1.millisecond + UUID.v7 + end + + uuids.should eq uuids.sort + end + end end diff --git a/src/uuid.cr b/src/uuid.cr index c7aaee0a605c..b1a043785472 100644 --- a/src/uuid.cr +++ b/src/uuid.cr @@ -23,6 +23,8 @@ struct UUID NCS # Reserved for RFC 4122 Specification (default). RFC4122 + # Reserved for RFC 9562 Specification (default for v7). + RFC9562 = RFC4122 # Reserved by Microsoft for backward compatibility. Microsoft # Reserved for future expansion. @@ -43,6 +45,8 @@ struct UUID V4 = 4 # SHA1 hash and namespace. V5 = 5 + # Prefixed with a UNIX timestamp with millisecond precision, filled in with randomness. + V7 = 7 end # A Domain represents a Version 2 domain (DCE security). @@ -80,7 +84,7 @@ struct UUID # do nothing when Variant::NCS @bytes[8] = (@bytes[8] & 0x7f) - when Variant::RFC4122 + when Variant::RFC4122, Variant::RFC9562 @bytes[8] = (@bytes[8] & 0x3f) | 0x80 when Variant::Microsoft @bytes[8] = (@bytes[8] & 0x1f) | 0xc0 @@ -321,6 +325,30 @@ struct UUID end {% end %} + # Generates an RFC9562-compatible v7 UUID, allowing the values to be sorted + # chronologically (with 1ms precision) by their raw or hexstring + # representation. + def self.v7(random r : Random = Random::Secure) + buffer = uninitialized UInt8[18] + value = buffer.to_slice + + # Generate the first 48 bits of the UUID with the current timestamp. We + # allocated enough room for a 64-bit timestamp to accommodate the + # NetworkEndian.encode call here, but we only need 48 bits of it so we chop + # off the first 2 bytes. + IO::ByteFormat::NetworkEndian.encode Time.utc.to_unix_ms, value + value = value[2..] + + # Fill in the rest with random bytes + r.random_bytes(value[6..]) + + # Set the version and variant + value[6] = (value[6] & 0x3F) | 0x70 + value[8] = (value[8] & 0x0F) | 0x80 + + new(value, variant: :rfc9562, version: :v7) + end + # Generates an empty UUID. # # ``` @@ -375,6 +403,7 @@ struct UUID when 3 then Version::V3 when 4 then Version::V4 when 5 then Version::V5 + when 7 then Version::V7 else Version::Unknown end end @@ -442,7 +471,7 @@ struct UUID class Error < Exception end - {% for v in %w(1 2 3 4 5) %} + {% for v in %w(1 2 3 4 5 7) %} # Returns `true` if UUID is a V{{ v.id }}, `false` otherwise. def v{{ v.id }}? variant == Variant::RFC4122 && version == Version::V{{ v.id }} From 1ac328023de8e2175309eaaa2d1194f6cc84ffa6 Mon Sep 17 00:00:00 2001 From: Anton Karankevich <83635660+anton7c3@users.noreply.github.com> Date: Wed, 26 Jun 2024 22:54:50 +0300 Subject: [PATCH 32/52] Allow parsing cookies with space in the value (#14455) This is an optional feature enhancement of https://datatracker.ietf.org/doc/html/rfc6265#section-5.2 --- spec/std/http/cookie_spec.cr | 61 +++++++++++++++++++++++++++++++----- src/http/cookie.cr | 22 +++++++++---- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/spec/std/http/cookie_spec.cr b/spec/std/http/cookie_spec.cr index 218bfd9c608e..1a29a3f56754 100644 --- a/spec/std/http/cookie_spec.cr +++ b/spec/std/http/cookie_spec.cr @@ -132,8 +132,8 @@ module HTTP it "raises on invalid value" do cookie = HTTP::Cookie.new("x", "") invalid_values = { - '"', ',', ';', '\\', # invalid printable ascii characters - ' ', '\r', '\t', '\n', # non-printable ascii characters + '"', ',', ';', '\\', # invalid printable ascii characters + '\r', '\t', '\n', # non-printable ascii characters }.map { |c| "foo#{c}bar" } invalid_values.each do |invalid_value| @@ -235,12 +235,6 @@ module HTTP cookie.to_set_cookie_header.should eq("key=value") end - it "parse_set_cookie with space" do - cookie = parse_set_cookie("key=value; path=/test") - parse_set_cookie("key=value;path=/test").should eq cookie - parse_set_cookie("key=value; \t\npath=/test").should eq cookie - end - it "parses key=" do cookie = parse_first_cookie("key=") cookie.name.should eq("key") @@ -285,9 +279,60 @@ module HTTP first.value.should eq("bar") second.value.should eq("baz") end + + it "parses cookie with spaces in value" do + parse_first_cookie(%[key=some value]).value.should eq "some value" + parse_first_cookie(%[key="some value"]).value.should eq "some value" + end + + it "strips spaces around value only when it's unquoted" do + parse_first_cookie(%[key= some value ]).value.should eq "some value" + parse_first_cookie(%[key=" some value "]).value.should eq " some value " + parse_first_cookie(%[key= " some value " ]).value.should eq " some value " + end end describe "parse_set_cookie" do + it "with space" do + cookie = parse_set_cookie("key=value; path=/test") + parse_set_cookie("key=value;path=/test").should eq cookie + parse_set_cookie("key=value; \t\npath=/test").should eq cookie + end + + it "parses cookie with spaces in value" do + parse_set_cookie(%[key=some value]).value.should eq "some value" + parse_set_cookie(%[key="some value"]).value.should eq "some value" + end + + it "removes leading and trailing whitespaces" do + cookie = parse_set_cookie(%[key= \tvalue \t; \t\npath=/test]) + cookie.name.should eq "key" + cookie.value.should eq "value" + cookie.path.should eq "/test" + + cookie = parse_set_cookie(%[ key\t =value \n;path=/test]) + cookie.name.should eq "key" + cookie.value.should eq "value" + cookie.path.should eq "/test" + end + + it "strips spaces around value only when it's unquoted" do + cookie = parse_set_cookie(%[key= value ; \tpath=/test]) + cookie.name.should eq "key" + cookie.value.should eq "value" + cookie.path.should eq "/test" + + cookie = parse_set_cookie(%[key=" value "; \tpath=/test]) + cookie.name.should eq "key" + cookie.value.should eq " value " + cookie.path.should eq "/test" + + cookie = parse_set_cookie(%[key= " value "\t ; \tpath=/test]) + cookie.name.should eq "key" + cookie.value.should eq " value " + cookie.path.should eq "/test" + end + it "parses path" do cookie = parse_set_cookie("key=value; path=/test") cookie.name.should eq("key") diff --git a/src/http/cookie.cr b/src/http/cookie.cr index 83b41297707e..8138249aa830 100644 --- a/src/http/cookie.cr +++ b/src/http/cookie.cr @@ -97,8 +97,8 @@ module HTTP private def validate_value(value) value.each_byte do |byte| # valid characters for cookie-value per https://tools.ietf.org/html/rfc6265#section-4.1.1 - # all printable ASCII characters except ' ', ',', '"', ';' and '\\' - if !byte.in?(0x21...0x7f) || byte.in?(0x22, 0x2c, 0x3b, 0x5c) + # all printable ASCII characters except ',', '"', ';' and '\\' + if !byte.in?(0x20...0x7f) || byte.in?(0x22, 0x2c, 0x3b, 0x5c) raise IO::Error.new("Invalid cookie value") end end @@ -196,9 +196,9 @@ module HTTP module Parser module Regex CookieName = /[^()<>@,;:\\"\/\[\]?={} \t\x00-\x1f\x7f]+/ - CookieOctet = /[!#-+\--:<-\[\]-~]/ + CookieOctet = /[!#-+\--:<-\[\]-~ ]/ CookieValue = /(?:"#{CookieOctet}*"|#{CookieOctet}*)/ - CookiePair = /(?#{CookieName})=(?#{CookieValue})/ + CookiePair = /\s*(?#{CookieName})\s*=\s*(?#{CookieValue})\s*/ DomainLabel = /[A-Za-z0-9\-]+/ DomainIp = /(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/ Time = /(?:\d{2}:\d{2}:\d{2})/ @@ -230,9 +230,11 @@ module HTTP def parse_cookies(header, &) header.scan(CookieString).each do |pair| value = pair["value"] - if value.starts_with?('"') + if value.starts_with?('"') && value.ends_with?('"') # Unwrap quoted cookie value value = value.byte_slice(1, value.bytesize - 2) + else + value = value.strip end yield Cookie.new(pair["name"], value) end @@ -251,8 +253,16 @@ module HTTP expires = parse_time(match["expires"]?) max_age = match["max_age"]?.try(&.to_i64.seconds) + # Unwrap quoted cookie value + cookie_value = match["value"] + if cookie_value.starts_with?('"') && cookie_value.ends_with?('"') + cookie_value = cookie_value.byte_slice(1, cookie_value.bytesize - 2) + else + cookie_value = cookie_value.strip + end + Cookie.new( - match["name"], match["value"], + match["name"], cookie_value, path: match["path"]?, expires: expires, domain: match["domain"]?, From b0fff7ee148f1237fe97049eda4561ec2c6b9636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 26 Jun 2024 21:55:26 +0200 Subject: [PATCH 33/52] Fix parser validate UTF-8 on first input byte (#14750) --- spec/compiler/crystal/tools/format_spec.cr | 6 +++--- spec/compiler/lexer/lexer_spec.cr | 9 +++++++++ src/compiler/crystal/syntax/lexer.cr | 7 +++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/spec/compiler/crystal/tools/format_spec.cr b/spec/compiler/crystal/tools/format_spec.cr index bde408afcc7b..c2d74ac45fdb 100644 --- a/spec/compiler/crystal/tools/format_spec.cr +++ b/spec/compiler/crystal/tools/format_spec.cr @@ -57,7 +57,7 @@ describe Crystal::Command::FormatCommand do format_command.run format_command.status_code.should eq(1) stdout.to_s.should be_empty - stderr.to_s.should contain("file 'STDIN' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8") + stderr.to_s.should contain("file 'STDIN' is not a valid Crystal source file: Unexpected byte 0xfe at position 0, malformed UTF-8") end it "formats stdin (bug)" do @@ -162,7 +162,7 @@ describe Crystal::Command::FormatCommand do format_command.status_code.should eq(1) stdout.to_s.should contain("Format #{Path[".", "format.cr"]}") stderr.to_s.should contain("syntax error in '#{Path[".", "syntax_error.cr"]}:1:3': unexpected token: EOF") - stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8") + stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xfe at position 0, malformed UTF-8") File.read(File.join(path, "format.cr")).should eq("if true\n 1\nend\n") end @@ -226,7 +226,7 @@ describe Crystal::Command::FormatCommand do stderr.to_s.should_not contain("not_format.cr") stderr.to_s.should contain("formatting '#{Path[".", "format.cr"]}' produced changes") stderr.to_s.should contain("syntax error in '#{Path[".", "syntax_error.cr"]}:1:3': unexpected token: EOF") - stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8") + stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xfe at position 0, malformed UTF-8") end end end diff --git a/spec/compiler/lexer/lexer_spec.cr b/spec/compiler/lexer/lexer_spec.cr index 6045635a603c..6813c1fe8df3 100644 --- a/spec/compiler/lexer/lexer_spec.cr +++ b/spec/compiler/lexer/lexer_spec.cr @@ -657,6 +657,15 @@ describe "Lexer" do assert_syntax_error "'\\u{DFFF}'", "invalid unicode codepoint (surrogate half)" assert_syntax_error ":+1", "unexpected token" + it "invalid byte sequence" do + expect_raises(InvalidByteSequenceError, "Unexpected byte 0xff at position 0, malformed UTF-8") do + parse "\xFF" + end + expect_raises(InvalidByteSequenceError, "Unexpected byte 0xff at position 1, malformed UTF-8") do + parse " \xFF" + end + end + assert_syntax_error "'\\1'", "invalid char escape sequence" it_lexes_string %("\\1"), String.new(Bytes[1]) diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 0f60e555cdac..dbca2448585d 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -59,6 +59,7 @@ module Crystal def initialize(string, string_pool : StringPool? = nil, warnings : WarningCollection? = nil) @warnings = warnings || WarningCollection.new @reader = Char::Reader.new(string) + check_reader_error @token = Token.new @temp_token = Token.new @line_number = 1 @@ -2754,11 +2755,13 @@ module Crystal end def next_char_no_column_increment - char = @reader.next_char + @reader.next_char.tap { check_reader_error } + end + + private def check_reader_error if error = @reader.error ::raise InvalidByteSequenceError.new("Unexpected byte 0x#{error.to_s(16)} at position #{@reader.pos}, malformed UTF-8") end - char end def next_char From 736f04c46f3ca584f856b0234961dcb074947c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 27 Jun 2024 10:18:35 +0200 Subject: [PATCH 34/52] Fix regression on `Socket#connect` timeout type restriction (#14755) The `timeout` parameter of `system_connect` has an implict type restriction of `Time::Span?` and we need to convert numeric arguments. This fixes a regression introduced in https://github.com/crystal-lang/crystal/pull/14643. --- src/socket.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/socket.cr b/src/socket.cr index dfad08d762cf..ca484c0140cc 100644 --- a/src/socket.cr +++ b/src/socket.cr @@ -110,6 +110,7 @@ class Socket < IO # Tries to connect to a remote address. Yields an `IO::TimeoutError` or an # `Socket::ConnectError` error if the connection failed. def connect(addr, timeout = nil, &) + timeout = timeout.seconds unless timeout.is_a?(::Time::Span?) result = system_connect(addr, timeout) yield result if result.is_a?(Exception) end From 912970ce9444245e2f00a6e0f219ba6293c92023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 27 Jun 2024 10:18:47 +0200 Subject: [PATCH 35/52] Drop default timeout for `Socket#connect` on Windows (#14756) IIRC the default timeout of was added when the IOCP event loop did not allow an unlimited timeout. This is possible now, and we can remove the workaround. --- src/crystal/system/win32/socket.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/win32/socket.cr b/src/crystal/system/win32/socket.cr index 2a540f4df88d..6a5d44ab5133 100644 --- a/src/crystal/system/win32/socket.cr +++ b/src/crystal/system/win32/socket.cr @@ -146,7 +146,7 @@ module Crystal::System::Socket return nil end - operation.wait_for_wsa_result(read_timeout || 1.seconds) do |error| + operation.wait_for_wsa_result(read_timeout) do |error| case error when .wsa_io_incomplete?, .wsaeconnrefused? return ::Socket::ConnectError.from_os_error(method, error) From 5cf6df9f8350957c892788e8ae74ca5475f62a92 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 27 Jun 2024 18:23:41 +0800 Subject: [PATCH 36/52] Remove incorrect uses of `describe` (#14757) Fixes a small number of specs which pass both the class name and the method name as separate arguments to `describe`. But it doesn't work like that and the second argument is treated as a file name instead. --- spec/std/benchmark_spec.cr | 126 +++---- spec/std/string_scanner_spec.cr | 568 ++++++++++++++++---------------- 2 files changed, 349 insertions(+), 345 deletions(-) diff --git a/spec/std/benchmark_spec.cr b/spec/std/benchmark_spec.cr index 1bd3738c0f3b..2f3c1fb06fd5 100644 --- a/spec/std/benchmark_spec.cr +++ b/spec/std/benchmark_spec.cr @@ -34,81 +34,83 @@ private def create_entry Benchmark::IPS::Entry.new("label", ->{ 1 + 1 }) end -describe Benchmark::IPS::Entry, "#set_cycles" do - it "sets the number of cycles needed to make 100ms" do - e = create_entry - e.set_cycles(2.seconds, 100) - e.cycles.should eq(5) - - e.set_cycles(100.milliseconds, 1) - e.cycles.should eq(1) - end - - it "sets the cycles to 1 no matter what" do - e = create_entry - e.set_cycles(2.seconds, 1) - e.cycles.should eq(1) - end +private def h_mean(mean) + create_entry.tap { |e| e.mean = mean }.human_mean end -describe Benchmark::IPS::Entry, "#calculate_stats" do - it "correctly calculates basic stats" do - e = create_entry - e.calculate_stats([2, 4, 4, 4, 5, 5, 7, 9]) +private def h_ips(seconds) + mean = 1.0 / seconds + create_entry.tap { |e| e.mean = mean }.human_iteration_time +end - e.size.should eq(8) - e.mean.should eq(5.0) - e.variance.should eq(4.0) - e.stddev.should eq(2.0) +describe Benchmark::IPS::Entry do + describe "#set_cycles" do + it "sets the number of cycles needed to make 100ms" do + e = create_entry + e.set_cycles(2.seconds, 100) + e.cycles.should eq(5) + + e.set_cycles(100.milliseconds, 1) + e.cycles.should eq(1) + end + + it "sets the cycles to 1 no matter what" do + e = create_entry + e.set_cycles(2.seconds, 1) + e.cycles.should eq(1) + end end -end -private def h_mean(mean) - create_entry.tap { |e| e.mean = mean }.human_mean -end + describe "#calculate_stats" do + it "correctly calculates basic stats" do + e = create_entry + e.calculate_stats([2, 4, 4, 4, 5, 5, 7, 9]) -describe Benchmark::IPS::Entry, "#human_mean" do - it { h_mean(0.01234567890123).should eq(" 12.35m") } - it { h_mean(0.12345678901234).should eq("123.46m") } + e.size.should eq(8) + e.mean.should eq(5.0) + e.variance.should eq(4.0) + e.stddev.should eq(2.0) + end + end - it { h_mean(1.23456789012345).should eq(" 1.23 ") } - it { h_mean(12.3456789012345).should eq(" 12.35 ") } - it { h_mean(123.456789012345).should eq("123.46 ") } + describe "#human_mean" do + it { h_mean(0.01234567890123).should eq(" 12.35m") } + it { h_mean(0.12345678901234).should eq("123.46m") } - it { h_mean(1234.56789012345).should eq(" 1.23k") } - it { h_mean(12345.6789012345).should eq(" 12.35k") } - it { h_mean(123456.789012345).should eq("123.46k") } + it { h_mean(1.23456789012345).should eq(" 1.23 ") } + it { h_mean(12.3456789012345).should eq(" 12.35 ") } + it { h_mean(123.456789012345).should eq("123.46 ") } - it { h_mean(1234567.89012345).should eq(" 1.23M") } - it { h_mean(12345678.9012345).should eq(" 12.35M") } - it { h_mean(123456789.012345).should eq("123.46M") } + it { h_mean(1234.56789012345).should eq(" 1.23k") } + it { h_mean(12345.6789012345).should eq(" 12.35k") } + it { h_mean(123456.789012345).should eq("123.46k") } - it { h_mean(1234567890.12345).should eq(" 1.23G") } - it { h_mean(12345678901.2345).should eq(" 12.35G") } - it { h_mean(123456789012.345).should eq("123.46G") } -end + it { h_mean(1234567.89012345).should eq(" 1.23M") } + it { h_mean(12345678.9012345).should eq(" 12.35M") } + it { h_mean(123456789.012345).should eq("123.46M") } -private def h_ips(seconds) - mean = 1.0 / seconds - create_entry.tap { |e| e.mean = mean }.human_iteration_time -end + it { h_mean(1234567890.12345).should eq(" 1.23G") } + it { h_mean(12345678901.2345).should eq(" 12.35G") } + it { h_mean(123456789012.345).should eq("123.46G") } + end -describe Benchmark::IPS::Entry, "#human_iteration_time" do - it { h_ips(1234.567_890_123).should eq("1,234.57s ") } - it { h_ips(123.456_789_012_3).should eq("123.46s ") } - it { h_ips(12.345_678_901_23).should eq(" 12.35s ") } - it { h_ips(1.234_567_890_123).should eq(" 1.23s ") } + describe "#human_iteration_time" do + it { h_ips(1234.567_890_123).should eq("1,234.57s ") } + it { h_ips(123.456_789_012_3).should eq("123.46s ") } + it { h_ips(12.345_678_901_23).should eq(" 12.35s ") } + it { h_ips(1.234_567_890_123).should eq(" 1.23s ") } - it { h_ips(0.123_456_789_012).should eq("123.46ms") } - it { h_ips(0.012_345_678_901).should eq(" 12.35ms") } - it { h_ips(0.001_234_567_890).should eq(" 1.23ms") } + it { h_ips(0.123_456_789_012).should eq("123.46ms") } + it { h_ips(0.012_345_678_901).should eq(" 12.35ms") } + it { h_ips(0.001_234_567_890).should eq(" 1.23ms") } - it { h_ips(0.000_123_456_789).should eq("123.46µs") } - it { h_ips(0.000_012_345_678).should eq(" 12.35µs") } - it { h_ips(0.000_001_234_567).should eq(" 1.23µs") } + it { h_ips(0.000_123_456_789).should eq("123.46µs") } + it { h_ips(0.000_012_345_678).should eq(" 12.35µs") } + it { h_ips(0.000_001_234_567).should eq(" 1.23µs") } - it { h_ips(0.000_000_123_456).should eq("123.46ns") } - it { h_ips(0.000_000_012_345).should eq(" 12.34ns") } - it { h_ips(0.000_000_001_234).should eq(" 1.23ns") } - it { h_ips(0.000_000_000_123).should eq(" 0.12ns") } + it { h_ips(0.000_000_123_456).should eq("123.46ns") } + it { h_ips(0.000_000_012_345).should eq(" 12.34ns") } + it { h_ips(0.000_000_001_234).should eq(" 1.23ns") } + it { h_ips(0.000_000_000_123).should eq(" 0.12ns") } + end end diff --git a/spec/std/string_scanner_spec.cr b/spec/std/string_scanner_spec.cr index 5513e44b7902..18a661b46638 100644 --- a/spec/std/string_scanner_spec.cr +++ b/spec/std/string_scanner_spec.cr @@ -1,356 +1,358 @@ require "spec" require "string_scanner" -describe StringScanner, "#scan" do - it "returns the string matched and advances the offset" do - s = StringScanner.new("this is a string") - s.scan(/\w+/).should eq("this") - s.scan(' ').should eq(" ") - s.scan("is ").should eq("is ") - s.scan(/\w+\s/).should eq("a ") - s.scan(/\w+/).should eq("string") +describe StringScanner do + describe "#scan" do + it "returns the string matched and advances the offset" do + s = StringScanner.new("this is a string") + s.scan(/\w+/).should eq("this") + s.scan(' ').should eq(" ") + s.scan("is ").should eq("is ") + s.scan(/\w+\s/).should eq("a ") + s.scan(/\w+/).should eq("string") + end + + it "returns nil if it can't match from the offset" do + s = StringScanner.new("test string") + s.scan(/\w+/).should_not be_nil # => "test" + s.scan(/\w+/).should be_nil + s.scan('s').should be_nil + s.scan("string").should be_nil + s.scan(/\s\w+/).should_not be_nil # => " string" + s.scan(/.*/).should_not be_nil # => "" + end end - it "returns nil if it can't match from the offset" do - s = StringScanner.new("test string") - s.scan(/\w+/).should_not be_nil # => "test" - s.scan(/\w+/).should be_nil - s.scan('s').should be_nil - s.scan("string").should be_nil - s.scan(/\s\w+/).should_not be_nil # => " string" - s.scan(/.*/).should_not be_nil # => "" + describe "#scan_until" do + it "returns the string matched and advances the offset" do + s = StringScanner.new("test string") + s.scan_until(/t /).should eq("test ") + s.offset.should eq(5) + s.scan_until("tr").should eq("str") + s.offset.should eq(8) + s.scan_until('n').should eq("in") + s.offset.should eq(10) + end + + it "returns nil if it can't match from the offset" do + s = StringScanner.new("test string") + s.offset = 8 + s.scan_until(/tr/).should be_nil + s.scan_until('r').should be_nil + s.scan_until("tr").should be_nil + end end -end - -describe StringScanner, "#scan_until" do - it "returns the string matched and advances the offset" do - s = StringScanner.new("test string") - s.scan_until(/t /).should eq("test ") - s.offset.should eq(5) - s.scan_until("tr").should eq("str") - s.offset.should eq(8) - s.scan_until('n').should eq("in") - s.offset.should eq(10) - end - - it "returns nil if it can't match from the offset" do - s = StringScanner.new("test string") - s.offset = 8 - s.scan_until(/tr/).should be_nil - s.scan_until('r').should be_nil - s.scan_until("tr").should be_nil - end -end -describe StringScanner, "#skip" do - it "advances the offset but does not returns the string matched" do - s = StringScanner.new("this is a string") + describe "#skip" do + it "advances the offset but does not returns the string matched" do + s = StringScanner.new("this is a string") - s.skip(/\w+\s/).should eq(5) - s.offset.should eq(5) - s[0]?.should_not be_nil + s.skip(/\w+\s/).should eq(5) + s.offset.should eq(5) + s[0]?.should_not be_nil - s.skip(/\d+/).should eq(nil) - s.offset.should eq(5) + s.skip(/\d+/).should eq(nil) + s.offset.should eq(5) - s.skip('i').should eq(1) - s.offset.should eq(6) + s.skip('i').should eq(1) + s.offset.should eq(6) - s.skip("s ").should eq(2) - s.offset.should eq(8) + s.skip("s ").should eq(2) + s.offset.should eq(8) - s.skip(/\w+\s/).should eq(2) - s.offset.should eq(10) + s.skip(/\w+\s/).should eq(2) + s.offset.should eq(10) - s.skip(/\w+/).should eq(6) - s.offset.should eq(16) + s.skip(/\w+/).should eq(6) + s.offset.should eq(16) + end end -end - -describe StringScanner, "#skip_until" do - it "advances the offset but does not returns the string matched" do - s = StringScanner.new("this is a string") - s.skip_until(/not/).should eq(nil) - s.offset.should eq(0) - s[0]?.should be_nil + describe "#skip_until" do + it "advances the offset but does not returns the string matched" do + s = StringScanner.new("this is a string") - s.skip_until(/\sis\s/).should eq(8) - s.offset.should eq(8) - s[0]?.should_not be_nil + s.skip_until(/not/).should eq(nil) + s.offset.should eq(0) + s[0]?.should be_nil - s.skip_until("st").should eq(4) - s.offset.should eq(12) - s[0]?.should_not be_nil + s.skip_until(/\sis\s/).should eq(8) + s.offset.should eq(8) + s[0]?.should_not be_nil - s.skip_until("ng").should eq(4) - s.offset.should eq(16) - s[0]?.should_not be_nil - end -end - -describe StringScanner, "#eos" do - it "it is true when the offset is at the end" do - s = StringScanner.new("this is a string") - s.eos?.should eq(false) - s.skip(/(\w+\s?){4}/) - s.eos?.should eq(true) - end -end + s.skip_until("st").should eq(4) + s.offset.should eq(12) + s[0]?.should_not be_nil -describe StringScanner, "#check" do - it "returns the string matched but does not advances the offset" do - s = StringScanner.new("this is a string") - s.offset = 5 - - s.check(/\w+\s/).should eq("is ") - s.offset.should eq(5) - s.check(/\w+\s/).should eq("is ") - s.offset.should eq(5) - s.check('i').should eq("i") - s.offset.should eq(5) - s.check("is ").should eq("is ") - s.offset.should eq(5) + s.skip_until("ng").should eq(4) + s.offset.should eq(16) + s[0]?.should_not be_nil + end end - it "returns nil if it can't match from the offset" do - s = StringScanner.new("test string") - s.check(/\d+/).should be_nil - s.check('0').should be_nil - s.check("01").should be_nil + describe "#eos" do + it "it is true when the offset is at the end" do + s = StringScanner.new("this is a string") + s.eos?.should eq(false) + s.skip(/(\w+\s?){4}/) + s.eos?.should eq(true) + end end -end -describe StringScanner, "#check_until" do - it "returns the string matched and advances the offset" do - s = StringScanner.new("test string") - s.check_until(/tr/).should eq("test str") - s.offset.should eq(0) - s.check_until('r').should eq("test str") - s.offset.should eq(0) - s.check_until("tr").should eq("test str") - s.offset.should eq(0) - s.check_until(/g/).should eq("test string") - s.offset.should eq(0) - s.check_until('g').should eq("test string") - s.offset.should eq(0) - s.check_until("ng").should eq("test string") - s.offset.should eq(0) + describe "#check" do + it "returns the string matched but does not advances the offset" do + s = StringScanner.new("this is a string") + s.offset = 5 + + s.check(/\w+\s/).should eq("is ") + s.offset.should eq(5) + s.check(/\w+\s/).should eq("is ") + s.offset.should eq(5) + s.check('i').should eq("i") + s.offset.should eq(5) + s.check("is ").should eq("is ") + s.offset.should eq(5) + end + + it "returns nil if it can't match from the offset" do + s = StringScanner.new("test string") + s.check(/\d+/).should be_nil + s.check('0').should be_nil + s.check("01").should be_nil + end end - it "returns nil if it can't match from the offset" do - s = StringScanner.new("test string") - s.offset = 8 - s.check_until(/tr/).should be_nil - s.check_until('r').should be_nil - s.check_until("tr").should be_nil + describe "#check_until" do + it "returns the string matched and advances the offset" do + s = StringScanner.new("test string") + s.check_until(/tr/).should eq("test str") + s.offset.should eq(0) + s.check_until('r').should eq("test str") + s.offset.should eq(0) + s.check_until("tr").should eq("test str") + s.offset.should eq(0) + s.check_until(/g/).should eq("test string") + s.offset.should eq(0) + s.check_until('g').should eq("test string") + s.offset.should eq(0) + s.check_until("ng").should eq("test string") + s.offset.should eq(0) + end + + it "returns nil if it can't match from the offset" do + s = StringScanner.new("test string") + s.offset = 8 + s.check_until(/tr/).should be_nil + s.check_until('r').should be_nil + s.check_until("tr").should be_nil + end end -end -describe StringScanner, "#rest" do - it "returns the remainder of the string from the offset" do - s = StringScanner.new("this is a string") - s.rest.should eq("this is a string") + describe "#rest" do + it "returns the remainder of the string from the offset" do + s = StringScanner.new("this is a string") + s.rest.should eq("this is a string") - s.scan(/this is a /) - s.rest.should eq("string") + s.scan(/this is a /) + s.rest.should eq("string") - s.scan(/string/) - s.rest.should eq("") + s.scan(/string/) + s.rest.should eq("") + end end -end -describe StringScanner, "#[]" do - it "allows access to subgroups of the last match" do - s = StringScanner.new("Fri Dec 12 1975 14:39") - regex = /(?\w+) (?\w+) (?\d+)/ - s.scan(regex).should eq("Fri Dec 12") - s[0].should eq("Fri Dec 12") - s[1].should eq("Fri") - s[2].should eq("Dec") - s[3].should eq("12") - s["wday"].should eq("Fri") - s["month"].should eq("Dec") - s["day"].should eq("12") - - s.scan(' ').should eq(" ") - s[0].should eq(" ") - s.scan("1975").should eq("1975") - s[0].should eq("1975") - end + describe "#[]" do + it "allows access to subgroups of the last match" do + s = StringScanner.new("Fri Dec 12 1975 14:39") + regex = /(?\w+) (?\w+) (?\d+)/ + s.scan(regex).should eq("Fri Dec 12") + s[0].should eq("Fri Dec 12") + s[1].should eq("Fri") + s[2].should eq("Dec") + s[3].should eq("12") + s["wday"].should eq("Fri") + s["month"].should eq("Dec") + s["day"].should eq("12") - it "raises when there is no last match" do - s = StringScanner.new("Fri Dec 12 1975 14:39") + s.scan(' ').should eq(" ") + s[0].should eq(" ") + s.scan("1975").should eq("1975") + s[0].should eq("1975") + end - s.scan(/this is not there/) - expect_raises(Exception, "Nil assertion failed") { s[0] } + it "raises when there is no last match" do + s = StringScanner.new("Fri Dec 12 1975 14:39") - s.scan('t') - expect_raises(Exception, "Nil assertion failed") { s[0] } + s.scan(/this is not there/) + expect_raises(Exception, "Nil assertion failed") { s[0] } - s.scan("this is not there") - expect_raises(Exception, "Nil assertion failed") { s[0] } - end + s.scan('t') + expect_raises(Exception, "Nil assertion failed") { s[0] } - it "raises when there is no subgroup" do - s = StringScanner.new("Fri Dec 12 1975 14:39") - regex = /(?\w+) (?\w+) (?\d+)/ + s.scan("this is not there") + expect_raises(Exception, "Nil assertion failed") { s[0] } + end - s.scan(regex) + it "raises when there is no subgroup" do + s = StringScanner.new("Fri Dec 12 1975 14:39") + regex = /(?\w+) (?\w+) (?\d+)/ - s[0].should_not be_nil - expect_raises(IndexError) { s[5] } - expect_raises(KeyError, "Capture group 'something' does not exist") { s["something"] } + s.scan(regex) - s.scan(' ') + s[0].should_not be_nil + expect_raises(IndexError) { s[5] } + expect_raises(KeyError, "Capture group 'something' does not exist") { s["something"] } - s[0].should_not be_nil - expect_raises(IndexError) { s[1] } - expect_raises(KeyError, "Capture group 'something' does not exist") { s["something"] } + s.scan(' ') - s.scan("1975") + s[0].should_not be_nil + expect_raises(IndexError) { s[1] } + expect_raises(KeyError, "Capture group 'something' does not exist") { s["something"] } - s[0].should_not be_nil - expect_raises(IndexError) { s[1] } - expect_raises(KeyError, "Capture group 'something' does not exist") { s["something"] } - end -end + s.scan("1975") -describe StringScanner, "#[]?" do - it "allows access to subgroups of the last match" do - s = StringScanner.new("Fri Dec 12 1975 14:39") - result = s.scan(/(?\w+) (?\w+) (?\d+)/) - - result.should eq("Fri Dec 12") - s[0]?.should eq("Fri Dec 12") - s[1]?.should eq("Fri") - s[2]?.should eq("Dec") - s[3]?.should eq("12") - s["wday"]?.should eq("Fri") - s["month"]?.should eq("Dec") - s["day"]?.should eq("12") - - s.scan(' ').should eq(" ") - s[0]?.should eq(" ") - s.scan("1975").should eq("1975") - s[0]?.should eq("1975") + s[0].should_not be_nil + expect_raises(IndexError) { s[1] } + expect_raises(KeyError, "Capture group 'something' does not exist") { s["something"] } + end end - it "returns nil when there is no last match" do - s = StringScanner.new("Fri Dec 12 1975 14:39") - s.scan(/this is not there/) + describe "#[]?" do + it "allows access to subgroups of the last match" do + s = StringScanner.new("Fri Dec 12 1975 14:39") + result = s.scan(/(?\w+) (?\w+) (?\d+)/) - s[0]?.should be_nil + result.should eq("Fri Dec 12") + s[0]?.should eq("Fri Dec 12") + s[1]?.should eq("Fri") + s[2]?.should eq("Dec") + s[3]?.should eq("12") + s["wday"]?.should eq("Fri") + s["month"]?.should eq("Dec") + s["day"]?.should eq("12") - s.scan('t') - s[0]?.should be_nil + s.scan(' ').should eq(" ") + s[0]?.should eq(" ") + s.scan("1975").should eq("1975") + s[0]?.should eq("1975") + end - s.scan("this is not there") - s[0]?.should be_nil - end + it "returns nil when there is no last match" do + s = StringScanner.new("Fri Dec 12 1975 14:39") + s.scan(/this is not there/) - it "raises when there is no subgroup" do - s = StringScanner.new("Fri Dec 12 1975 14:39") + s[0]?.should be_nil - s.scan(/(?\w+) (?\w+) (?\d+)/) + s.scan('t') + s[0]?.should be_nil - s[0]?.should_not be_nil - s[5]?.should be_nil - s["something"]?.should be_nil + s.scan("this is not there") + s[0]?.should be_nil + end - s.scan(' ') + it "raises when there is no subgroup" do + s = StringScanner.new("Fri Dec 12 1975 14:39") - s[0]?.should_not be_nil - s[1]?.should be_nil - s["something"]?.should be_nil + s.scan(/(?\w+) (?\w+) (?\d+)/) - s.scan("1975") + s[0]?.should_not be_nil + s[5]?.should be_nil + s["something"]?.should be_nil - s[0]?.should_not be_nil - s[1]?.should be_nil - s["something"]?.should be_nil - end -end + s.scan(' ') -describe StringScanner, "#string" do - it { StringScanner.new("foo").string.should eq("foo") } -end + s[0]?.should_not be_nil + s[1]?.should be_nil + s["something"]?.should be_nil -describe StringScanner, "#offset" do - it "returns the current position" do - s = StringScanner.new("this is a string") - s.offset.should eq(0) - s.scan(/\w+/) - s.offset.should eq(4) - end -end + s.scan("1975") -describe StringScanner, "#offset=" do - it "sets the current position" do - s = StringScanner.new("this is a string") - s.offset = 5 - s.scan(/\w+/).should eq("is") + s[0]?.should_not be_nil + s[1]?.should be_nil + s["something"]?.should be_nil + end end - it "raises on negative positions" do - s = StringScanner.new("this is a string") - expect_raises(IndexError) { s.offset = -2 } + describe "#string" do + it { StringScanner.new("foo").string.should eq("foo") } end -end -describe StringScanner, "#inspect" do - it "has information on the scanner" do - s = StringScanner.new("this is a string") - s.inspect.should eq(%(#)) - s.scan(/\w+\s/) - s.inspect.should eq(%(#)) - s.scan(/\w+\s/) - s.inspect.should eq(%(#)) - s.scan(/\w+\s\w+/) - s.inspect.should eq(%(#)) + describe "#offset" do + it "returns the current position" do + s = StringScanner.new("this is a string") + s.offset.should eq(0) + s.scan(/\w+/) + s.offset.should eq(4) + end end - it "works with small strings" do - s = StringScanner.new("hi") - s.inspect.should eq(%(#)) - s.scan(/\w\w/) - s.inspect.should eq(%(#)) + describe "#offset=" do + it "sets the current position" do + s = StringScanner.new("this is a string") + s.offset = 5 + s.scan(/\w+/).should eq("is") + end + + it "raises on negative positions" do + s = StringScanner.new("this is a string") + expect_raises(IndexError) { s.offset = -2 } + end end -end -describe StringScanner, "#peek" do - it "shows the next len characters without advancing the offset" do - s = StringScanner.new("this is a string") - s.offset.should eq(0) - s.peek(4).should eq("this") - s.offset.should eq(0) - s.peek(7).should eq("this is") - s.offset.should eq(0) + describe "#inspect" do + it "has information on the scanner" do + s = StringScanner.new("this is a string") + s.inspect.should eq(%(#)) + s.scan(/\w+\s/) + s.inspect.should eq(%(#)) + s.scan(/\w+\s/) + s.inspect.should eq(%(#)) + s.scan(/\w+\s\w+/) + s.inspect.should eq(%(#)) + end + + it "works with small strings" do + s = StringScanner.new("hi") + s.inspect.should eq(%(#)) + s.scan(/\w\w/) + s.inspect.should eq(%(#)) + end end -end - -describe StringScanner, "#reset" do - it "resets the scan offset to the beginning and clears the last match" do - s = StringScanner.new("this is a string") - s.scan_until(/str/) - s[0]?.should_not be_nil - s.offset.should_not eq(0) - s.reset - s[0]?.should be_nil - s.offset.should eq(0) + describe "#peek" do + it "shows the next len characters without advancing the offset" do + s = StringScanner.new("this is a string") + s.offset.should eq(0) + s.peek(4).should eq("this") + s.offset.should eq(0) + s.peek(7).should eq("this is") + s.offset.should eq(0) + end end -end -describe StringScanner, "#terminate" do - it "moves the scan offset to the end of the string and clears the last match" do - s = StringScanner.new("this is a string") - s.scan_until(/str/) - s[0]?.should_not be_nil - s.eos?.should eq(false) + describe "#reset" do + it "resets the scan offset to the beginning and clears the last match" do + s = StringScanner.new("this is a string") + s.scan_until(/str/) + s[0]?.should_not be_nil + s.offset.should_not eq(0) + + s.reset + s[0]?.should be_nil + s.offset.should eq(0) + end + end - s.terminate - s[0]?.should be_nil - s.eos?.should eq(true) + describe "#terminate" do + it "moves the scan offset to the end of the string and clears the last match" do + s = StringScanner.new("this is a string") + s.scan_until(/str/) + s[0]?.should_not be_nil + s.eos?.should eq(false) + + s.terminate + s[0]?.should be_nil + s.eos?.should eq(true) + end end end From 70ed2d0f3a80ddbdff0eb4cdb4de9f98998c6622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 1 Jul 2024 12:02:29 +0200 Subject: [PATCH 37/52] Fix `GC.malloc` for `gc_none` to clear memory (#14746) `GC.malloc` is supposed to clear the memory, but libc `malloc` does not do this. So we need to add it explicitly. --- src/gc/none.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gc/none.cr b/src/gc/none.cr index 1121caef1bf4..640e6e8f927d 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -10,7 +10,9 @@ module GC # :nodoc: def self.malloc(size : LibC::SizeT) : Void* Crystal.trace :gc, "malloc", size: size - LibC.malloc(size) + # libc malloc is not guaranteed to return cleared memory, so we need to + # explicitly clear it. Ref: https://github.com/crystal-lang/crystal/issues/14678 + LibC.malloc(size).tap(&.clear) end # :nodoc: From 057771fdd235b2f3b7d8ea9e22673a5c683bab94 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 1 Jul 2024 12:03:14 +0200 Subject: [PATCH 38/52] Prefer `strerror_r` over `strerror` for thread-safe errno (#14764) Prefer the thread safe `strerror_r` C call over the thread unsafe `strerror`. --- src/errno.cr | 13 +++++++++++-- src/lib_c/aarch64-android/c/string.cr | 1 + src/lib_c/aarch64-darwin/c/string.cr | 1 + src/lib_c/aarch64-linux-gnu/c/string.cr | 1 + src/lib_c/aarch64-linux-musl/c/string.cr | 1 + src/lib_c/arm-linux-gnueabihf/c/string.cr | 1 + src/lib_c/i386-linux-gnu/c/string.cr | 1 + src/lib_c/i386-linux-musl/c/string.cr | 1 + src/lib_c/wasm32-wasi/c/string.cr | 1 + src/lib_c/x86_64-darwin/c/string.cr | 1 + src/lib_c/x86_64-dragonfly/c/string.cr | 1 + src/lib_c/x86_64-freebsd/c/string.cr | 1 + src/lib_c/x86_64-linux-gnu/c/string.cr | 1 + src/lib_c/x86_64-linux-musl/c/string.cr | 1 + src/lib_c/x86_64-netbsd/c/string.cr | 1 + src/lib_c/x86_64-openbsd/c/string.cr | 1 + src/lib_c/x86_64-solaris/c/string.cr | 1 + 17 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/errno.cr b/src/errno.cr index 2a68371f4a19..9d608c80bc1b 100644 --- a/src/errno.cr +++ b/src/errno.cr @@ -45,8 +45,17 @@ enum Errno # :nodoc: def unsafe_message(&) - pointer = LibC.strerror(value) - yield Bytes.new(pointer, LibC.strlen(pointer)) + {% if LibC.has_method?(:strerror_r) %} + buffer = uninitialized UInt8[256] + if LibC.strerror_r(value, buffer, buffer.size) == 0 + yield Bytes.new(buffer.to_unsafe, LibC.strlen(buffer)) + else + yield "(???)".to_slice + end + {% else %} + pointer = LibC.strerror(value) + yield Bytes.new(pointer, LibC.strlen(pointer)) + {% end %} end # returns the value of libc's errno. diff --git a/src/lib_c/aarch64-android/c/string.cr b/src/lib_c/aarch64-android/c/string.cr index 5133435e13dc..583a40e7c7f1 100644 --- a/src/lib_c/aarch64-android/c/string.cr +++ b/src/lib_c/aarch64-android/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(__lhs : Void*, __rhs : Void*, __n : SizeT) : Int fun strcmp(__lhs : Char*, __rhs : Char*) : Int fun strerror(__errno_value : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(__s : Char*) : SizeT end diff --git a/src/lib_c/aarch64-darwin/c/string.cr b/src/lib_c/aarch64-darwin/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/aarch64-darwin/c/string.cr +++ b/src/lib_c/aarch64-darwin/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end diff --git a/src/lib_c/aarch64-linux-gnu/c/string.cr b/src/lib_c/aarch64-linux-gnu/c/string.cr index c583804acd98..0d012a54002b 100644 --- a/src/lib_c/aarch64-linux-gnu/c/string.cr +++ b/src/lib_c/aarch64-linux-gnu/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(s1 : Void*, s2 : Void*, n : SizeT) : Int fun strcmp(s1 : Char*, s2 : Char*) : Int fun strerror(errnum : Int) : Char* + fun strerror_r = __xpg_strerror_r(Int, Char*, SizeT) : Int fun strlen(s : Char*) : SizeT end diff --git a/src/lib_c/aarch64-linux-musl/c/string.cr b/src/lib_c/aarch64-linux-musl/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/aarch64-linux-musl/c/string.cr +++ b/src/lib_c/aarch64-linux-musl/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end diff --git a/src/lib_c/arm-linux-gnueabihf/c/string.cr b/src/lib_c/arm-linux-gnueabihf/c/string.cr index c583804acd98..0d012a54002b 100644 --- a/src/lib_c/arm-linux-gnueabihf/c/string.cr +++ b/src/lib_c/arm-linux-gnueabihf/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(s1 : Void*, s2 : Void*, n : SizeT) : Int fun strcmp(s1 : Char*, s2 : Char*) : Int fun strerror(errnum : Int) : Char* + fun strerror_r = __xpg_strerror_r(Int, Char*, SizeT) : Int fun strlen(s : Char*) : SizeT end diff --git a/src/lib_c/i386-linux-gnu/c/string.cr b/src/lib_c/i386-linux-gnu/c/string.cr index c583804acd98..0d012a54002b 100644 --- a/src/lib_c/i386-linux-gnu/c/string.cr +++ b/src/lib_c/i386-linux-gnu/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(s1 : Void*, s2 : Void*, n : SizeT) : Int fun strcmp(s1 : Char*, s2 : Char*) : Int fun strerror(errnum : Int) : Char* + fun strerror_r = __xpg_strerror_r(Int, Char*, SizeT) : Int fun strlen(s : Char*) : SizeT end diff --git a/src/lib_c/i386-linux-musl/c/string.cr b/src/lib_c/i386-linux-musl/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/i386-linux-musl/c/string.cr +++ b/src/lib_c/i386-linux-musl/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end diff --git a/src/lib_c/wasm32-wasi/c/string.cr b/src/lib_c/wasm32-wasi/c/string.cr index 5be77e03cf1c..e12128de9659 100644 --- a/src/lib_c/wasm32-wasi/c/string.cr +++ b/src/lib_c/wasm32-wasi/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : ULong end diff --git a/src/lib_c/x86_64-darwin/c/string.cr b/src/lib_c/x86_64-darwin/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/x86_64-darwin/c/string.cr +++ b/src/lib_c/x86_64-darwin/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end diff --git a/src/lib_c/x86_64-dragonfly/c/string.cr b/src/lib_c/x86_64-dragonfly/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/x86_64-dragonfly/c/string.cr +++ b/src/lib_c/x86_64-dragonfly/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end diff --git a/src/lib_c/x86_64-freebsd/c/string.cr b/src/lib_c/x86_64-freebsd/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/x86_64-freebsd/c/string.cr +++ b/src/lib_c/x86_64-freebsd/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end diff --git a/src/lib_c/x86_64-linux-gnu/c/string.cr b/src/lib_c/x86_64-linux-gnu/c/string.cr index c583804acd98..0d012a54002b 100644 --- a/src/lib_c/x86_64-linux-gnu/c/string.cr +++ b/src/lib_c/x86_64-linux-gnu/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(s1 : Void*, s2 : Void*, n : SizeT) : Int fun strcmp(s1 : Char*, s2 : Char*) : Int fun strerror(errnum : Int) : Char* + fun strerror_r = __xpg_strerror_r(Int, Char*, SizeT) : Int fun strlen(s : Char*) : SizeT end diff --git a/src/lib_c/x86_64-linux-musl/c/string.cr b/src/lib_c/x86_64-linux-musl/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/x86_64-linux-musl/c/string.cr +++ b/src/lib_c/x86_64-linux-musl/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end diff --git a/src/lib_c/x86_64-netbsd/c/string.cr b/src/lib_c/x86_64-netbsd/c/string.cr index 471d1ed82b36..ff94ee456646 100644 --- a/src/lib_c/x86_64-netbsd/c/string.cr +++ b/src/lib_c/x86_64-netbsd/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : ULong end diff --git a/src/lib_c/x86_64-openbsd/c/string.cr b/src/lib_c/x86_64-openbsd/c/string.cr index 471d1ed82b36..ff94ee456646 100644 --- a/src/lib_c/x86_64-openbsd/c/string.cr +++ b/src/lib_c/x86_64-openbsd/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : ULong end diff --git a/src/lib_c/x86_64-solaris/c/string.cr b/src/lib_c/x86_64-solaris/c/string.cr index 02e025ae4880..b9657fc871f7 100644 --- a/src/lib_c/x86_64-solaris/c/string.cr +++ b/src/lib_c/x86_64-solaris/c/string.cr @@ -5,5 +5,6 @@ lib LibC fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int fun strcmp(x0 : Char*, x1 : Char*) : Int fun strerror(x0 : Int) : Char* + fun strerror_r(Int, Char*, SizeT) : Int fun strlen(x0 : Char*) : SizeT end From e279b3c7f90a43ff654694d9f7726a244aeda988 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 1 Jul 2024 12:04:06 +0200 Subject: [PATCH 39/52] Detect and error on failed codegen process (#14762) Report an exception when it occurs in a codegen forked process, otherwise detects when a codegen process terminated early (which is what happens on LLVM error). In both cases a BUG message is printed on stderr and the main process exits. --- src/compiler/crystal/compiler.cr | 34 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index eec190e85eeb..1d540e02f2e9 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -533,6 +533,9 @@ module Crystal result = {name: unit.name, reused: unit.reused_previous_compilation?} output.puts result.to_json end + rescue ex + result = {exception: {name: ex.class.name, message: ex.message, backtrace: ex.backtrace}} + output.puts result.to_json end overqueue = 1 @@ -554,13 +557,21 @@ module Crystal while (index = indexes.add(1)) < units.size input.puts index - response = output.gets(chomp: true).not_nil! - channel.send response + if response = output.gets(chomp: true) + channel.send response + else + Crystal::System.print_error "\nBUG: a codegen process failed\n" + exit 1 + end end overqueued.times do - response = output.gets(chomp: true).not_nil! - channel.send response + if response = output.gets(chomp: true) + channel.send response + else + Crystal::System.print_error "\nBUG: a codegen process failed\n" + exit 1 + end end input << '\n' @@ -578,11 +589,18 @@ module Crystal end while response = channel.receive? - next unless wants_stats_or_progress - result = JSON.parse(response) - all_reused << result["name"].as_s if result["reused"].as_bool - @progress_tracker.stage_progress += 1 + + if ex = result["exception"]? + Crystal::System.print_error "\nBUG: a codegen process failed: %s (%s)\n", ex["message"].as_s, ex["name"].as_s + ex["backtrace"].as_a?.try(&.each { |frame| Crystal::System.print_error " from %s\n", frame }) + exit 1 + end + + if wants_stats_or_progress + all_reused << result["name"].as_s if result["reused"].as_bool + @progress_tracker.stage_progress += 1 + end end all_reused From 5b500882bb51cff0f38a51735bf008923cde2219 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 1 Jul 2024 17:31:15 +0200 Subject: [PATCH 40/52] Fix: don't hardcode alpn protocol byte size (openssl) (#14769) For some reason OpenSSL used to negotiate the protocol by itself, without invoking the select callback, or maybe it didn't respect the total bytesize when processing the alpn string. That changed in the 3.0.14 and other bugfix releases of OpenSSL, which exposed the bug. --- src/openssl/ssl/context.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openssl/ssl/context.cr b/src/openssl/ssl/context.cr index c7d5b5a0de2a..38e0054cba17 100644 --- a/src/openssl/ssl/context.cr +++ b/src/openssl/ssl/context.cr @@ -178,7 +178,7 @@ abstract class OpenSSL::SSL::Context {% if LibSSL.has_method?(:ssl_ctx_set_alpn_select_cb) %} alpn_cb = ->(ssl : LibSSL::SSL, o : LibC::Char**, olen : LibC::Char*, i : LibC::Char*, ilen : LibC::Int, data : Void*) { proto = Box(Bytes).unbox(data) - ret = LibSSL.ssl_select_next_proto(o, olen, proto, 2, i, ilen) + ret = LibSSL.ssl_select_next_proto(o, olen, proto, proto.size, i, ilen) if ret != LibSSL::OPENSSL_NPN_NEGOTIATED LibSSL::SSL_TLSEXT_ERR_NOACK else From 4d9a7e8410f0db731c2b39a62241c10e8f34a007 Mon Sep 17 00:00:00 2001 From: Hugo Parente Lima Date: Mon, 1 Jul 2024 19:24:46 -0300 Subject: [PATCH 41/52] =?UTF-8?q?Fix=20exa=C3=B6.e=20`WeakRef`=20example?= =?UTF-8?q?=20by=20removing=20ref.value=20call.=20(#10846)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `ref.value` call cause the GC to not collect `ref` until end of scope. Co-authored-by: Beta Ziliani --- src/weak_ref.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/weak_ref.cr b/src/weak_ref.cr index b5f7468383d0..518f51ff772d 100644 --- a/src/weak_ref.cr +++ b/src/weak_ref.cr @@ -7,10 +7,14 @@ # require "weak_ref" # # ref = WeakRef.new("oof".reverse) -# p ref.value # => "foo" +# p ref # => # # GC.collect +# p ref # => # # p ref.value # => nil # ``` +# +# Note that the collection of objects is not deterministic, and depends on many subtle aspects. For instance, +# if the example above is modified to print `ref.value` in the first print, then the collector will not collect it. class WeakRef(T) @target : Void* From 53c4991c9afbca84086c48be3d63fff59df04d37 Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Tue, 2 Jul 2024 00:25:40 +0200 Subject: [PATCH 42/52] Codegen: stats and progress issues (#14763) The `--stats` and `--progress` params had a couple issues: - codegen progress isn't updated when `--threads=1` (always the case on Windows); - only stats need to collect reused modules (progress doesn't). --- src/compiler/crystal/compiler.cr | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 1d540e02f2e9..b30b184e1023 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -505,8 +505,6 @@ module Crystal private def codegen_many_units(program, units, target_triple) all_reused = [] of String - wants_stats_or_progress = @progress_tracker.stats? || @progress_tracker.progress? - # Don't start more processes than compilation units n_threads = @n_threads.clamp(1..units.size) @@ -516,7 +514,12 @@ module Crystal if n_threads == 1 units.each do |unit| unit.compile - all_reused << unit.name if wants_stats_or_progress && unit.reused_previous_compilation? + @progress_tracker.stage_progress += 1 + end + if @progress_tracker.stats? + units.each do |unit| + all_reused << unit.name && unit.reused_previous_compilation? + end end return all_reused end @@ -597,10 +600,10 @@ module Crystal exit 1 end - if wants_stats_or_progress + if @progress_tracker.stats? all_reused << result["name"].as_s if result["reused"].as_bool - @progress_tracker.stage_progress += 1 end + @progress_tracker.stage_progress += 1 end all_reused From be46ba200c049772a625fb5c719ff7f9b198c0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 3 Jul 2024 13:34:42 +0200 Subject: [PATCH 43/52] Fix `IO::Delimited` reading into limited slice with peek (#14772) --- spec/std/io/delimited_spec.cr | 16 ++++++++++++++++ src/io/delimited.cr | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/spec/std/io/delimited_spec.cr b/spec/std/io/delimited_spec.cr index 63096322237d..b41af9ee5fdb 100644 --- a/spec/std/io/delimited_spec.cr +++ b/spec/std/io/delimited_spec.cr @@ -259,6 +259,22 @@ describe "IO::Delimited" do io.gets_to_end.should eq("hello") end + it "handles the case of peek matching first byte, not having enough room, but later not matching (limted slice)" do + # not a delimiter + # --- + io = MemoryIOWithFixedPeek.new("abcdefgwijkfghhello") + # ------- --- + # peek delimiter + io.peek_size = 7 + delimited = IO::Delimited.new(io, read_delimiter: "fgh") + + delimited.peek.should eq("abcde".to_slice) + delimited.read_string(6).should eq "abcdef" + delimited.read_string(5).should eq("gwijk") + delimited.gets_to_end.should eq("") + io.gets_to_end.should eq("hello") + end + it "handles the case of peek matching first byte, not having enough room, later only partially matching" do # delimiter # ------------ diff --git a/src/io/delimited.cr b/src/io/delimited.cr index b0e235881499..4da7074b52bb 100644 --- a/src/io/delimited.cr +++ b/src/io/delimited.cr @@ -111,11 +111,13 @@ class IO::Delimited < IO next_index = @active_delimiter_buffer.index(first_byte, 1) # We read up to that new match, if any, or the entire buffer - read_bytes = next_index || @active_delimiter_buffer.size + read_bytes = Math.min(next_index || @active_delimiter_buffer.size, slice.size) slice.copy_from(@active_delimiter_buffer[0, read_bytes]) slice += read_bytes @active_delimiter_buffer += read_bytes + + return read_bytes if slice.empty? return read_bytes + read_internal(slice) end end From cdf54629e65e77489076b277c52ed78785dde588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 4 Jul 2024 11:09:58 +0200 Subject: [PATCH 44/52] [CI] Update to Ruby 3 in macOS circleCI runner (#14777) --- .circleci/config.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf6d612d61b0..0c1227013673 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -296,12 +296,7 @@ jobs: command: | brew unlink python@2 || true - # We need ruby-install >= 0.8.3 - brew install ruby-install - - ruby-install ruby 2.7.3 - - brew install pkgconfig libtool + brew install ruby@3 libffi pkgconfig libtool sudo mkdir -p /opt/crystal sudo chown $(whoami) /opt/crystal/ @@ -312,7 +307,6 @@ jobs: - run: no_output_timeout: 40m command: | - echo "2.7.3" > /tmp/workspace/distribution-scripts/.ruby-version cd /tmp/workspace/distribution-scripts source build.env cd omnibus From b3412d2d40275067f5567e402583933911d3a2e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 4 Jul 2024 14:24:23 +0200 Subject: [PATCH 45/52] Update distribution-scripts (#14776) Updates `distribution-scripts` dependency to https://github.com/crystal-lang/distribution-scripts/commit/96e431e170979125018bd4fd90111a3147477eec This includes the following changes: * crystal-lang/distribution-scripts#320 * crystal-lang/distribution-scripts#319 * Install automake --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c1227013673..190695224419 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: distribution-scripts-version: description: "Git ref for version of https://github.com/crystal-lang/distribution-scripts/" type: string - default: "fe82a34ad7855ddb432a26ef7e48c46e7b440e49" + default: "96e431e170979125018bd4fd90111a3147477eec" previous_crystal_base_url: description: "Prefix for URLs to Crystal bootstrap compiler" type: string @@ -296,7 +296,7 @@ jobs: command: | brew unlink python@2 || true - brew install ruby@3 libffi pkgconfig libtool + brew install ruby@3 libffi pkgconfig libtool automake sudo mkdir -p /opt/crystal sudo chown $(whoami) /opt/crystal/ From 5ab6e4b4675ed5e13a7533a839800295044c4808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 4 Jul 2024 20:25:28 +0200 Subject: [PATCH 46/52] Fix changelog generator increase topic priority for `infrastructure` (#14781) --- scripts/github-changelog.cr | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/github-changelog.cr b/scripts/github-changelog.cr index 26fce69fd145..f7ae12e74dad 100755 --- a/scripts/github-changelog.cr +++ b/scripts/github-changelog.cr @@ -222,9 +222,10 @@ record PullRequest, topics.sort_by! { |parts| topic_priority = case parts[0] - when "tools" then 2 - when "lang" then 1 - else 0 + when "infrastructure" then 3 + when "tools" then 2 + when "lang" then 1 + else 0 end {-topic_priority, parts[0]} } From 074ec9992b99234050200b1cfcc524366ec59e36 Mon Sep 17 00:00:00 2001 From: Damir Sharipov Date: Fri, 5 Jul 2024 14:10:05 +0300 Subject: [PATCH 47/52] Fix JSON discriminator for Bool `false` value (#14779) --- spec/std/json/serializable_spec.cr | 13 ++++++++++--- src/json/serialization.cr | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/spec/std/json/serializable_spec.cr b/spec/std/json/serializable_spec.cr index 042e42ff5fd5..ca74c6e73e3e 100644 --- a/spec/std/json/serializable_spec.cr +++ b/spec/std/json/serializable_spec.cr @@ -419,7 +419,8 @@ class JSONVariableDiscriminatorValueType use_json_discriminator "type", { 0 => JSONVariableDiscriminatorNumber, "1" => JSONVariableDiscriminatorString, - true => JSONVariableDiscriminatorBool, + true => JSONVariableDiscriminatorBoolTrue, + false => JSONVariableDiscriminatorBoolFalse, JSONVariableDiscriminatorEnumFoo::Foo => JSONVariableDiscriminatorEnum, JSONVariableDiscriminatorEnumFoo8::Foo => JSONVariableDiscriminatorEnum8, } @@ -431,7 +432,10 @@ end class JSONVariableDiscriminatorString < JSONVariableDiscriminatorValueType end -class JSONVariableDiscriminatorBool < JSONVariableDiscriminatorValueType +class JSONVariableDiscriminatorBoolTrue < JSONVariableDiscriminatorValueType +end + +class JSONVariableDiscriminatorBoolFalse < JSONVariableDiscriminatorValueType end class JSONVariableDiscriminatorEnum < JSONVariableDiscriminatorValueType @@ -1130,7 +1134,10 @@ describe "JSON mapping" do object_string.should be_a(JSONVariableDiscriminatorString) object_bool = JSONVariableDiscriminatorValueType.from_json(%({"type": true})) - object_bool.should be_a(JSONVariableDiscriminatorBool) + object_bool.should be_a(JSONVariableDiscriminatorBoolTrue) + + object_bool = JSONVariableDiscriminatorValueType.from_json(%({"type": false})) + object_bool.should be_a(JSONVariableDiscriminatorBoolFalse) object_enum = JSONVariableDiscriminatorValueType.from_json(%({"type": 4})) object_enum.should be_a(JSONVariableDiscriminatorEnum) diff --git a/src/json/serialization.cr b/src/json/serialization.cr index 610979517a18..b1eb86d15082 100644 --- a/src/json/serialization.cr +++ b/src/json/serialization.cr @@ -448,7 +448,7 @@ module JSON end end - unless discriminator_value + if discriminator_value.nil? raise ::JSON::SerializableError.new("Missing JSON discriminator field '{{field.id}}'", to_s, nil, *location, nil) end From 45c9e6f5a337e0991071d8953def6e2c8a519e19 Mon Sep 17 00:00:00 2001 From: kojix2 <2xijok@gmail.com> Date: Sat, 6 Jul 2024 20:30:04 +0900 Subject: [PATCH 48/52] Fix `Compress::Gzip` extra field (#14550) --- spec/std/compress/gzip/gzip_spec.cr | 16 +++++++++++++++- spec/std/data/test.gz | Bin 0 -> 60 bytes src/compress/gzip/header.cr | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 spec/std/data/test.gz diff --git a/spec/std/compress/gzip/gzip_spec.cr b/spec/std/compress/gzip/gzip_spec.cr index 7c0262b5328d..8ffa0624bc6d 100644 --- a/spec/std/compress/gzip/gzip_spec.cr +++ b/spec/std/compress/gzip/gzip_spec.cr @@ -1,4 +1,4 @@ -require "spec" +require "../../spec_helper" require "compress/gzip" private SAMPLE_TIME = Time.utc(2016, 1, 2) @@ -57,4 +57,18 @@ describe Compress::Gzip do gzip.rewind gzip.gets_to_end.should eq(SAMPLE_CONTENTS) end + + it "reads file with extra fields from file system" do + File.open(datapath("test.gz")) do |file| + Compress::Gzip::Reader.open(file) do |gzip| + header = gzip.header.not_nil! + header.modification_time.to_utc.should eq(Time.utc(2012, 9, 4, 22, 6, 5)) + header.os.should eq(3_u8) + header.extra.should eq(Bytes[1, 2, 3, 4, 5]) + header.name.should eq("test.txt") + header.comment.should eq("happy birthday") + gzip.gets_to_end.should eq("One\nTwo") + end + end + end end diff --git a/spec/std/data/test.gz b/spec/std/data/test.gz new file mode 100644 index 0000000000000000000000000000000000000000..dd10b17c6a6061fa00a55c905a779e45f27c340a GIT binary patch literal 60 zcmb2|=8*TTb_-x)W@TVxVrF42Ni8nXE2$`9$Ve Date: Sun, 7 Jul 2024 22:56:36 +0800 Subject: [PATCH 49/52] Fix macro interpolation in `NamedTuple#from` (#14790) --- spec/std/named_tuple_spec.cr | 8 ++++++++ src/named_tuple.cr | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/spec/std/named_tuple_spec.cr b/spec/std/named_tuple_spec.cr index 4097215dfca3..34619b76fea4 100644 --- a/spec/std/named_tuple_spec.cr +++ b/spec/std/named_tuple_spec.cr @@ -33,6 +33,10 @@ describe "NamedTuple" do t.should eq({"foo bar": 1, "baz qux": 2}) t.class.should eq(NamedTuple("foo bar": Int32, "baz qux": Int32)) + t = NamedTuple("\"": Int32, "\#{exit}": Int32).from({"\"" => 2, "\#{exit}" => 3}) + t.should eq({"\"": 2, "\#{exit}": 3}) + t.class.should eq(NamedTuple("\"": Int32, "\#{exit}": Int32)) + expect_raises ArgumentError do NamedTuple(foo: Int32, bar: Int32).from({:foo => 1}) end @@ -74,6 +78,10 @@ describe "NamedTuple" do t = {foo: Int32, bar: Int32}.from({"foo" => 1, :bar => 2} of String | Int32 | Symbol => Int32) t.should eq({foo: 1, bar: 2}) t.class.should eq(NamedTuple(foo: Int32, bar: Int32)) + + t = {"\"": Int32, "\#{exit}": Int32}.from({"\"" => 2, "\#{exit}" => 3}) + t.should eq({"\"": 2, "\#{exit}": 3}) + t.class.should eq(NamedTuple("\"": Int32, "\#{exit}": Int32)) end it "gets size" do diff --git a/src/named_tuple.cr b/src/named_tuple.cr index 4ea9df02fd20..d147873e5341 100644 --- a/src/named_tuple.cr +++ b/src/named_tuple.cr @@ -119,7 +119,7 @@ struct NamedTuple {% begin %} NamedTuple.new( {% for key, value in T %} - {{key.stringify}}: self[{{key.symbolize}}].cast(hash.fetch({{key.symbolize}}) { hash["{{key}}"] }), + {{key.stringify}}: self[{{key.symbolize}}].cast(hash.fetch({{key.symbolize}}) { hash[{{key.stringify}}] }), {% end %} ) {% end %} From c0488512e3e8d00ab967e014ec041032670deab0 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Sun, 7 Jul 2024 10:57:00 -0400 Subject: [PATCH 50/52] Fix regression with `NamedTuple.new` when using key with a hyphen (#14785) --- spec/std/named_tuple_spec.cr | 4 ++++ src/named_tuple.cr | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/std/named_tuple_spec.cr b/spec/std/named_tuple_spec.cr index 34619b76fea4..f94078adaec6 100644 --- a/spec/std/named_tuple_spec.cr +++ b/spec/std/named_tuple_spec.cr @@ -20,6 +20,10 @@ describe "NamedTuple" do t.class.should_not eq(NamedTuple(foo: Int32, bar: String)) end + it "does NamedTuple.new, with hyphen in key" do + NamedTuple("a-b": String).new("a-b": "foo").should eq({"a-b": "foo"}) + end + it "does NamedTuple.from" do t = NamedTuple(foo: Int32, bar: Int32).from({:foo => 1, :bar => 2}) t.should eq({foo: 1, bar: 2}) diff --git a/src/named_tuple.cr b/src/named_tuple.cr index d147873e5341..f9d606baca68 100644 --- a/src/named_tuple.cr +++ b/src/named_tuple.cr @@ -70,7 +70,7 @@ struct NamedTuple {% begin %} { {% for key in T %} - {{ key.stringify }}: options[{{ key.symbolize }}].as(typeof(element_type({{ key }}))), + {{ key.stringify }}: options[{{ key.symbolize }}].as(typeof(element_type({{ key.symbolize }}))), {% end %} } {% end %} From 3d007b121a53b49b79c317cddd1b2b53039e4b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Cla=C3=9Fen?= Date: Mon, 8 Jul 2024 21:19:31 +0200 Subject: [PATCH 51/52] Fix code example for `Process.on_terminate` (#14798) --- src/process.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/process.cr b/src/process.cr index 045615c814a7..c8364196373f 100644 --- a/src/process.cr +++ b/src/process.cr @@ -89,7 +89,7 @@ class Process # end # end # - # wait_channel.receive + # wait_channel.receive? # puts "bye" # ``` def self.on_terminate(&handler : ::Process::ExitReason ->) : Nil From 0571f19bdbce008c2c969a4e74664ba6d082aabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 8 Jul 2024 21:19:49 +0200 Subject: [PATCH 52/52] Add spec for `Compress::Gzip::Writer` with `extra` (#14788) --- spec/std/compress/gzip/gzip_spec.cr | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/spec/std/compress/gzip/gzip_spec.cr b/spec/std/compress/gzip/gzip_spec.cr index 8ffa0624bc6d..675b704436ea 100644 --- a/spec/std/compress/gzip/gzip_spec.cr +++ b/spec/std/compress/gzip/gzip_spec.cr @@ -71,4 +71,27 @@ describe Compress::Gzip do end end end + + it "writes and reads file with extra fields" do + io = IO::Memory.new + Compress::Gzip::Writer.open(io) do |gzip| + header = gzip.header + header.modification_time = Time.utc(2012, 9, 4, 22, 6, 5) + header.os = 3_u8 + header.extra = Bytes[1, 2, 3, 4, 5] + header.name = "test.txt" + header.comment = "happy birthday" + gzip << "One\nTwo" + end + io.rewind + Compress::Gzip::Reader.open(io) do |gzip| + header = gzip.header.not_nil! + header.modification_time.to_utc.should eq(Time.utc(2012, 9, 4, 22, 6, 5)) + header.os.should eq(3_u8) + header.extra.should eq(Bytes[1, 2, 3, 4, 5]) + header.name.should eq("test.txt") + header.comment.should eq("happy birthday") + gzip.gets_to_end.should eq("One\nTwo") + end + end end