From 63fefe07350fc448d2e0585aa5f10be972284863 Mon Sep 17 00:00:00 2001 From: Steve Kelly Date: Wed, 5 Jul 2023 11:09:10 +0100 Subject: [PATCH 01/33] Add devdocs/jit.md to the menu (#50310) Added in https://github.com/JuliaLang/julia/pull/50168 --- doc/make.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/make.jl b/doc/make.jl index a9343a3133a63..d35d893143eda 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -154,6 +154,7 @@ DevDocs = [ "devdocs/EscapeAnalysis.md", "devdocs/gc-sa.md", "devdocs/gc.md", + "devdocs/jit.md", ], "Developing/debugging Julia's C code" => [ "devdocs/backtraces.md", From 1279de6daaea4c3dae46aa3a1be760836241ea94 Mon Sep 17 00:00:00 2001 From: Ujjwal Sarswat <76774914+vmpyr@users.noreply.github.com> Date: Wed, 5 Jul 2023 15:40:08 +0530 Subject: [PATCH 02/33] fix conversion of empty `Dates.CompoundPeriod()` to zero units (#50259) * fix(stdlib/Dates/periods.jl): conversion of empty CompoundPeriod to zero units * add(stdlib/Dates/test/periods.jl): add test for empty CompoundPeriod --- stdlib/Dates/src/periods.jl | 2 +- stdlib/Dates/test/periods.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/stdlib/Dates/src/periods.jl b/stdlib/Dates/src/periods.jl index 9b7e29496e642..d8995cc9066d1 100644 --- a/stdlib/Dates/src/periods.jl +++ b/stdlib/Dates/src/periods.jl @@ -325,7 +325,7 @@ end Base.show(io::IO,x::CompoundPeriod) = print(io, string(x)) Base.convert(::Type{T}, x::CompoundPeriod) where T<:Period = - isconcretetype(T) ? sum(T, x.periods) : throw(MethodError(convert,(T,x))) + isconcretetype(T) ? sum(T, x.periods; init = zero(T)) : throw(MethodError(convert,(T,x))) # E.g. Year(1) + Day(1) (+)(x::Period,y::Period) = CompoundPeriod(Period[x, y]) diff --git a/stdlib/Dates/test/periods.jl b/stdlib/Dates/test/periods.jl index 7b23ffcb5d4e1..0f00694e21f74 100644 --- a/stdlib/Dates/test/periods.jl +++ b/stdlib/Dates/test/periods.jl @@ -523,6 +523,7 @@ end @test convert(Second, Minute(1) + Second(30)) === Second(90) @test convert(Minute, Minute(1) + Second(60)) === Minute(2) @test convert(Millisecond, Minute(1) + Second(30)) === Millisecond(90_000) + @test convert(Millisecond, Dates.CompoundPeriod()) === Millisecond(0) @test_throws InexactError convert(Minute, Minute(1) + Second(30)) @test_throws MethodError convert(Month, Minute(1) + Second(30)) @test_throws MethodError convert(Second, Month(1) + Second(30)) From e025877105630311b7f07d470d3cecda11a3e5d7 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 5 Jul 2023 12:11:13 +0200 Subject: [PATCH 03/33] Remove dynamic dispatch from _wait/wait2 (#50202) Co-authored-by: Gabriel Baraldi --- base/task.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/base/task.jl b/base/task.jl index 4fbb51fde3e8e..f7a82b9c42ab9 100644 --- a/base/task.jl +++ b/base/task.jl @@ -303,13 +303,14 @@ end # just wait for a task to be done, no error propagation function _wait(t::Task) if !istaskdone(t) - lock(t.donenotify) + donenotify = t.donenotify::ThreadSynchronizer + lock(donenotify) try while !istaskdone(t) - wait(t.donenotify) + wait(donenotify) end finally - unlock(t.donenotify) + unlock(donenotify) end end nothing @@ -330,13 +331,14 @@ function _wait2(t::Task, waiter::Task) tid = Threads.threadid() ccall(:jl_set_task_tid, Cint, (Any, Cint), waiter, tid-1) end - lock(t.donenotify) + donenotify = t.donenotify::ThreadSynchronizer + lock(donenotify) if !istaskdone(t) - push!(t.donenotify.waitq, waiter) - unlock(t.donenotify) + push!(donenotify.waitq, waiter) + unlock(donenotify) return nothing else - unlock(t.donenotify) + unlock(donenotify) end end schedule(waiter) From 7e3c706a879ae6d381737e13ce9b3275d083a86a Mon Sep 17 00:00:00 2001 From: Loong Date: Wed, 5 Jul 2023 18:12:43 +0800 Subject: [PATCH 04/33] Check input expresion in numbered prompt (#50064) --- stdlib/REPL/src/REPL.jl | 1 + stdlib/REPL/test/repl.jl | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index f8bb442ad6ec4..1328a87f1a77d 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1418,6 +1418,7 @@ function out_transform(@nospecialize(x), n::Ref{Int}) end function get_usings!(usings, ex) + ex isa Expr || return usings # get all `using` and `import` statements which are at the top level for (i, arg) in enumerate(ex.args) if Base.isexpr(arg, :toplevel) diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 8a6c6a3445e0a..f0d5052ff9e32 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1652,6 +1652,10 @@ fake_repl() do stdin_write, stdout_read, repl @test !contains(s, "ERROR") @test contains(s, "Test Passed") + # Test for https://github.com/JuliaLang/julia/issues/49319 + s = sendrepl2("# comment", "In [16]") + @test !contains(s, "ERROR") + write(stdin_write, '\x04') Base.wait(repltask) end From 877b368d78f9fbb6bb698f40f6e9c725ef057bd0 Mon Sep 17 00:00:00 2001 From: Adnan Alhomssi Date: Wed, 5 Jul 2023 12:16:01 +0200 Subject: [PATCH 05/33] Use tempdir() to store heap snapshot files instead of abspatch ~= rootdir (#50026) --- stdlib/Profile/src/Profile.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 71bbfc70ee937..f0323d0334b1a 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -1234,7 +1234,7 @@ function take_heap_snapshot(filepath::String, all_one::Bool=false) return filepath end function take_heap_snapshot(all_one::Bool=false) - f = abspath("$(getpid())_$(time_ns()).heapsnapshot") + f = joinpath(tempdir(), "$(getpid())_$(time_ns()).heapsnapshot") return take_heap_snapshot(f, all_one) end From 929a84549691f998611cd7f7912b5f3c8e36c292 Mon Sep 17 00:00:00 2001 From: James Foster <38274066+jd-foster@users.noreply.github.com> Date: Wed, 5 Jul 2023 04:22:33 -0600 Subject: [PATCH 06/33] Docs: Windows build devdocs clean up (#49760) --- doc/src/devdocs/build/build.md | 2 +- doc/src/devdocs/build/windows.md | 133 +++++++++++++++++-------------- 2 files changed, 73 insertions(+), 62 deletions(-) diff --git a/doc/src/devdocs/build/build.md b/doc/src/devdocs/build/build.md index ad3871c2e70f0..51bcf7d0ee469 100644 --- a/doc/src/devdocs/build/build.md +++ b/doc/src/devdocs/build/build.md @@ -16,7 +16,7 @@ variables. When compiled the first time, the build will automatically download pre-built [external -dependencies](#required-build-tools-and-external-libraries). If you +dependencies](#Required-Build-Tools-and-External-Libraries). If you prefer to build all the dependencies on your own, or are building on a system that cannot access the network during the build process, add the following in `Make.user`: diff --git a/doc/src/devdocs/build/windows.md b/doc/src/devdocs/build/windows.md index 7192bb8a7a544..8f8f0c8bc676a 100644 --- a/doc/src/devdocs/build/windows.md +++ b/doc/src/devdocs/build/windows.md @@ -47,15 +47,18 @@ MinGW-w64 compilers available through Cygwin's package manager. either 32 or 64 bit Julia from either 32 or 64 bit Cygwin. 64 bit Cygwin has a slightly smaller but often more up-to-date selection of packages. - Advanced: you may skip steps 2-4 by running: + *Advanced*: you may skip steps 2-4 by running: - setup-x86_64.exe -s -q -P cmake,gcc-g++,git,make,patch,curl,m4,python3,p7zip,mingw64-i686-gcc-g++,mingw64-i686-gcc-fortran,mingw64-x86_64-gcc-g++,mingw64-x86_64-gcc-fortran - :: replace with a site from https://cygwin.com/mirrors.html - :: or run setup manually first and select a mirror + ```sh + setup-x86_64.exe -s -q -P cmake,gcc-g++,git,make,patch,curl,m4,python3,p7zip,mingw64-i686-gcc-g++,mingw64-i686-gcc-fortran,mingw64-x86_64-gcc-g++,mingw64-x86_64-gcc-fortran + ``` - 2. Select installation location and download mirror. + replacing `` with a site from [https://cygwin.com/mirrors.html](https://cygwin.com/mirrors.html) + or run setup manually first and select a mirror. - 3. At the '*Select Packages'* step, select the following: + 2. Select installation location and a mirror to download from. + + 3. At the *Select Packages* step, select the following: 1. From the *Devel* category: `cmake`, `gcc-g++`, `git`, `make`, `patch` 2. From the *Net* category: `curl` @@ -67,7 +70,7 @@ MinGW-w64 compilers available through Cygwin's package manager. `mingw64-x86_64-gcc-g++` and `mingw64-x86_64-gcc-fortran` 4. Allow Cygwin installation to finish, then start from the installed shortcut - a *'Cygwin Terminal'*, or *'Cygwin64 Terminal'*, respectively. + *'Cygwin Terminal'*, or *'Cygwin64 Terminal'*, respectively. 5. Build Julia and its dependencies from source: @@ -93,64 +96,67 @@ MinGW-w64 compilers available through Cygwin's package manager. make -j 4 # Adjust the number of threads (4) to match your build environment. make -j 4 debug # This builds julia-debug.exe ``` - - - > Protip: build both! - > ```sh - > make O=julia-win32 configure - > make O=julia-win64 configure - > echo 'XC_HOST = i686-w64-mingw32' > julia-win32/Make.user - > echo 'XC_HOST = x86_64-w64-mingw32' > julia-win64/Make.user - > echo 'ifeq ($(BUILDROOT),$(JULIAHOME)) - > $(error "in-tree build disabled") - > endif' >> Make.user - > make -C julia-win32 # build for Windows x86 in julia-win32 folder - > make -C julia-win64 # build for Windows x86-64 in julia-win64 folder - > ``` - 6. Run Julia using the Julia executables directly ```sh usr/bin/julia.exe usr/bin/julia-debug.exe ``` +!!! note "Pro tip: build both!" + ```sh + make O=julia-win32 configure + make O=julia-win64 configure + echo 'XC_HOST = i686-w64-mingw32' > julia-win32/Make.user + echo 'XC_HOST = x86_64-w64-mingw32' > julia-win64/Make.user + echo 'ifeq ($(BUILDROOT),$(JULIAHOME)) + $(error "in-tree build disabled") + endif' >> Make.user + make -C julia-win32 # build for Windows x86 in julia-win32 folder + make -C julia-win64 # build for Windows x86-64 in julia-win64 folder + ``` + ### Compiling with MinGW/MSYS2 -> MSYS2 provides a robust MSYS experience. +[MSYS2](https://www.msys2.org/) is a software distribution and build environment for Windows. Note: MSYS2 requires **64 bit** Windows 7 or newer. - 1. Install and configure [MSYS2](https://www.msys2.org/), Software Distribution - and Building Platform for Windows. + 1. Install and configure MSYS2. 1. Download and run the latest installer for the [64-bit](https://github.com/msys2/msys2-installer/releases/latest) distribution. The installer will have a name like `msys2-x86_64-yyyymmdd.exe`. - 2. Open MSYS2. Update package database and base packages: - ```sh + 2. Open the MSYS2 shell. Update the package database and base packages: + + ``` pacman -Syu ``` + 3. Exit and restart MSYS2. Update the rest of the base packages: - 3. Exit and restart MSYS2, Update the rest of the base packages: - ```sh + ``` pacman -Syu ``` - 3. Then install tools required to build julia: - ```sh - # tools + 4. Then install tools required to build julia: + + ``` pacman -S cmake diffutils git m4 make patch tar p7zip curl python + ``` + + For 64 bit Julia, install the x86_64 version: - # For 64 bit Julia, install x86_64 + ``` pacman -S mingw-w64-x86_64-gcc - # For 32 bit Julia, install i686 - pacman -S mingw-w64-i686-gcc ``` - 4. Configuration of MSYS2 is complete. Now `exit` the MSYS2 shell. + For 32 bit Julia, install the i686 version: + ``` + pacman -S mingw-w64-i686-gcc + ``` + 5. Configuration of MSYS2 is complete. Now `exit` the MSYS2 shell. 2. Build Julia and its dependencies with pre-build dependencies. 1. Open a new [**MINGW64/MINGW32 shell**](https://www.msys2.org/docs/environments/#overview). @@ -158,25 +164,27 @@ Note: MSYS2 requires **64 bit** Windows 7 or newer. so if you want to build the x86_64 and i686 versions, you'll need to build them in each environment separately. - 2. and clone the Julia sources - ```sh + 2. Clone the Julia sources: + + ``` git clone https://github.com/JuliaLang/julia.git cd julia ``` 3. Start the build - ```sh + + ``` make -j$(nproc) ``` - > Protip: build in dir - > ```sh - > make O=julia-mingw-w64 configure - > echo 'ifeq ($(BUILDROOT),$(JULIAHOME)) - > $(error "in-tree build disabled") - > endif' >> Make.user - > make -C julia-mingw-w64 - > ``` +!!! note "Pro tip: build in dir" + ```sh + make O=julia-mingw-w64 configure + echo 'ifeq ($(BUILDROOT),$(JULIAHOME)) + $(error "in-tree build disabled") + endif' >> Make.user + make -C julia-mingw-w64 + ``` ### Cross-compiling from Unix (Linux/Mac/WSL) @@ -185,7 +193,7 @@ You can also use MinGW-w64 cross compilers to build a Windows version of Julia f Linux, Mac, or the Windows Subsystem for Linux (WSL). First, you will need to ensure your system has the required dependencies. We -need wine (>=1.7.5), a system compiler, and some downloaders. Note: a cygwin install might +need wine (>=1.7.5), a system compiler, and some downloaders. Note: a Cygwin install might interfere with this method if using WSL. **On Ubuntu** (on other Linux systems the dependency names are likely to be similar): @@ -193,7 +201,9 @@ interfere with this method if using WSL. apt-get install wine-stable gcc wget p7zip-full winbind mingw-w64 gfortran-mingw-w64 dpkg --add-architecture i386 && apt-get update && apt-get install wine32 # add sudo to each if needed # switch all of the following to their "-posix" variants (interactively): -for pkg in i686-w64-mingw32-g++ i686-w64-mingw32-gcc i686-w64-mingw32-gfortran x86_64-w64-mingw32-g++ x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gfortran; do sudo update-alternatives --config $pkg; done +for pkg in i686-w64-mingw32-g++ i686-w64-mingw32-gcc i686-w64-mingw32-gfortran x86_64-w64-mingw32-g++ x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gfortran; do + sudo update-alternatives --config $pkg +done ``` **On Mac**: Install XCode, XCode command line tools, X11 (now @@ -211,19 +221,19 @@ install wine wget mingw-w64`, as appropriate. 6. `make binary-dist` then `make exe` to create the Windows installer. 7. move the `julia-*.exe` installer to the target machine -If you are building for 64-bit windows, the steps are essentially the same. -Just replace `i686` in `XC_HOST` with `x86_64`. (note: on Mac, wine only runs +If you are building for 64-bit Windows, the steps are essentially the same. +Just replace `i686` in `XC_HOST` with `x86_64`. (Note: on Mac, wine only runs in 32-bit mode). ## Debugging a cross-compiled build under wine The most effective way to debug a cross-compiled version of Julia on the cross-compilation -host is to install a windows version of gdb and run it under wine as usual. The pre-built +host is to install a Windows version of GDB and run it under wine as usual. The pre-built packages available [as part of the MSYS2 -project](https://sourceforge.net/projects/msys2/files/REPOS/MINGW/) are known to work. Apart +project](https://packages.msys2.org/) are known to work. Apart from the GDB package you may also need the python and termcap packages. Finally, GDB's -prompt may not work when launch from the command line. This can be worked around by +prompt may not work when launched from the command line. This can be worked around by prepending `wineconsole` to the regular GDB invocation. @@ -232,24 +242,25 @@ prepending `wineconsole` to the regular GDB invocation. Compiling using one of the options above creates a basic Julia build, but not some extra components that are included if you run the full Julia binary installer. If you need these components, the easiest way to get them is to build the installer -yourself using ```make win-extras``` followed by ```make binary-dist``` and ```make exe```. Then running the resulting installer. +yourself using ```make win-extras``` followed by ```make binary-dist``` and ```make exe```. +Then run the resulting installer. ## Windows Build Debugging -### GDB hangs with cygwin mintty +### GDB hangs with Cygwin mintty -- Run gdb under the windows console (cmd) instead. gdb [may not function +- Run GDB under the Windows console (cmd) instead. GDB [may not function properly](https://www.cygwin.com/ml/cygwin/2009-02/msg00531.html) under mintty with non- - cygwin applications. You can use `cmd /c start` to start the windows console from mintty + Cygwin applications. You can use `cmd /c start` to start the Windows console from mintty if necessary. ### GDB not attaching to the right process - - Use the PID from the windows task manager or `WINPID` from the `ps` command - instead of the PID from unix style command line tools (e.g. `pgrep`). You - may need to add the PID column if it is not shown by default in the windows + - Use the PID from the Windows task manager or `WINPID` from the `ps` command + instead of the PID from unix-style command line tools (e.g. `pgrep`). You + may need to add the PID column if it is not shown by default in the Windows task manager. ### GDB not showing the right backtrace From fb2ceeace759716e1512c9e5434d8d39052a4f6e Mon Sep 17 00:00:00 2001 From: Ben Baumgold <4933671+baumgold@users.noreply.github.com> Date: Wed, 5 Jul 2023 06:54:03 -0400 Subject: [PATCH 07/33] enhance Timer call taking callback to accept any `timeout` arg and kwargs (#50027) --- base/asyncevent.jl | 4 ++-- stdlib/Dates/src/periods.jl | 4 ++++ stdlib/Dates/src/types.jl | 6 +++--- stdlib/Dates/test/periods.jl | 9 +++++++++ stdlib/Dates/test/types.jl | 5 +++++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/base/asyncevent.jl b/base/asyncevent.jl index a26945bbb1105..498fb054ecd02 100644 --- a/base/asyncevent.jl +++ b/base/asyncevent.jl @@ -272,8 +272,8 @@ julia> begin 3 ``` """ -function Timer(cb::Function, timeout::Real; interval::Real=0.0) - timer = Timer(timeout, interval=interval) +function Timer(cb::Function, timeout; kwargs...) + timer = Timer(timeout; kwargs...) t = @task begin unpreserve_handle(timer) while _trywait(timer) diff --git a/stdlib/Dates/src/periods.jl b/stdlib/Dates/src/periods.jl index d8995cc9066d1..876680dd456a5 100644 --- a/stdlib/Dates/src/periods.jl +++ b/stdlib/Dates/src/periods.jl @@ -465,3 +465,7 @@ days(c::Year) = 365.2425 * value(c) days(c::Quarter) = 91.310625 * value(c) days(c::Month) = 30.436875 * value(c) days(c::CompoundPeriod) = isempty(c.periods) ? 0.0 : Float64(sum(days, c.periods)) +seconds(x::Nanosecond) = value(x) / 1000000000 +seconds(x::Microsecond) = value(x) / 1000000 +seconds(x::Millisecond) = value(x) / 1000 +seconds(x::Period) = value(Second(x)) diff --git a/stdlib/Dates/src/types.jl b/stdlib/Dates/src/types.jl index 1d9769a05bd3d..a96d183bbc590 100644 --- a/stdlib/Dates/src/types.jl +++ b/stdlib/Dates/src/types.jl @@ -460,14 +460,14 @@ Base.hash(x::Time, h::UInt) = hash(hour(x), hash(minute(x), hash(second(x), hash(millisecond(x), hash(microsecond(x), hash(nanosecond(x), h)))))) -Base.sleep(duration::Period) = sleep(toms(duration) / 1000) +Base.sleep(duration::Period) = sleep(seconds(duration)) function Base.Timer(delay::Period; interval::Period=Second(0)) - Timer(toms(delay) / 1000, interval=toms(interval) / 1000) + Timer(seconds(delay), interval=seconds(interval)) end function Base.timedwait(testcb, timeout::Period; pollint::Period=Millisecond(100)) - timedwait(testcb, toms(timeout) / 1000, pollint=toms(pollint) / 1000) + timedwait(testcb, seconds(timeout), pollint=seconds(pollint)) end Base.OrderStyle(::Type{<:AbstractTime}) = Base.Ordered() diff --git a/stdlib/Dates/test/periods.jl b/stdlib/Dates/test/periods.jl index 0f00694e21f74..82e7ca27ab6f5 100644 --- a/stdlib/Dates/test/periods.jl +++ b/stdlib/Dates/test/periods.jl @@ -343,6 +343,15 @@ end @test Dates.days(Dates.Hour(24)) == 1 @test Dates.days(d) == 1 @test Dates.days(w) == 7 + + @test Dates.seconds(ns) == 0.000000001 + @test Dates.seconds(us) == 0.000001 + @test Dates.seconds(ms) == 0.001 + @test Dates.seconds(s) == 1 + @test Dates.seconds(mi) == 60 + @test Dates.seconds(h) == 3600 + @test Dates.seconds(d) == 86400 + @test Dates.seconds(w) == 604800 end @testset "issue #9214" begin @test 2s + (7ms + 1ms) == (2s + 7ms) + 1ms == 1ms + (2s + 7ms) == 1ms + (1s + 7ms) + 1s == 1ms + (2s + 3d + 7ms) + (-3d) == (1ms + (2s + 3d)) + (7ms - 3d) == (1ms + (2s + 3d)) - (3d - 7ms) diff --git a/stdlib/Dates/test/types.jl b/stdlib/Dates/test/types.jl index 8823e56e41a2f..3bd11f80540d7 100644 --- a/stdlib/Dates/test/types.jl +++ b/stdlib/Dates/test/types.jl @@ -273,6 +273,11 @@ end end +@testset "timer" begin + @test hasmethod(Timer, (Period,)) + @test hasmethod(Timer, (Function, Period)) +end + @testset "timedwait" begin @test timedwait(() -> false, Second(0); pollint=Millisecond(1)) === :timed_out end From fcb31107a9515dda8b519e126c039c345ef7daf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Santtu=20S=C3=B6derholm?= Date: Wed, 5 Jul 2023 14:03:10 +0300 Subject: [PATCH 08/33] =?UTF-8?q?REPL/latex=5Fsymbols:=20define=20commands?= =?UTF-8?q?=20for=20=C2=AB=20and=20=C2=BB=20(#50399)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new commands are \guillemotleft and \guillemotright, respectively. These commands are in line with the corresponding commands defined in the LaTeΧ package csquotes. Co-authored-by: Steven G. Johnson --- stdlib/REPL/src/latex_symbols.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/latex_symbols.jl b/stdlib/REPL/src/latex_symbols.jl index 3c2be918d6bd2..9e71819f6562b 100644 --- a/stdlib/REPL/src/latex_symbols.jl +++ b/stdlib/REPL/src/latex_symbols.jl @@ -432,8 +432,10 @@ const latex_symbols = Dict( "\\pertenthousand" => "‱", "\\prime" => "′", "\\backprime" => "‵", - "\\guilsinglleft" => "‹", + "\\guilsinglleft" => "‹", # note: \guil* quote names follow the LaTeX csquotes package "\\guilsinglright" => "›", + "\\guillemotleft" => "«", + "\\guillemotright" => "»", "\\nolinebreak" => "\u2060", "\\pes" => "₧", "\\dddot" => "⃛", From 435c1c1a2c9db60be89b82bb9438916ac891a1ff Mon Sep 17 00:00:00 2001 From: William Moses Date: Wed, 5 Jul 2023 07:32:13 -0400 Subject: [PATCH 09/33] Add CPU feature helper function (#50402) --- src/jl_exported_funcs.inc | 1 + src/processor.h | 2 ++ src/processor_arm.cpp | 5 +++++ src/processor_fallback.cpp | 5 +++++ src/processor_x86.cpp | 5 +++++ test/sysinfo.jl | 3 +++ 6 files changed, 21 insertions(+) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index fd824131bdbda..6875e36d6da6a 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -212,6 +212,7 @@ XX(jl_get_binding_or_error) \ XX(jl_get_binding_wr) \ XX(jl_get_cpu_name) \ + XX(jl_get_cpu_features) \ XX(jl_get_current_task) \ XX(jl_get_default_sysimg_path) \ XX(jl_get_excstack) \ diff --git a/src/processor.h b/src/processor.h index 3e83bbb2247d6..2255cf4c10daa 100644 --- a/src/processor.h +++ b/src/processor.h @@ -221,6 +221,8 @@ jl_image_t jl_init_processor_pkgimg(void *hdl); // Return the name of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void); +// Return the features of the host CPU as a julia string. +JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void); // Dump the name and feature set of the host CPU // For debugging only JL_DLLEXPORT void jl_dump_host_cpu(void); diff --git a/src/processor_arm.cpp b/src/processor_arm.cpp index 0797fa4381f9d..0a8090a8a6d9c 100644 --- a/src/processor_arm.cpp +++ b/src/processor_arm.cpp @@ -1802,6 +1802,11 @@ JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) return jl_cstr_to_string(host_cpu_name().c_str()); } +JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void) +{ + return jl_cstr_to_string(jl_get_cpu_features_llvm().c_str()); +} + jl_image_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) diff --git a/src/processor_fallback.cpp b/src/processor_fallback.cpp index 1aebde6dab90a..d50edc8e9b621 100644 --- a/src/processor_fallback.cpp +++ b/src/processor_fallback.cpp @@ -164,6 +164,11 @@ JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) return jl_cstr_to_string(host_cpu_name().c_str()); } +JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void) +{ + return jl_cstr_to_string(jl_get_cpu_features_llvm().c_str()); +} + JL_DLLEXPORT void jl_dump_host_cpu(void) { jl_safe_printf("CPU: %s\n", host_cpu_name().c_str()); diff --git a/src/processor_x86.cpp b/src/processor_x86.cpp index e129b1239c7df..b9e7d8c0f0daf 100644 --- a/src/processor_x86.cpp +++ b/src/processor_x86.cpp @@ -1042,6 +1042,11 @@ JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) return jl_cstr_to_string(host_cpu_name().c_str()); } +JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void) +{ + return jl_cstr_to_string(jl_get_cpu_features_llvm().c_str()); +} + jl_image_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) diff --git a/test/sysinfo.jl b/test/sysinfo.jl index 3a16dc73b4f6a..cb943cfd38843 100644 --- a/test/sysinfo.jl +++ b/test/sysinfo.jl @@ -10,6 +10,9 @@ Base.Sys.loadavg() @test Base.libllvm_path() isa Symbol @test contains(String(Base.libllvm_path()), "LLVM") +@test length(ccall(:jl_get_cpu_name, String, ())) != 0 +@test length(ccall(:jl_get_cpu_features, String, ())) >= 0 + if Sys.isunix() mktempdir() do tempdir firstdir = joinpath(tempdir, "first") From 23c0418899637464b7619bd57932b635c2c80d94 Mon Sep 17 00:00:00 2001 From: woclass Date: Wed, 5 Jul 2023 21:30:42 +0800 Subject: [PATCH 10/33] gc: fix time unit in jl_print_gc_stats (#50417) --- src/gc-debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index bab2c5b0fa607..0e51d625da74a 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -784,11 +784,12 @@ void jl_print_gc_stats(JL_STREAM *s) malloc_stats(); #endif double ptime = jl_hrtime() - process_t0; - jl_safe_printf("exec time\t%.5f sec\n", ptime); + double exec_time = jl_ns2s(ptime); + jl_safe_printf("exec time\t%.5f sec\n", exec_time); if (gc_num.pause > 0) { jl_safe_printf("gc time \t%.5f sec (%2.1f%%) in %d (%d full) collections\n", jl_ns2s(gc_num.total_time), - jl_ns2s(gc_num.total_time) / ptime * 100, + jl_ns2s(gc_num.total_time) / exec_time * 100, gc_num.pause, gc_num.full_sweep); jl_safe_printf("gc pause \t%.2f ms avg\n\t\t%2.0f ms max\n", jl_ns2ms(gc_num.total_time) / gc_num.pause, From 8d62b4017e032066d731809a045ba7ab63020453 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 5 Jul 2023 20:21:40 +0000 Subject: [PATCH 11/33] Lower inlining cost of floating point div Our inlining cost model is extremely primitive, though surprisingly functional given its limitations. The basic idea for it was just that we'd give every intrinsic the approximate cost in cycles, such that for sufficiently large functions (>100 cycles), the cost of the extra call would be dwarfed by the cost of the function. However, there's a few problems with this. For one, the real issue is usually not the extra overhead of the call (which is small and well-predicated), but rather the inhibition of optimizations that inlining might have allowed. Additionally, the relevant cost comparison is not generally latency, but rather the size of the resulting binary. Lastly, the latency metric is misleading on modern superscalar architectures, because the core will perform other tasks while the operation is executing. In fact, somewhat counter-intuitively, this means that it is *more* important to inline high-latency instructions to allow the compiler to perform better latency hiding by spreading out the high-latency instructions. We probably need a full-on rethink of the inlining model at some point, but for the time being, this fixes a problem that I ran into in real code by reducing the inlining cost for floating point division to be the same as that of floating point multiplication. The particular case where I saw this was the batched forward AD rule for division, which had 6 calls to div_float. Inlining these provided substantially better performance. --- base/compiler/tfuncs.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index f5690f4e5b8d6..117f5288418e1 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -184,7 +184,7 @@ add_tfunc(neg_float, 1, 1, math_tfunc, 1) add_tfunc(add_float, 2, 2, math_tfunc, 1) add_tfunc(sub_float, 2, 2, math_tfunc, 1) add_tfunc(mul_float, 2, 2, math_tfunc, 4) -add_tfunc(div_float, 2, 2, math_tfunc, 20) +add_tfunc(div_float, 2, 2, math_tfunc, 4) add_tfunc(fma_float, 3, 3, math_tfunc, 5) add_tfunc(muladd_float, 3, 3, math_tfunc, 5) @@ -193,7 +193,7 @@ add_tfunc(neg_float_fast, 1, 1, math_tfunc, 1) add_tfunc(add_float_fast, 2, 2, math_tfunc, 1) add_tfunc(sub_float_fast, 2, 2, math_tfunc, 1) add_tfunc(mul_float_fast, 2, 2, math_tfunc, 2) -add_tfunc(div_float_fast, 2, 2, math_tfunc, 10) +add_tfunc(div_float_fast, 2, 2, math_tfunc, 2) # bitwise operators # ----------------- From c09a199ead2c5301d721369fe87a3d11c99ed8f1 Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Wed, 5 Jul 2023 17:33:43 -0300 Subject: [PATCH 12/33] update halfpages pointer after actually sweeping pages (#50387) --- src/gc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/gc.c b/src/gc.c index 9fd93b7340d56..edc0fdb49c4a5 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1542,7 +1542,6 @@ static void gc_sweep_pool(int sweep_full) pg->nfree = (GC_PAGE_SZ - (last_p - gc_page_data(last_p - 1))) / p->osize; pg->has_young = 1; } - p->newpages = NULL; } jl_gc_pagemeta_t *pg = ptls2->page_metadata_lazily_freed; while (pg != NULL) { @@ -1564,6 +1563,10 @@ static void gc_sweep_pool(int sweep_full) pg = pg2; } ptls2->page_metadata_allocd = allocd; + for (int i = 0; i < JL_GC_N_POOLS; i++) { + jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; + p->newpages = NULL; + } } } From 0b54ded93b693189c8efe25ac0bc4efbce4e0a06 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Thu, 6 Jul 2023 03:58:09 -0400 Subject: [PATCH 13/33] avoid potential type-instability in _replace_(str, ...) (#50424) --- base/strings/util.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/strings/util.jl b/base/strings/util.jl index c818d723d688f..c77d45255a735 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -778,11 +778,11 @@ end # note: leave str untyped here to make it easier for packages like StringViews to hook in function _replace_(str, pat_repl::NTuple{N, Pair}, count::Int) where N - count == 0 && return str + count == 0 && return String(str) e1, patterns, replaces, rs, notfound = _replace_init(str, pat_repl, count) if notfound foreach(_free_pat_replacer, patterns) - return str + return String(str) end out = IOBuffer(sizehint=floor(Int, 1.2sizeof(str))) return String(take!(_replace_finish(out, str, count, e1, patterns, replaces, rs))) From 55795662ab52617557a16b77d31826a2722ccea0 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:01:18 +0200 Subject: [PATCH 14/33] Add paragraph on abstract `Function` fields to performance tips (#50421) Co-authored-by: Steven G. Johnson --- doc/src/manual/performance-tips.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index c86630ce2a8f1..069c3b2d424e5 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -358,6 +358,27 @@ julia> !isconcretetype(Array), !isabstracttype(Array), isstructtype(Array), !isc ``` In this case, it would be better to avoid declaring `MyType` with a field `a::Array` and instead declare the field as `a::Array{T,N}` or as `a::A`, where `{T,N}` or `A` are parameters of `MyType`. +The previous advice is especially useful when the fields of a struct are meant to be functions, or more generally callable objects. +It is very tempting to define a struct as follows: + +```julia +struct MyCallableWrapper + f::Function +end +``` + +But since `Function` is an abstract type, every call to `wrapper.f` will require dynamic dispatch, due to the type instability of accessing the field `f`. +Instead, you should write something like: + +```julia +struct MyCallableWrapper{F} + f::F +end +``` + +which has nearly identical behavior but will be much faster (because the type instability is eliminated). +Note that we do not impose `F<:Function`: this means callable objects which do not subtype `Function` are also allowed for the field `f`. + ### Avoid fields with abstract containers The same best practices also work for container types: From 77ce3439c0fccf22c44f539c19f086a3303c2998 Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Thu, 6 Jul 2023 14:01:12 +0530 Subject: [PATCH 15/33] Fix some inference checks in reduce tests (#50437) --- test/reduce.jl | 66 +++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/test/reduce.jl b/test/reduce.jl index 4c05b179edcff..aea1f1e60f6cd 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -439,39 +439,39 @@ end # any & all -@test @inferred any([]) == false -@test @inferred any(Bool[]) == false -@test @inferred any([true]) == true -@test @inferred any([false, false]) == false -@test @inferred any([false, true]) == true -@test @inferred any([true, false]) == true -@test @inferred any([true, true]) == true -@test @inferred any([true, true, true]) == true -@test @inferred any([true, false, true]) == true -@test @inferred any([false, false, false]) == false - -@test @inferred all([]) == true -@test @inferred all(Bool[]) == true -@test @inferred all([true]) == true -@test @inferred all([false, false]) == false -@test @inferred all([false, true]) == false -@test @inferred all([true, false]) == false -@test @inferred all([true, true]) == true -@test @inferred all([true, true, true]) == true -@test @inferred all([true, false, true]) == false -@test @inferred all([false, false, false]) == false - -@test @inferred any(x->x>0, []) == false -@test @inferred any(x->x>0, Int[]) == false -@test @inferred any(x->x>0, [-3]) == false -@test @inferred any(x->x>0, [4]) == true -@test @inferred any(x->x>0, [-3, 4, 5]) == true - -@test @inferred all(x->x>0, []) == true -@test @inferred all(x->x>0, Int[]) == true -@test @inferred all(x->x>0, [-3]) == false -@test @inferred all(x->x>0, [4]) == true -@test @inferred all(x->x>0, [-3, 4, 5]) == false +@test @inferred(Union{Missing,Bool}, any([])) == false +@test @inferred(any(Bool[])) == false +@test @inferred(any([true])) == true +@test @inferred(any([false, false])) == false +@test @inferred(any([false, true])) == true +@test @inferred(any([true, false])) == true +@test @inferred(any([true, true])) == true +@test @inferred(any([true, true, true])) == true +@test @inferred(any([true, false, true])) == true +@test @inferred(any([false, false, false])) == false + +@test @inferred(Union{Missing,Bool}, all([])) == true +@test @inferred(all(Bool[])) == true +@test @inferred(all([true])) == true +@test @inferred(all([false, false])) == false +@test @inferred(all([false, true])) == false +@test @inferred(all([true, false])) == false +@test @inferred(all([true, true])) == true +@test @inferred(all([true, true, true])) == true +@test @inferred(all([true, false, true])) == false +@test @inferred(all([false, false, false])) == false + +@test @inferred(Union{Missing,Bool}, any(x->x>0, [])) == false +@test @inferred(any(x->x>0, Int[])) == false +@test @inferred(any(x->x>0, [-3])) == false +@test @inferred(any(x->x>0, [4])) == true +@test @inferred(any(x->x>0, [-3, 4, 5])) == true + +@test @inferred(Union{Missing,Bool}, all(x->x>0, [])) == true +@test @inferred(all(x->x>0, Int[])) == true +@test @inferred(all(x->x>0, [-3])) == false +@test @inferred(all(x->x>0, [4])) == true +@test @inferred(all(x->x>0, [-3, 4, 5])) == false @test reduce((a, b) -> a .| b, fill(trues(5), 24)) == trues(5) @test reduce((a, b) -> a .| b, fill(falses(5), 24)) == falses(5) From feb2988b3a21968410267378b910ce67726a51d8 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Thu, 6 Jul 2023 01:41:15 -0700 Subject: [PATCH 16/33] remove type parameter from AbstractTriangular (#26307) Co-authored-by: Daniel Karrasch --- stdlib/LinearAlgebra/src/special.jl | 2 +- stdlib/LinearAlgebra/src/triangular.jl | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/stdlib/LinearAlgebra/src/special.jl b/stdlib/LinearAlgebra/src/special.jl index 1744a2301f48a..22567c2a8ef96 100644 --- a/stdlib/LinearAlgebra/src/special.jl +++ b/stdlib/LinearAlgebra/src/special.jl @@ -334,7 +334,7 @@ end const _SpecialArrays = Union{Diagonal, Bidiagonal, Tridiagonal, SymTridiagonal} const _Symmetric_DenseArrays{T,A<:Matrix} = Symmetric{T,A} const _Hermitian_DenseArrays{T,A<:Matrix} = Hermitian{T,A} -const _Triangular_DenseArrays{T,A<:Matrix} = AbstractTriangular{T,A} +const _Triangular_DenseArrays{T,A<:Matrix} = UpperOrLowerTriangular{T,A} const _Annotated_DenseArrays = Union{_SpecialArrays, _Triangular_DenseArrays, _Symmetric_DenseArrays, _Hermitian_DenseArrays} const _Annotated_Typed_DenseArrays{T} = Union{_Triangular_DenseArrays{T}, _Symmetric_DenseArrays{T}, _Hermitian_DenseArrays{T}} const _DenseConcatGroup = Union{Number, Vector, Adjoint{<:Any,<:Vector}, Transpose{<:Any,<:Vector}, Matrix, _Annotated_DenseArrays} diff --git a/stdlib/LinearAlgebra/src/triangular.jl b/stdlib/LinearAlgebra/src/triangular.jl index 295a46f1522a5..7c8985385f220 100644 --- a/stdlib/LinearAlgebra/src/triangular.jl +++ b/stdlib/LinearAlgebra/src/triangular.jl @@ -3,13 +3,12 @@ ## Triangular # could be renamed to Triangular when that name has been fully deprecated -abstract type AbstractTriangular{T,S<:AbstractMatrix} <: AbstractMatrix{T} end +abstract type AbstractTriangular{T} <: AbstractMatrix{T} end # First loop through all methods that don't need special care for upper/lower and unit diagonal -for t in (:LowerTriangular, :UnitLowerTriangular, :UpperTriangular, - :UnitUpperTriangular) +for t in (:LowerTriangular, :UnitLowerTriangular, :UpperTriangular, :UnitUpperTriangular) @eval begin - struct $t{T,S<:AbstractMatrix{T}} <: AbstractTriangular{T,S} + struct $t{T,S<:AbstractMatrix{T}} <: AbstractTriangular{T} data::S function $t{T,S}(data) where {T,S<:AbstractMatrix{T}} @@ -854,9 +853,9 @@ for t in (:LowerTriangular, :UnitLowerTriangular, :UpperTriangular, :UnitUpperTr end end -errorbounds(A::AbstractTriangular{T,<:AbstractMatrix}, X::AbstractVecOrMat{T}, B::AbstractVecOrMat{T}) where {T<:Union{BigFloat,Complex{BigFloat}}} = +errorbounds(A::AbstractTriangular{T}, X::AbstractVecOrMat{T}, B::AbstractVecOrMat{T}) where {T<:Union{BigFloat,Complex{BigFloat}}} = error("not implemented yet! Please submit a pull request.") -function errorbounds(A::AbstractTriangular{TA,<:AbstractMatrix}, X::AbstractVecOrMat{TX}, B::AbstractVecOrMat{TB}) where {TA<:Number,TX<:Number,TB<:Number} +function errorbounds(A::AbstractTriangular{TA}, X::AbstractVecOrMat{TX}, B::AbstractVecOrMat{TB}) where {TA<:Number,TX<:Number,TB<:Number} TAXB = promote_type(TA, TB, TX, Float32) errorbounds(convert(AbstractMatrix{TAXB}, A), convert(AbstractArray{TAXB}, X), convert(AbstractArray{TAXB}, B)) end From 64ab5379094fe0dab89cabc269cb4fa830a11bbc Mon Sep 17 00:00:00 2001 From: Malte Sandstede Date: Thu, 6 Jul 2023 20:45:02 +0200 Subject: [PATCH 17/33] Add GC metric `last_incremental_sweep` (#50190) * Add GC metric `last_incremental_sweep` * Update gc.c * Update gc.c --- base/timing.jl | 1 + src/gc.c | 3 +++ src/gc.h | 1 + 3 files changed, 5 insertions(+) diff --git a/base/timing.jl b/base/timing.jl index d166b4162db59..154951d031af5 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -25,6 +25,7 @@ struct GC_Num total_sweep_time ::Int64 total_mark_time ::Int64 last_full_sweep ::Int64 + last_incremental_sweep ::Int64 end gc_num() = ccall(:jl_gc_num, GC_Num, ()) diff --git a/src/gc.c b/src/gc.c index edc0fdb49c4a5..cb86d5d4535a7 100644 --- a/src/gc.c +++ b/src/gc.c @@ -3274,6 +3274,9 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) if (sweep_full) { gc_num.last_full_sweep = gc_end_time; } + else { + gc_num.last_incremental_sweep = gc_end_time; + } // sweeping is over // 7. if it is a quick sweep, put back the remembered objects in queued state diff --git a/src/gc.h b/src/gc.h index b1eee5a1d5bda..83c803c3cb8aa 100644 --- a/src/gc.h +++ b/src/gc.h @@ -82,6 +82,7 @@ typedef struct { uint64_t total_sweep_time; uint64_t total_mark_time; uint64_t last_full_sweep; + uint64_t last_incremental_sweep; } jl_gc_num_t; // Array chunks (work items representing suffixes of From c14d4bbe0c6b8a19c620e6fd3680f8ef55cd458d Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Thu, 6 Jul 2023 15:50:32 -0400 Subject: [PATCH 18/33] copyuntil(out::IO, in::IO, delim) (#48273) Note that this defines the lock order as `out` then `in` for streams which may try to take both locks. This is now a mandatory API convention for all future streams. Co-authored-by: Rafael Fourquet --- NEWS.md | 1 + base/exports.jl | 2 + base/io.jl | 163 ++++++++++++++++++++++++++++--------- base/iobuffer.jl | 66 +++++++++------ base/iostream.jl | 35 ++++++++ doc/src/base/io-network.md | 2 + src/flisp/iostream.c | 4 +- src/support/ios.c | 8 +- src/support/ios.h | 2 +- src/sys.c | 46 ++++++++++- test/read.jl | 57 ++++++++++++- 11 files changed, 315 insertions(+), 71 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8907275925155..52180a852b0e7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -21,6 +21,7 @@ Build system changes New library functions --------------------- +* `copyuntil(out, io, delim)` and `copyline(out, io)` copy data into an `out::IO` stream ([#48273]). New library features -------------------- diff --git a/base/exports.jl b/base/exports.jl index 10f43825e12df..0959fa1c391e2 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -857,6 +857,8 @@ export readline, readlines, readuntil, + copyuntil, + copyline, redirect_stdio, redirect_stderr, redirect_stdin, diff --git a/base/io.jl b/base/io.jl index 60a24831587cb..c62d6393d12ec 100644 --- a/base/io.jl +++ b/base/io.jl @@ -440,10 +440,10 @@ for f in ( end read(io::AbstractPipe, byte::Type{UInt8}) = read(pipe_reader(io)::IO, byte)::UInt8 unsafe_read(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt) = unsafe_read(pipe_reader(io)::IO, p, nb) -readuntil(io::AbstractPipe, arg::UInt8; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) -readuntil(io::AbstractPipe, arg::AbstractChar; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) -readuntil(io::AbstractPipe, arg::AbstractString; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) -readuntil(io::AbstractPipe, arg::AbstractVector; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::UInt8; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::AbstractChar; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::AbstractString; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::AbstractVector; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) readuntil_vector!(io::AbstractPipe, target::AbstractVector, keep::Bool, out) = readuntil_vector!(pipe_reader(io)::IO, target, keep, out) readbytes!(io::AbstractPipe, target::AbstractVector{UInt8}, n=length(target)) = readbytes!(pipe_reader(io)::IO, target, n) peek(io::AbstractPipe, ::Type{T}) where {T} = peek(pipe_reader(io)::IO, T)::T @@ -499,11 +499,15 @@ read!(filename::AbstractString, a) = open(io->read!(io, a), convert(String, file readuntil(stream::IO, delim; keep::Bool = false) readuntil(filename::AbstractString, delim; keep::Bool = false) -Read a string from an I/O stream or a file, up to the given delimiter. +Read a string from an I/O `stream` or a file, up to the given delimiter. The delimiter can be a `UInt8`, `AbstractChar`, string, or vector. Keyword argument `keep` controls whether the delimiter is included in the result. The text is assumed to be encoded in UTF-8. +Return a `String` if `delim` is an `AbstractChar` or a string +or otherwise return a `Vector{typeof(delim)}`. See also [`copyuntil`](@ref) +to instead write in-place to another stream (which can be a preallocated [`IOBuffer`](@ref)). + # Examples ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); @@ -517,7 +521,39 @@ julia> readuntil("my_file.txt", '.', keep = true) julia> rm("my_file.txt") ``` """ -readuntil(filename::AbstractString, args...; kw...) = open(io->readuntil(io, args...; kw...), convert(String, filename)::String) +readuntil(filename::AbstractString, delim; kw...) = open(io->readuntil(io, delim; kw...), convert(String, filename)::String) +readuntil(stream::IO, delim::UInt8; kw...) = _unsafe_take!(copyuntil(IOBuffer(sizehint=70), stream, delim; kw...)) +readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = String(_unsafe_take!(copyuntil(IOBuffer(sizehint=70), stream, delim; kw...))) + + +""" + copyuntil(out::IO, stream::IO, delim; keep::Bool = false) + copyuntil(out::IO, filename::AbstractString, delim; keep::Bool = false) + +Copy a string from an I/O `stream` or a file, up to the given delimiter, to +the `out` stream, returning `out`. +The delimiter can be a `UInt8`, `AbstractChar`, string, or vector. +Keyword argument `keep` controls whether the delimiter is included in the result. +The text is assumed to be encoded in UTF-8. + +Similar to [`readuntil`](@ref), which returns a `String`; in contrast, +`copyuntil` writes directly to `out`, without allocating a string. +(This can be used, for example, to read data into a pre-allocated [`IOBuffer`](@ref).) + +# Examples +```jldoctest +julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); + +julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", 'L'))) +"Julia" + +julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true))) +"JuliaLang is a GitHub organization." + +julia> rm("my_file.txt") +``` +""" +copyuntil(out::IO, filename::AbstractString, delim; kw...) = open(io->copyuntil(out, io, delim; kw...), convert(String, filename)::String) """ readline(io::IO=stdin; keep::Bool=false) @@ -530,6 +566,11 @@ false (as it is by default), these trailing newline characters are removed from line before it is returned. When `keep` is true, they are returned as part of the line. +Return a `String`. See also [`copyline`](@ref) to instead write in-place +to another stream (which can be a preallocated [`IOBuffer`](@ref)). + +See also [`readuntil`](@ref) for reading until more general delimiters. + # Examples ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); @@ -551,21 +592,63 @@ Logan "Logan" ``` """ -function readline(filename::AbstractString; keep::Bool=false) - open(filename) do f - readline(f, keep=keep) - end -end +readline(filename::AbstractString; keep::Bool=false) = + open(io -> readline(io; keep), filename) +readline(s::IO=stdin; keep::Bool=false) = + String(_unsafe_take!(copyline(IOBuffer(sizehint=70), s; keep))) -function readline(s::IO=stdin; keep::Bool=false) - line = readuntil(s, 0x0a, keep=true)::Vector{UInt8} - i = length(line) - if keep || i == 0 || line[i] != 0x0a - return String(line) - elseif i < 2 || line[i-1] != 0x0d - return String(resize!(line,i-1)) +""" + copyline(out::IO, io::IO=stdin; keep::Bool=false) + copyline(out::IO, filename::AbstractString; keep::Bool=false) + +Copy a single line of text from an I/O `stream` or a file to the `out` stream, +returning `out`. + +When reading from a file, the text is assumed to be encoded in UTF-8. Lines in the +input end with `'\\n'` or `"\\r\\n"` or the end of an input stream. When `keep` is +false (as it is by default), these trailing newline characters are removed from the +line before it is returned. When `keep` is true, they are returned as part of the +line. + +Similar to [`readline`](@ref), which returns a `String`; in contrast, +`copyline` writes directly to `out`, without allocating a string. +(This can be used, for example, to read data into a pre-allocated [`IOBuffer`](@ref).) + +See also [`copyuntil`](@ref) for reading until more general delimiters. + +# Examples +```jldoctest +julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); + +julia> String(take!(copyline(IOBuffer(), "my_file.txt"))) +"JuliaLang is a GitHub organization." + +julia> String(take!(copyline(IOBuffer(), "my_file.txt", keep=true))) +"JuliaLang is a GitHub organization.\\n" + +julia> rm("my_file.txt") +``` +""" +copyline(out::IO, filename::AbstractString; keep::Bool=false) = + open(io -> copyline(out, io; keep), filename) + +# fallback to optimized methods for IOBuffer in iobuffer.jl +function copyline(out::IO, s::IO; keep::Bool=false) + if keep + return copyuntil(out, s, 0x0a, keep=true) else - return String(resize!(line,i-2)) + # more complicated to deal with CRLF logic + while !eof(s) + b = read(s, UInt8) + b == 0x0a && break + if b == 0x0d && !eof(s) + b = read(s, UInt8) + b == 0x0a && break + write(out, 0x0d) + end + write(out, b) + end + return out end end @@ -816,15 +899,10 @@ end # read(io, T) is not defined for other AbstractChar: implementations # must provide their own encoding-specific method. -# readuntil_string is useful below since it has -# an optimized method for s::IOStream -readuntil_string(s::IO, delim::UInt8, keep::Bool) = String(readuntil(s, delim, keep=keep))::String - -function readuntil(s::IO, delim::AbstractChar; keep::Bool=false) +function copyuntil(out::IO, s::IO, delim::AbstractChar; keep::Bool=false) if delim ≤ '\x7f' - return readuntil_string(s, delim % UInt8, keep) + return copyuntil(out, s, delim % UInt8; keep) end - out = IOBuffer() for c in readeach(s, Char) if c == delim keep && write(out, c) @@ -832,20 +910,27 @@ function readuntil(s::IO, delim::AbstractChar; keep::Bool=false) end write(out, c) end - return String(take!(out)) + return out end -function readuntil(s::IO, delim::T; keep::Bool=false) where T - out = (T === UInt8 ? StringVector(0) : Vector{T}()) +# note: optimized methods of copyuntil for IOStreams and delim::UInt8 in iostream.jl +function _copyuntil(out, s::IO, delim::T, keep::Bool) where T + output! = isa(out, IO) ? write : push! for c in readeach(s, T) if c == delim - keep && push!(out, c) + keep && output!(out, c) break end - push!(out, c) + output!(out, c) end return out end +readuntil(s::IO, delim::T; keep::Bool=false) where T = + _copyuntil(Vector{T}(), s, delim, keep) +readuntil(s::IO, delim::UInt8; keep::Bool=false) = + _copyuntil(resize!(StringVector(70), 0), s, delim, keep) +copyuntil(out::IO, s::IO, delim::T; keep::Bool=false) where T = + _copyuntil(out, s, delim, keep) # requires that indices for target are the integer unit range from firstindex to lastindex # returns whether the delimiter was matched @@ -933,27 +1018,29 @@ function readuntil_vector!(io::IO, target::AbstractVector{T}, keep::Bool, out) w return false end -function readuntil(io::IO, target::AbstractString; keep::Bool=false) +function copyuntil(out::IO, io::IO, target::AbstractString; keep::Bool=false) # small-string target optimizations x = Iterators.peel(target) - isnothing(x) && return "" + isnothing(x) && return out c, rest = x if isempty(rest) && c <= '\x7f' - return readuntil_string(io, c % UInt8, keep) + return copyuntil(out, io, c % UInt8; keep) end # convert String to a utf8-byte-iterator if !(target isa String) && !(target isa SubString{String}) target = String(target) end target = codeunits(target)::AbstractVector - return String(readuntil(io, target, keep=keep)) + return copyuntil(out, io, target, keep=keep) end function readuntil(io::IO, target::AbstractVector{T}; keep::Bool=false) where T - out = (T === UInt8 ? StringVector(0) : Vector{T}()) + out = (T === UInt8 ? resize!(StringVector(70), 0) : Vector{T}()) readuntil_vector!(io, target, keep, out) return out end +copyuntil(out::IO, io::IO, target::AbstractVector; keep::Bool=false) = + (readuntil_vector!(io, target, keep, out); out) """ readchomp(x) @@ -1128,7 +1215,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) empty!(chunks) # will cause next iteration to terminate seekend(r.itr.stream) # reposition to end of stream for isdone - s = String(take!(buf)) + s = String(_unsafe_take!(buf)) else # extract the string from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline] if ichunk == jchunk # common case: current and previous newline in same chunk @@ -1145,7 +1232,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) end write(buf, view(chunks[jchunk], 1:jnewline)) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) - s = String(take!(buf)) + s = String(_unsafe_take!(buf)) # overwrite obsolete chunks (ichunk+1:jchunk) i = jchunk diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 6c95285f232f2..deb86e774f4e4 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -516,33 +516,53 @@ function occursin(delim::UInt8, buf::GenericIOBuffer) return false end -function readuntil(io::GenericIOBuffer, delim::UInt8; keep::Bool=false) - lb = 70 - A = StringVector(lb) - nread = 0 - nout = 0 - data = io.data - for i = io.ptr : io.size - @inbounds b = data[i] - nread += 1 - if keep || b != delim - nout += 1 - if nout > lb - lb = nout*2 - resize!(A, lb) - end - @inbounds A[nout] = b - end - if b == delim - break - end +function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) + data = view(io.data, io.ptr:io.size) + # note: findfirst + copyto! is much faster than a single loop + # except for nout ≲ 20. A single loop is 2x faster for nout=5. + nout = nread = something(findfirst(==(delim), data), length(data)) + if !keep && nout > 0 && data[nout] == delim + nout -= 1 end + write(out, view(io.data, io.ptr:io.ptr+nout-1)) io.ptr += nread - if lb != nout - resize!(A, nout) + return out +end + +function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) + copyuntil(out, s, 0x0a, keep=true) + line = out.data + i = out.size + if keep || i == 0 || line[i] != 0x0a + return out + elseif i < 2 || line[i-1] != 0x0d + i -= 1 + else + i -= 2 end - A + out.size = i + if !out.append + out.ptr = i+1 + end + return out +end + +function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) + data = view(io.data, io.ptr:io.size) + # note: findfirst + copyto! is much faster than a single loop + # except for nout ≲ 20. A single loop is 2x faster for nout=5. + nout = nread = something(findfirst(==(0x0a), data), length(data)) + if !keep && nout > 0 && data[nout] == 0x0a + nout -= 1 + nout > 0 && data[nout] == 0x0d && (nout -= 1) + end + write(out, view(io.data, io.ptr:io.ptr+nout-1)) + io.ptr += nread + return out end +copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) +copyline(out::GenericIOBuffer, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) + # copy-free crc32c of IOBuffer: function _crc32c(io::IOBuffer, nb::Integer, crc::UInt32=0x00000000) diff --git a/base/iostream.jl b/base/iostream.jl index 23dfb53256e82..f5a8c0a8dffc8 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -443,11 +443,46 @@ end function readuntil_string(s::IOStream, delim::UInt8, keep::Bool) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 1, !keep) end +readuntil(s::IOStream, delim::AbstractChar; keep::Bool=false) = + delim ≤ '\x7f' ? readuntil_string(s, delim % UInt8, keep) : + String(unsafe_take!(copyuntil(IOBuffer(sizehint=70), s, delim; keep))) function readline(s::IOStream; keep::Bool=false) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, keep ? 0 : 2) end +function copyuntil(out::IOBuffer, s::IOStream, delim::UInt8; keep::Bool=false) + ensureroom(out, 16) + ptr = (out.append ? out.size+1 : out.ptr) + d = out.data + len = length(d) + while true + GC.@preserve d @_lock_ios s n= + Int(ccall(:jl_readuntil_buf, Csize_t, (Ptr{Cvoid}, UInt8, Ptr{UInt8}, Csize_t), + s.ios, delim, pointer(d, ptr), (len - ptr + 1) % Csize_t)) + iszero(n) && break + ptr += n + if d[ptr-1] == delim + keep || (ptr -= 1) + break + end + (eof(s) || len == out.maxsize) && break + len = min(2len + 64, out.maxsize) + resize!(d, len) + end + out.size = max(out.size, ptr - 1) + if !out.append + out.ptr = ptr + end + return out +end + +function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) + @_lock_ios out @_lock_ios s ccall(:ios_copyuntil, Csize_t, + (Ptr{Cvoid}, Ptr{Cvoid}, UInt8, Cint), out.ios, s.ios, delim, keep) + return out +end + function readbytes_all!(s::IOStream, b::Union{Array{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}}, nb::Integer) diff --git a/doc/src/base/io-network.md b/doc/src/base/io-network.md index 4e371039f1a9b..68f144427a892 100644 --- a/doc/src/base/io-network.md +++ b/doc/src/base/io-network.md @@ -71,6 +71,8 @@ Base.readline Base.readuntil Base.readlines Base.eachline +Base.copyline +Base.copyuntil Base.displaysize ``` diff --git a/src/flisp/iostream.c b/src/flisp/iostream.c index b2b2477bb43c6..c1c6d965d2917 100644 --- a/src/flisp/iostream.c +++ b/src/flisp/iostream.c @@ -354,7 +354,7 @@ value_t fl_ioreaduntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_setbuf(&dest, data, 80, 0); char delim = get_delim_arg(fl_ctx, args[1], "io.readuntil"); ios_t *src = toiostream(fl_ctx, args[0], "io.readuntil"); - size_t n = ios_copyuntil(&dest, src, delim); + size_t n = ios_copyuntil(&dest, src, delim, 1); cv->len = n; if (dest.buf != data) { // outgrew initial space @@ -376,7 +376,7 @@ value_t fl_iocopyuntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_t *dest = toiostream(fl_ctx, args[0], "io.copyuntil"); ios_t *src = toiostream(fl_ctx, args[1], "io.copyuntil"); char delim = get_delim_arg(fl_ctx, args[2], "io.copyuntil"); - return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim)); + return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim, 1)); } value_t fl_iocopy(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) diff --git a/src/support/ios.c b/src/support/ios.c index b5a168f705603..c98c529991642 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -832,7 +832,7 @@ size_t ios_copyall(ios_t *to, ios_t *from) #define LINE_CHUNK_SIZE 160 -size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) +size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, int keep) { size_t total = 0, avail = (size_t)(from->size - from->bpos); while (!ios_eof(from)) { @@ -850,9 +850,9 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) avail = 0; } else { - size_t ntowrite = pd - (from->buf+from->bpos) + 1; + size_t ntowrite = pd - (from->buf+from->bpos) + (keep != 0); written = ios_write(to, from->buf+from->bpos, ntowrite); - from->bpos += ntowrite; + from->bpos += ntowrite + (keep == 0); total += written; return total; } @@ -1217,7 +1217,7 @@ char *ios_readline(ios_t *s) { ios_t dest; ios_mem(&dest, 0); - ios_copyuntil(&dest, s, '\n'); + ios_copyuntil(&dest, s, '\n', 1); size_t n; return ios_take_buffer(&dest, &n); } diff --git a/src/support/ios.h b/src/support/ios.h index 2547555b5585d..6eab9e21c45b6 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -108,7 +108,7 @@ JL_DLLEXPORT int ios_get_writable(ios_t *s); JL_DLLEXPORT void ios_set_readonly(ios_t *s); JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes); JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from); -JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) JL_NOTSAFEPOINT; +JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, int keep) JL_NOTSAFEPOINT; JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite); // ensure at least n bytes are buffered if possible. returns # available. JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); diff --git a/src/sys.c b/src/sys.c index 2de4bc61a20b8..d55c5df7ab066 100644 --- a/src/sys.c +++ b/src/sys.c @@ -288,7 +288,7 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint ios_t dest; ios_mem(&dest, 0); ios_setbuf(&dest, (char*)a->data, 80, 0); - size_t n = ios_copyuntil(&dest, s, delim); + size_t n = ios_copyuntil(&dest, s, delim, 1); if (chomp && n > 0 && dest.buf[n - 1] == delim) { n--; if (chomp == 2 && n > 0 && dest.buf[n - 1] == '\r') { @@ -316,6 +316,50 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint return (jl_value_t*)a; } +// read up to buflen bytes, including delim, into buf. returns number of bytes read. +JL_DLLEXPORT size_t jl_readuntil_buf(ios_t *s, uint8_t delim, uint8_t *buf, size_t buflen) +{ + // manually inlined common case + size_t avail = (size_t)(s->size - s->bpos); + if (avail > buflen) avail = buflen; + char *pd = (char*)memchr(s->buf + s->bpos, delim, avail); + if (pd) { + size_t n = pd - (s->buf + s->bpos) + 1; + memcpy(buf, s->buf + s->bpos, n); + s->bpos += n; + return n; + } + else { + size_t total = avail; + memcpy(buf, s->buf + s->bpos, avail); + s->bpos += avail; + if (avail == buflen) return total; + + // code derived from ios_copyuntil + while (!ios_eof(s)) { + avail = ios_readprep(s, 160); // read LINE_CHUNK_SIZE + if (avail == 0) break; + if (total+avail > buflen) avail = buflen-total; + char *pd = (char*)memchr(s->buf+s->bpos, delim, avail); + if (pd == NULL) { + memcpy(buf+total, s->buf+s->bpos, avail); + s->bpos += avail; + total += avail; + if (buflen == total) return total; + } + else { + size_t ntowrite = pd - (s->buf+s->bpos) + 1; + memcpy(buf+total, s->buf+s->bpos, ntowrite); + s->bpos += ntowrite; + total += ntowrite; + return total; + } + } + s->_eof = 1; + return total; + } +} + JL_DLLEXPORT int jl_ios_buffer_n(ios_t *s, const size_t n) { size_t space, ret; diff --git a/test/read.jl b/test/read.jl index b8060a023333f..ff0952b1495da 100644 --- a/test/read.jl +++ b/test/read.jl @@ -145,6 +145,7 @@ for (name, f) in l verbose && println("$name readuntil...") for (t, s, m, kept) in [ + ("a", "", "", ""), ("a", "ab", "a", "a"), ("b", "ab", "b", "b"), ("α", "αγ", "α", "α"), @@ -152,16 +153,19 @@ for (name, f) in l ("bc", "abc", "bc", "bc"), ("αβ", "αβγ", "αβ", "αβ"), ("aaabc", "ab", "aa", "aaab"), + ("aaabc", "b", "aaa", "aaab"), ("aaabc", "ac", "aaabc", "aaabc"), ("aaabc", "aab", "a", "aaab"), ("aaabc", "aac", "aaabc", "aaabc"), ("αααβγ", "αβ", "αα", "αααβ"), + ("αααβγ", "β", "ααα", "αααβ"), ("αααβγ", "ααβ", "α", "αααβ"), ("αααβγ", "αγ", "αααβγ", "αααβγ"), ("barbarbarians", "barbarian", "bar", "barbarbarian"), ("abcaabcaabcxl", "abcaabcx", "abca", "abcaabcaabcx"), ("abbaabbaabbabbaax", "abbaabbabbaax", "abba", "abbaabbaabbabbaax"), ("abbaabbabbaabbaabbabbaax", "abbaabbabbaax", "abbaabbabba", "abbaabbabbaabbaabbabbaax"), + ('a'^500 * 'x' * "bbbb", "x", 'a'^500, 'a'^500 * 'x'), ] local t, s, m, kept @test readuntil(io(t), s) == m @@ -174,6 +178,18 @@ for (name, f) in l @test readuntil(io(t), unsafe_wrap(Vector{UInt8},s), keep=true) == unsafe_wrap(Vector{UInt8},kept) @test readuntil(io(t), collect(s)::Vector{Char}) == Vector{Char}(m) @test readuntil(io(t), collect(s)::Vector{Char}, keep=true) == Vector{Char}(kept) + + buf = IOBuffer() + @test String(take!(copyuntil(buf, io(t), s))) == m + @test String(take!(copyuntil(buf, io(t), s, keep=true))) == kept + file = tempname() + for (k,m) in ((false, m), (true, kept)) + open(file, "w") do f + @test f == copyuntil(f, io(t), s, keep=k) + end + @test read(file, String) == m + end + rm(file) end cleanup() @@ -281,8 +297,45 @@ for (name, f) in l cleanup() verbose && println("$name readline...") - @test readline(io(), keep=true) == readline(IOBuffer(text), keep=true) - @test readline(io(), keep=true) == readline(filename, keep=true) + file = tempname() + for lineending in ("\n", "\r\n", "") + kept = "foo bar" * lineending + t = isempty(lineending) ? "foo bar" : kept * "baz\n" + write(file, t) + @test readline(io(t)) == readline(file) == "foo bar" + @test readline(io(t), keep=true) == readline(file, keep=true) == kept + + @test String(take!(copyline(IOBuffer(), file))) == "foo bar" + @test String(take!(copyline(IOBuffer(), file, keep=true))) == kept + + cleanup() + + buf = IOBuffer() + @test buf === copyline(buf, io(t)) + @test String(take!(buf)) == "foo bar" + @test String(take!(copyline(buf, file, keep=true))) == kept + for keep in (true, false) + open(file, "w") do f + @test f === copyline(f, io(t), keep=keep) + end + @test read(file, String) == (keep ? kept : "foo bar") + end + + cleanup() + + write(file, lineending) + @test readline(IOBuffer(lineending)) == "" + @test readline(IOBuffer(lineending), keep=true) == lineending + @test String(take!(copyline(IOBuffer(), IOBuffer(lineending)))) == "" + @test String(take!(copyline(IOBuffer(), IOBuffer(lineending), keep=true))) == lineending + @test readline(file) == "" + @test readline(file, keep=true) == lineending + @test String(take!(copyline(IOBuffer(), file))) == "" + @test String(take!(copyline(IOBuffer(), file, keep=true))) == lineending + + cleanup() + end + rm(file) verbose && println("$name readlines...") @test readlines(io(), keep=true) == readlines(IOBuffer(text), keep=true) From b6bfe988ef6098ae98ba8d38094839b302f388e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Riedemann?= <38795484+longemen3000@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:05:43 -0400 Subject: [PATCH 19/33] Define `Base.isstored` for Diagonals and Triangular matrices (#50391) X-ref #50377 --- stdlib/LinearAlgebra/src/bidiag.jl | 13 +++++++++++++ stdlib/LinearAlgebra/src/diagonal.jl | 10 ++++++++++ stdlib/LinearAlgebra/src/triangular.jl | 9 +++++++++ stdlib/LinearAlgebra/src/tridiag.jl | 26 ++++++++++++++++++++++++++ stdlib/LinearAlgebra/test/bidiag.jl | 15 +++++++++++++++ stdlib/LinearAlgebra/test/diagonal.jl | 3 +++ 6 files changed, 76 insertions(+) diff --git a/stdlib/LinearAlgebra/src/bidiag.jl b/stdlib/LinearAlgebra/src/bidiag.jl index 192272cc61e98..0014ba1ec8ab0 100644 --- a/stdlib/LinearAlgebra/src/bidiag.jl +++ b/stdlib/LinearAlgebra/src/bidiag.jl @@ -143,6 +143,19 @@ end end end +@inline function Base.isstored(A::Bidiagonal, i::Int, j::Int) + @boundscheck checkbounds(A, i, j) + if i == j + return @inbounds Base.isstored(A.dv, i) + elseif A.uplo == 'U' && (i == j - 1) + return @inbounds Base.isstored(A.ev, i) + elseif A.uplo == 'L' && (i == j + 1) + return @inbounds Base.isstored(A.ev, j) + else + return false + end +end + @inline function getindex(A::Bidiagonal{T}, i::Integer, j::Integer) where T @boundscheck checkbounds(A, i, j) if i == j diff --git a/stdlib/LinearAlgebra/src/diagonal.jl b/stdlib/LinearAlgebra/src/diagonal.jl index 29c190e87df72..37359fd1074f8 100644 --- a/stdlib/LinearAlgebra/src/diagonal.jl +++ b/stdlib/LinearAlgebra/src/diagonal.jl @@ -150,6 +150,16 @@ end r end +@inline function Base.isstored(D::Diagonal, i::Int, j::Int) + @boundscheck checkbounds(D, i, j) + if i == j + @inbounds r = Base.isstored(D.diag, i) + else + r = false + end + r +end + @inline function getindex(D::Diagonal, i::Int, j::Int) @boundscheck checkbounds(D, i, j) if i == j diff --git a/stdlib/LinearAlgebra/src/triangular.jl b/stdlib/LinearAlgebra/src/triangular.jl index 7c8985385f220..807ba5619f7c8 100644 --- a/stdlib/LinearAlgebra/src/triangular.jl +++ b/stdlib/LinearAlgebra/src/triangular.jl @@ -231,6 +231,15 @@ Base.isassigned(A::UnitUpperTriangular, i::Int, j::Int) = Base.isassigned(A::UpperTriangular, i::Int, j::Int) = i <= j ? isassigned(A.data, i, j) : true +Base.isstored(A::UnitLowerTriangular, i::Int, j::Int) = + i > j ? Base.isstored(A.data, i, j) : false +Base.isstored(A::LowerTriangular, i::Int, j::Int) = + i >= j ? Base.isstored(A.data, i, j) : false +Base.isstored(A::UnitUpperTriangular, i::Int, j::Int) = + i < j ? Base.isstored(A.data, i, j) : false +Base.isstored(A::UpperTriangular, i::Int, j::Int) = + i <= j ? Base.isstored(A.data, i, j) : false + getindex(A::UnitLowerTriangular{T}, i::Integer, j::Integer) where {T} = i > j ? A.data[i,j] : ifelse(i == j, oneunit(T), zero(T)) getindex(A::LowerTriangular, i::Integer, j::Integer) = diff --git a/stdlib/LinearAlgebra/src/tridiag.jl b/stdlib/LinearAlgebra/src/tridiag.jl index 13f6a1bb70756..a53bb3815a481 100644 --- a/stdlib/LinearAlgebra/src/tridiag.jl +++ b/stdlib/LinearAlgebra/src/tridiag.jl @@ -427,6 +427,19 @@ logabsdet(A::SymTridiagonal; shift::Number=false) = logabsdet(ldlt(A; shift=shif end end +@inline function Base.isstored(A::SymTridiagonal, i::Int, j::Int) + @boundscheck checkbounds(A, i, j) + if i == j + return @inbounds Base.isstored(A.dv, i) + elseif i == j + 1 + return @inbounds Base.isstored(A.ev, j) + elseif i + 1 == j + return @inbounds Base.isstored(A.ev, i) + else + return false + end +end + @inline function getindex(A::SymTridiagonal{T}, i::Integer, j::Integer) where T @boundscheck checkbounds(A, i, j) if i == j @@ -632,6 +645,19 @@ end end end +@inline function Base.isstored(A::Tridiagonal, i::Int, j::Int) + @boundscheck checkbounds(A, i, j) + if i == j + return @inbounds Base.isstored(A.d, i) + elseif i == j + 1 + return @inbounds Base.isstored(A.dl, j) + elseif i + 1 == j + return @inbounds Base.isstored(A.du, i) + else + return false + end +end + @inline function getindex(A::Tridiagonal{T}, i::Integer, j::Integer) where T @boundscheck checkbounds(A, i, j) if i == j diff --git a/stdlib/LinearAlgebra/test/bidiag.jl b/stdlib/LinearAlgebra/test/bidiag.jl index d13009780b975..2306b46b1315e 100644 --- a/stdlib/LinearAlgebra/test/bidiag.jl +++ b/stdlib/LinearAlgebra/test/bidiag.jl @@ -120,6 +120,21 @@ Random.seed!(1) @test_throws ArgumentError Bl[4, 5] = 1 end + @testset "isstored" begin + ubd = Bidiagonal(dv, ev, :U) + lbd = Bidiagonal(dv, ev, :L) + # bidiagonal isstored / upper & lower + @test_throws BoundsError Base.isstored(ubd, n + 1, 1) + @test_throws BoundsError Base.isstored(ubd, 1, n + 1) + @test Base.isstored(ubd, 2, 2) + # bidiagonal isstored / upper + @test Base.isstored(ubd, 2, 3) + @test !Base.isstored(ubd, 3, 2) + # bidiagonal isstored / lower + @test Base.isstored(lbd, 3, 2) + @test !Base.isstored(lbd, 2, 3) + end + @testset "show" begin BD = Bidiagonal(dv, ev, :U) dstring = sprint(Base.print_matrix,BD.dv') diff --git a/stdlib/LinearAlgebra/test/diagonal.jl b/stdlib/LinearAlgebra/test/diagonal.jl index 2a8248d9ca716..61045a957cfed 100644 --- a/stdlib/LinearAlgebra/test/diagonal.jl +++ b/stdlib/LinearAlgebra/test/diagonal.jl @@ -78,6 +78,9 @@ Random.seed!(1) @test !istril(D, -1) @test istril(D, 1) @test istril(Diagonal(zero(diag(D))), -1) + @test Base.isstored(D,1,1) + @test !Base.isstored(D,1,2) + @test_throws BoundsError Base.isstored(D, n + 1, 1) if elty <: Real @test ishermitian(D) end From 6d4470753620fd332f8a4ae21b5f9c9805ba114e Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:36:37 -0400 Subject: [PATCH 20/33] retrieve `splice!` item via `only` rather than `getindex` (#50430) Because `only` uses iteration, similar to the other branches here, rather than assuming it supports one-based-indexing. --- base/array.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index 3a12b38c5bc26..15545c2b1144a 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1757,7 +1757,7 @@ function splice!(a::Vector, i::Integer, ins=_default_splice) if m == 0 _deleteat!(a, i, 1) elseif m == 1 - a[i] = ins[1] + a[i] = only(ins) else _growat!(a, i, m-1) k = 1 From c09efd934df375a291a2c9ca33e3bac7c811f4c6 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Thu, 6 Jul 2023 15:42:14 -0500 Subject: [PATCH 21/33] More consistent findall output type (take 2) (#48976) Fixup for #45538 Helps with #45495 --- base/array.jl | 3 +-- test/functional.jl | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index 15545c2b1144a..b99ec7ee2b015 100644 --- a/base/array.jl +++ b/base/array.jl @@ -2445,9 +2445,8 @@ julia> findall(x -> x >= 0, d) ``` """ function findall(testf::Function, A) - T = eltype(keys(A)) gen = (first(p) for p in pairs(A) if testf(last(p))) - isconcretetype(T) ? collect(T, gen) : collect(gen) + @default_eltype(gen) === Union{} ? collect(@default_eltype(keys(A)), gen) : collect(gen) end # Broadcasting is much faster for small testf, and computing diff --git a/test/functional.jl b/test/functional.jl index 19355d13ff335..fce64c0e5720a 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -145,6 +145,13 @@ let gen = (i for i in 1:3); @test @inferred(findall(x -> false, gen))::Vector{Int} == Int[] @test @inferred(findall(x -> x < 0, gen))::Vector{Int} == Int[] end +let d = Dict() + d[7]=2 + d[3]=6 + @test @inferred(sort(findall(x -> true, d)))::Vector{Int} == [3, 7] + @test @inferred(sort(findall(x -> false, d)))::Vector{Any} == [] + @test @inferred(sort(findall(x -> x < 0, d)))::Vector{Any} == [] +end # inference on vararg generator of a type (see #22907 comments) let f(x) = collect(Base.Generator(=>, x, x)) From 46477cc9efd70c1332c73471d038d1e0755e1a96 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 6 Jul 2023 17:18:59 -0400 Subject: [PATCH 22/33] Delete `trust_inference` option (#50432) * Flip trust_inference option I think the time has come to flip this. This was added when the type system was much less reliable at producing intersections. It is true that we still have the occasional type sytem bug, but we're already trusting inference in a bunch of other places. At the same time, the cost of this has grown in terms of bloated IR needing to be visited in places like irinterp, so let's flip the bit and we'll deal with type system bugs the way we usually due. * refactor to remove trust_inference entirely --------- Co-authored-by: oscarddssmith --- base/compiler/ssair/inlining.jl | 13 +++---------- base/compiler/types.jl | 10 ---------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 170725f231761..137473271f749 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -236,7 +236,7 @@ function cfg_inline_unionsplit!(ir::IRCode, idx::Int, push!(from_bbs, length(state.new_cfg_blocks)) # TODO: Right now we unconditionally generate a fallback block # in case of subtyping errors - This is probably unnecessary. - if i != length(cases) || (!fully_covered || (!params.trust_inference)) + if i != length(cases) || (!fully_covered) # This block will have the next condition or the final else case push!(state.new_cfg_blocks, BasicBlock(StmtRange(idx, idx))) push!(state.new_cfg_blocks[cond_bb].succs, length(state.new_cfg_blocks)) @@ -575,7 +575,7 @@ function ir_inline_unionsplit!(compact::IncrementalCompact, idx::Int, cond = true nparams = fieldcount(atype) @assert nparams == fieldcount(mtype) - if i != ncases || !fully_covered || !params.trust_inference + if i != ncases || !fully_covered for i = 1:nparams a, m = fieldtype(atype, i), fieldtype(mtype, i) # If this is always true, we don't need to check for it @@ -630,14 +630,7 @@ function ir_inline_unionsplit!(compact::IncrementalCompact, idx::Int, end bb += 1 # We're now in the fall through block, decide what to do - if fully_covered - if !params.trust_inference - e = Expr(:call, GlobalRef(Core, :throw), FATAL_TYPE_BOUND_ERROR) - insert_node_here!(compact, NewInstruction(e, Union{}, line)) - insert_node_here!(compact, NewInstruction(ReturnNode(), Union{}, line)) - finish_current_bb!(compact, 0) - end - else + if !fully_covered ssa = insert_node_here!(compact, NewInstruction(stmt, typ, line)) push!(pn.edges, bb) push!(pn.values, ssa) diff --git a/base/compiler/types.jl b/base/compiler/types.jl index 14f1c90dca0e9..c53256c61ace9 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -247,10 +247,6 @@ Parameters that control optimizer operation. generating `:invoke` expression based on the [`@nospecialize`](@ref) annotation, in order to avoid over-specialization. --- -- `opt_params.trust_inference::Bool = false`\\ - If `false`, the inliner will unconditionally generate a fallback block when union-splitting - a callsite, in case of existing subtyping bugs. This option may be removed in the future. ---- - `opt_params.assume_fatal_throw::Bool = false`\\ If `true`, gives the optimizer license to assume that any `throw` is fatal and thus the state after a `throw` is not externally observable. In particular, this gives the @@ -266,7 +262,6 @@ struct OptimizationParams inline_error_path_cost::Int max_tuple_splat::Int compilesig_invokes::Bool - trust_inference::Bool assume_fatal_throw::Bool function OptimizationParams( @@ -277,7 +272,6 @@ struct OptimizationParams inline_error_path_cost::Int, max_tuple_splat::Int, compilesig_invokes::Bool, - trust_inference::Bool, assume_fatal_throw::Bool) return new( inlining, @@ -287,7 +281,6 @@ struct OptimizationParams inline_error_path_cost, max_tuple_splat, compilesig_invokes, - trust_inference, assume_fatal_throw) end end @@ -300,7 +293,6 @@ function OptimizationParams( #=inline_error_path_cost::Int=# 20, #=max_tuple_splat::Int=# 32, #=compilesig_invokes::Bool=# true, - #=trust_inference::Bool=# false, #=assume_fatal_throw::Bool=# false); inlining::Bool = params.inlining, inline_cost_threshold::Int = params.inline_cost_threshold, @@ -309,7 +301,6 @@ function OptimizationParams( inline_error_path_cost::Int = params.inline_error_path_cost, max_tuple_splat::Int = params.max_tuple_splat, compilesig_invokes::Bool = params.compilesig_invokes, - trust_inference::Bool = params.trust_inference, assume_fatal_throw::Bool = params.assume_fatal_throw) return OptimizationParams( inlining, @@ -319,7 +310,6 @@ function OptimizationParams( inline_error_path_cost, max_tuple_splat, compilesig_invokes, - trust_inference, assume_fatal_throw) end From 2a406b243730a0a50aa99355786b3ac41340a386 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 6 Jul 2023 17:20:40 -0400 Subject: [PATCH 23/33] Add pattern matching for `typeof` into field type tparam (#50422) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add pattern matching for `typeof` into field type tparam This PR allows full elimination of the following, even in ill-typed code. ``` struct TParamTypeofTest{T} x::T @eval TParamTypeofTest(x) = $(Expr(:new, :(TParamTypeofTest{typeof(x)}), :x)) end function tparam_typeof_test_elim(x) TParamTypeofTest(x).x end ``` Before this PR, we would get: ``` julia> code_typed(tparam_typeof_test_elim, Tuple{Any}) 1-element Vector{Any}: CodeInfo( 1 ─ %1 = Main.typeof(x)::DataType │ %2 = Core.apply_type(Main.TParamTypeofTest, %1)::Type{TParamTypeofTest{_A}} where _A │ %new(%2, x)::TParamTypeofTest └── return x ``` Where the `new` is non-eliminable, because the compiler did not know that `x::_A`. Fix this by pattern matching this particular pattern (where the condition is guaranteed, because we computed `_A` by `typeof`). This is not particularly general, but this pattern comes up a lot, so it's surprisingly effective. * add test case for optimizing multiple abstract fields * improve robustness --------- Co-authored-by: Shuhei Kadowaki --- base/compiler/optimize.jl | 66 +++++++++++++++++++---------------- base/compiler/ssair/passes.jl | 58 ++++++++++++++++++++++++++++++ test/compiler/irpasses.jl | 16 +++++++++ 3 files changed, 110 insertions(+), 30 deletions(-) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 3a8de06811cc2..95feb7cb4ea31 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -215,6 +215,41 @@ is_stmt_inline(stmt_flag::UInt8) = stmt_flag & IR_FLAG_INLINE ≠ 0 is_stmt_noinline(stmt_flag::UInt8) = stmt_flag & IR_FLAG_NOINLINE ≠ 0 is_stmt_throw_block(stmt_flag::UInt8) = stmt_flag & IR_FLAG_THROW_BLOCK ≠ 0 +function new_expr_effect_flags(𝕃ₒ::AbstractLattice, args::Vector{Any}, src::Union{IRCode,IncrementalCompact}, pattern_match=nothing) + Targ = args[1] + atyp = argextype(Targ, src) + # `Expr(:new)` of unknown type could raise arbitrary TypeError. + typ, isexact = instanceof_tfunc(atyp) + if !isexact + atyp = unwrap_unionall(widenconst(atyp)) + if isType(atyp) && isTypeDataType(atyp.parameters[1]) + typ = atyp.parameters[1] + else + return (false, false, false) + end + isabstracttype(typ) && return (false, false, false) + else + isconcretedispatch(typ) || return (false, false, false) + end + typ = typ::DataType + fcount = datatype_fieldcount(typ) + fcount === nothing && return (false, false, false) + fcount >= length(args) - 1 || return (false, false, false) + for fidx in 1:(length(args) - 1) + farg = args[fidx + 1] + eT = argextype(farg, src) + fT = fieldtype(typ, fidx) + if !isexact && has_free_typevars(fT) + if pattern_match !== nothing && pattern_match(src, typ, fidx, Targ, farg) + continue + end + return (false, false, false) + end + ⊑(𝕃ₒ, eT, fT) || return (false, false, false) + end + return (false, true, true) +end + """ stmt_effect_flags(stmt, rt, src::Union{IRCode,IncrementalCompact}) -> (consistent::Bool, effect_free_and_nothrow::Bool, nothrow::Bool) @@ -264,36 +299,7 @@ function stmt_effect_flags(𝕃ₒ::AbstractLattice, @nospecialize(stmt), @nospe nothrow = is_nothrow(effects) return (consistent, effect_free & nothrow, nothrow) elseif head === :new - atyp = argextype(args[1], src) - # `Expr(:new)` of unknown type could raise arbitrary TypeError. - typ, isexact = instanceof_tfunc(atyp) - if !isexact - atyp = unwrap_unionall(widenconst(atyp)) - if isType(atyp) && isTypeDataType(atyp.parameters[1]) - typ = atyp.parameters[1] - else - return (false, false, false) - end - isabstracttype(typ) && return (false, false, false) - else - isconcretedispatch(typ) || return (false, false, false) - end - typ = typ::DataType - fcount = datatype_fieldcount(typ) - fcount === nothing && return (false, false, false) - fcount >= length(args) - 1 || return (false, false, false) - for fld_idx in 1:(length(args) - 1) - eT = argextype(args[fld_idx + 1], src) - fT = fieldtype(typ, fld_idx) - # Currently, we cannot represent any type equality constraints - # in the lattice, so if we see any type of type parameter, - # there is very little we can say about it - if !isexact && has_free_typevars(fT) - return (false, false, false) - end - ⊑(𝕃ₒ, eT, fT) || return (false, false, false) - end - return (false, true, true) + return new_expr_effect_flags(𝕃ₒ, args, src) elseif head === :foreigncall effects = foreigncall_effects(stmt) do @nospecialize x argextype(x, src) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 42e1f747974a3..cf3a2118743a0 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -908,6 +908,62 @@ end return nothing end +struct IsEgal <: Function + x::Any + IsEgal(@nospecialize(x)) = new(x) +end +(x::IsEgal)(@nospecialize(y)) = x.x === y + +# This tries to match patterns of the form +# %ft = typeof(%farg) +# %Targ = apply_type(Foo, ft) +# %x = new(%Targ, %farg) +# +# and if possible refines the nothrowness of the new expr based on it. +function pattern_match_typeof(compact::IncrementalCompact, typ::DataType, fidx::Int, + @nospecialize(Targ), @nospecialize(farg)) + isa(Targ, SSAValue) || return false + + Tdef = compact[Targ][:inst] + is_known_call(Tdef, Core.apply_type, compact) || return false + length(Tdef.args) ≥ 2 || return false + + applyT = argextype(Tdef.args[2], compact) + isa(applyT, Const) || return false + + applyT = applyT.val + tvars = Any[] + while isa(applyT, UnionAll) + applyTvar = applyT.var + applyT = applyT.body + push!(tvars, applyTvar) + end + + applyT.name === typ.name || return false + fT = fieldtype(applyT, fidx) + idx = findfirst(IsEgal(fT), tvars) + idx === nothing && return false + checkbounds(Bool, Tdef.args, 2+idx) || return false + valarg = Tdef.args[2+idx] + isa(valarg, SSAValue) || return false + valdef = compact[valarg][:inst] + is_known_call(valdef, typeof, compact) || return false + + return valdef.args[2] === farg +end + +function refine_new_effects!(𝕃ₒ::AbstractLattice, compact::IncrementalCompact, idx::Int, stmt::Expr) + (consistent, effect_free_and_nothrow, nothrow) = new_expr_effect_flags(𝕃ₒ, stmt.args, compact, pattern_match_typeof) + if consistent + compact[SSAValue(idx)][:flag] |= IR_FLAG_CONSISTENT + end + if effect_free_and_nothrow + compact[SSAValue(idx)][:flag] |= IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW + elseif nothrow + compact[SSAValue(idx)][:flag] |= IR_FLAG_NOTHROW + end +end + # NOTE we use `IdSet{Int}` instead of `BitSet` for in these passes since they work on IR after inlining, # which can be very large sometimes, and program counters in question are often very sparse const SPCSet = IdSet{Int} @@ -1037,6 +1093,8 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) lift_comparison!(===, compact, idx, stmt, lifting_cache, 𝕃ₒ) elseif is_known_call(stmt, isa, compact) lift_comparison!(isa, compact, idx, stmt, lifting_cache, 𝕃ₒ) + elseif isexpr(stmt, :new) && (compact[SSAValue(idx)][:flag] & IR_FLAG_NOTHROW) == 0x00 + refine_new_effects!(𝕃ₒ, compact, idx, stmt) end continue end diff --git a/test/compiler/irpasses.jl b/test/compiler/irpasses.jl index a1738b52161bf..68eb2e7137796 100644 --- a/test/compiler/irpasses.jl +++ b/test/compiler/irpasses.jl @@ -1355,3 +1355,19 @@ let src = code_typed1(mut50285, Tuple{Bool, Int, Float64}) @test count(isnew, src.code) == 0 @test count(iscall((src, typeassert)), src.code) == 0 end + +# Test that we can eliminate new{typeof(x)}(x) +struct TParamTypeofTest1{T} + x::T + @eval TParamTypeofTest1(x) = $(Expr(:new, :(TParamTypeofTest1{typeof(x)}), :x)) +end +tparam_typeof_test_elim1(x) = TParamTypeofTest1(x).x +@test fully_eliminated(tparam_typeof_test_elim1, Tuple{Any}) + +struct TParamTypeofTest2{S,T} + x::S + y::T + @eval TParamTypeofTest2(x, y) = $(Expr(:new, :(TParamTypeofTest2{typeof(x),typeof(y)}), :x, :y)) +end +tparam_typeof_test_elim2(x, y) = TParamTypeofTest2(x, y).x +@test fully_eliminated(tparam_typeof_test_elim2, Tuple{Any,Any}) From d9ad6d2e122ab4efcf8ddc773c1a3a421b64cb25 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Fri, 7 Jul 2023 10:23:35 +1200 Subject: [PATCH 24/33] docs: fix link in `AbstractString` docstring (#50436) --- base/strings/basic.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 2609edeaaaa18..8be33c4fb6406 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -16,9 +16,7 @@ about strings: * Each `AbstractChar` in a string is encoded by one or more code units * Only the index of the first code unit of an `AbstractChar` is a valid index * The encoding of an `AbstractChar` is independent of what precedes or follows it - * String encodings are [self-synchronizing] – i.e. `isvalid(s, i)` is O(1) - -[self-synchronizing]: https://en.wikipedia.org/wiki/Self-synchronizing_code + * String encodings are [self-synchronizing](https://en.wikipedia.org/wiki/Self-synchronizing_code) – i.e. `isvalid(s, i)` is O(1) Some string functions that extract code units, characters or substrings from strings error if you pass them out-of-bounds or invalid string indices. This From 4cce2a2603eb837e5dbf23429b4577725e3cde46 Mon Sep 17 00:00:00 2001 From: pchintalapudi <34727397+pchintalapudi@users.noreply.github.com> Date: Fri, 7 Jul 2023 13:34:55 +0000 Subject: [PATCH 25/33] Add some documentation about ahead of time compilation mechanics (#50376) --- doc/make.jl | 1 + doc/src/devdocs/aot.md | 76 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 doc/src/devdocs/aot.md diff --git a/doc/make.jl b/doc/make.jl index d35d893143eda..c3e7f2d690e1e 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -152,6 +152,7 @@ DevDocs = [ "devdocs/inference.md", "devdocs/ssair.md", "devdocs/EscapeAnalysis.md", + "devdocs/aot.md", "devdocs/gc-sa.md", "devdocs/gc.md", "devdocs/jit.md", diff --git a/doc/src/devdocs/aot.md b/doc/src/devdocs/aot.md new file mode 100644 index 0000000000000..75ce6092d49b3 --- /dev/null +++ b/doc/src/devdocs/aot.md @@ -0,0 +1,76 @@ +# Ahead of Time Compilation + +This document describes the design and structure of the ahead-of-time (AOT) compilation system in Julia. This system is used when generating system images and package images. Much of the implementation described here is located in `aotcompile.cpp`, `staticdata.c`, and `processor.cpp` + +## Introduction + +Though Julia normally compiles code just-in-time (JIT), it is possible to compile code ahead of time and save the resulting code to a file. This can be useful for a number of reasons: +1. To reduce the time it takes to start a Julia process. +2. To reduce the time spent in the JIT compiler instead of executing code (time to first execution, TTFX). +3. To reduce the amount of memory used by the JIT compiler. + +## High-Level Overview + +The following descriptions are a snapshot of the current implementation details of the end-to-end pipeline that happens internally when the user compiles a new AOT module, such as occurs when they type `using Foo`. These details are likely to change over time as we implement better ways to handle them, so current implementations may not exactly match the dataflow and functions described below. + +### Compiling Code Images + +Firstly, the methods that need to be compiled to native code must be identified. This can only be done by actually executing the code to be compiled, as the set of methods that need to be compiled depends on the types of the arguments passed to the methods, and method invocations with certain combinations of types may not be known until runtime. During this process, the exact methods that the compiler sees are tracked for later compilation, producing a compilation trace. + +!!! note + + Currently when compiling images, Julia runs the trace generation in a different process than the process performing the AOT compilation. This can have impacts when attempting to use a debugger during precompilation. The best way to debug precompilation with a debugger is to use the rr debugger, record the entire process tree, use `rr ps` to identify the relevant failing process, and then use `rr replay -p PID` to replay just the failing process. + +Once the methods to be compiled have been identified, they are passed to the `jl_create_system_image` function. This function sets up a number of data structures that will be used when serializing native code to a file, and then calls `jl_create_native` with the array of methods. `jl_create_native` runs codegen on the methods produces one or more LLVM modules. `jl_create_system_image` then records some useful information about what codegen produced from the module(s). + +The module(s) are then passed to `jl_dump_native`, along with the information recorded by `jl_create_system_image`. `jl_dump_native` contains the code necessary to serialize the module(s) to bitcode, object, or assembly files depending on the command-line options passed to Julia. The serialized code and information is then written to a file as an archive. + +The final step is to run a system linker on the object files in the archive produced by `jl_dump_native`. Once this step is complete, a shared library containing the compiled code is produced. + +### Loading Code Images + +When loading a code image, the shared library produced by the linker is loaded into memory. The system image data is then loaded from the shared library. This data contains information about the types, methods, and code instances that were compiled into the shared library. This data is used to restore the state of the runtime to what it was when the code image was compiled. + +If the code image was compiled with multiversioning, the loader will pick the appropriate version of each function to use based on the CPU features available on the current machine. + +For system images, since no other code has been loaded, the state of the runtime is now the same as it was when the code image was compiled. For package images, the environment may have changed compared to when the code was compiled, so each method must be checked against the global method table to determine if it is still valid code. + +## Compiling Methods + +### Tracing Compiled Methods + +Julia has a command-line flag to record all of the methods that are compiled by the JIT compiler, `--trace-compile=filename`. When a function is compiled and this flag has a filename, Julia will print out a precompile statement to that file with the method and argument types it was called with. This therefore generates a precompile script that can be used later in the AOT compilation process. The [PrecompileTools](https://julialang.github.io/PrecompileTools.jl/stable/) package has tooling that can make taking advantage of this functionality easier for package developers. + +### `jl_create_system_image` + +`jl_create_system_image` saves all of the Julia-specific metadata necessary to later restore the state of the runtime. This includes data such as code instances, method instances, method tables, and type information. This function also sets up the data structures necessary to serialize the native code to a file. Finally, it calls `jl_create_native` to create one or more LLVM modules containing the native code for the methods passed to it. `jl_create_native` is responsible for running codegen on the methods passed to it. + +### `jl_dump_native` + +`jl_dump_native` is responsible for serializing the LLVM module containing the native code to a file. In addition to the module, the system image data produced by `jl_create_system_image` is compiled as a global variable. The output of this method is bitcode, object, and/or assembly archives containing the code and system image data. + +`jl_dump_native` is typically one of the larger time sinks when emitting native code, with much of the time spent in optimizing LLVM IR and emitting machine code. Therefore, this function is capable of multithreading the optimization and machine code emission steps. This multithreading is parameterized on the size of the module, but can be explicitly overriden by setting the `JULIA_IMAGE_THREADS` environment variable. The default maximum number of threads is half the number of available threads, but setting it to be lower can reduce peak memory usage during compilation. + +`jl_dump_native` can also produce native code optimized for multiple architectures, when integrated with the Julia loader. This is triggered by setting the `JULIA_CPU_TARGET` environment variable and mediated by the multiversioning pass in the optimization pipeline. To make this work with multithreading, an annotation step is added before the module is split into submodules that are emitted on their own threads, and this annotation step uses information available throughout the entire module to decide what functions are cloned for different architectures. Once the annotation has happened, individual threads can emit code for different architectures in parallel, knowing that a different submodule is guaranteed to produce the necessary functions that will be called by a cloned function. + +Some other metadata about how the module was serialized is also stored in the archive, such as the number of threads used to serialize the module and the number of functions that were compiled. + +### Static Linking + +The final step in the AOT compilation process is to run a linker on the object files in the archive produced by `jl_dump_native`. This produces a shared library containing the compiled code. This shared library can then be loaded by Julia to restore the state of the runtime. When compiling a system image, the native linker used by a C compiler is used to produce the final shared library. For package images, the LLVM linker LLD is used to provide a more consistent linking interface. + +## Loading Code Images + +### Loading the Shared Library + +The first step in loading a code image is to load the shared library produced by the linker. This is done by calling `jl_dlopen` on the path to the shared library. This function is responsible for loading the shared library and resolving all of the symbols in the library. + +### Loading Native Code + +The loader first needs to identify whether the native code that was compiled is valid for the architecture that the loader is running on. This is necessary to avoid executing instructions that older CPUs do not recognize. This is done by checking the CPU features available on the current machine against the CPU features that the code was compiled for. When multiversioning is enabled, the loader will pick the appropriate version of each function to use based on the CPU features available on the current machine. If none of the feature sets that were multiversioned, the loader will throw an error. + +Part of the multiversioning pass creates a number of global arrays of all of the functions in the module. When this process is multithreaded, an array of arrays is created, which the loader reorganizes into one large array with all of the functions that were compiled for this architecture. A similar process occurs for the global variables in the module. + +### Setting Up Julia State + +The loader then uses the global variables and functions produced from loading native code to set up Julia runtime core data structures in the current process. This setup involves adding types and methods to the Julia runtime, and making the cached native code available for use by other Julia functions and the interpreter. For package images, each method must be validated, in that the global method table's state must match the state that the package image was compiled for. In particular, if a different set of methods exists at the load time compared to compile time of the package image, the method must be invalidated and recompiled on first use. This is necessary to ensure that execution semantics remain the same regardless of if a package was precompiled or if the code was directly executed. System images do not need to perform this validation, since the global method table is empty at load time. Thus, system images have faster load times than package images. From 0718995f97c0d3df7f3f488523dd305c942efcf7 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Fri, 7 Jul 2023 11:55:56 -0300 Subject: [PATCH 26/33] Optimize getfield lowering to avoid boxing in some cases (#50444) --- src/codegen.cpp | 39 +++++++++++++++++++++++++++++++++++++++ src/datatype.c | 3 +-- src/jl_exported_funcs.inc | 1 + src/julia.h | 1 + src/rtutils.c | 5 +++++ test/compiler/codegen.jl | 12 ++++++++++++ 6 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 122170ae3fa97..04f7564dd3e33 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -739,6 +739,13 @@ static const auto jlundefvarerror_func = new JuliaFunction<>{ {PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted)}, false); }, get_attrs_noreturn, }; +static const auto jlhasnofield_func = new JuliaFunction<>{ + XSTR(jl_has_no_field_error), + [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), + {PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted), + PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted)}, false); }, + get_attrs_noreturn, +}; static const auto jlboundserrorv_func = new JuliaFunction{ XSTR(jl_bounds_error_ints), [](LLVMContext &C, Type *T_size) { return FunctionType::get(getVoidTy(C), @@ -3318,6 +3325,8 @@ static jl_llvm_functions_t jl_value_t *jlrettype, jl_codegen_params_t ¶ms); +static void emit_hasnofield_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *type, jl_cgval_t name); + static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_cgval_t *argv, size_t nargs, jl_value_t *rt, jl_expr_t *ex, bool is_promotable) @@ -3819,6 +3828,19 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } } + else if (fld.typ == (jl_value_t*)jl_symbol_type) { + if (jl_is_datatype(utt) && !jl_is_namedtuple_type(utt)) { // TODO: Look into this for NamedTuple + if (jl_struct_try_layout(utt) && (jl_datatype_nfields(utt) == 1)) { + jl_svec_t *fn = jl_field_names(utt); + assert(jl_svec_len(fn) == 1); + Value *typ_sym = literal_pointer_val(ctx, jl_svecref(fn, 0)); + Value *cond = ctx.builder.CreateICmpEQ(mark_callee_rooted(ctx, typ_sym), mark_callee_rooted(ctx, boxed(ctx, fld))); + emit_hasnofield_error_ifnot(ctx, cond, utt->name->name, fld); + *ret = emit_getfield_knownidx(ctx, obj, 0, utt, order); + return true; + } + } + } // TODO: generic getfield func with more efficient calling convention return false; } @@ -4612,6 +4634,22 @@ static void undef_var_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *name) ctx.builder.SetInsertPoint(ifok); } +static void emit_hasnofield_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *type, jl_cgval_t name) +{ + ++EmittedUndefVarErrors; + assert(name.typ == (jl_value_t*)jl_symbol_type); + BasicBlock *err = BasicBlock::Create(ctx.builder.getContext(), "err", ctx.f); + BasicBlock *ifok = BasicBlock::Create(ctx.builder.getContext(), "ok"); + ctx.builder.CreateCondBr(ok, ifok, err); + ctx.builder.SetInsertPoint(err); + ctx.builder.CreateCall(prepare_call(jlhasnofield_func), + {mark_callee_rooted(ctx, literal_pointer_val(ctx, (jl_value_t*)type)), + mark_callee_rooted(ctx, boxed(ctx, name))}); + ctx.builder.CreateUnreachable(); + ctx.f->getBasicBlockList().push_back(ifok); + ctx.builder.SetInsertPoint(ifok); +} + // returns a jl_ppvalue_t location for the global variable m.s // if the reference currently bound or assign == true, // pbnd will also be assigned with the binding address @@ -9011,6 +9049,7 @@ static void init_jit_functions(void) add_named_global(jlatomicerror_func, &jl_atomic_error); add_named_global(jlthrow_func, &jl_throw); add_named_global(jlundefvarerror_func, &jl_undefined_var_error); + add_named_global(jlhasnofield_func, &jl_has_no_field_error); add_named_global(jlboundserrorv_func, &jl_bounds_error_ints); add_named_global(jlboundserror_func, &jl_bounds_error_int); add_named_global(jlvboundserror_func, &jl_bounds_error_tuple_int); diff --git a/src/datatype.c b/src/datatype.c index 95c3b11c9abdc..905959fb80e0a 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -1558,8 +1558,7 @@ JL_DLLEXPORT int jl_field_index(jl_datatype_t *t, jl_sym_t *fld, int err) } } if (err) - jl_errorf("type %s has no field %s", jl_symbol_name(t->name->name), - jl_symbol_name(fld)); + jl_has_no_field_error(t->name->name, fld); return -1; } diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 6875e36d6da6a..c2b2a1578fd76 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -510,6 +510,7 @@ XX(jl_uncompress_argname_n) \ XX(jl_uncompress_ir) \ XX(jl_undefined_var_error) \ + XX(jl_has_no_field_error) \ XX(jl_value_ptr) \ XX(jl_ver_is_release) \ XX(jl_ver_major) \ diff --git a/src/julia.h b/src/julia.h index d2eb9a98a4a42..5af8a5bc1a170 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1790,6 +1790,7 @@ JL_DLLEXPORT void JL_NORETURN jl_type_error_rt(const char *fname, jl_value_t *ty JL_MAYBE_UNROOTED, jl_value_t *got JL_MAYBE_UNROOTED); JL_DLLEXPORT void JL_NORETURN jl_undefined_var_error(jl_sym_t *var); +JL_DLLEXPORT void JL_NORETURN jl_has_no_field_error(jl_sym_t *type_name, jl_sym_t *var); JL_DLLEXPORT void JL_NORETURN jl_atomic_error(char *str); JL_DLLEXPORT void JL_NORETURN jl_bounds_error(jl_value_t *v JL_MAYBE_UNROOTED, jl_value_t *t JL_MAYBE_UNROOTED); diff --git a/src/rtutils.c b/src/rtutils.c index 01ea11014a6db..eefd1b25f9bc4 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -134,6 +134,11 @@ JL_DLLEXPORT void JL_NORETURN jl_undefined_var_error(jl_sym_t *var) jl_throw(jl_new_struct(jl_undefvarerror_type, var)); } +JL_DLLEXPORT void JL_NORETURN jl_has_no_field_error(jl_sym_t *type_name, jl_sym_t *var) +{ + jl_errorf("type %s has no field %s", jl_symbol_name(type_name), jl_symbol_name(var)); +} + JL_DLLEXPORT void JL_NORETURN jl_atomic_error(char *str) // == jl_exceptionf(jl_atomicerror_type, "%s", str) { jl_value_t *msg = jl_pchar_to_string((char*)str, strlen(str)); diff --git a/test/compiler/codegen.jl b/test/compiler/codegen.jl index e93ecd232498f..85013ce30d2ca 100644 --- a/test/compiler/codegen.jl +++ b/test/compiler/codegen.jl @@ -820,3 +820,15 @@ end # issue 48917, hoisting load to above the parent f48917(x, w) = (y = (a=1, b=x); z = (; a=(a=(1, w), b=(3, y)))) @test f48917(1,2) == (a = (a = (1, 2), b = (3, (a = 1, b = 1))),) + +# https://github.com/JuliaLang/julia/issues/50317 getproperty allocation on struct with 1 field +struct Wrapper50317 + lock::ReentrantLock +end +const MONITOR50317 = Wrapper50317(ReentrantLock()) +issue50317() = @noinline MONITOR50317.lock +issue50317() +let res = @timed issue50317() + @test res.bytes == 0 + return res # must return otherwise the compiler may eliminate the result entirely +end From 21bb0c78a5496e96f80555d6b6f42d9e6057318d Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 7 Jul 2023 11:43:52 -0400 Subject: [PATCH 27/33] Remove union penalties for inlining cost (#50429) I added this code back in #27057, when I first made Union-full signatures inlineable. The justification was to try to encourage the union splitting to happen on the outside. However (and I believe this changed since this code was introduced), these days inference is in complete control of union splitting and we do not take inlineability or non-inlineability of the non-unionsplit function into account when deciding how to inline. As a result, the only effect of the union split penalties was to prevent inlining of functions that are not union-split eligible (e.g. `+(::Vararg{Union{Int, Missing}, 3})`), but are nevertheless cheap by our inlining metric. There is really no reason not to try to inline such functions, so delete this logic. --- base/compiler/optimize.jl | 36 ++++++++++-------------------------- base/reflection.jl | 2 +- test/offsetarray.jl | 8 ++++---- 3 files changed, 15 insertions(+), 31 deletions(-) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 95feb7cb4ea31..dd50f8c9d47e1 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -406,18 +406,9 @@ function finish(interp::AbstractInterpreter, opt::OptimizationState, opt.ir = ir # determine and cache inlineability - union_penalties = false if !force_noinline sig = unwrap_unionall(specTypes) - if isa(sig, DataType) && sig.name === Tuple.name - for P in sig.parameters - P = unwrap_unionall(P) - if isa(P, Union) - union_penalties = true - break - end - end - else + if !(isa(sig, DataType) && sig.name === Tuple.name) force_noinline = true end if !is_declared_inline(src) && result === Bottom @@ -448,7 +439,7 @@ function finish(interp::AbstractInterpreter, opt::OptimizationState, cost_threshold += 4*default end end - src.inlining_cost = inline_cost(ir, params, union_penalties, cost_threshold) + src.inlining_cost = inline_cost(ir, params, cost_threshold) end end return nothing @@ -645,7 +636,7 @@ plus_saturate(x::Int, y::Int) = max(x, y, x+y) isknowntype(@nospecialize T) = (T === Union{}) || isa(T, Const) || isconcretetype(widenconst(T)) function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptypes::Vector{VarState}, - union_penalties::Bool, params::OptimizationParams, error_path::Bool = false) + params::OptimizationParams, error_path::Bool = false) head = ex.head if is_meta_expr_head(head) return 0 @@ -683,13 +674,6 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp return isknowntype(atyp) ? 4 : error_path ? params.inline_error_path_cost : params.inline_nonleaf_penalty elseif f === typeassert && isconstType(widenconst(argextype(ex.args[3], src, sptypes))) return 1 - elseif f === Core.isa - # If we're in a union context, we penalize type computations - # on union types. In such cases, it is usually better to perform - # union splitting on the outside. - if union_penalties && isa(argextype(ex.args[2], src, sptypes), Union) - return params.inline_nonleaf_penalty - end end fidx = find_tfunc(f) if fidx === nothing @@ -720,7 +704,7 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp end a = ex.args[2] if a isa Expr - cost = plus_saturate(cost, statement_cost(a, -1, src, sptypes, union_penalties, params, error_path)) + cost = plus_saturate(cost, statement_cost(a, -1, src, sptypes, params, error_path)) end return cost elseif head === :copyast @@ -736,11 +720,11 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp end function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{CodeInfo, IRCode}, sptypes::Vector{VarState}, - union_penalties::Bool, params::OptimizationParams) + params::OptimizationParams) thiscost = 0 dst(tgt) = isa(src, IRCode) ? first(src.cfg.blocks[tgt].stmts) : tgt if stmt isa Expr - thiscost = statement_cost(stmt, line, src, sptypes, union_penalties, params, + thiscost = statement_cost(stmt, line, src, sptypes, params, is_stmt_throw_block(isa(src, IRCode) ? src.stmts.flag[line] : src.ssaflags[line]))::Int elseif stmt isa GotoNode # loops are generally always expensive @@ -753,24 +737,24 @@ function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{Cod return thiscost end -function inline_cost(ir::IRCode, params::OptimizationParams, union_penalties::Bool=false, +function inline_cost(ir::IRCode, params::OptimizationParams, cost_threshold::Integer=params.inline_cost_threshold)::InlineCostType bodycost::Int = 0 for line = 1:length(ir.stmts) stmt = ir.stmts[line][:inst] - thiscost = statement_or_branch_cost(stmt, line, ir, ir.sptypes, union_penalties, params) + thiscost = statement_or_branch_cost(stmt, line, ir, ir.sptypes, params) bodycost = plus_saturate(bodycost, thiscost) bodycost > cost_threshold && return MAX_INLINE_COST end return inline_cost_clamp(bodycost) end -function statement_costs!(cost::Vector{Int}, body::Vector{Any}, src::Union{CodeInfo, IRCode}, sptypes::Vector{VarState}, unionpenalties::Bool, params::OptimizationParams) +function statement_costs!(cost::Vector{Int}, body::Vector{Any}, src::Union{CodeInfo, IRCode}, sptypes::Vector{VarState}, params::OptimizationParams) maxcost = 0 for line = 1:length(body) stmt = body[line] thiscost = statement_or_branch_cost(stmt, line, src, sptypes, - unionpenalties, params) + params) cost[line] = thiscost if thiscost > maxcost maxcost = thiscost diff --git a/base/reflection.jl b/base/reflection.jl index bbcd6cad27128..05ffb3a6e9211 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1669,7 +1669,7 @@ function print_statement_costs(io::IO, @nospecialize(tt::Type); empty!(cst) resize!(cst, length(code.code)) sptypes = Core.Compiler.VarState[Core.Compiler.VarState(sp, false) for sp in match.sparams] - maxcost = Core.Compiler.statement_costs!(cst, code.code, code, sptypes, false, params) + maxcost = Core.Compiler.statement_costs!(cst, code.code, code, sptypes, params) nd = ndigits(maxcost) irshow_config = IRShow.IRShowConfig() do io, linestart, idx print(io, idx > 0 ? lpad(cst[idx], nd+1) : " "^(nd+1), " ") diff --git a/test/offsetarray.jl b/test/offsetarray.jl index c447c6d420f2a..257e91db5f49e 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -627,15 +627,15 @@ end B = OffsetArray(reshape(1:24, 4, 3, 2), -5, 6, -7) for R in (fill(0, -4:-1), fill(0, -4:-1, 7:7), fill(0, -4:-1, 7:7, -6:-6)) @test @inferred(maximum!(R, B)) == reshape(maximum(B, dims=(2,3)), axes(R)) == reshape(21:24, axes(R)) - @test @allocated(maximum!(R, B)) <= 800 + @test @allocated(maximum!(R, B)) <= 1300 @test @inferred(minimum!(R, B)) == reshape(minimum(B, dims=(2,3)), axes(R)) == reshape(1:4, axes(R)) - @test @allocated(minimum!(R, B)) <= 800 + @test @allocated(minimum!(R, B)) <= 1300 end for R in (fill(0, -4:-4, 7:9), fill(0, -4:-4, 7:9, -6:-6)) @test @inferred(maximum!(R, B)) == reshape(maximum(B, dims=(1,3)), axes(R)) == reshape(16:4:24, axes(R)) - @test @allocated(maximum!(R, B)) <= 800 + @test @allocated(maximum!(R, B)) <= 1300 @test @inferred(minimum!(R, B)) == reshape(minimum(B, dims=(1,3)), axes(R)) == reshape(1:4:9, axes(R)) - @test @allocated(minimum!(R, B)) <= 800 + @test @allocated(minimum!(R, B)) <= 1300 end @test_throws DimensionMismatch maximum!(fill(0, -4:-1, 7:7, -6:-6, 1:1), B) @test_throws DimensionMismatch minimum!(fill(0, -4:-1, 7:7, -6:-6, 1:1), B) From 930838bd912538283654647a0e43d255b2a3df44 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Fri, 7 Jul 2023 19:59:39 +0200 Subject: [PATCH 28/33] Add exports to `Example` module for the test workflow in the docs (#50459) The tests are not supposed to work if the toy module `Example` does not export its functions See https://discourse.julialang.org/t/workflow-for-testing-packages/101305 --- stdlib/Test/docs/src/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/Test/docs/src/index.md b/stdlib/Test/docs/src/index.md index 1c9a55480d2c9..f1142409747bd 100644 --- a/stdlib/Test/docs/src/index.md +++ b/stdlib/Test/docs/src/index.md @@ -380,6 +380,8 @@ function type_multiply(a::Float64, b::Float64) a * b end +export greet, simple_add, type_multiply + end ``` From b99f251e86c7c09b957a1b362b6408dbba106ff0 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Fri, 7 Jul 2023 15:11:29 -0400 Subject: [PATCH 29/33] Merge new `reinterpret` with essentials.jl `reinterpret` (#50367) --- base/essentials.jl | 32 ++++++++++++++++++++++++++------ base/reinterpretarray.jl | 36 ++---------------------------------- test/core.jl | 2 +- test/numbers.jl | 4 ++-- 4 files changed, 31 insertions(+), 43 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index 97f32483a6b14..7b70c0dff074d 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -543,21 +543,41 @@ unsafe_convert(::Type{T}, x::T) where {T<:Ptr} = x # to resolve ambiguity with unsafe_convert(::Type{P}, x::Ptr) where {P<:Ptr} = convert(P, x) """ - reinterpret(type, x) + reinterpret(::Type{Out}, x::In) -Change the type-interpretation of the binary data in the primitive value `x` -to that of the primitive type `type`. -The size of `type` has to be the same as that of the type of `x`. +Change the type-interpretation of the binary data in the isbits value `x` +to that of the isbits type `Out`. +The size (ignoring padding) of `Out` has to be the same as that of the type of `x`. For example, `reinterpret(Float32, UInt32(7))` interprets the 4 bytes corresponding to `UInt32(7)` as a [`Float32`](@ref). -# Examples ```jldoctest julia> reinterpret(Float32, UInt32(7)) 1.0f-44 + +julia> reinterpret(NTuple{2, UInt8}, 0x1234) +(0x34, 0x12) + +julia> reinterpret(UInt16, (0x34, 0x12)) +0x1234 + +julia> reinterpret(Tuple{UInt16, UInt8}, (0x01, 0x0203)) +(0x0301, 0x02) ``` + +!!! warning + + Use caution if some combinations of bits in `Out` are not considered valid and would + otherwise be prevented by the type's constructors and methods. Unexpected behavior + may result without additional validation. """ -reinterpret(::Type{T}, x) where {T} = bitcast(T, x) +function reinterpret(Out::Type, x::In) where {In} + if isprimitivetype(Out) && isprimitivetype(In) + return bitcast(Out, x) + end + # only available when Base is fully loaded. + return _reinterpret(Out, x) +end """ sizeof(T::DataType) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index d33c127b78c76..74b888a39fd76 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -795,42 +795,10 @@ function _copyfrompacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} end end -""" - reinterpret(::Type{Out}, x::In) - -Reinterpret the valid non-padding bytes of an isbits value `x` as isbits type `Out`. - -Both types must have the same amount of non-padding bytes. This operation is guaranteed -to be reversible. - -```jldoctest -julia> reinterpret(NTuple{2, UInt8}, 0x1234) -(0x34, 0x12) - -julia> reinterpret(UInt16, (0x34, 0x12)) -0x1234 - -julia> reinterpret(Tuple{UInt16, UInt8}, (0x01, 0x0203)) -(0x0301, 0x02) -``` - -!!! warning - - Use caution if some combinations of bits in `Out` are not considered valid and would - otherwise be prevented by the type's constructors and methods. Unexpected behavior - may result without additional validation. -""" -@inline function reinterpret(::Type{Out}, x::In) where {Out, In} +@inline function _reinterpret(::Type{Out}, x::In) where {Out, In} + # handle non-primitive types isbitstype(Out) || throw(ArgumentError("Target type for `reinterpret` must be isbits")) isbitstype(In) || throw(ArgumentError("Source type for `reinterpret` must be isbits")) - if isprimitivetype(Out) && isprimitivetype(In) - outsize = sizeof(Out) - insize = sizeof(In) - outsize == insize || - throw(ArgumentError("Sizes of types $Out and $In do not match; got $outsize \ - and $insize, respectively.")) - return bitcast(Out, x) - end inpackedsize = packedsize(In) outpackedsize = packedsize(Out) inpackedsize == outpackedsize || diff --git a/test/core.jl b/test/core.jl index f0439afeed23c..a87c45b698e49 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1897,7 +1897,7 @@ function f4528(A, B) end end @test f4528(false, Int32(12)) === nothing -@test_throws ArgumentError f4528(true, Int32(12)) +@test_throws ErrorException f4528(true, Int32(12)) # issue #4518 f4518(x, y::Union{Int32,Int64}) = 0 diff --git a/test/numbers.jl b/test/numbers.jl index d7fd6531b157d..e89dffd8e33cf 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2216,11 +2216,11 @@ end @test round(Int16, -32768.1) === Int16(-32768) end # issue #7508 -@test_throws ArgumentError reinterpret(Int, 0x01) +@test_throws ErrorException reinterpret(Int, 0x01) @testset "issue #12832" begin @test_throws ArgumentError reinterpret(Float64, Complex{Int64}(1)) - @test_throws ArgumentError reinterpret(Int32, false) + @test_throws ErrorException reinterpret(Int32, false) end # issue #41 ndigf(n) = Float64(log(Float32(n))) From a660798e47ed38c1f8039b0d7af3ff5a451f53e8 Mon Sep 17 00:00:00 2001 From: Jeremie Knuesel Date: Sat, 8 Jul 2023 00:14:40 +0200 Subject: [PATCH 30/33] Improve documentation of sort-related functions (#48387) * document the `order` keyword in `sort!` * list explicitly the required properties of `lt` in `sort!` * clarify the sequence of "by" transformations if both `by` and `order` are given * show default values in the signatures for `searchsorted` and related functions * note that `by` is also applied to searched value in `searchsorted` and related * add `isunordered` to the manual (it's already exported) --------- Co-authored-by: Lilith Orion Hafner --- base/operators.jl | 12 +-- base/ordering.jl | 8 +- base/sort.jl | 209 +++++++++++++++++++++++++++----------- doc/src/base/base.md | 1 + doc/src/base/sort.md | 2 +- doc/src/manual/missing.md | 2 +- 6 files changed, 161 insertions(+), 73 deletions(-) diff --git a/base/operators.jl b/base/operators.jl index 3f51be737ca5c..3f0f8bc49b164 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -143,7 +143,7 @@ isequal(x::AbstractFloat, y::Real ) = (isnan(x) & isnan(y)) | signequal( isless(x, y) Test whether `x` is less than `y`, according to a fixed total order (defined together with -[`isequal`](@ref)). `isless` is not defined on all pairs of values `(x, y)`. However, if it +[`isequal`](@ref)). `isless` is not defined for pairs `(x, y)` of all types. However, if it is defined, it is expected to satisfy the following: - If `isless(x, y)` is defined, then so is `isless(y, x)` and `isequal(x, y)`, and exactly one of those three yields `true`. @@ -154,13 +154,13 @@ Values that are normally unordered, such as `NaN`, are ordered after regular values. [`missing`](@ref) values are ordered last. -This is the default comparison used by [`sort`](@ref). +This is the default comparison used by [`sort!`](@ref). # Implementation Non-numeric types with a total order should implement this function. Numeric types only need to implement it if they have special values such as `NaN`. Types with a partial order should implement [`<`](@ref). -See the documentation on [Alternate orderings](@ref) for how to define alternate +See the documentation on [Alternate Orderings](@ref) for how to define alternate ordering methods that can be used in sorting and related functions. # Examples @@ -335,6 +335,8 @@ New types with a canonical partial order should implement this function for two arguments of the new type. Types with a canonical total order should implement [`isless`](@ref) instead. +See also [`isunordered`](@ref). + # Examples ```jldoctest julia> 'a' < 'b' @@ -1352,7 +1354,7 @@ corresponding position in `collection`. To get a vector indicating whether each in `items` is in `collection`, wrap `collection` in a tuple or a `Ref` like this: `in.(items, Ref(collection))` or `items .∈ Ref(collection)`. -See also: [`∉`](@ref). +See also: [`∉`](@ref), [`insorted`](@ref), [`contains`](@ref), [`occursin`](@ref), [`issubset`](@ref). # Examples ```jldoctest @@ -1390,8 +1392,6 @@ julia> [1, 2] .∈ ([2, 3],) 0 1 ``` - -See also: [`insorted`](@ref), [`contains`](@ref), [`occursin`](@ref), [`issubset`](@ref). """ in diff --git a/base/ordering.jl b/base/ordering.jl index d0c9cb99f9c72..5383745b1dd1f 100644 --- a/base/ordering.jl +++ b/base/ordering.jl @@ -87,8 +87,8 @@ By(by) = By(by, Forward) """ Lt(lt) -`Ordering` which calls `lt(a, b)` to compare elements. `lt` should -obey the same rules as implementations of [`isless`](@ref). +`Ordering` that calls `lt(a, b)` to compare elements. `lt` must +obey the same rules as the `lt` parameter of [`sort!`](@ref). """ struct Lt{T} <: Ordering lt::T @@ -146,8 +146,8 @@ Construct an [`Ordering`](@ref) object from the same arguments used by Elements are first transformed by the function `by` (which may be [`identity`](@ref)) and are then compared according to either the function `lt` or an existing ordering `order`. `lt` should be [`isless`](@ref) or a function -which obeys similar rules. Finally, the resulting order is reversed if -`rev=true`. +that obeys the same rules as the `lt` parameter of [`sort!`](@ref). Finally, +the resulting order is reversed if `rev=true`. Passing an `lt` other than `isless` along with an `order` other than [`Base.Order.Forward`](@ref) or [`Base.Order.Reverse`](@ref) is not permitted, diff --git a/base/sort.jl b/base/sort.jl index 90f8755d3b1a4..786d8e110e6e2 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -65,8 +65,8 @@ end """ issorted(v, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) -Test whether a vector is in sorted order. The `lt`, `by` and `rev` keywords modify what -order is considered to be sorted just as they do for [`sort`](@ref). +Test whether a collection is in sorted order. The keywords modify what +order is considered sorted, as described in the [`sort!`](@ref) documentation. # Examples ```jldoctest @@ -81,6 +81,9 @@ false julia> issorted([(1, "b"), (2, "a")], by = x -> x[2], rev=true) true + +julia> issorted([1, 2, -2, 3], by=abs) +true ``` """ issorted(itr; @@ -96,14 +99,17 @@ maybeview(v, k) = view(v, k) maybeview(v, k::Integer) = v[k] """ - partialsort!(v, k; by=, lt=, rev=false) + partialsort!(v, k; by=identity, lt=isless, rev=false) -Partially sort the vector `v` in place, according to the order specified by `by`, `lt` and -`rev` so that the value at index `k` (or range of adjacent values if `k` is a range) occurs +Partially sort the vector `v` in place so that the value at index `k` (or +range of adjacent values if `k` is a range) occurs at the position where it would appear if the array were fully sorted. If `k` is a single index, that value is returned; if `k` is a range, an array of values at those indices is returned. Note that `partialsort!` may not fully sort the input array. +For the keyword arguments, see the documentation of [`sort!`](@ref). + + # Examples ```jldoctest julia> a = [1, 2, 4, 3, 4] @@ -150,9 +156,9 @@ partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}; partialsort!(v, k, ord(lt,by,rev,order)) """ - partialsort(v, k, by=, lt=, rev=false) + partialsort(v, k, by=identity, lt=isless, rev=false) -Variant of [`partialsort!`](@ref) which copies `v` before partially sorting it, thereby returning the +Variant of [`partialsort!`](@ref) that copies `v` before partially sorting it, thereby returning the same thing as `partialsort!` but leaving `v` unmodified. """ partialsort(v::AbstractVector, k::Union{Integer,OrdinalRange}; kws...) = @@ -161,7 +167,7 @@ partialsort(v::AbstractVector, k::Union{Integer,OrdinalRange}; kws...) = # reference on sorted binary search: # http://www.tbray.org/ongoing/When/200x/2003/03/22/Binary -# index of the first value of vector a that is greater than or equal to x; +# index of the first value of vector a that is greater than or equivalent to x; # returns lastindex(v)+1 if x is greater than all values in v. function searchsortedfirst(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer hi = hi + T(1) @@ -180,7 +186,7 @@ function searchsortedfirst(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::key return lo end -# index of the last value of vector a that is less than or equal to x; +# index of the last value of vector a that is less than or equivalent to x; # returns firstindex(v)-1 if x is less than all values of v. function searchsortedlast(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer u = T(1) @@ -197,7 +203,7 @@ function searchsortedlast(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keyt return lo end -# returns the range of indices of v equal to x +# returns the range of indices of v equivalent to x # if v does not contain x, returns a 0-length range # indicating the insertion point of x function searchsorted(v::AbstractVector, x, ilo::T, ihi::T, o::Ordering)::UnitRange{keytype(v)} where T<:Integer @@ -290,16 +296,19 @@ for s in [:searchsortedfirst, :searchsortedlast, :searchsorted] end """ - searchsorted(a, x; by=, lt=, rev=false) + searchsorted(v, x; by=identity, lt=isless, rev=false) -Return the range of indices of `a` which compare as equal to `x` (using binary search) -according to the order specified by the `by`, `lt` and `rev` keywords, assuming that `a` -is already sorted in that order. Return an empty range located at the insertion point -if `a` does not contain values equal to `x`. +Return the range of indices in `v` where values are equivalent to `x`, or an +empty range located at the insertion point if `v` does not contain values +equivalent to `x`. The vector `v` must be sorted according to the order defined +by the keywords. Refer to [`sort!`](@ref) for the meaning of the keywords and +the definition of equivalence. Note that the `by` function is applied to the +searched value `x` as well as the values in `v`. -See [`sort!`](@ref) for an explanation of the keyword arguments `by`, `lt` and `rev`. +The range is generally found using binary search, but there are optimized +implementations for some inputs. -See also: [`insorted`](@ref), [`searchsortedfirst`](@ref), [`sort`](@ref), [`findall`](@ref). +See also: [`searchsortedfirst`](@ref), [`sort!`](@ref), [`insorted`](@ref), [`findall`](@ref). # Examples ```jldoctest @@ -324,15 +333,19 @@ julia> searchsorted([1=>"one", 2=>"two", 2=>"two", 4=>"four"], 2=>"two", by=firs """ searchsorted """ - searchsortedfirst(a, x; by=, lt=, rev=false) + searchsortedfirst(v, x; by=identity, lt=isless, rev=false) -Return the index of the first value in `a` greater than or equal to `x`, according to the -specified order. Return `lastindex(a) + 1` if `x` is greater than all values in `a`. -`a` is assumed to be sorted. +Return the index of the first value in `v` greater than or equivalent to `x`. +If `x` is greater than all values in `v`, return `lastindex(v) + 1`. -`insert!`ing `x` at this index will maintain sorted order. +The vector `v` must be sorted according to the order defined by the keywords. +`insert!`ing `x` at the returned index will maintain the sorted order. Refer to +[`sort!`](@ref) for the meaning of the keywords and the definition of +"greater than" and equivalence. Note that the `by` function is applied to the +searched value `x` as well as the values in `v`. -See [`sort!`](@ref) for an explanation of the keyword arguments `by`, `lt` and `rev`. +The index is generally found using binary search, but there are optimized +implementations for some inputs. See also: [`searchsortedlast`](@ref), [`searchsorted`](@ref), [`findfirst`](@ref). @@ -353,19 +366,24 @@ julia> searchsortedfirst([1, 2, 4, 5, 5, 7], 9) # no match, insert at end julia> searchsortedfirst([1, 2, 4, 5, 5, 7], 0) # no match, insert at start 1 -julia> searchsortedfirst([1=>"one", 2=>"two", 4=>"four"], 3=>"three", by=first) # Compare the keys of the pairs +julia> searchsortedfirst([1=>"one", 2=>"two", 4=>"four"], 3=>"three", by=first) # compare the keys of the pairs 3 ``` """ searchsortedfirst """ - searchsortedlast(a, x; by=, lt=, rev=false) + searchsortedlast(v, x; by=identity, lt=isless, rev=false) + +Return the index of the last value in `v` less than or equivalent to `x`. +If `x` is less than all values in `v` the function returns `firstindex(v) - 1`. -Return the index of the last value in `a` less than or equal to `x`, according to the -specified order. Return `firstindex(a) - 1` if `x` is less than all values in `a`. `a` is -assumed to be sorted. +The vector `v` must be sorted according to the order defined by the keywords. +Refer to [`sort!`](@ref) for the meaning of the keywords and the definition of +"less than" and equivalence. Note that the `by` function is applied to the +searched value `x` as well as the values in `v`. -See [`sort!`](@ref) for an explanation of the keyword arguments `by`, `lt` and `rev`. +The index is generally found using binary search, but there are optimized +implementations for some inputs # Examples ```jldoctest @@ -390,12 +408,16 @@ julia> searchsortedlast([1=>"one", 2=>"two", 4=>"four"], 3=>"three", by=first) # """ searchsortedlast """ - insorted(x, a; by=, lt=, rev=false) -> Bool + insorted(x, v; by=identity, lt=isless, rev=false) -> Bool + +Determine whether a vector `v` contains any value equivalent to `x`. +The vector `v` must be sorted according to the order defined by the keywords. +Refer to [`sort!`](@ref) for the meaning of the keywords and the definition of +equivalence. Note that the `by` function is applied to the searched value `x` +as well as the values in `v`. -Determine whether an item `x` is in the sorted collection `a`, in the sense that -it is [`==`](@ref) to one of the values of the collection according to the order -specified by the `by`, `lt` and `rev` keywords, assuming that `a` is already -sorted in that order, see [`sort`](@ref) for the keywords. +The check is generally done using binary search, but there are optimized +implementations for some inputs. See also [`in`](@ref). @@ -415,6 +437,9 @@ false julia> insorted(0, [1, 2, 4, 5, 5, 7]) # no match false + +julia> insorted(2=>"TWO", [1=>"one", 2=>"two", 4=>"four"], by=first) # compare the keys of the pairs +true ``` !!! compat "Julia 1.6" @@ -741,8 +766,8 @@ Insertion sort traverses the collection one element at a time, inserting each element into its correct, sorted position in the output vector. Characteristics: -* *stable*: preserves the ordering of elements which compare equal -(e.g. "a" and "A" in a sort of letters which ignores case). +* *stable*: preserves the ordering of elements that compare equal +(e.g. "a" and "A" in a sort of letters that ignores case). * *in-place* in memory. * *quadratic performance* in the number of elements to be sorted: it is well-suited to small collections but should not be used for large ones. @@ -981,8 +1006,8 @@ is treated as the first or last index of the input, respectively. `lo` and `hi` may be specified together as an `AbstractUnitRange`. Characteristics: - * *stable*: preserves the ordering of elements which compare equal - (e.g. "a" and "A" in a sort of letters which ignores case). + * *stable*: preserves the ordering of elements that compare equal + (e.g. "a" and "A" in a sort of letters that ignores case). * *not in-place* in memory. * *divide-and-conquer*: sort strategy similar to [`QuickSort`](@ref). * *linear runtime* if `length(lo:hi)` is constant @@ -1330,16 +1355,54 @@ defalg(v::AbstractArray{Union{}}) = DEFAULT_UNSTABLE # for method disambiguation """ sort!(v; alg::Algorithm=defalg(v), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) -Sort the vector `v` in place. A stable algorithm is used by default. You can select a -specific algorithm to use via the `alg` keyword (see [Sorting Algorithms](@ref) for -available algorithms). The `by` keyword lets you provide a function that will be applied to -each element before comparison; the `lt` keyword allows providing a custom "less than" -function (note that for every `x` and `y`, only one of `lt(x,y)` and `lt(y,x)` can return -`true`); use `rev=true` to reverse the sorting order. `rev=true` preserves forward stability: -Elements that compare equal are not reversed. These options are independent and can -be used together in all possible combinations: if both `by` and `lt` are specified, the `lt` -function is applied to the result of the `by` function; `rev=true` reverses whatever -ordering specified via the `by` and `lt` keywords. +Sort the vector `v` in place. A stable algorithm is used by default: the +ordering of elements that compare equal is preserved. A specific algorithm can +be selected via the `alg` keyword (see [Sorting Algorithms](@ref) for available +algorithms). + +Elements are first transformed with the function `by` and then compared +according to either the function `lt` or the ordering `order`. Finally, the +resulting order is reversed if `rev=true` (this preserves forward stability: +elements that compare equal are not reversed). The current implemention applies +the `by` transformation before each comparison rather than once per element. + +Passing an `lt` other than `isless` along with an `order` other than +[`Base.Order.Forward`](@ref) or [`Base.Order.Reverse`](@ref) is not permitted, +otherwise all options are independent and can be used together in all possible +combinations. Note that `order` can also include a "by" transformation, in +which case it is applied after that defined with the `by` keyword. For more +information on `order` values see the documentation on [Alternate +Orderings](@ref). + +Relations between two elements are defined as follows (with "less" and +"greater" exchanged when `rev=true`): + +* `x` is less than `y` if `lt(by(x), by(y))` (or `Base.Order.lt(order, by(x), by(y))`) yields true. +* `x` is greater than `y` if `y` is less than `x`. +* `x` and `y` are equivalent if neither is less than the other ("incomparable" + is sometimes used as a synonym for "equivalent"). + +The result of `sort!` is sorted in the sense that every element is greater than +or equivalent to the previous one. + +The `lt` function must define a strict weak order, that is, it must be + +* irreflexive: `lt(x, x)` always yields `false`, +* asymmetric: if `lt(x, y)` yields `true` then `lt(y, x)` yields `false`, +* transitive: `lt(x, y) && lt(y, z)` implies `lt(x, z)`, +* transitive in equivalence: `!lt(x, y) && !lt(y, x)` and `!lt(y, z) && !lt(z, + y)` together imply `!lt(x, z) && !lt(z, x)`. In words: if `x` and `y` are + equivalent and `y` and `z` are equivalent then `x` and `z` must be + equivalent. + +For example `<` is a valid `lt` function for `Int` values but `≤` is not: it +violates irreflexivity. For `Float64` values even `<` is invalid as it violates +the fourth condition: `1.0` and `NaN` are equivalent and so are `NaN` and `2.0` +but `1.0` and `2.0` are not equivalent. + +See also [`sort`](@ref), [`sortperm`](@ref), [`sortslices`](@ref), +[`partialsort!`](@ref), [`partialsortperm`](@ref), [`issorted`](@ref), +[`searchsorted`](@ref), [`insorted`](@ref), [`Base.Order.ord`](@ref). # Examples ```jldoctest @@ -1366,6 +1429,29 @@ julia> v = [(1, "c"), (3, "a"), (2, "b")]; sort!(v, by = x -> x[2]); v (3, "a") (2, "b") (1, "c") + +julia> sort(0:3, by=x->x-2, order=Base.Order.By(abs)) # same as sort(0:3, by=abs(x->x-2)) +4-element Vector{Int64}: + 2 + 1 + 3 + 0 + +julia> sort([2, NaN, 1, NaN, 3]) # correct sort with default lt=isless +5-element Vector{Float64}: + 1.0 + 2.0 + 3.0 + NaN + NaN + +julia> sort([2, NaN, 1, NaN, 3], lt=<) # wrong sort due to invalid lt. This behavior is undefined. +5-element Vector{Float64}: + 2.0 + NaN + 1.0 + NaN + 3.0 ``` """ function sort!(v::AbstractVector{T}; @@ -1443,15 +1529,15 @@ merge(x::NTuple, y::NTuple, o::Ordering) = ## partialsortperm: the permutation to sort the first k elements of an array ## """ - partialsortperm(v, k; by=, lt=, rev=false) + partialsortperm(v, k; by=ientity, lt=isless, rev=false) Return a partial permutation `I` of the vector `v`, so that `v[I]` returns values of a fully sorted version of `v` at index `k`. If `k` is a range, a vector of indices is returned; if `k` is an integer, a single index is returned. The order is specified using the same -keywords as `sort!`. The permutation is stable, meaning that indices of equal elements -appear in ascending order. +keywords as `sort!`. The permutation is stable: the indices of equal elements +will appear in ascending order. -Note that this function is equivalent to, but more efficient than, calling `sortperm(...)[k]`. +This function is equivalent to, but more efficient than, calling `sortperm(...)[k]`. # Examples ```jldoctest @@ -1477,7 +1563,7 @@ partialsortperm(v::AbstractVector, k::Union{Integer,OrdinalRange}; kwargs...) = partialsortperm!(similar(Vector{eltype(k)}, axes(v,1)), v, k; kwargs...) """ - partialsortperm!(ix, v, k; by=, lt=, rev=false) + partialsortperm!(ix, v, k; by=identity, lt=isless, rev=false) Like [`partialsortperm`](@ref), but accepts a preallocated index vector `ix` the same size as `v`, which is used to store (a permutation of) the indices of `v`. @@ -1543,7 +1629,7 @@ end Return a permutation vector or array `I` that puts `A[I]` in sorted order along the given dimension. If `A` has more than one dimension, then the `dims` keyword argument must be specified. The order is specified using the same keywords as [`sort!`](@ref). The permutation is guaranteed to be stable even -if the sorting algorithm is unstable, meaning that indices of equal elements appear in +if the sorting algorithm is unstable: the indices of equal elements will appear in ascending order. See also [`sortperm!`](@ref), [`partialsortperm`](@ref), [`invperm`](@ref), [`indexin`](@ref). @@ -1777,7 +1863,8 @@ end sort!(A; dims::Integer, alg::Algorithm=defalg(A), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) Sort the multidimensional array `A` along dimension `dims`. -See [`sort!`](@ref) for a description of possible keyword arguments. +See the one-dimensional version of [`sort!`](@ref) for a description of +possible keyword arguments. To sort slices of an array, refer to [`sortslices`](@ref). @@ -1931,8 +2018,8 @@ algorithm. Partial quick sort returns the smallest `k` elements sorted from smal to largest, finding them and sorting them using [`QuickSort`](@ref). Characteristics: - * *not stable*: does not preserve the ordering of elements which - compare equal (e.g. "a" and "A" in a sort of letters which + * *not stable*: does not preserve the ordering of elements that + compare equal (e.g. "a" and "A" in a sort of letters that ignores case). * *in-place* in memory. * *divide-and-conquer*: sort strategy similar to [`MergeSort`](@ref). @@ -1969,8 +2056,8 @@ Indicate that a sorting function should use the quick sort algorithm, which is *not* stable. Characteristics: - * *not stable*: does not preserve the ordering of elements which - compare equal (e.g. "a" and "A" in a sort of letters which + * *not stable*: does not preserve the ordering of elements that + compare equal (e.g. "a" and "A" in a sort of letters that ignores case). * *in-place* in memory. * *divide-and-conquer*: sort strategy similar to [`MergeSort`](@ref). @@ -1988,8 +2075,8 @@ subcollection at each step, until the entire collection has been recombined in sorted form. Characteristics: - * *stable*: preserves the ordering of elements which compare - equal (e.g. "a" and "A" in a sort of letters which ignores + * *stable*: preserves the ordering of elements that compare + equal (e.g. "a" and "A" in a sort of letters that ignores case). * *not in-place* in memory. * *divide-and-conquer* sort strategy. diff --git a/doc/src/base/base.md b/doc/src/base/base.md index bb72b94293c6b..aa5d23281dd6b 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -129,6 +129,7 @@ Core.:(===) Core.isa Base.isequal Base.isless +Base.isunordered Base.ifelse Core.typeassert Core.typeof diff --git a/doc/src/base/sort.md b/doc/src/base/sort.md index 16e1839cf64a2..455a95d39617c 100644 --- a/doc/src/base/sort.md +++ b/doc/src/base/sort.md @@ -163,7 +163,7 @@ Base.Sort.defalg(::AbstractArray{<:Union{SmallInlineStrings, Missing}}) = Inline be stable since Julia 1.9. Previous versions had unstable edge cases when sorting numeric arrays. -## Alternate orderings +## Alternate Orderings By default, `sort` and related functions use [`isless`](@ref) to compare two elements in order to determine which should come first. The diff --git a/doc/src/manual/missing.md b/doc/src/manual/missing.md index 9bddcdfbb2ac2..8c8e801ccac9a 100644 --- a/doc/src/manual/missing.md +++ b/doc/src/manual/missing.md @@ -88,7 +88,7 @@ true ``` The [`isless`](@ref) operator is another exception: `missing` is considered -as greater than any other value. This operator is used by [`sort`](@ref), +as greater than any other value. This operator is used by [`sort!`](@ref), which therefore places `missing` values after all other values: ```jldoctest From 085c3d1ce5fdfada26219dcc88a46d3302fc7bf2 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Fri, 7 Jul 2023 17:16:25 -0500 Subject: [PATCH 31/33] Make `(1:3:4)[6148914691236517207]` throw (#50118) --- base/range.jl | 16 ++++++++++------ test/ranges.jl | 8 ++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/base/range.jl b/base/range.jl index 6b701d31b0358..e6341003d0c5a 100644 --- a/base/range.jl +++ b/base/range.jl @@ -938,12 +938,16 @@ end function getindex(v::AbstractRange{T}, i::Integer) where T @inline i isa Bool && throw(ArgumentError("invalid index: $i of type Bool")) - ret = convert(T, first(v) + (i - oneunit(i))*step_hp(v)) - ok = ifelse(step(v) > zero(step(v)), - (ret <= last(v)) & (ret >= first(v)), - (ret <= first(v)) & (ret >= last(v))) - @boundscheck ((i > 0) & ok) || throw_boundserror(v, i) - ret + @boundscheck checkbounds(v, i) + convert(T, first(v) + (i - oneunit(i))*step_hp(v)) +end + +let BitInteger64 = Union{Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64} # for bootstrapping + function checkbounds(::Type{Bool}, v::StepRange{<:BitInteger64, <:BitInteger64}, i::BitInteger64) + @inline + res = widemul(step(v), i-oneunit(i)) + first(v) + (0 < i) & ifelse(0 < step(v), res <= last(v), res >= last(v)) + end end function getindex(r::Union{StepRangeLen,LinRange}, i::Integer) diff --git a/test/ranges.jl b/test/ranges.jl index b263e6d4d530d..0e195f5dde24e 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -2490,3 +2490,11 @@ function check_ranges(rx, ry) end @test Core.Compiler.is_foldable(Base.infer_effects(check_ranges, (UnitRange{Int},UnitRange{Int}))) # TODO JET.@test_opt check_ranges(1:2, 3:4) + +@testset "checkbounds overflow (#26623)" begin + # the reported issue: + @test_throws BoundsError (1:3:4)[typemax(Int)÷3*2+3] + + # a case that using mul_with_overflow & add_with_overflow might get wrong: + @test (-10:2:typemax(Int))[typemax(Int)÷2+2] == typemax(Int)-9 +end From e20274f8dc6ee971ece421d320fd223078eac175 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 8 Jul 2023 04:34:18 +0200 Subject: [PATCH 32/33] Remove FATAL_TYPE_BOUND_ERROR (#50465) --- base/compiler/ssair/inlining.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 137473271f749..add390f7d454f 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -494,8 +494,6 @@ function fix_va_argexprs!(insert_node!::Inserter, inline_target::Union{IRCode, I return newargexprs end -const FATAL_TYPE_BOUND_ERROR = ErrorException("fatal error in type inference (type bound)") - """ ir_inline_unionsplit! From d60f9b3b47ee585cc1d8a836bb0d7acab81a9b6e Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 8 Jul 2023 02:52:22 -0400 Subject: [PATCH 33/33] Refactor and pass correct interpreter to typeinf finish loop (#50469) When we have an inference loop with different interpreters, the current code was trying to cache everything with the top level interpreter of the loop, yielding some unexpected behavior. I don't think that it's necessarily super well defined what should happen here, as it depends on the interpreters, in question, but I think it's better to try to cache each frame with the interpreter that created it, since they may have different lattices, etc. Doing this fixes an error I saw downstream that had just such a situation. --------- Co-authored-by: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> --- base/compiler/typeinfer.jl | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 77e1fd02de8d0..dfeaba18321d1 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -257,33 +257,28 @@ function _typeinf(interp::AbstractInterpreter, frame::InferenceState) end for caller in frames caller.valid_worlds = valid_worlds - finish(caller, interp) + finish(caller, caller.interp) end - # collect results for the new expanded frame - results = Tuple{InferenceResult, Vector{Any}, Bool}[ - ( frames[i].result, - frames[i].stmt_edges[1]::Vector{Any}, - frames[i].cached ) - for i in 1:length(frames) ] - empty!(frames) - for (caller, _, _) in results - opt = caller.src - if opt isa OptimizationState{typeof(interp)} # implies `may_optimize(interp) === true` - optimize(interp, opt, caller) + for caller in frames + opt = caller.result.src + if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` + optimize(caller.interp, opt, caller.result) end end - for (caller, edges, cached) in results - valid_worlds = caller.valid_worlds + for caller in frames + (; result ) = caller + valid_worlds = result.valid_worlds if last(valid_worlds) >= get_world_counter() # if we aren't cached, we don't need this edge # but our caller might, so let's just make it anyways - store_backedges(caller, edges) + store_backedges(result, caller.stmt_edges[1]) end - if cached - cache_result!(interp, caller) + if caller.cached + cache_result!(caller.interp, result) end - finish!(interp, caller) + finish!(caller.interp, result) end + empty!(frames) return true end