reverse iteration for eachline (JuliaLang#42225)

NEWS.md

@@ -80,6 +80,7 @@ Standard library changes
 
 * `range` accepts either `stop` or `length` as a sole keyword argument ([#39241])
 * `precision` and `setprecision` now accept a `base` keyword ([#42428]).
+* `Iterators.reverse` (and hence `last`) now supports `eachline` iterators ([#42225]).
 * The `length` function on certain ranges of certain specific element types no longer checks for integer
   overflow in most cases. The new function `checked_length` is now available, which will try to use checked
   arithmetic to error if the result may be wrapping. Or use a package such as SaferIntegers.jl when

base/io.jl

@@ -1019,6 +1019,11 @@ closed when the `EachLine` object is garbage collected.
 
 To iterate over each line of a `String`, `eachline(IOBuffer(str))` can be used.
 
+[`Iterators.reverse`](@ref) can be used on an `EachLine` object to read the lines
+in reverse order (for files, buffers, and other I/O streams supporting [`seek`](@ref)),
+and [`first`](@ref) or [`last`](@ref) can be used to extract the initial or final
+lines, respectively.
+
 # Examples
 ```jldoctest
 julia> open("my_file.txt", "w") do io
@@ -1032,6 +1037,9 @@ JuliaLang is a GitHub organization. It has many members.
 
 julia> rm("my_file.txt");
 ```
+
+!!! compat "Julia 1.8"
+       Julia 1.8 is required to use `Iterators.reverse` or `last` with `eachline` iterators.
 """
 function eachline(stream::IO=stdin; keep::Bool=false)
     EachLine(stream, keep=keep)::EachLine
@@ -1053,6 +1061,115 @@ IteratorSize(::Type{<:EachLine}) = SizeUnknown()
 
 isdone(itr::EachLine, state...) = eof(itr.stream)
 
+# Reverse-order iteration for the EachLine iterator for seekable streams,
+# which works by reading the stream from the end in 4kiB chunks.
+function iterate(r::Iterators.Reverse{<:EachLine})
+    p0 = position(r.itr.stream)
+    seekend(r.itr.stream) # may throw if io is non-seekable
+    p = position(r.itr.stream)
+    # chunks = circular buffer of 4kiB blocks read from end of stream
+    chunks = empty!(Vector{Vector{UInt8}}(undef, 2)) # allocate space for 2 buffers (common case)
+    inewline = jnewline = 0
+    while p > p0 && inewline == 0 # read chunks until we find a newline or we read whole file
+        chunk = Vector{UInt8}(undef, min(4096, p-p0))
+        p -= length(chunk)
+        readbytes!(seek(r.itr.stream, p), chunk)
+        pushfirst!(chunks, chunk)
+        inewline = something(findlast(==(UInt8('\n')), chunk), 0)
+        if length(chunks) == 1 && inewline == length(chunks[1])
+            # found newline at end of file … keep looking
+            jnewline = inewline
+            inewline = something(findprev(==(UInt8('\n')), chunk, inewline-1), 0)
+        end
+    end
+    return iterate(r, (; p0, p, chunks, ichunk=1, inewline, jchunk=length(chunks), jnewline = jnewline == 0 && !isempty(chunks) ? length(chunks[end]) : jnewline))
+end
+function iterate(r::Iterators.Reverse{<:EachLine}, state)
+    function _stripnewline(keep, pos, data)
+        # strip \n or \r\n from data[pos] by decrementing pos
+        if !keep && pos > 0 && data[pos] == UInt8('\n')
+            pos -= 1
+            pos -= pos > 0 && data[pos] == UInt8('\r')
+        end
+        return pos
+    end
+    # state tuple: p0 = initial file position, p = current position,
+    #              chunks = circular array of chunk buffers,
+    #              current line is from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline]
+    p0, p, chunks, ichunk, inewline, jchunk, jnewline = state
+    if inewline == 0 # no newline found, remaining line = rest of chunks (if any)
+        isempty(chunks) && return (r.itr.ondone(); nothing)
+        buf = IOBuffer(sizehint = ichunk==jchunk ? jnewline : 4096)
+        while ichunk != jchunk
+            write(buf, chunks[ichunk])
+            ichunk = ichunk == length(chunks) ? 1 : ichunk + 1
+        end
+        chunk = chunks[jchunk]
+        write(buf, view(chunk, 1:jnewline))
+        buf.size = _stripnewline(r.itr.keep, buf.size, buf.data)
+        empty!(chunks) # will cause next iteration to terminate
+        seekend(r.itr.stream) # reposition to end of stream for isdone
+        s = String(take!(buf))
+    else
+        # extract the string from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline]
+        if ichunk == jchunk # common case: current and previous newline in same chunk
+            chunk = chunks[ichunk]
+            s = String(view(chunk, inewline+1:_stripnewline(r.itr.keep, jnewline, chunk)))
+        else
+            buf = IOBuffer(sizehint=max(128, length(chunks[ichunk])-inewline+jnewline))
+            write(buf, view(chunks[ichunk], inewline+1:length(chunks[ichunk])))
+            i = ichunk
+            while true
+                i = i == length(chunks) ? 1 : i + 1
+                i == jchunk && break
+                write(buf, chunks[i])
+            end
+            write(buf, view(chunks[jchunk], 1:jnewline))
+            buf.size = _stripnewline(r.itr.keep, buf.size, buf.data)
+            s = String(take!(buf))
+
+            # overwrite obsolete chunks (ichunk+1:jchunk)
+            i = jchunk
+            while i != ichunk
+                chunk = chunks[i]
+                p -= length(resize!(chunk, min(4096, p-p0)))
+                readbytes!(seek(r.itr.stream, p), chunk)
+                i = i == 1 ? length(chunks) : i - 1
+            end
+        end
+
+        # find the newline previous to inewline
+        jchunk = ichunk
+        jnewline = inewline
+        while true
+            inewline = something(findprev(==(UInt8('\n')), chunks[ichunk], inewline-1), 0)
+            inewline > 0 && break
+            ichunk = ichunk == 1 ? length(chunks) : ichunk - 1
+            ichunk == jchunk && break # found nothing — may need to read more chunks
+            inewline = length(chunks[ichunk])+1 # start for next findprev
+        end
+
+        # read more chunks to look for a newline (should rarely happen)
+        if inewline == 0 && p > p0
+            ichunk = jchunk + 1
+            while true
+                chunk = Vector{UInt8}(undef, min(4096, p-p0))
+                p -= length(chunk)
+                readbytes!(seek(r.itr.stream, p), chunk)
+                insert!(chunks, ichunk, chunk)
+                inewline = something(findlast(==(UInt8('\n')), chunk), 0)
+                (p == p0 || inewline > 0) && break
+            end
+        end
+    end
+    return (s, (; p0, p, chunks, ichunk, inewline, jchunk, jnewline))
+end
+isdone(r::Iterators.Reverse{<:EachLine}, state) = isempty(state.chunks)
+isdone(r::Iterators.Reverse{<:EachLine}) = isdone(r.itr)
+
+# use reverse iteration to get end of EachLines (if possible)
+last(itr::EachLine) = first(Iterators.reverse(itr))
+
 struct ReadEachIterator{T, IOT <: IO}
     stream::IOT
 end

base/iterators.jl

@@ -103,7 +103,6 @@ size(r::Reverse) = size(r.itr)
 IteratorSize(::Type{Reverse{T}}) where {T} = IteratorSize(T)
 IteratorEltype(::Type{Reverse{T}}) where {T} = IteratorEltype(T)
 last(r::Reverse) = first(r.itr) # the first shall be last
-first(r::Reverse) = last(r.itr) # and the last shall be first
 
 # reverse-order array iterators: assumes more-specialized Reverse for eachindex
 @propagate_inbounds function iterate(A::Reverse{<:AbstractArray}, state=(reverse(eachindex(A.itr)),))

test/read.jl

@@ -293,6 +293,14 @@ for (name, f) in l
         @test collect(eachline(io(), keep=true)) == collect(eachline(filename, keep=true))
         @test collect(eachline(io())) == collect(eachline(IOBuffer(text)))
         @test collect(@inferred(eachline(io()))) == collect(@inferred(eachline(filename))) #20351
+        if try; seekend(io()); true; catch; false; end # reverse iteration only supports seekable streams
+            for keep in (true, false)
+                lines = readlines(io(); keep)
+                @test last(lines) == last(eachline(io(); keep))
+                @test last(lines,2) == last(eachline(io(); keep),2)
+                @test reverse!(lines) == collect(Iterators.reverse(eachline(io(); keep))) == collect(Iterators.reverse(eachline(IOBuffer(text); keep)))
+            end
+        end
 
         cleanup()
 
@@ -621,3 +629,33 @@ end
     first(itr) # consume the iterator
     @test  isempty(itr) # now it is empty
 end
+
+# more tests for reverse(eachline)
+@testset "reverse(eachline)" begin
+    lines = vcat(repr.(1:4), ' '^50000 .* repr.(5:10), repr.(11:10^5))
+    for lines in (lines, reverse(lines)), finalnewline in (true, false), eol in ("\n", "\r\n")
+        buf = IOBuffer(join(lines, eol) * (finalnewline ? eol : ""))
+        @test reverse!(collect(Iterators.reverse(eachline(seekstart(buf))))) == lines
+        @test last(eachline(seekstart(buf))) == last(lines)
+        @test last(eachline(seekstart(buf)),10^4) == last(lines,10^4)
+        @test last(eachline(seekstart(buf)),length(lines)*2) == lines
+        @test reverse!(collect(Iterators.reverse(eachline(seek(buf, sum(sizeof, lines[1:100]) + 100*sizeof(eol)))))) == lines[101:end]
+        @test isempty(Iterators.reverse(eachline(buf)))
+    end
+
+    let rempty = Iterators.reverse(eachline(IOBuffer()))
+        @test isempty(rempty)
+        @test isempty(collect(rempty))
+    end
+
+    let buf = IOBuffer("foo\nbar")
+        @test readline(buf) == "foo"
+        r = Iterators.reverse(eachline(buf))
+        line, state = iterate(r)
+        @test line == "bar"
+        @test Base.isdone(r, state)
+        @test Base.isdone(r)
+        @test isempty(r) && isempty(collect(r))
+    end
+end
+