Skip to content

Commit

Permalink
strip CRLF, not just LF
Browse files Browse the repository at this point in the history
  • Loading branch information
stevengj committed Oct 1, 2021
1 parent cbc8468 commit 905ba00
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 7 deletions.
18 changes: 14 additions & 4 deletions base/io.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1079,9 +1079,17 @@ function iterate(r::Iterators.Reverse{<:EachLine})
inewline = something(findprev(==(UInt8('\n')), chunk, inewline-1), 0)
end
end
return iterate(r, (p0, p, chunks, 1, inewline, length(chunks), jnewline == 0 && !isempty(chunks) ? length(chunks[end])+1 : jnewline))
return iterate(r, (p0, p, chunks, 1, inewline, length(chunks), jnewline == 0 && !isempty(chunks) ? length(chunks[end]) : jnewline))
end
function iterate(r::Iterators.Reverse{<:EachLine}, state)
function _stripnewline(keep, pos, data)
# strip \n or \r\n from data[pos] by decrementing pos
if !keep && pos > 0 && data[pos] == UInt8('\n')
pos -= 1
pos -= pos > 0 && data[pos] == UInt8('\r')
end
return pos
end
# state tuple: p0 = initial file position, p = current position,
# chunks = circular array of chunk buffers,
# current line is from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline]
Expand All @@ -1094,15 +1102,16 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state)
ichunk = ichunk == length(chunks) ? 1 : ichunk + 1
end
chunk = chunks[jchunk]
write(buf, view(chunk, 1:min(length(chunk), jnewline - 1 + r.itr.keep)))
write(buf, view(chunk, 1:jnewline))
buf.size = _stripnewline(r.itr.keep, buf.size, buf.data)
empty!(chunks) # will cause next iteration to terminate
seekend(r.itr.stream) # reposition to end of stream for isdone
s = String(take!(buf))
else
# extract the string from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline]
jnewline = min(length(chunks[jchunk]), jnewline - 1 + r.itr.keep)
if ichunk == jchunk # common case: current and previous newline in same chunk
s = String(view(chunks[ichunk], inewline+1:jnewline))
chunk = chunks[ichunk]
s = String(view(chunk, inewline+1:_stripnewline(r.itr.keep, jnewline, chunk)))
else
buf = IOBuffer(sizehint=max(128, length(chunks[ichunk])-inewline+jnewline))
write(buf, view(chunks[ichunk], inewline+1:length(chunks[ichunk])))
Expand All @@ -1113,6 +1122,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state)
write(buf, chunks[i])
end
write(buf, view(chunks[jchunk], 1:jnewline))
buf.size = _stripnewline(r.itr.keep, buf.size, buf.data)
s = String(take!(buf))

# overwrite obsolete chunks (ichunk+1:jchunk)
Expand Down
6 changes: 3 additions & 3 deletions test/read.jl
Original file line number Diff line number Diff line change
Expand Up @@ -633,13 +633,13 @@ end
# more tests for reverse(eachline)
@testset "reverse(eachline)" begin
lines = vcat(repr.(1:4), ' '^50000 .* repr.(5:10), repr.(11:10^5))
for lines in (lines, reverse(lines)), finalnewline in (true, false)
buf = IOBuffer(join(lines, '\n') * (finalnewline ? "\n" : ""))
for lines in (lines, reverse(lines)), finalnewline in (true, false), eol in ("\n", "\r\n")
buf = IOBuffer(join(lines, eol) * (finalnewline ? eol : ""))
@test reverse!(collect(Iterators.reverse(eachline(seekstart(buf))))) == lines
@test last(eachline(seekstart(buf))) == last(lines)
@test last(eachline(seekstart(buf)),10^4) == last(lines,10^4)
@test last(eachline(seekstart(buf)),length(lines)*2) == lines
@test reverse!(collect(Iterators.reverse(eachline(seek(buf, sum(sizeof, lines[1:100]) + 100))))) == lines[101:end]
@test reverse!(collect(Iterators.reverse(eachline(seek(buf, sum(sizeof, lines[1:100]) + 100*sizeof(eol)))))) == lines[101:end]
@test isempty(Iterators.reverse(eachline(buf)))
end

Expand Down

0 comments on commit 905ba00

Please sign in to comment.