Skip to content

Commit

Permalink
[FileWatching] fix PollingFileWatcher design and add workaround for a…
Browse files Browse the repository at this point in the history 
… stat bug

What started as an innocent fix for a stat bug on Apple (#48667) turned
into a full blown investigation into the design problems with the libuv
backend for PollingFileWatcher, and writing my own implementation of it
instead which could avoid those singled-threaded concurrency bugs.
  • Loading branch information
@vtjnash
vtjnash committed Sep 26, 2024
1 parent e4b29f7 commit 6da6acf
Show file tree
Hide file tree
Showing 7 changed files with 215 additions and 167 deletions.
8 changes: 4 additions & 4 deletions base/libuv.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ for r in uv_req_types
@eval const $(Symbol("_sizeof_", lowercase(string(r)))) = uv_sizeof_req($r)
end

uv_handle_data(handle) = ccall(:jl_uv_handle_data, Ptr{Cvoid}, (Ptr{Cvoid},), handle)
uv_req_data(handle) = ccall(:jl_uv_req_data, Ptr{Cvoid}, (Ptr{Cvoid},), handle)
uv_req_set_data(req, data) = ccall(:jl_uv_req_set_data, Cvoid, (Ptr{Cvoid}, Any), req, data)
uv_req_set_data(req, data::Ptr{Cvoid}) = ccall(:jl_uv_req_set_data, Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), req, data)
uv_handle_data(handle) = ccall(:uv_handle_get_data, Ptr{Cvoid}, (Ptr{Cvoid},), handle)
uv_req_data(handle) = ccall(:uv_req_get_data, Ptr{Cvoid}, (Ptr{Cvoid},), handle)
uv_req_set_data(req, data) = ccall(:uv_req_set_data, Cvoid, (Ptr{Cvoid}, Any), req, data)
uv_req_set_data(req, data::Ptr{Cvoid}) = ccall(:uv_handle_set_data, Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), req, data)

macro handle_as(hand, typ)
return quote
Expand Down
3 changes: 2 additions & 1 deletion base/reflection.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -964,7 +964,7 @@ use it in the following manner to summarize information about a struct:
julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:fieldcount(T)];
julia> structinfo(Base.Filesystem.StatStruct)
13-element Vector{Tuple{UInt64, Symbol, Type}}:
14-element Vector{Tuple{UInt64, Symbol, Type}}:
(0x0000000000000000, :desc, Union{RawFD, String})
(0x0000000000000008, :device, UInt64)
(0x0000000000000010, :inode, UInt64)
Expand All @@ -978,6 +978,7 @@ julia> structinfo(Base.Filesystem.StatStruct)
(0x0000000000000050, :blocks, Int64)
(0x0000000000000058, :mtime, Float64)
(0x0000000000000060, :ctime, Float64)
(0x0000000000000068, :ioerrno, Int32)
```
"""
fieldoffset(x::DataType, idx::Integer) = (@_foldable_meta; ccall(:jl_get_field_offset, Csize_t, (Any, Cint), x, idx))
Expand Down
111 changes: 57 additions & 54 deletions base/stat.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ struct StatStruct
blocks :: Int64
mtime :: Float64
ctime :: Float64
ioerrno :: Int32
end

@eval function Base.:(==)(x::StatStruct, y::StatStruct) # do not include `desc` in equality or hash
Expand All @@ -80,22 +81,23 @@ end
end)
end

StatStruct() = StatStruct("", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
StatStruct(buf::Union{Vector{UInt8},Ptr{UInt8}}) = StatStruct("", buf)
StatStruct(desc::Union{AbstractString, OS_HANDLE}, buf::Union{Vector{UInt8},Ptr{UInt8}}) = StatStruct(
StatStruct() = StatStruct("", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Base.UV_ENOENT)
StatStruct(buf::Union{Memory{UInt8},Vector{UInt8},Ptr{UInt8}}, ioerrno::Int32) = StatStruct("", buf, ioerrno)
StatStruct(desc::Union{AbstractString, OS_HANDLE}, buf::Union{Memory{UInt8},Vector{UInt8},Ptr{UInt8}}, ioerrno::Int32) = StatStruct(
desc isa OS_HANDLE ? desc : String(desc),
ccall(:jl_stat_dev, UInt32, (Ptr{UInt8},), buf),
ccall(:jl_stat_ino, UInt32, (Ptr{UInt8},), buf),
ccall(:jl_stat_mode, UInt32, (Ptr{UInt8},), buf),
ccall(:jl_stat_nlink, UInt32, (Ptr{UInt8},), buf),
ccall(:jl_stat_uid, UInt32, (Ptr{UInt8},), buf),
ccall(:jl_stat_gid, UInt32, (Ptr{UInt8},), buf),
ccall(:jl_stat_rdev, UInt32, (Ptr{UInt8},), buf),
ccall(:jl_stat_size, UInt64, (Ptr{UInt8},), buf),
ccall(:jl_stat_blksize, UInt64, (Ptr{UInt8},), buf),
ccall(:jl_stat_blocks, UInt64, (Ptr{UInt8},), buf),
ccall(:jl_stat_mtime, Float64, (Ptr{UInt8},), buf),
ccall(:jl_stat_ctime, Float64, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt32) : ccall(:jl_stat_dev, UInt32, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt32) : ccall(:jl_stat_ino, UInt32, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt32) : ccall(:jl_stat_mode, UInt32, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt32) : ccall(:jl_stat_nlink, UInt32, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt32) : ccall(:jl_stat_uid, UInt32, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt32) : ccall(:jl_stat_gid, UInt32, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt32) : ccall(:jl_stat_rdev, UInt32, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt64) : ccall(:jl_stat_size, UInt64, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt64) : ccall(:jl_stat_blksize, UInt64, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(UInt64) : ccall(:jl_stat_blocks, UInt64, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(Float64) : ccall(:jl_stat_mtime, Float64, (Ptr{UInt8},), buf),
ioerrno != 0 ? zero(Float64) : ccall(:jl_stat_ctime, Float64, (Ptr{UInt8},), buf),
ioerrno
)

function iso_datetime_with_relative(t, tnow)
Expand Down Expand Up @@ -130,35 +132,41 @@ end
function show_statstruct(io::IO, st::StatStruct, oneline::Bool)
print(io, oneline ? "StatStruct(" : "StatStruct for ")
show(io, st.desc)
oneline || print(io, "\n ")
print(io, " size: ", st.size, " bytes")
oneline || print(io, "\n")
print(io, " device: ", st.device)
oneline || print(io, "\n ")
print(io, " inode: ", st.inode)
oneline || print(io, "\n ")
print(io, " mode: 0o", string(filemode(st), base = 8, pad = 6), " (", filemode_string(st), ")")
oneline || print(io, "\n ")
print(io, " nlink: ", st.nlink)
oneline || print(io, "\n ")
print(io, " uid: $(st.uid)")
username = getusername(st.uid)
username === nothing || print(io, " (", username, ")")
oneline || print(io, "\n ")
print(io, " gid: ", st.gid)
groupname = getgroupname(st.gid)
groupname === nothing || print(io, " (", groupname, ")")
oneline || print(io, "\n ")
print(io, " rdev: ", st.rdev)
oneline || print(io, "\n ")
print(io, " blksz: ", st.blksize)
oneline || print(io, "\n")
print(io, " blocks: ", st.blocks)
tnow = round(UInt, time())
oneline || print(io, "\n ")
print(io, " mtime: ", iso_datetime_with_relative(st.mtime, tnow))
oneline || print(io, "\n ")
print(io, " ctime: ", iso_datetime_with_relative(st.ctime, tnow))
code = st.ioerrno
if code != 0
print(io, oneline ? " " : "\n ")
print(io, Base.uverrorname(code), ": ", Base.struverror(code))
else
oneline || print(io, "\n ")
print(io, " size: ", st.size, " bytes")
oneline || print(io, "\n")
print(io, " device: ", st.device)
oneline || print(io, "\n ")
print(io, " inode: ", st.inode)
oneline || print(io, "\n ")
print(io, " mode: 0o", string(filemode(st), base = 8, pad = 6), " (", filemode_string(st), ")")
oneline || print(io, "\n ")
print(io, " nlink: ", st.nlink)
oneline || print(io, "\n ")
print(io, " uid: $(st.uid)")
username = getusername(st.uid)
username === nothing || print(io, " (", username, ")")
oneline || print(io, "\n ")
print(io, " gid: ", st.gid)
groupname = getgroupname(st.gid)
groupname === nothing || print(io, " (", groupname, ")")
oneline || print(io, "\n ")
print(io, " rdev: ", st.rdev)
oneline || print(io, "\n ")
print(io, " blksz: ", st.blksize)
oneline || print(io, "\n")
print(io, " blocks: ", st.blocks)
tnow = round(UInt, time())
oneline || print(io, "\n ")
print(io, " mtime: ", iso_datetime_with_relative(st.mtime, tnow))
oneline || print(io, "\n ")
print(io, " ctime: ", iso_datetime_with_relative(st.ctime, tnow))
end
oneline && print(io, ")")
return nothing
end
Expand All @@ -168,18 +176,13 @@ show(io::IO, ::MIME"text/plain", st::StatStruct) = show_statstruct(io, st, false

# stat & lstat functions

checkstat(s::StatStruct) = Int(s.ioerrno) in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL) ? s : uv_error(string("stat(", repr(s.desc), ")"), s.ioerrno)

macro stat_call(sym, arg1type, arg)
return quote
stat_buf = zeros(UInt8, Int(ccall(:jl_sizeof_stat, Int32, ())))
stat_buf = fill!(Memory{UInt8}(undef, Int(ccall(:jl_sizeof_stat, Int32, ()))), 0x00)
r = ccall($(Expr(:quote, sym)), Int32, ($(esc(arg1type)), Ptr{UInt8}), $(esc(arg)), stat_buf)
if !(r in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL))
uv_error(string("stat(", repr($(esc(arg))), ")"), r)
end
st = StatStruct($(esc(arg)), stat_buf)
if ispath(st) != (r == 0)
error("stat returned zero type for a valid path")
end
return st
return checkstat(StatStruct($(esc(arg)), stat_buf, r))
end
end

Expand Down Expand Up @@ -334,7 +337,7 @@ Return `true` if a valid filesystem entity exists at `path`,
otherwise returns `false`.
This is the generalization of [`isfile`](@ref), [`isdir`](@ref) etc.
"""
ispath(st::StatStruct) = filemode(st) & 0xf000 != 0x0000
ispath(st::StatStruct) = st.ioerrno == 0
function ispath(path::String)
# We use `access()` and `F_OK` to determine if a given path exists. `F_OK` comes from `unistd.h`.
F_OK = 0x00
Expand Down
1 change: 0 additions & 1 deletion src/sys.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ JL_DLLEXPORT int32_t jl_nb_available(ios_t *s)

// --- dir/file stuff ---

JL_DLLEXPORT int jl_sizeof_uv_fs_t(void) { return sizeof(uv_fs_t); }
JL_DLLEXPORT char *jl_uv_fs_t_ptr(uv_fs_t *req) { return (char*)req->ptr; }
JL_DLLEXPORT char *jl_uv_fs_t_path(uv_fs_t *req) { return (char*)req->path; }

Expand Down
Loading

0 comments on commit 6da6acf

Please sign in to comment.