Skip to content

Commit

Permalink
Make the "second Base copy" trick actually work
Browse files Browse the repository at this point in the history
As was mentioned in #25988, there is a handy trick where you can load
a second copy of Base on top of an existing copy. This is useful for
at least two reasons:
1. Base printing is available, so things like MethodErrors print nicely
2. Even if the load fails, the resulting (broken) copy of base is inspectable
   by standard introspection tools from the REPL, as long as you're a bit
   careful not to mix types from the two copies of Base.

However, as I mentioned in #26079, this only actually works until about version.jl,
at which point things crash. This is because at that point it tries to use PCRE
which uses `Ref(0)`, which is actually an abstract type in Core, even though the
type of the constructed object (`RefValue`) is in Base. As a result, the new Base
gets the wrong kind of `RefValue` (the one from the original `Base`) and things break.

Luckily this is easily fixed by using an explicit `RefValue` call in the relevant places.

A second problem we run into is that `module`s nested under our new `Base`, get a default
import of the old `Base` (unless we declare the new Base to be the global top module, but
that would break the REPL subsequent to loading the new Base, which breaks reason 2 above).
I suggest (and implement in this PR) to have the default import be the next topmodule along
the parent link chain (as we already do for syntax defined in Base), which makes this work.
A small related detail is that in all such modules `import Base: x`, needs to be replaced by
`import .Base: x`, to make sure we resolve the identifier `Base` (as imported from our
new top module) rather than the global name `Base` (which still refers to the old module).

I changed sysimg.jl to avoid loading stdlibs in second Base mode, to avoid having to implement
the same changes there. Since the stdlibs are already decoupled from Base, they can already
be developed separately fairly easily, so there's not much reason to include them in this trick.

For completeness, there's a couple of ways to use this trick, but perhaps the simplest is:
```
cd("base")
baremodule NotBase
    Core.include(NotBase, "sysimg.jl")
end
```
from the REPL.
  • Loading branch information
Keno committed Feb 19, 2018
1 parent 90d0ff5 commit d774e40
Show file tree
Hide file tree
Showing 20 changed files with 102 additions and 84 deletions.
4 changes: 2 additions & 2 deletions base/atomics.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

using Core.Intrinsics: llvmcall

import Base: setindex!, getindex, unsafe_convert
import Base.Sys: ARCH, WORD_SIZE
import .Base: setindex!, getindex, unsafe_convert
import .Base.Sys: ARCH, WORD_SIZE

export
Atomic,
Expand Down
6 changes: 3 additions & 3 deletions base/broadcast.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

module Broadcast

using Base.Cartesian
using Base: Indices, OneTo, linearindices, tail, to_shape,
using .Base.Cartesian
using .Base: Indices, OneTo, linearindices, tail, to_shape,
_msk_end, unsafe_bitgetindex, bitcache_chunks, bitcache_size, dumpbitcache,
isoperator, promote_typejoin
import Base: broadcast, broadcast!
import .Base: broadcast, broadcast!
export BroadcastStyle, broadcast_indices, broadcast_similar,
broadcast_getindex, broadcast_setindex!, dotview, @__dot__

Expand Down
7 changes: 4 additions & 3 deletions base/docs/Docs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,9 @@ function.

include("bindings.jl")

import Base.Meta: quot, isexpr
import Base: Callable, with_output_color
import .Base.Meta: quot, isexpr
import .Base: Callable, with_output_color
using .Base: RefValue
import ..CoreDocs: lazy_iterpolate

export doc
Expand Down Expand Up @@ -390,7 +391,7 @@ end
function multidoc(__source__, __module__, meta, ex, define)
out = Expr(:toplevel)
str = docexpr(__source__, __module__, lazy_iterpolate(meta), metadata(__source__, __module__, ex, false))
ref = Ref{DocStr}()
ref = RefValue{DocStr}()
for (n, arg) in enumerate(ex.args)
# The first `arg` to be documented needs to also create the docstring for the group.
# Subsequent `arg`s just need `ref` to be able to find the docstring without having
Expand Down
2 changes: 1 addition & 1 deletion base/docs/utils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Text / HTML objects

import Base: print, show, ==, hash
import .Base: print, show, ==, hash

export HTML, @html_str

Expand Down
4 changes: 3 additions & 1 deletion base/file.jl
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ export
unlink,
walkdir

import .Base.RefValue

# get and set current directory

"""
Expand All @@ -36,7 +38,7 @@ Get the current working directory.
"""
function pwd()
b = Vector{UInt8}(uninitialized, 1024)
len = Ref{Csize_t}(length(b))
len = RefValue{Csize_t}(length(b))
uv_error(:getcwd, ccall(:uv_cwd, Cint, (Ptr{UInt8}, Ptr{Csize_t}), b, len))
String(b[1:len[]])
end
Expand Down
4 changes: 2 additions & 2 deletions base/filesystem.jl
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@ export File,
S_IRGRP, S_IWGRP, S_IXGRP, S_IRWXG,
S_IROTH, S_IWOTH, S_IXOTH, S_IRWXO

import Base:
import .Base:
UVError, _sizeof_uv_fs, check_open, close, eof, eventloop, fd, isopen,
bytesavailable, position, read, read!, readavailable, seek, seekend, show,
skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error,
rawhandle, OS_HANDLE, INVALID_OS_HANDLE

if Sys.iswindows()
import Base: cwstring
import .Base: cwstring
end

include("path.jl")
Expand Down
4 changes: 2 additions & 2 deletions base/gmp.jl
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ module GMP

export BigInt

import Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), xor,
import .Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), xor,
binomial, cmp, convert, div, divrem, factorial, fld, gcd, gcdx, lcm, mod,
ndigits, promote_rule, rem, show, isqrt, string, powermod,
sum, trailing_zeros, trailing_ones, count_ones, base, tryparse_internal,
Expand Down Expand Up @@ -110,7 +110,7 @@ module MPZ
# - a method modifying its input has a "!" appendend to its name, according to Julia's conventions
# - some convenient methods are added (in addition to the pure MPZ ones), e.g. `add(a, b) = add!(BigInt(), a, b)`
# and `add!(x, a) = add!(x, x, a)`.
using Base.GMP: BigInt, Limb
using .Base.GMP: BigInt, Limb

const mpz_t = Ref{BigInt}
const bitcnt_t = Culong
Expand Down
2 changes: 1 addition & 1 deletion base/locks.jl
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

import Base: _uv_hook_close, unsafe_convert,
import .Base: _uv_hook_close, unsafe_convert,
lock, trylock, unlock, islocked

export SpinLock, RecursiveSpinLock, Mutex
Expand Down
6 changes: 3 additions & 3 deletions base/math.jl
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@ export sin, cos, sincos, tan, sinh, cosh, tanh, asin, acos, atan,
clamp, clamp!, modf, ^, mod2pi, rem2pi,
beta, lbeta, @evalpoly

import Base: log, exp, sin, cos, tan, sinh, cosh, tanh, asin,
import .Base: log, exp, sin, cos, tan, sinh, cosh, tanh, asin,
acos, atan, asinh, acosh, atanh, sqrt, log2, log10,
max, min, minmax, ^, exp2, muladd, rem,
exp10, expm1, log1p

using Base: sign_mask, exponent_mask, exponent_one,
using .Base: sign_mask, exponent_mask, exponent_one,
exponent_half, uinttype, significand_mask

using Core.Intrinsics: sqrt_llvm

using Base: IEEEFloat
using .Base: IEEEFloat

@noinline function throw_complex_domainerror(f, x)
throw(DomainError(x, string("$f will only return a complex result if called with a ",
Expand Down
16 changes: 8 additions & 8 deletions base/mpfr.jl
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export
setprecision

import
Base: *, +, -, /, <, <=, ==, >, >=, ^, ceil, cmp, convert, copysign, div,
.Base: *, +, -, /, <, <=, ==, >, >=, ^, ceil, cmp, convert, copysign, div,
inv, exp, exp2, exponent, factorial, floor, fma, hypot, isinteger,
isfinite, isinf, isnan, ldexp, log, log2, log10, max, min, mod, modf,
nextfloat, prevfloat, promote_rule, rem, rem2pi, round, show, float,
Expand All @@ -17,15 +17,15 @@ import
cosh, sinh, tanh, sech, csch, coth, acosh, asinh, atanh, atan2,
cbrt, typemax, typemin, unsafe_trunc, realmin, realmax, rounding,
setrounding, maxintfloat, widen, significand, frexp, tryparse, iszero,
isone, big, beta
isone, big, beta, RefValue

import Base.Rounding: rounding_raw, setrounding_raw
import .Base.Rounding: rounding_raw, setrounding_raw

import Base.GMP: ClongMax, CulongMax, CdoubleMax, Limb
import .Base.GMP: ClongMax, CulongMax, CdoubleMax, Limb

import Base.Math.lgamma_r
import .Base.Math.lgamma_r

import Base.FastMath.sincos_fast
import .Base.FastMath.sincos_fast

version() = VersionNumber(unsafe_string(ccall((:mpfr_get_version,:libmpfr), Ptr{Cchar}, ())))
patches() = split(unsafe_string(ccall((:mpfr_get_patches,:libmpfr), Ptr{Cchar}, ())),' ')
Expand All @@ -40,8 +40,8 @@ function __init__()
end
end

const ROUNDING_MODE = Ref{Cint}(0)
const DEFAULT_PRECISION = Ref(256)
const ROUNDING_MODE = RefValue{Cint}(0)
const DEFAULT_PRECISION = RefValue(256)

# Basic type and initialization definitions

Expand Down
10 changes: 5 additions & 5 deletions base/multidimensional.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@

### Multidimensional iterators
module IteratorsMD
import Base: eltype, length, size, start, done, next, first, last, in, getindex,
import .Base: eltype, length, size, start, done, next, first, last, in, getindex,
setindex!, IndexStyle, min, max, zero, one, isless, eachindex,
ndims, IteratorSize, convert, show

import Base: +, -, *
import Base: simd_outer_range, simd_inner_length, simd_index
using Base: IndexLinear, IndexCartesian, AbstractCartesianIndex, fill_to_length, tail
using Base.Iterators: Reverse
import .Base: +, -, *
import .Base: simd_outer_range, simd_inner_length, simd_index
using .Base: IndexLinear, IndexCartesian, AbstractCartesianIndex, fill_to_length, tail
using .Base.Iterators: Reverse

export CartesianIndex, CartesianIndices, LinearIndices

Expand Down
4 changes: 2 additions & 2 deletions base/multimedia.jl
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ export AbstractDisplay, display, pushdisplay, popdisplay, displayable, redisplay
# defined in sysimg.jl for bootstrapping:
# struct MIME{mime} end
# macro MIME_str(s)
import Base: MIME, @MIME_str
import Base: show, print, string, convert, repr
import .Base: MIME, @MIME_str
import .Base: show, print, string, convert, repr
MIME(s) = MIME{Symbol(s)}()
show(io::IO, ::MIME{mime}) where {mime} = print(io, "MIME type ", string(mime))
print(io::IO, ::MIME{mime}) where {mime} = print(io, mime)
Expand Down
2 changes: 1 addition & 1 deletion base/path.jl
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ Return the current user's home directory.
function homedir()
path_max = 1024
buf = Vector{UInt8}(uninitialized, path_max)
sz = Ref{Csize_t}(path_max + 1)
sz = RefValue{Csize_t}(path_max + 1)
while true
rc = ccall(:uv_os_homedir, Cint, (Ptr{UInt8}, Ptr{Csize_t}), buf, sz)
if rc == 0
Expand Down
16 changes: 9 additions & 7 deletions base/pcre.jl
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@

module PCRE

import ..RefValue

include(string(length(Core.ARGS) >= 2 ? Core.ARGS[2] : "", "pcre_h.jl")) # include($BUILDROOT/base/pcre_h.jl)

const PCRE_LIB = "libpcre2-8"

const JIT_STACK = Ref{Ptr{Cvoid}}(C_NULL)
const MATCH_CONTEXT = Ref{Ptr{Cvoid}}(C_NULL)
const JIT_STACK = RefValue{Ptr{Cvoid}}(C_NULL)
const MATCH_CONTEXT = RefValue{Ptr{Cvoid}}(C_NULL)

function __init__()
try
Expand Down Expand Up @@ -70,7 +72,7 @@ const OPTIONS_MASK = COMPILE_MASK | EXECUTE_MASK
const UNSET = ~Csize_t(0) # Indicates that an output vector element is unset

function info(regex::Ptr{Cvoid}, what::Integer, ::Type{T}) where T
buf = Ref{T}()
buf = RefValue{T}()
ret = ccall((:pcre2_pattern_info_8, PCRE_LIB), Int32,
(Ptr{Cvoid}, Int32, Ptr{Cvoid}),
regex, what, buf)
Expand All @@ -92,8 +94,8 @@ function get_ovec(match_data)
end

function compile(pattern::AbstractString, options::Integer)
errno = Ref{Cint}(0)
erroff = Ref{Csize_t}(0)
errno = RefValue{Cint}(0)
erroff = RefValue{Csize_t}(0)
re_ptr = ccall((:pcre2_compile_8, PCRE_LIB), Ptr{Cvoid},
(Ptr{UInt8}, Csize_t, UInt32, Ref{Cint}, Ref{Csize_t}, Ptr{Cvoid}),
pattern, sizeof(pattern), options, errno, erroff, C_NULL)
Expand Down Expand Up @@ -146,15 +148,15 @@ function substring_number_from_name(re, name)
end

function substring_length_bynumber(match_data, number)
s = Ref{Csize_t}()
s = RefValue{Csize_t}()
rc = ccall((:pcre2_substring_length_bynumber_8, PCRE_LIB), Cint,
(Ptr{Cvoid}, UInt32, Ref{Csize_t}), match_data, number, s)
rc < 0 && error("PCRE error: $(err_message(rc))")
convert(Int, s[])
end

function substring_copy_bynumber(match_data, number, buf, buf_size)
s = Ref{Csize_t}(buf_size)
s = RefValue{Csize_t}(buf_size)
rc = ccall((:pcre2_substring_copy_bynumber_8, PCRE_LIB), Cint,
(Ptr{Cvoid}, UInt32, Ptr{UInt8}, Ref{Csize_t}),
match_data, number, buf, s)
Expand Down
4 changes: 2 additions & 2 deletions base/printf.jl
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

module Printf
using Base.Grisu
using Base.GMP
using .Base.Grisu
using .Base.GMP

### printf formatter generation ###
const SmallFloatingPoint = Union{Float64,Float32,Float16}
Expand Down
53 changes: 29 additions & 24 deletions base/refpointer.jl
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,8 @@ end
RefValue(x::T) where {T} = RefValue{T}(x)
isassigned(x::RefValue) = isdefined(x, :x)

Ref(x::Any) = RefValue(x)
Ref{T}() where {T} = RefValue{T}() # Ref{T}()
Ref{T}(x) where {T} = RefValue{T}(x) # Ref{T}(x)
convert(::Type{Ref{T}}, x) where {T} = RefValue{T}(x)

Ref(x::Ref, i::Integer) = (i != 1 && error("Ref only has one element"); x)
Ref(x::Ptr{T}, i::Integer) where {T} = x + (i - 1) * Core.sizeof(T)

function unsafe_convert(P::Type{Ptr{T}}, b::RefValue{T}) where T
if datatype_pointerfree(RefValue{T})
p = pointer_from_objref(b)
Expand Down Expand Up @@ -87,7 +81,6 @@ end
RefArray(x::AbstractArray{T}, i::Int, roots::Any) where {T} = RefArray{T,typeof(x),Any}(x, i, roots)
RefArray(x::AbstractArray{T}, i::Int=1, roots::Nothing=nothing) where {T} = RefArray{T,typeof(x),Nothing}(x, i, nothing)
convert(::Type{Ref{T}}, x::AbstractArray{T}) where {T} = RefArray(x, 1)
Ref(x::AbstractArray, i::Integer) = RefArray(x, i)

function unsafe_convert(P::Type{Ptr{T}}, b::RefArray{T}) where T
if datatype_pointerfree(RefValue{T})
Expand All @@ -105,26 +98,38 @@ function unsafe_convert(P::Type{Ptr{Any}}, b::RefArray{Any})
end
unsafe_convert(::Type{Ptr{Cvoid}}, b::RefArray{T}) where {T} = convert(Ptr{Cvoid}, unsafe_convert(Ptr{T}, b))

# convert Arrays to pointer arrays for ccall
function Ref{P}(a::Array{<:Union{Ptr,Cwstring,Cstring}}) where P<:Union{Ptr,Cwstring,Cstring}
return RefArray(a) # effectively a no-op
end
function Ref{P}(a::Array{T}) where P<:Union{Ptr,Cwstring,Cstring} where T
if (!isbits(T) && T <: eltype(P))
# this Array already has the right memory layout for the requested Ref
return RefArray(a,1,false) # root something, so that this function is type-stable
else
ptrs = Vector{P}(uninitialized, length(a)+1)
roots = Vector{Any}(uninitialized, length(a))
for i = 1:length(a)
root = cconvert(P, a[i])
ptrs[i] = unsafe_convert(P, root)::P
roots[i] = root
###
if parentmodule(@__MODULE__) === Main
Ref(x::Any) = RefValue(x)
Ref{T}() where {T} = RefValue{T}() # Ref{T}()
Ref{T}(x) where {T} = RefValue{T}(x) # Ref{T}(x)

Ref(x::Ref, i::Integer) = (i != 1 && error("Ref only has one element"); x)
Ref(x::Ptr{T}, i::Integer) where {T} = x + (i - 1) * Core.sizeof(T)

# convert Arrays to pointer arrays for ccall
function Ref{P}(a::Array{<:Union{Ptr,Cwstring,Cstring}}) where P<:Union{Ptr,Cwstring,Cstring}
return RefArray(a) # effectively a no-op
end
function Ref{P}(a::Array{T}) where P<:Union{Ptr,Cwstring,Cstring} where T
if (!isbits(T) && T <: eltype(P))
# this Array already has the right memory layout for the requested Ref
return RefArray(a,1,false) # root something, so that this function is type-stable
else
ptrs = Vector{P}(uninitialized, length(a)+1)
roots = Vector{Any}(uninitialized, length(a))
for i = 1:length(a)
root = cconvert(P, a[i])
ptrs[i] = unsafe_convert(P, root)::P
roots[i] = root
end
ptrs[length(a)+1] = C_NULL
return RefArray(ptrs,1,roots)
end
ptrs[length(a)+1] = C_NULL
return RefArray(ptrs,1,roots)
end
Ref(x::AbstractArray, i::Integer) = RefArray(x, i)
end

cconvert(::Type{Ptr{P}}, a::Array{<:Ptr}) where {P<:Ptr} = a
cconvert(::Type{Ref{P}}, a::Array{<:Ptr}) where {P<:Ptr} = a
cconvert(::Type{Ptr{P}}, a::Array) where {P<:Union{Ptr,Cwstring,Cstring}} = Ref{P}(a)
Expand Down
20 changes: 10 additions & 10 deletions base/sort.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@

module Sort

using Base.Order, Base.Checked
using Base: copymutable, linearindices, IndexStyle, viewindexing, IndexLinear, _length

import
Base.sort,
Base.sort!,
Base.issorted,
Base.sortperm,
Base.Slice,
Base.to_indices
using .Base.Order, .Base.Checked
using .Base: copymutable, linearindices, IndexStyle, viewindexing, IndexLinear, _length

import .Base:
sort,
sort!,
issorted,
sortperm,
Slice,
to_indices

export # also exported by Base
# order-only:
Expand Down
Loading

0 comments on commit d774e40

Please sign in to comment.