Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Coherent map and comprehension, the imminent death of type_goto #11034

Closed
wants to merge 1 commit into from
Closed

Coherent map and comprehension, the imminent death of type_goto #11034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 27 additions & 15 deletions base/abstractarray.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1345,14 +1345,20 @@ function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray)
return dest
end

freeze!{T}(A::Array{T,1}) = ccall(:jl_array_freeze, Void, (Any,), A)
freeze!(::AbstractArray) = nothing

function map(f, A::AbstractArray)
if isempty(A)
return isa(f,Type) ? similar(A,f) : similar(A)
dest = isa(f,Type) ? similar(A,f) : similar(A,Union())
else
first = f(A[1])
dest = similar(A, typeof(first))
dest[1] = first
dest = map_to!(f, 2, dest, A)
end
first = f(A[1])
dest = similar(A, typeof(first))
dest[1] = first
return map_to!(f, 2, dest, A)
freeze!(dest)
dest
end

## 2 argument
Expand Down Expand Up @@ -1382,12 +1388,15 @@ end
function map(f, A::AbstractArray, B::AbstractArray)
shp = promote_shape(size(A),size(B))
if prod(shp) == 0
return similar(A, promote_type(eltype(A),eltype(B)), shp)
dest = similar(A, Union(), shp)
else
first = f(A[1], B[1])
dest = similar(A, typeof(first), shp)
dest[1] = first
dest = map_to!(f, 2, dest, A, B)
end
first = f(A[1], B[1])
dest = similar(A, typeof(first), shp)
dest[1] = first
return map_to!(f, 2, dest, A, B)
freeze!(dest)
dest
end

## N argument
Expand Down Expand Up @@ -1422,12 +1431,15 @@ end
function map(f, As::AbstractArray...)
shape = mapreduce(size, promote_shape, As)
if prod(shape) == 0
return similar(As[1], promote_eltype(As...), shape)
dest = similar(As[1], Union(), shape)
else
first = f(map(a->a[1], As)...)
dest = similar(As[1], typeof(first), shape)
dest[1] = first
dest = map_to!(f, 2, dest, As...)
end
first = f(map(a->a[1], As)...)
dest = similar(As[1], typeof(first), shape)
dest[1] = first
return map_to!(f, 2, dest, As...)
freeze!(dest)
dest
end

# multi-item push!, unshift! (built on top of type-specific 1-item version)
Expand Down
16 changes: 9 additions & 7 deletions base/array.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,15 +305,15 @@ function getindex(A::Array, I::UnitRange{Int})
return X
end

function getindex{T<:Real}(A::Array, I::AbstractVector{T})
return [ A[i] for i in to_index(I) ]
function getindex{T<:Real,V}(A::Array{V}, I::AbstractVector{T})
return V[ A[i] for i in to_index(I) ]
end
function getindex{T<:Real}(A::Range, I::AbstractVector{T})
return [ A[i] for i in to_index(I) ]
function getindex{T<:Real,V}(A::Range{V}, I::AbstractVector{T})
return V[ A[i] for i in to_index(I) ]
end
function getindex(A::Range, I::AbstractVector{Bool})
function getindex{V}(A::Range{V}, I::AbstractVector{Bool})
checkbounds(A, I)
return [ A[i] for i in to_index(I) ]
return V[ A[i] for i in to_index(I) ]
end


Expand Down Expand Up @@ -505,7 +505,9 @@ end

function push!{T}(a::Array{T,1}, item)
# convert first so we don't grow the array if the assignment won't work
item = convert(T, item)
if T !== Union()
item = convert(T, item)
end
ccall(:jl_array_grow_end, Void, (Any, UInt), a, 1)
a[end] = item
return a
Expand Down
20 changes: 10 additions & 10 deletions base/sparse/sparsematrix.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2232,8 +2232,8 @@ end

function vcat(X::SparseMatrixCSC...)
num = length(X)
mX = [ size(x, 1) for x in X ]
nX = [ size(x, 2) for x in X ]
mX = Int[ size(x, 1) for x in X ]
nX = Int[ size(x, 2) for x in X ]
m = sum(mX)
n = nX[1]

Expand All @@ -2250,7 +2250,7 @@ function vcat(X::SparseMatrixCSC...)
Ti = promote_type(Ti, eltype(X[i].rowval))
end

nnzX = [ nnz(x) for x in X ]
nnzX = Int[ nnz(x) for x in X ]
nnz_res = sum(nnzX)
colptr = Array(Ti, n + 1)
rowval = Array(Ti, nnz_res)
Expand Down Expand Up @@ -2292,8 +2292,8 @@ end

function hcat(X::SparseMatrixCSC...)
num = length(X)
mX = [ size(x, 1) for x in X ]
nX = [ size(x, 2) for x in X ]
mX = Int[ size(x, 1) for x in X ]
nX = Int[ size(x, 2) for x in X ]
m = mX[1]
for i = 2 : num
if mX[i] != m; throw(DimensionMismatch("")); end
Expand All @@ -2304,7 +2304,7 @@ function hcat(X::SparseMatrixCSC...)
Ti = promote_type(map(x->eltype(x.rowval), X)...)

colptr = Array(Ti, n + 1)
nnzX = [ nnz(x) for x in X ]
nnzX = Int[ nnz(x) for x in X ]
nnz_res = sum(nnzX)
rowval = Array(Ti, nnz_res)
nzval = Array(Tv, nnz_res)
Expand Down Expand Up @@ -2342,16 +2342,16 @@ end

function blkdiag(X::SparseMatrixCSC...)
num = length(X)
mX = [ size(x, 1) for x in X ]
nX = [ size(x, 2) for x in X ]
mX = Int[ size(x, 1) for x in X ]
nX = Int[ size(x, 2) for x in X ]
m = sum(mX)
n = sum(nX)

Tv = promote_type(map(x->eltype(x.nzval), X)...)
Ti = promote_type(map(x->eltype(x.rowval), X)...)

colptr = Array(Ti, n + 1)
nnzX = [ nnz(x) for x in X ]
nnzX = Int[ nnz(x) for x in X ]
nnz_res = sum(nnzX)
rowval = Array(Ti, nnz_res)
nzval = Array(Tv, nnz_res)
Expand Down Expand Up @@ -2519,7 +2519,7 @@ function trace{Tv}(A::SparseMatrixCSC{Tv})
s
end

diag(A::SparseMatrixCSC) = [d for d in SpDiagIterator(A)]
diag{Tv}(A::SparseMatrixCSC{Tv}) = Tv[d for d in SpDiagIterator(A)]

function diagm{Tv,Ti}(v::SparseMatrixCSC{Tv,Ti})
if (size(v,1) != 1 && size(v,2) != 1)
Expand Down
14 changes: 13 additions & 1 deletion src/array.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
JL_GC_POP();
}
a->pooled = tsz <= GC_MAX_SZCLASS;

a->fixed_len = 0;
a->data = data;
if (elsz == 1) ((char*)data)[tot-1] = '\0';
#ifdef STORE_ARRAY_LEN
Expand Down Expand Up @@ -154,6 +154,7 @@ jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_value_t *di
a = (jl_array_t*)allocobj(tsz);
jl_set_typeof(a, atype);
a->pooled = tsz <= GC_MAX_SZCLASS;
a->fixed_len = 0;
a->ndims = ndims;
a->offset = 0;
a->data = NULL;
Expand Down Expand Up @@ -222,6 +223,7 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel,
a = (jl_array_t*)allocobj(tsz);
jl_set_typeof(a, atype);
a->pooled = tsz <= GC_MAX_SZCLASS;
a->fixed_len = 0;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
Expand Down Expand Up @@ -273,6 +275,7 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *dims,
a = (jl_array_t*)allocobj(tsz);
jl_set_typeof(a, atype);
a->pooled = tsz <= GC_MAX_SZCLASS;
a->fixed_len = 0;
a->data = data;
#ifdef STORE_ARRAY_LEN
a->length = nel;
Expand Down Expand Up @@ -526,6 +529,11 @@ void jl_arrayunset(jl_array_t *a, size_t i)
memset(ptail, 0, a->elsize);
}

void jl_array_freeze(jl_array_t *a)
{
a->fixed_len = 1;
}

// at this size and bigger, allocate resized array data with malloc
#define MALLOC_THRESH 1048576

Expand All @@ -540,6 +548,10 @@ static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, siz
size_t offsnb = offs * es;
size_t oldnbytes = oldlen * es;
size_t oldoffsnb = a->offset * es;

if (a->fixed_len) {
jl_error("cannot resize array of fixed length");
}
if (es == 1)
nbytes++;
assert(!a->isshared || a->how==3);
Expand Down
1 change: 1 addition & 0 deletions src/julia-syntax.scm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2158,6 +2158,7 @@
,@lengths))
(= ,ri 1)
,(construct-loops (reverse ranges) (reverse rv) is states (reverse lengths))
,@(if atype '() `((call (top freeze!) ,result)))
,result))))))

(define (lower-dict-comprehension expr ranges)
Expand Down
4 changes: 3 additions & 1 deletion src/julia.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ typedef struct {
size_t length;
#endif

unsigned short ndims:10;
unsigned short ndims:9;
unsigned short pooled:1;
unsigned short ptrarray:1; // representation is pointer array
/*
Expand All @@ -157,6 +157,7 @@ typedef struct {
unsigned short how:2;
unsigned short isshared:1; // data is shared by multiple Arrays
unsigned short isaligned:1; // data allocated with memalign
unsigned short fixed_len:1; // data cannot be resized (1d only)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we set isshared instead, which I think also prevents growing?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure. I did this quickly mostly to see how much of Base would break. Turns out not too much. I'm not sure how much people would miss dimensions 512-1023 but no reason to waste bits.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might want to rename the field to resizeable then since it will no longer actually mean shared. The error message can say that the array either has shared data or came from a map or comprehension.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

827 is my all-time favorite array dimensionality. Still, I imagine I'll somehow make it through.

uint16_t elsize;
uint32_t offset; // for 1-d only. does not need to get big.

Expand Down Expand Up @@ -1040,6 +1041,7 @@ DLLEXPORT void jl_array_del_beg(jl_array_t *a, size_t dec);
DLLEXPORT void jl_array_sizehint(jl_array_t *a, size_t sz);
DLLEXPORT void jl_cell_1d_push(jl_array_t *a, jl_value_t *item);
DLLEXPORT jl_value_t *jl_apply_array_type(jl_datatype_t *type, size_t dim);
DLLEXPORT void jl_array_freeze(jl_array_t*);
// property access
DLLEXPORT void *jl_array_ptr(jl_array_t *a);
DLLEXPORT void *jl_array_eltype(jl_value_t *a);
Expand Down