Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix #28990, improve top-level location info #30641

Merged
merged 1 commit into from
Jan 9, 2019
Merged

fix #28990, improve top-level location info #30641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions base/client.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,15 +146,30 @@ function eval_user_input(@nospecialize(ast), show_value::Bool)
nothing
end

function _parse_input_line_core(s::String, filename::String)
ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
s, sizeof(s), filename, sizeof(filename))
if ex isa Expr && ex.head === :toplevel
if isempty(ex.args)
return nothing
end
last = ex.args[end]
if last isa Expr && (last.head === :error || last.head === :incomplete)
# if a parse error happens in the middle of a multi-line input
# return only the error, so that none of the input is evaluated.
return last
end
end
return ex
end

function parse_input_line(s::String; filename::String="none", depwarn=true)
# For now, assume all parser warnings are depwarns
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think the parser has any deprecations, so this support code is dead and should be removed

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We might add more parser deprecations in the future.

ex = if depwarn
ccall(:jl_parse_input_line, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
s, sizeof(s), filename, sizeof(filename))
_parse_input_line_core(s, filename)
else
with_logger(NullLogger()) do
ccall(:jl_parse_input_line, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
s, sizeof(s), filename, sizeof(filename))
_parse_input_line_core(s, filename)
end
end
return ex
Expand Down
4 changes: 0 additions & 4 deletions base/meta.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,6 @@ function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool
if raise && isa(ex,Expr) && ex.head === :error
throw(ParseError(ex.args[1]))
end
if ex === ()
raise && throw(ParseError("end of input"))
ex = Expr(:error, "end of input")
end
return ex, pos+1 # C is zero-based, Julia is 1-based
end

Expand Down
54 changes: 43 additions & 11 deletions src/ast.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -763,20 +763,26 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v)
return julia_to_scm_noalloc2(fl_ctx, v);
}

// this is used to parse a line of repl input
JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const char *filename, size_t filename_len)
// parse an entire string like a file, reading multiple expressions
JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len, const char *filename, size_t filename_len)
{
JL_TIMING(PARSING);
jl_ast_context_t *ctx = jl_ast_ctx_enter();
fl_context_t *fl_ctx = &ctx->fl;
value_t s = cvalue_static_cstrn(fl_ctx, str, len);
value_t files = cvalue_static_cstrn(fl_ctx, filename, filename_len);
value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string")), s, files);
value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), s, files);
jl_value_t *res = e == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, e, NULL);
jl_ast_ctx_leave(ctx);
return res;
}

// for backwards compat
JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const char *filename, size_t filename_len)
{
return jl_parse_all(str, len, filename, filename_len);
}

// this is for parsing one expression out of a string, keeping track of
// the current position.
JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
Expand All @@ -792,7 +798,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
jl_ast_context_t *ctx = jl_ast_ctx_enter();
fl_context_t *fl_ctx = &ctx->fl;
value_t s = cvalue_static_cstrn(fl_ctx, str, len);
value_t p = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-parse-one-string")),
value_t p = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-parse-one")),
s, fixnum(pos0), greedy?fl_ctx->T:fl_ctx->F);
jl_value_t *expr=NULL, *pos1=NULL;
JL_GC_PUSH2(&expr, &pos1);
Expand Down Expand Up @@ -828,7 +834,7 @@ jl_value_t *jl_parse_eval_all(const char *fname,
JL_TIMING(PARSING);
value_t t = cvalue_static_cstrn(fl_ctx, content, contentlen);
fl_gc_handle(fl_ctx, &t);
ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f);
ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), t, f);
fl_free_gc_handles(fl_ctx, 1);
}
else {
Expand Down Expand Up @@ -869,9 +875,9 @@ jl_value_t *jl_parse_eval_all(const char *fname,
}
// expand non-final expressions in statement position (value unused)
expression =
fl_applyn(fl_ctx, 1,
fl_applyn(fl_ctx, 3,
symbol_value(symbol(fl_ctx, iscons(cdr_(ast)) ? "jl-expand-to-thunk-stmt" : "jl-expand-to-thunk")),
expression);
expression, symbol(fl_ctx, jl_filename), fixnum(jl_lineno));
}
jl_get_ptls_states()->world_age = jl_world_counter;
form = scm_to_julia(fl_ctx, expression, inmodule);
Expand Down Expand Up @@ -929,6 +935,21 @@ jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module
return result;
}

jl_value_t *jl_call_scm_on_ast_and_loc(const char *funcname, jl_value_t *expr, jl_module_t *inmodule,
const char *file, int line)
{
jl_ast_context_t *ctx = jl_ast_ctx_enter();
fl_context_t *fl_ctx = &ctx->fl;
JL_AST_PRESERVE_PUSH(ctx, old_roots, inmodule);
value_t arg = julia_to_scm(fl_ctx, expr);
value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, funcname)), arg,
symbol(fl_ctx, file), fixnum(line));
jl_value_t *result = scm_to_julia(fl_ctx, e, inmodule);
JL_AST_PRESERVE_POP(ctx, old_roots);
jl_ast_ctx_leave(ctx);
return result;
}

// syntax tree accessors

JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
Expand Down Expand Up @@ -1165,29 +1186,40 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
return expr;
}

JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule,
const char *file, int line)
{
JL_TIMING(LOWERING);
JL_GC_PUSH1(&expr);
expr = jl_copy_ast(expr);
expr = jl_expand_macros(expr, inmodule, NULL, 0);
expr = jl_call_scm_on_ast("jl-expand-to-thunk", expr, inmodule);
expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk", expr, inmodule, file, line);
JL_GC_POP();
return expr;
}

JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
{
return jl_expand_with_loc(expr, inmodule, "none", 0);
}

// expand in a context where the expression value is unused
JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule,
const char *file, int line)
{
JL_TIMING(LOWERING);
JL_GC_PUSH1(&expr);
expr = jl_copy_ast(expr);
expr = jl_expand_macros(expr, inmodule, NULL, 0);
expr = jl_call_scm_on_ast("jl-expand-to-thunk-stmt", expr, inmodule);
expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk-stmt", expr, inmodule, file, line);
JL_GC_POP();
return expr;
}

JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
{
return jl_expand_stmt_with_loc(expr, inmodule, "none", 0);
}

#ifdef __cplusplus
}
Expand Down
2 changes: 1 addition & 1 deletion src/jlapi.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str)
jl_value_t *r;
JL_TRY {
const char filename[] = "none";
jl_value_t *ast = jl_parse_input_line(str, strlen(str),
jl_value_t *ast = jl_parse_all(str, strlen(str),
filename, strlen(filename));
JL_GC_PUSH1(&ast);
r = jl_toplevel_eval_in(jl_main_module, ast);
Expand Down
71 changes: 29 additions & 42 deletions src/jlfrontend.scm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
;; return a lambda expression representing a thunk for a top-level expression
;; note: expansion of stuff inside module is delayed, so the contents obey
;; toplevel expansion order (don't expand until stuff before is evaluated).
(define (expand-toplevel-expr-- e)
(define (expand-toplevel-expr-- e file line)
(let ((ex0 (julia-expand-macroscope e)))
(if (and (pair? ex0) (eq? (car ex0) 'toplevel))
ex0
Expand All @@ -94,7 +94,8 @@
(scope-block
(block ,@(map (lambda (v) `(implicit-global ,v)) existing-gv)
,@(map (lambda (v) `(implicit-global ,v)) gv)
,ex))))))
,ex)))
file line)))
(if (and (null? (cdadr (caddr th)))
(and (length= (lam:body th) 2)
(let ((retval (cadadr (lam:body th))))
Expand All @@ -114,7 +115,7 @@
(and (eq? (car e) 'global) (every symbol? (cdr e))
(every (lambda (x) (not (memq x '(true false)))) (cdr e))))))

(define (expand-toplevel-expr e)
(define (expand-toplevel-expr e file line)
(cond ((or (atom? e) (toplevel-only-expr? e))
(if (underscore-symbol? e)
(error "all-underscore identifier used as rvalue"))
Expand All @@ -124,7 +125,7 @@
(if (not last)
(begin (reset-gensyms)
(set! *in-expand* #t)))
(begin0 (expand-toplevel-expr-- e)
(begin0 (expand-toplevel-expr-- e file line)
(set! *in-expand* last))))))

;; construct default definitions of `eval` for non-bare modules
Expand All @@ -146,10 +147,11 @@
(= (call include ,x)
(block
,@loc
(call (top include) ,name ,x)))))))
(call (top include) ,name ,x)))))
'none 0))

;; parse only, returning end position, no expansion.
(define (jl-parse-one-string s pos0 greedy)
;; parse one expression (if greedy) or atom, returning end position
(define (jl-parse-one s pos0 greedy)
(let ((inp (open-input-string s)))
(io.seek inp pos0)
(let ((expr (error-wrap (lambda ()
Expand All @@ -158,25 +160,7 @@
(julia-parse inp parse-atom))))))
(cons expr (io.pos inp)))))

(define (jl-parse-string s filename)
(with-bindings ((current-filename (symbol filename)))
(error-wrap (lambda ()
(let ((inp (make-token-stream (open-input-string s))))
;; parse all exprs into a (toplevel ...) form
(let loop ((exprs '()))
;; delay expansion so macros run in the Task executing
;; the input, not the task parsing it (issue #2378)
;; used to be (expand-toplevel-expr expr)
(let ((expr (julia-parse inp)))
(if (eof-object? expr)
(cond ((null? exprs) expr)
((length= exprs 1) (car exprs))
(else (cons 'toplevel (reverse! exprs))))
(if (and (pair? expr) (eq? (car expr) 'toplevel))
(loop (nreconc (cdr expr) exprs))
(loop (cons expr exprs)))))))))))

(define (jl-parse-all io filename)
(define (parse-all- io filename)
(unwind-protect
(with-bindings ((current-filename (symbol filename)))
(let ((stream (make-token-stream io)))
Expand All @@ -192,44 +176,47 @@
(julia-parse stream)))))
(if (eof-object? expr)
(cons 'toplevel (reverse! exprs))
(let* ((iserr (and (pair? expr) (eq? (car expr) 'error)))
(next (list* expr
;; for error, get most recent line number (#16720)
(if iserr
`(line ,(input-port-line io))
`(line ,lineno))
exprs)))
(let* ((iserr (and (pair? expr) (eq? (car expr) 'error)))
;; for error, get most recent line number (#16720)
(lineno (if iserr (input-port-line io) lineno))
(next (list* expr
;; include filename in first line node
(if (null? exprs)
`(line ,lineno ,(symbol filename))
`(line ,lineno))
exprs)))
(if iserr
(cons 'toplevel (reverse! next))
(loop next))))))))))
(io.close io)))

;; parse file-in-a-string
(define (jl-parse-string-stream str filename)
(jl-parse-all (open-input-string str) filename))
;; parse all expressions in a string, the same way files are parsed
(define (jl-parse-all str filename)
(parse-all- (open-input-string str) filename))

(define (jl-parse-file filename)
(trycatch
(jl-parse-all (open-input-file filename) filename)
(parse-all- (open-input-file filename) filename)
(lambda (e) #f)))

; expand a piece of raw surface syntax to an executable thunk
(define (jl-expand-to-thunk expr)
(define (jl-expand-to-thunk expr file line)
(error-wrap (lambda ()
(expand-toplevel-expr expr))))
(expand-toplevel-expr expr file line))))

(define (jl-expand-to-thunk-stmt expr)
(define (jl-expand-to-thunk-stmt expr file line)
(jl-expand-to-thunk (if (toplevel-only-expr? expr)
expr
`(block ,expr (null)))))
`(block ,expr (null)))
file line))

(define (jl-expand-macroscope expr)
(error-wrap (lambda ()
(julia-expand-macroscope expr))))

; run whole frontend on a string. useful for testing.
(define (fe str)
(expand-toplevel-expr (julia-parse str)))
(expand-toplevel-expr (julia-parse str) 'none 0))

(define (profile-e s)
(with-exception-catcher
Expand Down
Loading