Add public keyword (#320)

Add public as a contextual keyword that is parsed as a keyword only when it is both at the top-level and not followed by `(`, `=`, or `[`. Aside from this, the `public` keyword uses the same syntax as the `export` keyword and lowers analogously. Emit a warning when parsing `public` at the top-level followed by a `(`, `=`, or `[`.

Co-authored-by: Claire Foster <[email protected]>

src/kinds.jl

@@ -69,6 +69,7 @@ const _kind_names =
             "mutable"
             "outer"
             "primitive"
+            "public"
             "type"
             "var"
         "END_CONTEXTUAL_KEYWORDS"

src/parser.jl

@@ -482,7 +482,7 @@ end
 # flisp: parse-stmts
 function parse_stmts(ps::ParseState)
     mark = position(ps)
-    do_emit = parse_Nary(ps, parse_docstring, (K";",), (K"NewlineWs",))
+    do_emit = parse_Nary(ps, parse_public, (K";",), (K"NewlineWs",))
     # check for unparsed junk after an expression
     junk_mark = position(ps)
     while peek(ps) ∉ KSet"EndMarker NewlineWs"
@@ -499,6 +499,24 @@ function parse_stmts(ps::ParseState)
     end
 end
 
+# Parse `public foo, bar`
+#
+# We *only* call this from toplevel contexts (file and module level) for
+# compatibility. In the future we should probably make public a full fledged
+# keyword like `export`.
+function parse_public(ps::ParseState)
+    if ps.stream.version >= (1, 11) && peek(ps) == K"public"
+        if peek(ps, 2) ∈ KSet"( = ["
+            # this branch is for compatibility with use of public as a non-keyword.
+            # it should be removed at some point.
+            emit_diagnostic(ps, warning="using public as an identifier is deprecated")
+        else
+            return parse_resword(ps)
+        end
+    end
+    parse_docstring(ps)
+end
+
 # Parse docstrings attached by a space or single newline
 #
 # flisp: parse-docstring
@@ -1966,11 +1984,11 @@ function parse_resword(ps::ParseState)
         end
         # module A \n a \n b \n end  ==>  (module A (block a b))
         # module A \n "x"\na \n end  ==>  (module A (block (doc (string "x") a)))
-        parse_block(ps, parse_docstring)
+        parse_block(ps, parse_public)
         bump_closing_token(ps, K"end")
         emit(ps, mark, K"module",
              word == K"baremodule" ? BARE_MODULE_FLAG : EMPTY_FLAGS)
-    elseif word == K"export"
+    elseif word in KSet"export public"
         # export a         ==>  (export a)
         # export @a        ==>  (export @a)
         # export a, \n @b  ==>  (export a @b)
@@ -1979,7 +1997,7 @@ function parse_resword(ps::ParseState)
         # export \$a, \$(a*b) ==> (export (\$ a) (\$ (parens (call-i a * b))))
         bump(ps, TRIVIA_FLAG)
         parse_comma_separated(ps, x->parse_atsym(x, false))
-        emit(ps, mark, K"export")
+        emit(ps, mark, word)
     elseif word in KSet"import using"
         parse_imports(ps)
     elseif word == K"do"

src/tokenize.jl

@@ -1344,6 +1344,7 @@ K"let",
 K"local",
 K"macro",
 K"module",
+K"public",
 K"quote",
 K"return",
 K"struct",

test/diagnostics.jl

@@ -1,5 +1,5 @@
-function diagnostic(str; only_first=false, allow_multiple=false, rule=:all)
-    stream = ParseStream(str)
+function diagnostic(str; only_first=false, allow_multiple=false, rule=:all, version=v"1.6")
+    stream = ParseStream(str; version=version)
     parse!(stream, rule=rule)
     if allow_multiple
         stream.diagnostics
@@ -127,8 +127,12 @@ end
         Diagnostic(10, 13, :warning, "parentheses are not required here")
     @test diagnostic("export (x)") ==
         Diagnostic(8, 10, :warning, "parentheses are not required here")
-    @test diagnostic("export :x") == 
+    @test diagnostic("export :x") ==
         Diagnostic(8, 9, :error, "expected identifier")
+    @test diagnostic("public = 4", version=v"1.11") ==
+        diagnostic("public[7] = 5", version=v"1.11") ==
+        diagnostic("public() = 6", version=v"1.11") ==
+        Diagnostic(1, 6, :warning, "using public as an identifier is deprecated")
 end
 
 @testset "diagnostics for literal parsing" begin

test/parser.jl

@@ -17,13 +17,22 @@ function test_parse(production, input, output)
     else
         opts = NamedTuple()
     end
-    @test parse_to_sexpr_str(production, input; opts...) == output
+    parsed = parse_to_sexpr_str(production, input; opts...)
+    if output isa Regex # Could be AbstractPattern, but that type was added in Julia 1.6.
+        @test match(output, parsed) !== nothing
+    else
+        @test parsed == output
+    end
 end
 
 function test_parse(inout::Pair)
     test_parse(JuliaSyntax.parse_toplevel, inout...)
 end
 
+const PARSE_ERROR = r"\(error-t "
+
+with_version(v::VersionNumber, (i,o)::Pair) = ((;v=v), i) => o
+
 # TODO:
 # * Extract the following test cases from the source itself.
 # * Use only the green tree to generate the S-expressions
@@ -434,7 +443,7 @@ tests = [
         "x\"s\"in"   => """(macrocall @x_str (string-r "s") "in")"""
         "x\"s\"2"    => """(macrocall @x_str (string-r "s") 2)"""
         "x\"s\"10.0" => """(macrocall @x_str (string-r "s") 10.0)"""
-        # 
+        #
     ],
     JuliaSyntax.parse_resword => [
         # In normal_context
@@ -933,6 +942,26 @@ tests = [
         "10.0e1000'"  =>  "(ErrorNumericOverflow)"
         "10.0f100'"   =>  "(ErrorNumericOverflow)"
     ],
+    JuliaSyntax.parse_stmts => with_version.(v"1.11", [
+        "function f(public)\n    public + 3\nend"       => "(function (call f public) (block (call-i public + 3)))"
+        "public A, B"                                   => "(public A B)"
+        "if true \n public *= 4 \n end"                 => "(if true (block (*= public 4)))"
+        "module Mod\n public A, B \n end"               => "(module Mod (block (public A B)))"
+        "module Mod2\n a = 3; b = 6; public a, b\n end" => "(module Mod2 (block (= a 3) (= b 6) (public a b)))"
+        "a = 3; b = 6; public a, b"                     => "(toplevel-; (= a 3) (= b 6) (public a b))"
+        "begin \n public A, B \n end"                   => PARSE_ERROR
+        "if true \n public A, B \n end"                 => PARSE_ERROR
+        "public export=true foo, bar"                   => PARSE_ERROR # but these may be
+        "public experimental=true foo, bar"             => PARSE_ERROR # supported soon ;)
+        "public(x::String) = false"                     => "(= (call public (::-i x String)) false)"
+        "module M; export @a; end"                      => "(module M (block (export @a)))"
+        "module M; public @a; end"                      => "(module M (block (public @a)))"
+        "module M; export ⤈; end"                       => "(module M (block (export ⤈)))"
+        "module M; public ⤈; end"                       => "(module M (block (public ⤈)))"
+        "public = 4"                                    => "(= public 4)"
+        "public[7] = 5"                                 => "(= (ref public 7) 5)"
+        "public() = 6"                                  => "(= (call public) 6)"
+    ]),
     JuliaSyntax.parse_docstring => [
         """ "notdoc" ]        """ => "(string \"notdoc\")"
         """ "notdoc" \n]      """ => "(string \"notdoc\")"

test/tokenize.jl

@@ -932,6 +932,7 @@ const all_kws = Set([
     "local",
     "macro",
     "module",
+    "public",
     "quote",
     "return",
     "struct",