New/better macro pragmas, make some experimental

fix nim-lang#15920, close nim-lang#18212, close nim-lang#14781, close nim-lang#6696,
close nim-lang/RFCs#220

Variable macro pragmas have been changed to
only take a unary section node.
They can now also be applied in sections with multiple variables,
as well as `const` sections. They also accept arguments.

Templates now support macro pragmas, mirroring other routine types.

Type and variable macro pragmas have been made experimental.
Symbols without parentheses instatiating nullary macros or templates
has also been documented in the experimental manual.

A check for a redefinition error based on the left hand side of variable
definitions when using variable macro pragmas was disabled.
This nerfs `byaddr` specifically, however this has been documented as
a consequence of the experimental features `byaddr` uses.

Given how simple these changes are I'm worried if I'm missing something.

compiler/semstmts.nim

@@ -341,8 +341,12 @@ proc checkNilable(c: PContext; v: PSym) =
 
 #include liftdestructors
 
-proc addToVarSection(c: PContext; result: PNode; orig, identDefs: PNode) =
-  let value = identDefs[^1]
+proc addToVarSection(c: PContext; result: var PNode; n: PNode) =
+  if result.kind != nkStmtList:
+    result = makeStmtList(result)
+  result.add n
+
+proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   if result.kind == nkStmtList:
     let o = copyNode(orig)
     o.add identDefs
@@ -445,55 +449,111 @@ proc setVarType(c: PContext; v: PSym, typ: PType) =
         "; new type is: " & typeToString(typ, preferDesc))
   v.typ = typ
 
-proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode =
-  var b = a[0]
+proc copyExcept(n: PNode, i: int): PNode =
+  result = copyNode(n)
+  for j in 0..<n.len:
+    if j != i: result.add(n[j])
+
+proc semVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode =
+  result = nil
+  if a.len > 3:
+    # a, b {.prag.}: int = 3 not allowed
+    return
+  const lhsPos = 0
+  var b = a[lhsPos]
   if b.kind == nkPragmaExpr:
-    if b[1].len != 1:
-      # we could in future support pragmas w args e.g.: `var foo {.bar:"goo".} = expr`
-      return nil
-    let nodePragma = b[1][0]
-    # see: `singlePragma`
-
-    var amb = false
-    var sym: PSym = nil
-    case nodePragma.kind
-    of nkIdent, nkAccQuoted:
-      let ident = considerQuotedIdent(c, nodePragma)
-      var userPragma = strTableGet(c.userPragmas, ident)
-      if userPragma != nil: return nil
-      let w = nodePragma.whichPragma
-      if n.kind == nkVarSection and w in varPragmas or
-        n.kind == nkLetSection and w in letPragmas or
-        n.kind == nkConstSection and w in constPragmas:
-        return nil
-      sym = searchInScopes(c, ident, amb)
-      # XXX what if amb is true?
-      # CHECKME: should that test also apply to `nkSym` case?
-      if sym == nil or sfCustomPragma in sym.flags: return nil
-    of nkSym:
-      sym = nodePragma.sym
-    else:
-      return nil
-      # skip if not in scope; skip `template myAttr() {.pragma.}`
-
-    let lhs = b[0]
-    let clash = strTableGet(c.currentScope.symbols, lhs.ident)
-    if clash != nil:
-      # refs https://github.com/nim-lang/Nim/issues/8275
-      wrongRedefinition(c, lhs.info, lhs.ident.s, clash.info)
-
-    result = newTree(nkCall)
-    result.add nodePragma
-    result.add lhs
-    if a[1].kind != nkEmpty:
-      result.add a[1]
-    else:
-      result.add newNodeIT(nkNilLit, a.info, c.graph.sysTypes[tyNil])
-    result.add a[2]
-    result.info = a.info
-    let ret = newNodeI(nkStmtList, a.info)
-    ret.add result
-    result = semExprNoType(c, ret)
+    const
+      namePos = 0
+      pragmaPos = 1
+    let pragmas = b[pragmaPos]
+    for i in 0 ..< pragmas.len:
+      # Mirrored with semProcAnnotation
+
+      let it = pragmas[i]
+      let key = if it.kind in nkPragmaCallKinds and it.len >= 1: it[0] else: it
+
+      when false:
+        let lhs = b[0]
+        let clash = strTableGet(c.currentScope.symbols, lhs.ident)
+        if clash != nil:
+          # refs https://github.com/nim-lang/Nim/issues/8275
+          wrongRedefinition(c, lhs.info, lhs.ident.s, clash.info)
+
+      if whichPragma(it) != wInvalid:
+        # Not a custom pragma
+        continue
+      else:
+        let ident = considerQuotedIdent(c, key)
+        if strTableGet(c.userPragmas, ident) != nil:
+          continue # User defined pragma
+        else:
+          var amb = false
+          let sym = searchInScopes(c, ident, amb)
+          if sym != nil and sfCustomPragma in sym.flags:
+            continue # User custom pragma
+
+      # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and
+      # let the semantic checker deal with it:
+      var x = newNodeI(nkCall, key.info)
+      x.add(key)
+
+      if it.kind in nkPragmaCallKinds and it.len > 1:
+        # pass pragma arguments to the macro too:
+        for i in 1..<it.len:
+          x.add(it[i])
+
+      # Drop the pragma from the list, this prevents getting caught in endless
+      # recursion when the nkCall is semanticized
+      let oldExpr = a[lhsPos]
+      let newPragmas = copyExcept(pragmas, i)
+      if newPragmas.kind != nkEmpty and newPragmas.len == 0:
+        a[lhsPos] = oldExpr[namePos]
+      else:
+        a[lhsPos] = copyNode(oldExpr)
+        a[lhsPos].add(oldExpr[namePos])
+        a[lhsPos].add(newPragmas)
+
+      var unarySection = newNodeI(n.kind, a.info)
+      unarySection.add(a)
+      x.add(unarySection)
+
+      # recursion assures that this works for multiple macro annotations too:
+      var r = semOverloadedCall(c, x, x, {skMacro, skTemplate}, {efNoUndeclared})
+      if r == nil:
+        # Restore the old list of pragmas since we couldn't process this
+        a[lhsPos] = oldExpr
+        # No matching macro was found but there's always the possibility this may
+        # be a .pragma. template instead
+        continue
+
+      doAssert r[0].kind == nkSym
+      let m = r[0].sym
+      case m.kind
+      of skMacro: result = semMacroExpr(c, r, r, m, {})
+      of skTemplate: result = semTemplateExpr(c, r, m, {})
+      else:
+        a[lhsPos] = oldExpr
+        continue
+
+      doAssert result != nil
+
+      # since a macro pragma can set pragmas, we process these here again.
+      # This is required for SqueakNim-like export pragmas.
+      if result.kind in {nkVarSection, nkLetSection, nkConstSection}:
+        let validPragmas =
+          case result.kind
+          of nkVarSection: varPragmas
+          of nkLetSection: letPragmas
+          of nkConstSection: constPragmas
+          else: return # unreachable
+        for defs in result:
+          for i in 0 ..< defs.len - 2:
+            let ex = defs[i]
+            if ex.kind == nkPragmaExpr and
+              ex[namePos].kind == nkSym and
+              ex[pragmaPos].kind != nkEmpty:
+              pragma(c, defs[lhsPos][namePos].sym, defs[lhsPos][pragmaPos], validPragmas)
+      return
 
 proc errorSymChoiceUseQualifier(c: PContext; n: PNode) =
   assert n.kind in nkSymChoices
@@ -508,10 +568,6 @@ proc errorSymChoiceUseQualifier(c: PContext; n: PNode) =
   localError(c.config, n.info, errGenerated, err)
 
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
-  if n.len == 1:
-    result = semLowerLetVarCustomPragma(c, n[0], n)
-    if result != nil: return result
-
   var b: PNode
   result = copyNode(n)
   for i in 0..<n.len:
@@ -521,6 +577,11 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
     if a.kind notin {nkIdentDefs, nkVarTuple}: illFormedAst(a, c.config)
     checkMinSonsLen(a, 3, c.config)
 
+    b = semVarCustomPragma(c, a, n)
+    if b != nil:
+      addToVarSection(c, result, b)
+      continue
+
     var typ: PType = nil
     if a[^2].kind != nkEmpty:
       typ = semTypeNode(c, a[^2], nil)
@@ -663,13 +724,19 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
 proc semConst(c: PContext, n: PNode): PNode =
   result = copyNode(n)
   inc c.inStaticContext
+  var b: PNode
   for i in 0..<n.len:
     var a = n[i]
     if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
     if a.kind == nkCommentStmt: continue
     if a.kind notin {nkConstDef, nkVarTuple}: illFormedAst(a, c.config)
     checkMinSonsLen(a, 3, c.config)
 
+    b = semVarCustomPragma(c, a, n)
+    if b != nil:
+      addToVarSection(c, result, b)
+      continue
+
     var typ: PType = nil
     if a[^2].kind != nkEmpty:
       typ = semTypeNode(c, a[^2], nil)
@@ -704,7 +771,6 @@ proc semConst(c: PContext, n: PNode): PNode =
         typFlags.incl taConcept
       typeAllowedCheck(c, a.info, typ, skConst, typFlags)
 
-    var b: PNode
     if a.kind == nkVarTuple:
       if typ.kind != tyTuple:
         localError(c.config, a.info, errXExpected, "tuple")
@@ -735,7 +801,7 @@ proc semConst(c: PContext, n: PNode): PNode =
         v.ast = if def[j].kind != nkExprColonExpr: def[j]
                 else: def[j][1]
         b[j] = newSymNode(v)
-    result.add b
+    addToVarSection(c, result, n, b)
   dec c.inStaticContext
 
 include semfields
@@ -1519,16 +1585,13 @@ proc addResult(c: PContext, n: PNode, t: PType, owner: TSymKind) =
       n.add newSymNode(c.p.resultSym)
     addParamOrResult(c, c.p.resultSym, owner)
 
-proc copyExcept(n: PNode, i: int): PNode =
-  result = copyNode(n)
-  for j in 0..<n.len:
-    if j != i: result.add(n[j])
-
 proc semProcAnnotation(c: PContext, prc: PNode;
                        validPragmas: TSpecialWords): PNode =
   var n = prc[pragmasPos]
   if n == nil or n.kind == nkEmpty: return
   for i in 0..<n.len:
+    # Mirrored with semVarCustomPragma
+
     let it = n[i]
     let key = if it.kind in nkPragmaCallKinds and it.len >= 1: it[0] else: it
 

compiler/semtempl.nim

@@ -603,7 +603,12 @@ proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode =
     for i in 0..<n.len:
       result[i] = semTemplBodyDirty(c, n[i])
 
+# in semstmts.nim:
+proc semProcAnnotation(c: PContext, prc: PNode; validPragmas: TSpecialWords): PNode
+
 proc semTemplateDef(c: PContext, n: PNode): PNode =
+  result = semProcAnnotation(c, n, templatePragmas)
+  if result != nil: return result
   result = n
   var s: PSym
   if isTopLevel(c):

doc/manual.rst

@@ -7781,9 +7781,10 @@ More examples with custom pragmas:
 Macro pragmas
 -------------
 
-All macros and templates can also be used as pragmas. They can be attached
-to routines (procs, iterators, etc), type names, or type expressions. The
-compiler will perform the following simple syntactic transformations:
+Macros and templates can sometimes be called with the pragma syntax. Cases
+where this is possible include when attached to routine (procs, iterators, etc)
+declarations or routine type expressions. The compiler will perform the
+following simple syntactic transformations:
 
 .. code-block:: nim
   template command(name: string, def: untyped) = discard
@@ -7810,20 +7811,14 @@ This is translated to:
 
 ------
 
-.. code-block:: nim
-  type
-    MyObject {.schema: "schema.protobuf".} = object
-
-This is translated to a call to the `schema` macro with a `nnkTypeDef`
-AST node capturing both the left-hand side and right-hand side of the
-definition. The macro can return a potentially modified `nnkTypeDef` tree
-or multiple `nnkTypeDef` trees contained in a `nnkTypeSection` node
-which will replace the original row in the type section.
-
 When multiple macro pragmas are applied to the same definition, the
 compiler will apply them consequently from left to right. Each macro
 will receive as input the output of the previous one.
 
+There are a few more applications of macro pragmas, such as in type,
+variable and constant declarations, but this behavior is considered to be
+experimental and is documented in the `experimental manual
+<manual_experimental.html#extended-macro-pragmas>` instead.
 
 
 Foreign function interface

doc/manual_experimental.rst

@@ -404,6 +404,76 @@ to use this operator.
     doAssert (a.b)(c) == `()`(a.b, c)
 
 
+Extended macro pragmas
+======================
+
+Macro pragmas as described in `the manual <manual.html#userminusdefined-pragmas-macro-pragmas>`_
+can also be applied to type, variable and constant declarations.
+
+For types:
+
+.. code-block:: nim
+  type
+    MyObject {.schema: "schema.protobuf".} = object
+
+This is translated to a call to the `schema` macro with a `nnkTypeDef`
+AST node capturing the left-hand side, remaining pragmas and the right-hand
+side of the definition. The macro can return either a type section or
+another `nnkTypeDef` node, both of which will replace the original row
+in the type section.
+
+In the future, this `nnkTypeDef` argument may be replaced with a unary
+type section node containing the type definition, or some other node that may
+be more convenient to work with. The ability to return nodes other than type
+definitions may also be supported, however currently this is not convenient
+when dealing with mutual type recursion. For now, macros can return an unused
+type definition where the right-hand node is of kind `nnkStmtListType`.
+Declarations in this node will be attached to the same scope as
+the parent scope of the type section.
+
+------
+
+For variables and constants, it is largely the same, except a unary node with
+the same kind as the section containing a single definition is passed to macros,
+and macros can return any expression.
+
+.. code-block:: nim
+  var
+    a = ...
+    b {.importc, foo, nodecl.} = ...
+    c = ...
+
+Assuming `foo` is a macro or a template, this is roughly equivalent to:
+
+.. code-block:: nim
+  var a = ...
+  foo:
+    var b {.importc, nodecl.} = ...
+  var c = ...
+
+
+Symbols as template/macro calls
+===============================
+
+Templates and macros that take no arguments can be called as lone symbols,
+i.e. without parentheses. This is useful for repeated uses of complex
+expressions that cannot conveniently be represented as runtime values.
+
+.. code-block:: nim
+  type Foo = object
+    bar: int
+  
+  var foo = Foo(bar: 10)
+  template bar: untyped = foo.bar
+  assert bar == 10
+  bar = 15
+  assert bar == 15
+
+In the future, this may require more specific information on template or macro
+signatures to be used. Specializations for some applications of this may also
+be introduced to guarantee consistency and circumvent bugs.
+
+
 Not nil annotation
 ==================
 
@@ -613,7 +683,7 @@ has `source` as the owner. A path expression `e` is defined recursively:
 
 If a view type is used as a return type, the location must borrow from a location
 that is derived from the first parameter that is passed to the proc.
-See `the manual <https://nim-lang.org/docs/manual.html#procedures-var-return-type>`_
+See `the manual <manual.html#procedures-var-return-type>`_
 for details about how this is done for `var T`.
 
 A mutable view can borrow from a mutable location, an immutable view can borrow