let/var, block, stmt list semantics fixed

- let and var section semantics clarified:
  - section normalization rules
  - macro pragma processing
  - nkError generation
  - normalize symbol pragma data storage
- block semantic analysis uses case/let style
- stmt list semantic analysis clarified:
  - collapses nested stmtLists, unless a block arg
- pragma block analysis less likely to break defer
- number of other fixes

let and var section semantics clarifications:
- section asts are normalized per identdef and var tuple
- macro pragmas are processed on the normalized ast
- macro pragmas receive one normalized section at a time
- then each normalized section is semantically analysed
- identifier related pragmas are processed here
- other pragmas remain and are processed on the symbol during codegen
- identdef symbols always have a pragmaexpr, even if empty

let and var section internal change remarks:
- semVarOrLet renamed to semLetOrVar
- uses nkError exclusively, resulting in some cascade changes
- production nodes (result) are now entirely distinct from input nodes
  this results in more allocations but it'll make DOD easier in the end
- introduced `ast.newTypeError` to create error types with the `n` field
  containing an nkError, establishing a tyError convention
- documented analysis in proc comment

cascade internal changes:
- vmgen and transf handling of let/var sections and normalized data
- `msgs.illformedAstReport` introduced to stop forced `localReport`

cascade changes template evaluation:
- template evaluation watches and reports nkErrors -- not ideal

cascade changes block semantic analysis:
- `semBlock`: now produces nkErrors
- production and input nodes are now separated
- added some docs

cascade changes stmtlist semantic analysis:
- `semStmtList`: now produces nkErrors
- production and input nodes are now separated
- flattens nested stmts, unless a `do:` `nfBlockArg`
- added some docs

cascade changes type sections:
- semTypeNode and semTypeOf/2 are becoming nk/tyError aware

cascade changes semPragmaBlock:
- pragmas blocks are less likely to screw up defer processing
- pragmas block items disappear after processing from the production
- updated `exception/tdefer1` test with a `{.line.}` pragma for this

miscellaneous changes:
- adding tracing to more sem procs (block, stmtlist, when, const expr, ...)
- added many more asserts and checks to clarify input expectations
- tracing output is more compact and also faster
- introduced `ast.newSymNode2` that is skError aware, future refactors
  will likely have this used nearly everywhere in the compiler
- slightly more precise error messages for let and var sections
- fixed bug in astrepr leading to NPE for type printing
- fixed bug so error nodes now generate node ids
- fixed bug in stdlib.macros to handle empty nkPragmaExprs

next steps for later:
- bring semConstSection in line with this section
- `nfBlockArg` is a hack, should be removed for a node
- error msg in `errmsgs/twrongcolon` wasn't good before not great now

compiler/ast/ast.nim

@@ -106,16 +106,21 @@ when false:
       echo v
 
 proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
-             info: TLineInfo; options: TOptions = {}): PSym =
+             info: TLineInfo, typ: PType; options: TOptions = {}): PSym =
   # generates a symbol and initializes the hash field too
   result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id,
-                options: options, owner: owner, offset: defaultOffset)
+                typ: typ, options: options, owner: owner, offset: defaultOffset)
   when false:
     if id.module == 48 and id.item == 39:
       writeStackTrace()
       echo "kind ", symKind, " ", name.s
       if owner != nil: echo owner.name.s
 
+proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
+             info: TLineInfo; options: TOptions = {}): PSym {.inline.} =
+  # generates a symbol and initializes the hash field too
+  result = newSym(symKind, name, id, owner, info, typ = nil, options)
+
 proc linkTo*(t: PType, s: PSym): PType {.discardable.} =
   t.sym = s
   s.typ = t
@@ -168,6 +173,33 @@ proc newIdentNode*(ident: PIdent, info: TLineInfo): PNode =
   result.ident = ident
   result.info = info
 
+proc newSymNode2*(sym: PSym): PNode =
+  ## creates a new `nkSym` node, unless sym.kind is an skError where an nkError
+  ## is extracted from the sym and returned instead.
+  # TODO replace newSymNode with this
+  if sym.isError:
+    result = sym.ast
+  else:
+    result = newNode(nkSym)
+    result.sym = sym
+    result.typ = sym.typ
+    result.info = sym.info
+
+proc newSymNode2*(sym: PSym, info: TLineInfo): PNode =
+  ## creates a new `nkSym` node, unless sym.kind is an skError where an nkError
+  ## is extracted from the sym and returned instead. In either case sets the
+  ## node info to the one provided
+
+  # TODO replace newSymNode with this
+  if sym.isError:
+    result = sym.ast
+    result.info = info
+  else:
+    result = newNode(nkSym)
+    result.sym = sym
+    result.typ = sym.typ
+    result.info = info
+
 proc newSymNode*(sym: PSym): PNode =
   result = newNode(nkSym)
   result.sym = sym
@@ -235,6 +267,17 @@ proc newProcNode*(kind: TNodeKind, info: TLineInfo, body: PNode,
   result.sons = @[name, pattern, genericParams, params,
                   pragmas, exceptions, body]
 
+proc newTypeError*(prev: PType,
+                   id: ItemId, 
+                   owner: PSym = if prev.isNil: nil else: prev.owner,
+                   err: PNode): PType =
+  ## create a new error type, with an optional `prev`ious type (can be nil) and
+  ## `err`or node for the error msg
+  result = PType(kind: tyError, owner: owner, size: defaultSize,
+                 align: defaultAlignment, itemId: id,
+                 lockLevel: UnspecifiedLockLevel, uniqueId: id, n: err)
+  result.typeInst = prev
+
 proc newType*(kind: TTypeKind, id: ItemId; owner: PSym): PType =
   result = PType(kind: kind, owner: owner, size: defaultSize,
                  align: defaultAlignment, itemId: id,

compiler/ast/ast_types.nim

@@ -346,6 +346,9 @@ const
   effectListLen* = 3    ## list of effects list
   nkLastBlockStmts* = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt}
                         ## these must be last statements in a block
+  nkVariableSections* = {nkLetSection, nkVarSection}
+                        # xxx: doesn't include const because const section
+                        #      analysis isn't unified with them
 
 type
   TTypeKind* = enum  # order is important!
@@ -971,6 +974,7 @@ type
                               ## for procs and tyGenericBody, it's the
                               ## formal param list
                               ## for concepts, the concept body
+                              ## for errors, nkError or nil if legacy
                               ## else: unused
     owner*: PSym              ## the 'owner' of the type
     sym*: PSym                ## types have the sym associated with them
@@ -982,7 +986,7 @@ type
     lockLevel*: TLockLevel    ## lock level as required for deadlock checking
     loc*: TLoc
     typeInst*: PType          ## for generic instantiations the tyGenericInst that led to this
-                              ## type.
+                              ## type; for tyError the previous type if avaiable
     uniqueId*: ItemId         ## due to a design mistake, we need to keep the real ID here as it
                               ## is required by the --incremental:on mode.
 

compiler/ast/errorhandling.nim

@@ -88,12 +88,12 @@ proc newError*(
   assert wrongNode != nil, "can't have a nil node for `wrongNode`"
   assert not report.isEmpty(), $report
 
-  result = PNode(
-    kind: nkError,
-    info: wrongNode.info,
-    typ: newType(tyError, ItemId(module: -2, item: -1), nil),
-    reportId: report
+  result = newNodeIT(
+    nkError,
+    wrongNode.info,
+    newType(tyError, ItemId(module: -2, item: -1), nil)
   )
+  result.reportId = report
 
   addInNimDebugUtilsError(conf, wrongNode, result)
 

compiler/front/cli_reporter.nim

@@ -3135,13 +3135,19 @@ proc reportBody*(conf: ConfigRef, r: DebugReport): string =
         )
 
       proc render(node: PNode): string =
-        conf.wrap(conf.treeRepr(node, indent = indent + 2))
+        conf.wrap(conf.treeRepr(node,
+                                indent = indent + 2,
+                                rconf = compilerTraceReprConf))
 
       proc render(typ: PType): string =
-        conf.wrap(conf.treeRepr(typ, indent = indent + 2))
+        conf.wrap(conf.treeRepr(typ,
+                                indent = indent + 2,
+                                rconf = compilerTraceReprConf))
 
       proc render(sym: PSym): string =
-        conf.wrap(conf.treeRepr(sym, indent = indent + 2))
+        conf.wrap(conf.treeRepr(sym,
+                                indent = indent + 2,
+                                rconf = compilerTraceReprConf))
 
       result.addf("$1]", align($s.level, 2, '#'))
       result.add(
@@ -3240,10 +3246,12 @@ proc reportBody*(conf: ConfigRef, r: DebugReport): string =
               if s.candidate.call.isNil:
                 field("mismatch kind", $s.candidate.error.firstMismatch.kind)
               else:
-                field("callee")
-                result.add render(s.candidate.callee)
-                field("calleeSym")
-                result.add render(s.candidate.calleeSym)
+                if s.candidate.calleeSym.isNil:
+                  field("callee")
+                  result.add render(s.candidate.callee)
+                else:
+                  field("calleeSym")
+                  result.add render(s.candidate.calleeSym)
                 field("call")
                 result.add render(s.candidate.call)
 

compiler/front/msgs.nim

@@ -572,12 +572,15 @@ proc semReportCountMismatch*(
   result = SemReport(kind: kind, ast: node)
   result.countMismatch = (toInt128(expected), toInt128(got))
 
+proc illformedAstReport*(node: PNode, explain: string): SemReport {.inline.} =
+  SemReport(kind: rsemIllformedAst, ast: node, str: explain)
+
 template semReportIllformedAst*(
     conf: ConfigRef, node: PNode, explain: string): untyped =
   handleReport(
     conf,
     wrap(
-      SemReport(kind: rsemIllformedAst, ast: node, str: explain),
+      illformedAstReport(node, explain),
       instLoc(),
       node.info),
     instLoc(),
@@ -612,6 +615,10 @@ template createSemIllformedAstMsg*(node: PNode,
 
   "Expected $1, but found $2" % [joinAnyOf(exp), $node.kind]
 
+proc illformedAstReport*(node: PNode, 
+                         expected: set[TNodeKind]): SemReport {.inline.} =
+  illformedAstReport(node, createSemIllformedAstMsg(node, expected))
+
 template semReportIllformedAst*(
   conf: ConfigRef, node: PNode, expected: set[TNodeKind]): untyped =
   semReportIllformedAst(conf, node, createSemIllformedAstMsg(node, expected))

compiler/sem/evaltempl.nim

@@ -17,7 +17,8 @@ import
     idents,
     renderer,
     reports,
-    errorhandling
+    errorhandling,
+    errorreporting
   ],
   compiler/front/[
     options,
@@ -197,6 +198,9 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
     ctx.instLines = sfCallsite in tmpl.flags
     result = copyNode(ctx, body, n)
     for i in 0..<body.safeLen:
+      # xxx: likely should simply emit the error and wrap the body
+      if body[i].kind == nkError:
+        conf.localReport(body[i])
       evalTemplateAux(body[i], args, ctx, result)
   result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, tmpl, result)

compiler/sem/pragmas.nim

@@ -37,7 +37,7 @@ import
   compiler/utils/[
     ropes,
     pathutils,
-    debugUtils
+    debugUtils,
   ],
   compiler/sem/[
     semdata,
@@ -984,40 +984,53 @@ proc markCompilerProc(c: PContext; s: PSym): PNode =
 proc deprecatedStmt(c: PContext; outerPragma: PNode): PNode =
   result = outerPragma
   let pragma = outerPragma[1]
+
   if pragma.kind in {nkStrLit..nkTripleStrLit}:
     incl(c.module.flags, sfDeprecated)
     c.module.constraint = getStrLitNode(c, outerPragma)
+
     if c.module.constraint.kind == nkError:
       result = wrapError(c.config, outerPragma)
+
     return
   elif pragma.kind != nkBracket:
     result = c.config.newError(pragma, reportStr(
       rsemBadDeprecatedArgs, "list of key:value pairs expected"))
+
     return
+
   for n in pragma:
     if n.kind in nkPragmaCallKinds and n.len == 2:
       let dest = qualifiedLookUp(c, n[1], {checkUndeclared})
+
       if dest == nil or dest.kind in routineKinds or dest.kind == skError:
         # xxx: warnings need to be figured out, also this is just silly, why
         #      are they unreliable?
         localReport(c.config, n.info, SemReport(kind: rsemUserWarning))
+
       let (src, err) = considerQuotedIdent(c, n[0])
+
       if err.isNil:
         let alias = newSym(skAlias, src, nextSymId(c.idgen), dest, n[0].info, c.config.options)
         incl(alias.flags, sfExported)
+
         if sfCompilerProc in dest.flags:
           let e = markCompilerProc(c, alias)
+
           if e != nil:
             result = e
             return
+
         addInterfaceDecl(c, alias)
         n[1] = newSymNode(dest)
       else:
         result = err
+
         return
     else:
       result = c.config.newError(n, reportStr(
         rsemBadDeprecatedArgs, "key:value pair expected"))
+
       return
 
 proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =

compiler/sem/sem.nim

@@ -117,6 +117,7 @@ proc semStaticType(c: PContext, childNode: PNode, prev: PType): PType
 proc semTypeOf(c: PContext; n: PNode): PNode
 proc computeRequiresInit(c: PContext, t: PType): bool
 proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo)
+proc defaultConstructionError2(c: PContext, t: PType, n: PNode): PNode
 proc hasUnresolvedArgs(c: PContext, n: PNode): bool
 proc isArrayConstr(n: PNode): bool {.inline.} =
   result = n.kind == nkBracket and
@@ -314,7 +315,8 @@ proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
 
 proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
   # like newSymS, but considers gensym'ed symbols
-  if n.kind == nkSym:
+  case n.kind
+  of nkSym:
     # and sfGenSym in n.sym.flags:
     result = n.sym
     if result.kind notin {kind, skTemp}:
@@ -329,11 +331,12 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
         result = copySym(result)
         result.ast = n.sym.ast
         put(c.p, n.sym, result)
+
     # when there is a nested proc inside a template, semtmpl
     # will assign a wrong owner during the first pass over the
     # template; we must fix it here: see #909
     result.owner = getCurrOwner(c)
-  else:
+  else: # xxx: should know the kinds and error out if not valid
     let (ident, err) = considerQuotedIdent(c, n)
     if err != nil:
       localReport(c.config, err)
@@ -447,18 +450,25 @@ proc tryConstExpr(c: PContext, n: PNode): PNode =
   c.config.m.errorOutputs = oldErrorOutputs
 
 proc semConstExpr(c: PContext, n: PNode): PNode =
+  addInNimDebugUtils(c.config, "semConstExpr", n, result)
+
   var e = semExprWithType(c, n)
   if e == nil:
     localReport(c.config, n.info, reportAst(rsemConstExprExpected, n))
 
     return n
+
   if e.kind in nkSymChoices and e[0].typ.skipTypes(abstractInst).kind == tyEnum:
     return e
+
   result = getConstExpr(c.module, e, c.idgen, c.graph)
+
   if result == nil:
     #if e.kind == nkEmpty: globalReport(n.info, errConstExprExpected)
     result = evalConstExpr(c.module, c.idgen, c.graph, e)
+
     assert result != nil
+
     case result.kind
     of {nkEmpty, nkError}:
       let withContext = e.info != n.info

compiler/sem/semdata.nim

@@ -43,6 +43,7 @@ import
   ],
   compiler/utils/[
     pathutils,
+    astrepr,
   ]
 
 export TExprFlag, TExprFlags
@@ -1036,6 +1037,7 @@ proc errorType*(c: PContext): PType =
   result.flags.incl tfCheckedForDestructor
 
 proc errorNode*(c: PContext, n: PNode): PNode =
+  # xxx: convert to `nkError` instead of `nkEmpty`
   result = newNodeI(nkEmpty, n.info)
   result.typ = errorType(c)
 
@@ -1135,14 +1137,20 @@ proc extractPragma(s: PSym): PNode =
 
 proc warnAboutDeprecated(conf: ConfigRef; info: TLineInfo; s: PSym) =
   var pragmaNode: PNode
-  pragmaNode = if s.kind == skEnumField: extractPragma(s.owner) else: extractPragma(s)
+  pragmaNode =
+    if s.kind == skEnumField:
+      extractPragma(s.owner)
+    else:
+      extractPragma(s)
+
   if pragmaNode != nil:
     for it in pragmaNode:
       if whichPragma(it) == wDeprecated and it.safeLen == 2 and
           it[1].kind in {nkStrLit..nkTripleStrLit}:
         localReport(conf, info, reportSym(
           rsemDeprecated, s, str = it[1].strVal))
         return
+
   localReport(conf, info, reportSym(rsemDeprecated, s))
 
 proc userError(conf: ConfigRef; info: TLineInfo; s: PSym) =