decouple parser from PNode

compiler/ast/ast.nim

@@ -16,6 +16,7 @@ import
     ast_types, # Main ast type definitions
     ast_idgen, # Per module Id generation
     ast_query, # querying/reading the ast
+    ast_parsed_types, # Data types for the parsed node
   ],
   compiler/front/[
     options
@@ -29,7 +30,7 @@ import
     tables # For symbol table mapping
   ]
 
-export ast_types, ast_idgen, ast_query, int128
+export ast_types, ast_idgen, ast_query, int128, ast_parsed_types
 
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
 
@@ -604,3 +605,14 @@ proc toHumanStr*(kind: TSymKind): string =
 proc toHumanStr*(kind: TTypeKind): string =
   ## strips leading `tk`
   result = toHumanStrImpl(kind, 2)
+
+
+func toPNode*(n: ParsedNode): PNode =
+  result = newNodeI(n.kind, n.info)
+  case n.kind:
+    of nkFloatKinds: result.floatVal = n.token.fNumber
+    of nkIntKinds: result.intVal = n.token.iNumber
+    of nkStrKinds: result.strVal = n.token.literal
+    else:
+      for sub in items(n):
+        result.add sub.toPNode()

compiler/ast/ast_parsed_types.nim

@@ -0,0 +1,104 @@
+## Data structure for the parser results
+
+import
+  compiler/ast/[
+    ast_types, # For the node kinds
+    lexer # For the token type definition
+  ]
+
+# NOTE further refactoring considerations for the parser
+#
+# - store everything in tokens, do not require identifier interning for any
+#   purposes during the parsing stage, it must be done later, during
+#   conversion to a PNode. This will simplify some parts of the type
+#   definition.
+# - remove nim"pretty" - this is an absolute joke of implementation and
+#   it should not be placed where it is now.
+
+type
+  ParsedNode* = ref object
+    # NOTE next two fields are very large combined, but further plans will
+    # deal with that problem - current implementation is easier to write
+    # and it is just a transition point.
+    info*: TLineInfo # TODO replace line and separate token with index to
+                     # the token, which in turn will store information
+                     # about global positioning (tuple made up of a token
+                     # id and a file ID)
+                     #
+                     # NOTE technically this is not really necessary even
+                     # with the current implementation, but the parser
+                     # consistently copies this information around anyway,
+                     # so I will let it stay this way for now.
+    token*: Token # TODO Replace full token value with an index information
+    kind*: TNodeKind # NOTE/QUESTION - for now the same kind of nodes is
+                     # reused as the main parser, to ease the transition,
+                     # but in the future two different sets of node kinds
+                     # might(?) be introduced.
+
+    # TODO replace `ref` object tree with begin/end ranges for the nested
+    # trees in the linearized structure.
+    subnodes*: seq[ParsedNode]
+    comment*: string # TODO this should either be a token or a sequence of
+                     # tokens.
+
+    # HACK explicit flags in order to track down all 'extra' information
+    # that is collected during parsing.
+    isBlockArg*: bool # QUESTION add 'nkStmtListBlockArg' or similar node
+                      # and convert it to the `nkStmtList` + `nfBlocArg`
+                      # flags later on? Why do we need the `nfBlockArg`
+                      # flag in the first place?
+
+func len*(node: ParsedNode): int =
+  ## Number of the subnodes of a parsed node
+  return node.subnodes.len()
+
+func `[]`*(node: ParsedNode, idx: int | BackwardsIndex): ParsedNode =
+  return node.subnodes[idx]
+
+func `[]=`*(node: ParsedNode, idx: int | BackwardsIndex, other: ParsedNode) =
+  node.subnodes[idx] = other
+
+iterator items*(node: ParsedNode): ParsedNode =
+  for item in node.subnodes.items():
+    yield item
+
+proc add*(node: ParsedNode, other: ParsedNode) =
+  ## Add new element to the subnodes
+  node.subnodes.add(other)
+
+proc transitionSonsKind*(n: ParsedNode, kind: TNodeKind) =
+  n.kind = kind
+
+proc transitionIntKind*(n: ParsedNode, kind: TNodeKind) =
+  n.kind = kind
+
+proc transitionNoneToSym*(n: ParsedNode) =
+  n.kind = nkSym
+
+func newParsedNode*(kind: TNodeKind): ParsedNode =
+  ## Create a new parsed node without any location or token information
+  return ParsedNode(kind: kind)
+
+func newParsedNode*(
+  kind: TNodeKind, info: TLineInfo, subnodes: seq[ParsedNode] = @[]): ParsedNode =
+  ## Create a new non-leaf parsed node with a specified location
+  ## information and subnodes.
+  return ParsedNode(kind: kind, info: info, subnodes: subnodes)
+
+func newParsedNode*(kind: TNodeKind, info: TLineInfo, token: Token): ParsedNode =
+  ## Create a new leaf parsed node with the specified location information
+  ## and token kind.
+  return ParsedNode(kind: kind, info: info, token: token)
+
+
+proc newProcNode*(
+    kind: TNodeKind,
+    info: TLineInfo,
+    body, params, name, pattern, genericParams,
+    pragmas, exceptions: ParsedNode
+  ): ParsedNode =
+
+  result = newParsedNode(
+    kind,
+    info,
+    @[name, pattern, genericParams, params, pragmas, exceptions, body])

compiler/ast/ast_query.nim

@@ -97,6 +97,8 @@ const
   callableDefs* = nkLambdaKinds + routineDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
+  nkFloatKinds* = nkFloatLiterals # QUESTION remove float literals
+                                  # altogether?
   nkStrKinds* = {nkStrLit..nkTripleStrLit}
   nkIntKinds* = {nkCharLit .. nkUInt64Lit}