Merge #525

525: stdlib: escape strings of S-Expressions r=saem a=zerbina

## Summary
- use `strutils.escape` to escape the string when rendering an S-Expression via `toLines` and `treeRepr`
- make the S-Expression parser support `\x` escape sequences

Together, this allows the output of `treeRepr` and `toLines` to be parsed back into `SexpNode`s without losing information.

Fixes #263



Co-authored-by: zerbina <[email protected]>

lib/experimental/sexp.nim

@@ -485,9 +485,8 @@ proc treeRepr*(node: SexpNode): string =
       res.add " "
       res.add $node.getFNum()
     of SString:
-      res.add " \""
-      res.add node.getStr()
-      res.add "\""
+      res.add " "
+      res.add escapeJson(node.getStr())
     of SList:
       for item in node:
         res.add "\n"

lib/experimental/sexp_diff.nim

@@ -331,7 +331,7 @@ proc toLine*(s: SexpNode, sortfield: bool = false): ColText =
     if s.isNil: return
     case s.kind:
       of SInt:    add $s.getNum() + fgCyan
-      of SString: add ("\"" & s.getStr() & "\"") + fgYellow
+      of SString: add escapeJson(s.getStr()) + fgYellow
       of SFloat:  add $s.getFNum() + fgMagenta
       of SNil:    add "nil"
       of SSymbol: add s.getSymbol() + fgCyan

lib/experimental/sexp_parse.nim

@@ -125,6 +125,12 @@ proc parseString(parser: var SexpParser): TTokKind =
         if handleHexChar(parser.buf[pos], r): inc(pos)
         if handleHexChar(parser.buf[pos], r): inc(pos)
         add(parser.str, toUTF8(Rune(r)))
+      of 'x':
+        inc(pos, 2)
+        var r: int
+        if handleHexChar(parser.buf[pos], r): inc(pos)
+        if handleHexChar(parser.buf[pos], r): inc(pos)
+        add(parser.str, char(r))
       else:
         # don't bother with the error
         add(parser.str, parser.buf[pos])

tests/stdlib/algorithm/tsexp_diff.nim

@@ -222,6 +222,16 @@ block:
     eq d1.expected.getSymbol(), "R"
     eq d1.found.getSymbol(), "E"
 
+block string_roundtrip:
+  # a string containing control character must roundtrip through
+  # stringification
+  var str = newString(128) # only test characters in the ASCII range
+  for i, c in str.mpairs:
+    c = char(i)
+
+  let asText = toLine(newSString(str)).toString(false)
+  doAssert str == parseSexp(asText).getStr()
+
 if false:
   # Don't delete this section, it is used for print-debugging expected
   # formatting. And yes, 'if' is intentional as well - code needs to

tests/stdlib/experimental/tsexp.nim

@@ -55,6 +55,14 @@ suite "parse s-expressions - string atoms":
     let parsed = parseSexp("\"look at me \\\"escaping\\\"\"")
     check parsed == newSString("look at me \"escaping\"")
 
+  test "a '\\x' escape sequence encodes an ASCII character as a hexeadecimal number":
+    let parsed = parseSexp("\"the character 'a': \\x61\"")
+    check parsed == newSString("the character 'a': a")
+
+  test "a '\\u' escape sequence encodes a unicode character":
+    let parsed = parseSexp("\"the character 'a': \\u0061\"")
+    check parsed == newSString("the character 'a': a")
+
 suite "parse s-expressions - numeric atoms":
   test "integers are parsed into an integer cell":
     # xxx: not sure if BiggestInt should be part of the spec