[Review] Jinja2-like whitespace trim behavior

Sources/Lexer.swift

@@ -5,18 +5,47 @@ struct Lexer {
     self.templateString = templateString
   }
 
-  func createToken(string: String) -> Token {
-    func strip() -> String {
+  private func whiteSpaceBehavior(string: String, tagLength: Int) -> WhitespaceBehavior {
+    func behavior(string: String) -> WhitespaceBehavior.Behavior {
+      switch string {
+      case "+": return .keep
+      case "-": return .trim
+      default: return .unspecified
+      }
+    }
+    let leftIndex = string.index(string.startIndex, offsetBy: tagLength, limitedBy: string.endIndex)
+    let rightIndex = string.index(string.endIndex, offsetBy: -(tagLength + 1), limitedBy: string.startIndex)
+    let leftIndicator = leftIndex.map { ind in string[ind...ind] }
+    let rightIndicator = rightIndex.map { ind in string[ind...ind] }
+    return WhitespaceBehavior(
+      leading: behavior(string: leftIndicator ?? ""),
+      trailing: behavior(string: rightIndicator ?? "")
+    )
+
+  }
+  private static let TagLength = 2
+  func createToken(string:String) -> Token {
+    func strip(length: (Int, Int) = (Lexer.TagLength, Lexer.TagLength)) -> String {
       guard string.characters.count > 4 else { return "" }
-      let start = string.index(string.startIndex, offsetBy: 2)
-      let end = string.index(string.endIndex, offsetBy: -2)
+      let start = string.index(string.startIndex, offsetBy: length.0)
+      let end = string.index(string.endIndex, offsetBy: -length.1)
       return String(string[start..<end]).trim(character: " ")
     }
+    func additionalTagLength(b: WhitespaceBehavior.Behavior) -> Int {
+      switch b {
+      case .keep, .trim:
+        return 1
+      case .unspecified:
+        return 0
+      }
+    }
 
     if string.hasPrefix("{{") {
       return .variable(value: strip())
     } else if string.hasPrefix("{%") {
-      return .block(value: strip())
+      let behavior = whiteSpaceBehavior(string: string, tagLength: Lexer.TagLength)
+      let stripLengths = (Lexer.TagLength + additionalTagLength(b: behavior.leading),Lexer.TagLength +  additionalTagLength(b: behavior.trailing))
+      return .block(value: strip(length: stripLengths), newline: behavior)
     } else if string.hasPrefix("{#") {
       return .comment(value: strip())
     }

Sources/Node.swift

@@ -38,19 +38,48 @@ public class SimpleNode : NodeType {
   }
 }
 
+#if os(Linux)
+  #if swift(>=3.1)
+  #else
+    typealias NSRegularExpression = RegularExpression
+  #endif
+#endif
+
 
 public class TextNode : NodeType {
+  private static let leadingWhiteSpace = try! NSRegularExpression(pattern: "^\\s+", options: [])
+  private static let trailingWhiteSpace = try! NSRegularExpression(pattern: "\\s+$", options: [])
+  public struct TrimBehavior {
+    let trimLeft: Bool
+    let trimRight: Bool
+  }
   public let text:String
+  public let trimBehavior:TrimBehavior
 
-  public init(text:String) {
+  public init(text:String, tBehavior:TrimBehavior = TrimBehavior(trimLeft: false, trimRight: false)) {
     self.text = text
+    self.trimBehavior = tBehavior
   }
 
   public func render(_ context:Context) throws -> String {
-    return self.text
+    var string = self.text
+    if trimBehavior.trimLeft {
+      let range = NSMakeRange(0, string.characters.count)
+      string = TextNode.leadingWhiteSpace.stringByReplacingMatches(in: string, options: [], range: range, withTemplate: "")
+    }
+    if trimBehavior.trimRight {
+      let range = NSMakeRange(0, string.characters.count)
+      string = TextNode.trailingWhiteSpace.stringByReplacingMatches(in: string, options: [], range: range, withTemplate: "")
+    }
+    return string
   }
 }
 
+extension TextNode.TrimBehavior: Equatable {}
+public func == (lhs: TextNode.TrimBehavior, rhs: TextNode.TrimBehavior) -> Bool {
+  return (lhs.trimLeft == rhs.trimLeft) && (lhs.trimRight == rhs.trimRight)
+}
+
 
 public protocol Resolvable {
   func resolve(_ context: Context) throws -> Any?

Sources/Parser.swift

@@ -19,6 +19,7 @@ public class TokenParser {
 
   fileprivate var tokens: [Token]
   fileprivate let environment: Environment
+  private var previousWhiteSpace: WhitespaceBehavior.Behavior?
 
   public init(tokens: [Token], environment: Environment) {
     self.tokens = tokens
@@ -38,10 +39,14 @@ public class TokenParser {
 
       switch token {
       case .text(let text):
-        nodes.append(TextNode(text: text))
+        let leadingTrim = (previousWhiteSpace ?? .unspecified) == .trim
+        let trailingTrim = (peekWhitespace() ?? .unspecified) == .trim
+        let behavior = TextNode.TrimBehavior(trimLeft: leadingTrim, trimRight: trailingTrim)
+        nodes.append(TextNode(text: text, tBehavior: behavior))
       case .variable:
         nodes.append(VariableNode(variable: try compileFilter(token.contents)))
       case .block:
+        previousWhiteSpace = token.whitespace?.trailing
         if let parse_until = parse_until , parse_until(self, token) {
           prependToken(token)
           return nodes
@@ -67,6 +72,10 @@ public class TokenParser {
     return nil
   }
 
+  func peekWhitespace() -> WhitespaceBehavior.Behavior? {
+    return tokens.first?.whitespace?.leading
+  }
+
   public func prependToken(_ token:Token) {
     tokens.insert(token, at: 0)
   }

Sources/Tokenizer.swift

@@ -40,6 +40,23 @@ extension String {
   }
 }
 
+public struct WhitespaceBehavior: Equatable {
+  public enum Behavior {
+    case unspecified
+    case trim
+    case keep
+  }
+  let leading: Behavior
+  let trailing: Behavior
+  static func defaultBehavior() -> WhitespaceBehavior {
+    return WhitespaceBehavior(leading: .unspecified, trailing: .unspecified)
+  }
+}
+
+public func == (lhs: WhitespaceBehavior, rhs: WhitespaceBehavior) -> Bool {
+    return (lhs.leading == rhs.leading) && (lhs.trailing == rhs.trailing)
+}
+
 
 public enum Token : Equatable {
   /// A token representing a piece of text.
@@ -52,25 +69,20 @@ public enum Token : Equatable {
   case comment(value: String)
 
   /// A token representing a template block.
-  case block(value: String)
+  case block(value: String, newline: WhitespaceBehavior)
 
   /// Returns the underlying value as an array seperated by spaces
   public func components() -> [String] {
-    switch self {
-    case .block(let value):
-      return value.smartSplit()
-    case .variable(let value):
-      return value.smartSplit()
-    case .text(let value):
-      return value.smartSplit()
-    case .comment(let value):
-      return value.smartSplit()
-    }
+    return contents.smartSplit()
+  }
+
+  public static func mkBlock(_ value: String) -> Token {
+    return .block(value: value, newline: WhitespaceBehavior.defaultBehavior())
   }
 
   public var contents: String {
     switch self {
-    case .block(let value):
+    case .block(let value, _):
       return value
     case .variable(let value):
       return value
@@ -80,6 +92,12 @@ public enum Token : Equatable {
       return value
     }
   }
+  public var whitespace: WhitespaceBehavior? {
+    switch self {
+    case .variable, .comment, .text: return nil
+    case .block(_, let ws): return ws
+    }
+  }
 }
 
 
@@ -89,8 +107,8 @@ public func == (lhs: Token, rhs: Token) -> Bool {
     return lhsValue == rhsValue
   case (.variable(let lhsValue), .variable(let rhsValue)):
     return lhsValue == rhsValue
-  case (.block(let lhsValue), .block(let rhsValue)):
-    return lhsValue == rhsValue
+  case (.block(let lhsValue, let lhsBehavior), .block(let rhsValue, let rhsBehavior)):
+    return (lhsValue == rhsValue) && (lhsBehavior == rhsBehavior)
   case (.comment(let lhsValue), .comment(let rhsValue)):
     return lhsValue == rhsValue
   default:

Tests/StencilTests/ForNodeSpec.swift

@@ -155,7 +155,7 @@ func testForNode() {
 
     $0.it("handles invalid input") {
       let tokens: [Token] = [
-          .block(value: "for i"),
+          Token.mkBlock("for i"),
       ]
       let parser = TokenParser(tokens: tokens, environment: Environment())
       let error = TemplateSyntaxError("'for' statements should use the following 'for x in y where condition' `for i`.")

Tests/StencilTests/IfNodeSpec.swift

@@ -7,9 +7,9 @@ func testIfNode() {
     $0.describe("parsing") {
       $0.it("can parse an if block") {
         let tokens: [Token] = [
-          .block(value: "if value"),
+          Token.mkBlock("if value"),
           .text(value: "true"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -25,11 +25,11 @@ func testIfNode() {
 
       $0.it("can parse an if with else block") {
         let tokens: [Token] = [
-          .block(value: "if value"),
+          Token.mkBlock("if value"),
           .text(value: "true"),
-          .block(value: "else"),
+          Token.mkBlock("else"),
           .text(value: "false"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -50,13 +50,13 @@ func testIfNode() {
 
       $0.it("can parse an if with elif block") {
         let tokens: [Token] = [
-          .block(value: "if value"),
+          Token.mkBlock("if value"),
           .text(value: "true"),
-          .block(value: "elif something"),
+          Token.mkBlock("elif something"),
           .text(value: "some"),
-          .block(value: "else"),
+          Token.mkBlock("else"),
           .text(value: "false"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -81,11 +81,11 @@ func testIfNode() {
 
       $0.it("can parse an if with elif block without else") {
         let tokens: [Token] = [
-          .block(value: "if value"),
+          Token.mkBlock("if value"),
           .text(value: "true"),
-          .block(value: "elif something"),
+          Token.mkBlock("elif something"),
           .text(value: "some"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -106,15 +106,15 @@ func testIfNode() {
 
       $0.it("can parse an if with multiple elif block") {
         let tokens: [Token] = [
-          .block(value: "if value"),
+          Token.mkBlock("if value"),
           .text(value: "true"),
-          .block(value: "elif something1"),
+          Token.mkBlock("elif something1"),
           .text(value: "some1"),
-          .block(value: "elif something2"),
+          Token.mkBlock("elif something2"),
           .text(value: "some2"),
-          .block(value: "else"),
+          Token.mkBlock("else"),
           .text(value: "false"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -144,9 +144,9 @@ func testIfNode() {
 
       $0.it("can parse an if with complex expression") {
         let tokens: [Token] = [
-          .block(value: "if value == \"test\" and not name"),
+          Token.mkBlock("if value == \"test\" and not name"),
           .text(value: "true"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -156,11 +156,11 @@ func testIfNode() {
 
       $0.it("can parse an ifnot block") {
         let tokens: [Token] = [
-          .block(value: "ifnot value"),
+          Token.mkBlock("ifnot value"),
           .text(value: "false"),
-          .block(value: "else"),
+          Token.mkBlock("else"),
           .text(value: "true"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -180,7 +180,7 @@ func testIfNode() {
 
       $0.it("throws an error when parsing an if block without an endif") {
         let tokens: [Token] = [
-          .block(value: "if value"),
+          Token.mkBlock("if value"),
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -190,7 +190,7 @@ func testIfNode() {
 
       $0.it("throws an error when parsing an ifnot without an endif") {
         let tokens: [Token] = [
-            .block(value: "ifnot value"),
+            Token.mkBlock("ifnot value"),
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -242,9 +242,9 @@ func testIfNode() {
 
     $0.it("supports variable filters in the if expression") {
         let tokens: [Token] = [
-          .block(value: "if value|uppercase == \"TEST\""),
+          Token.mkBlock("if value|uppercase == \"TEST\""),
           .text(value: "true"),
-          .block(value: "endif")
+          Token.mkBlock("endif")
         ]
 
         let parser = TokenParser(tokens: tokens, environment: Environment())
@@ -256,9 +256,9 @@ func testIfNode() {
 
     $0.it("evaluates nil properties as false") {
       let tokens: [Token] = [
-        .block(value: "if instance.value"),
+        Token.mkBlock("if instance.value"),
         .text(value: "true"),
-        .block(value: "endif")
+        Token.mkBlock("endif")
       ]
 
       let parser = TokenParser(tokens: tokens, environment: Environment())

Tests/StencilTests/IncludeSpec.swift

@@ -11,15 +11,15 @@ func testInclude() {
 
     $0.describe("parsing") {
       $0.it("throws an error when no template is given") {
-        let tokens: [Token] = [ .block(value: "include") ]
+        let tokens: [Token] = [ Token.mkBlock("include") ]
         let parser = TokenParser(tokens: tokens, environment: Environment())
 
         let error = TemplateSyntaxError("'include' tag takes one argument, the template file to be included")
         try expect(try parser.parse()).toThrow(error)
       }
 
       $0.it("can parse a valid include block") {
-        let tokens: [Token] = [ .block(value: "include \"test.html\"") ]
+        let tokens: [Token] = [ Token.mkBlock("include \"test.html\"") ]
         let parser = TokenParser(tokens: tokens, environment: Environment())
 
         let nodes = try parser.parse()