Trim whitespace

CHANGELOG.md

@@ -18,6 +18,14 @@
   [Ilya Puchka](https://github.com/ilyapuchka)
   [#219](https://github.com/stencilproject/Stencil/issues/219)
   [#246](https://github.com/stencilproject/Stencil/pull/246)
+- Added support for trimming whitespace around blocks with Jinja2 whitespace control symbols. eg `{%- if value +%}`.  
+  [Miguel Bejar](https://github.com/bejar37)
+  [Yonas Kolb](https://github.com/yonaskolb)
+  [#92](https://github.com/stencilproject/Stencil/pull/92)
+  [#287](https://github.com/stencilproject/Stencil/pull/287)
+- Added support for adding default whitespace trimming behaviour to an environment.  
+  [Yonas Kolb](https://github.com/yonaskolb)
+  [#287](https://github.com/stencilproject/Stencil/pull/287)
 
 ### Deprecations
 

Sources/Stencil/Environment.swift

@@ -4,6 +4,8 @@ public struct Environment {
   public let templateClass: Template.Type
   /// List of registered extensions
   public var extensions: [Extension]
+  /// How to handle whitespace
+  public var trimBehaviour: TrimBehaviour
   /// Mechanism for loading new files
   public var loader: Loader?
 
@@ -13,14 +15,17 @@ public struct Environment {
   ///  - loader: Mechanism for loading new files
   ///  - extensions: List of extension containers
   ///  - templateClass: Class for newly loaded templates
+  ///  - trimBehaviour: How to handle whitespace
   public init(
     loader: Loader? = nil,
     extensions: [Extension] = [],
-    templateClass: Template.Type = Template.self
+    templateClass: Template.Type = Template.self,
+    trimBehaviour: TrimBehaviour = .nothing
   ) {
     self.templateClass = templateClass
     self.loader = loader
     self.extensions = extensions + [DefaultExtension()]
+    self.trimBehaviour = trimBehaviour
   }
 
   /// Load a template with the given name

Sources/Stencil/Lexer.swift

@@ -11,6 +11,9 @@ struct Lexer {
   /// `{` character, for example `{{`, `{%`, `{#`, ...
   private static let tokenChars: [Unicode.Scalar] = ["{", "%", "#"]
 
+  /// The minimum length of a tag
+  private static let tagLength = 2
+
   /// The token end characters, corresponding to their token start characters.
   /// For example, a variable token starts with `{{` and ends with `}}`
   private static let tokenCharMap: [Unicode.Scalar: Unicode.Scalar] = [
@@ -19,6 +22,12 @@ struct Lexer {
     "#": "#"
   ]
 
+  /// Characters controlling whitespace trimming behaviour
+  private static let behaviourMap: [Character: WhitespaceBehaviour.Behaviour] = [
+    "+": .keep,
+    "-": .trim
+  ]
+
   init(templateName: String? = nil, templateString: String) {
     self.templateName = templateName
     self.templateString = templateString
@@ -30,6 +39,16 @@ struct Lexer {
     }
   }
 
+  private func behaviour(string: String, tagLength: Int) -> WhitespaceBehaviour {
+    let leftIndex = string.index(string.startIndex, offsetBy: tagLength, limitedBy: string.endIndex)
+    let rightIndex = string.index(string.endIndex, offsetBy: -(tagLength + 1), limitedBy: string.startIndex)
+
+    return WhitespaceBehaviour(
+      leading: Self.behaviourMap[leftIndex.map { string[$0] } ?? " "] ?? .unspecified,
+      trailing: Self.behaviourMap[rightIndex.map { string[$0] } ?? " "] ?? .unspecified
+    )
+  }
+
   /// Create a token that will be passed on to the parser, with the given
   /// content and a range. The content will be tested to see if it's a
   /// `variable`, a `block` or a `comment`, otherwise it'll default to a simple
@@ -40,9 +59,9 @@ struct Lexer {
   ///   - range: The range within the template content, used for smart
   ///            error reporting
   func createToken(string: String, at range: Range<String.Index>) -> Token {
-    func strip() -> String {
-      guard string.count > 4 else { return "" }
-      let trimmed = String(string.dropFirst(2).dropLast(2))
+    func strip(length: (Int, Int) = (Self.tagLength, Self.tagLength)) -> String {
+      guard string.count > (length.0 + length.1) else { return "" }
+      let trimmed = String(string.dropFirst(length.0).dropLast(length.1))
         .components(separatedBy: "\n")
         .filter { !$0.isEmpty }
         .map { $0.trim(character: " ") }
@@ -51,15 +70,21 @@ struct Lexer {
     }
 
     if string.hasPrefix("{{") || string.hasPrefix("{%") || string.hasPrefix("{#") {
-      let value = strip()
+      let behaviour = string.hasPrefix("{%") ? behaviour(string: string, tagLength: Self.tagLength) : .unspecified
+      let stripLengths = (
+        Self.tagLength + (behaviour.leading != .unspecified ? 1 : 0),
+        Self.tagLength + (behaviour.trailing != .unspecified ? 1 : 0)
+      )
+
+      let value = strip(length: stripLengths)
       let range = templateString.range(of: value, range: range) ?? range
       let location = rangeLocation(range)
       let sourceMap = SourceMap(filename: templateName, location: location)
 
       if string.hasPrefix("{{") {
         return .variable(value: value, at: sourceMap)
       } else if string.hasPrefix("{%") {
-        return .block(value: value, at: sourceMap)
+        return .block(value: strip(length: stripLengths), at: sourceMap, whitespace: behaviour)
       } else if string.hasPrefix("{#") {
         return .comment(value: value, at: sourceMap)
       }

Sources/Stencil/Node.swift

@@ -41,14 +41,27 @@ public class SimpleNode: NodeType {
 public class TextNode: NodeType {
   public let text: String
   public let token: Token?
+  public let trimBehaviour: TrimBehaviour
 
-  public init(text: String) {
+  public init(text: String, trimBehaviour: TrimBehaviour = .nothing) {
     self.text = text
     self.token = nil
+    self.trimBehaviour = trimBehaviour
   }
 
   public func render(_ context: Context) throws -> String {
-    self.text
+    var string = self.text
+    if trimBehaviour.leading != .nothing, !string.isEmpty {
+      let range = NSRange(..<string.endIndex, in: string)
+      string = TrimBehaviour.leadingRegex(trim: trimBehaviour.leading)
+        .stringByReplacingMatches(in: string, options: [], range: range, withTemplate: "")
+    }
+    if trimBehaviour.trailing != .nothing, !string.isEmpty {
+      let range = NSRange(..<string.endIndex, in: string)
+      string = TrimBehaviour.trailingRegex(trim: trimBehaviour.trailing)
+        .stringByReplacingMatches(in: string, options: [], range: range, withTemplate: "")
+    }
+    return string
   }
 }
 

Sources/Stencil/Parser.swift

@@ -19,6 +19,7 @@ public class TokenParser {
 
   fileprivate var tokens: [Token]
   fileprivate let environment: Environment
+  fileprivate var previousWhiteSpace: WhitespaceBehaviour.Behaviour?
 
   /// Simple initializer
   public init(tokens: [Token], environment: Environment) {
@@ -41,10 +42,12 @@ public class TokenParser {
 
       switch token.kind {
       case .text:
-        nodes.append(TextNode(text: token.contents))
+        nodes.append(TextNode(text: token.contents, trimBehaviour: trimBehaviour))
       case .variable:
+        previousWhiteSpace = nil
         try nodes.append(VariableNode.parse(self, token: token))
       case .block:
+        previousWhiteSpace = token.whitespace?.trailing
         if let parseUntil = parseUntil, parseUntil(self, token) {
           prependToken(token)
           return nodes
@@ -60,6 +63,7 @@ public class TokenParser {
           }
         }
       case .comment:
+        previousWhiteSpace = nil
         continue
       }
     }
@@ -76,6 +80,10 @@ public class TokenParser {
     return nil
   }
 
+  func peekWhitespace() -> WhitespaceBehaviour.Behaviour? {
+    tokens.first?.whitespace?.leading
+  }
+
   /// Insert a token 
   public func prependToken(_ token: Token) {
     tokens.insert(token, at: 0)
@@ -95,6 +103,27 @@ public class TokenParser {
   public func compileResolvable(_ token: String, containedIn containingToken: Token) throws -> Resolvable {
     try environment.compileResolvable(token, containedIn: containingToken)
   }
+
+  private var trimBehaviour: TrimBehaviour {
+    var behaviour: TrimBehaviour = .nothing
+
+    if let leading = previousWhiteSpace {
+      if leading == .unspecified {
+        behaviour.leading = environment.trimBehaviour.trailing
+      } else {
+        behaviour.leading = leading == .trim ? .whitespaceAndNewLines : .nothing
+      }
+    }
+    if let trailing = peekWhitespace() {
+      if trailing == .unspecified {
+        behaviour.trailing = environment.trimBehaviour.leading
+      } else {
+        behaviour.trailing = trailing == .trim ? .whitespaceAndNewLines : .nothing
+      }
+    }
+
+    return behaviour
+  }
 }
 
 extension Environment {

Sources/Stencil/Tokenizer.swift

@@ -79,6 +79,19 @@ public struct SourceMap: Equatable {
   }
 }
 
+public struct WhitespaceBehaviour: Equatable {
+  public enum Behaviour {
+    case unspecified
+    case trim
+    case keep
+  }
+
+  let leading: Behaviour
+  let trailing: Behaviour
+
+  public static let unspecified = WhitespaceBehaviour(leading: .unspecified, trailing: .unspecified)
+}
+
 public class Token: Equatable {
   public enum Kind: Equatable {
     /// A token representing a piece of text.
@@ -94,14 +107,16 @@ public class Token: Equatable {
   public let contents: String
   public let kind: Kind
   public let sourceMap: SourceMap
+  public var whitespace: WhitespaceBehaviour?
 
   /// Returns the underlying value as an array seperated by spaces
   public private(set) lazy var components: [String] = self.contents.smartSplit()
 
-  init(contents: String, kind: Kind, sourceMap: SourceMap) {
+  init(contents: String, kind: Kind, sourceMap: SourceMap, whitespace: WhitespaceBehaviour? = nil) {
     self.contents = contents
     self.kind = kind
     self.sourceMap = sourceMap
+    self.whitespace = whitespace
   }
 
   /// A token representing a piece of text.
@@ -120,8 +135,12 @@ public class Token: Equatable {
   }
 
   /// A token representing a template block.
-  public static func block(value: String, at sourceMap: SourceMap) -> Token {
-    Token(contents: value, kind: .block, sourceMap: sourceMap)
+  public static func block(
+    value: String,
+    at sourceMap: SourceMap,
+    whitespace: WhitespaceBehaviour = .unspecified
+  ) -> Token {
+    Token(contents: value, kind: .block, sourceMap: sourceMap, whitespace: whitespace)
   }
 
   public static func == (lhs: Token, rhs: Token) -> Bool {

Sources/Stencil/TrimBehaviour.swift

@@ -0,0 +1,70 @@
+import Foundation
+
+public struct TrimBehaviour: Equatable {
+  var leading: Trim
+  var trailing: Trim
+
+  public enum Trim {
+    /// nothing
+    case nothing
+
+    /// tabs and spaces
+    case whitespace
+
+    /// tabs and spaces and a single new line
+    case whitespaceAndOneNewLine
+
+    /// all tabs spaces and newlines
+    case whitespaceAndNewLines
+  }
+
+  public init(leading: Trim, trailing: Trim) {
+    self.leading = leading
+    self.trailing = trailing
+  }
+
+  /// doesn't touch newlines
+  public static let nothing = TrimBehaviour(leading: .nothing, trailing: .nothing)
+
+  /// removes whitespace before a block and whitespace and a single newline after a block
+  public static let smart = TrimBehaviour(leading: .whitespace, trailing: .whitespaceAndOneNewLine)
+
+  /// removes all whitespace and newlines before and after a block
+  public static let all = TrimBehaviour(leading: .whitespaceAndNewLines, trailing: .whitespaceAndNewLines)
+
+  static func leadingRegex(trim: Trim) -> NSRegularExpression {
+    switch trim {
+    case .nothing:
+      fatalError("No RegularExpression for none")
+    case .whitespace:
+      return Self.leadingWhitespace
+    case .whitespaceAndOneNewLine:
+      return Self.leadingWhitespaceAndOneNewLine
+    case .whitespaceAndNewLines:
+      return Self.leadingWhitespaceAndNewlines
+    }
+  }
+
+  static func trailingRegex(trim: Trim) -> NSRegularExpression {
+    switch trim {
+    case .nothing:
+      fatalError("No RegularExpression for none")
+    case .whitespace:
+      return Self.trailingWhitespace
+    case .whitespaceAndOneNewLine:
+      return Self.trailingWhitespaceAndOneNewLine
+    case .whitespaceAndNewLines:
+      return Self.trailingWhitespaceAndNewLines
+    }
+  }
+
+  // swiftlint:disable force_try
+  private static let leadingWhitespaceAndNewlines = try! NSRegularExpression(pattern: "^\\s+")
+  private static let trailingWhitespaceAndNewLines = try! NSRegularExpression(pattern: "\\s+$")
+
+  private static let leadingWhitespaceAndOneNewLine = try! NSRegularExpression(pattern: "^[ \t]*\n")
+  private static let trailingWhitespaceAndOneNewLine = try! NSRegularExpression(pattern: "\n[ \t]*$")
+
+  private static let leadingWhitespace = try! NSRegularExpression(pattern: "^[ \t]*")
+  private static let trailingWhitespace = try! NSRegularExpression(pattern: "[ \t]*$")
+}