diff --git a/Sources/Lexer.swift b/Sources/Lexer.swift index 5bd590da..d25cde37 100644 --- a/Sources/Lexer.swift +++ b/Sources/Lexer.swift @@ -5,18 +5,47 @@ struct Lexer { self.templateString = templateString } - func createToken(string: String) -> Token { - func strip() -> String { + private func whiteSpaceBehavior(string: String, tagLength: Int) -> WhitespaceBehavior { + func behavior(string: String) -> WhitespaceBehavior.Behavior { + switch string { + case "+": return .keep + case "-": return .trim + default: return .unspecified + } + } + let leftIndex = string.index(string.startIndex, offsetBy: tagLength, limitedBy: string.endIndex) + let rightIndex = string.index(string.endIndex, offsetBy: -(tagLength + 1), limitedBy: string.startIndex) + let leftIndicator = leftIndex.map { ind in string[ind...ind] } + let rightIndicator = rightIndex.map { ind in string[ind...ind] } + return WhitespaceBehavior( + leading: behavior(string: leftIndicator ?? ""), + trailing: behavior(string: rightIndicator ?? "") + ) + + } + private static let TagLength = 2 + func createToken(string:String) -> Token { + func strip(length: (Int, Int) = (Lexer.TagLength, Lexer.TagLength)) -> String { guard string.characters.count > 4 else { return "" } - let start = string.index(string.startIndex, offsetBy: 2) - let end = string.index(string.endIndex, offsetBy: -2) + let start = string.index(string.startIndex, offsetBy: length.0) + let end = string.index(string.endIndex, offsetBy: -length.1) return String(string[start.. Int { + switch b { + case .keep, .trim: + return 1 + case .unspecified: + return 0 + } + } if string.hasPrefix("{{") { return .variable(value: strip()) } else if string.hasPrefix("{%") { - return .block(value: strip()) + let behavior = whiteSpaceBehavior(string: string, tagLength: Lexer.TagLength) + let stripLengths = (Lexer.TagLength + additionalTagLength(b: behavior.leading),Lexer.TagLength + additionalTagLength(b: behavior.trailing)) + return .block(value: strip(length: stripLengths), newline: behavior) } else if string.hasPrefix("{#") { return .comment(value: strip()) } diff --git a/Sources/Node.swift b/Sources/Node.swift index 5b47177a..60a24d01 100644 --- a/Sources/Node.swift +++ b/Sources/Node.swift @@ -38,19 +38,48 @@ public class SimpleNode : NodeType { } } +#if os(Linux) + #if swift(>=3.1) + #else + typealias NSRegularExpression = RegularExpression + #endif +#endif + public class TextNode : NodeType { + private static let leadingWhiteSpace = try! NSRegularExpression(pattern: "^\\s+", options: []) + private static let trailingWhiteSpace = try! NSRegularExpression(pattern: "\\s+$", options: []) + public struct TrimBehavior { + let trimLeft: Bool + let trimRight: Bool + } public let text:String + public let trimBehavior:TrimBehavior - public init(text:String) { + public init(text:String, tBehavior:TrimBehavior = TrimBehavior(trimLeft: false, trimRight: false)) { self.text = text + self.trimBehavior = tBehavior } public func render(_ context:Context) throws -> String { - return self.text + var string = self.text + if trimBehavior.trimLeft { + let range = NSMakeRange(0, string.characters.count) + string = TextNode.leadingWhiteSpace.stringByReplacingMatches(in: string, options: [], range: range, withTemplate: "") + } + if trimBehavior.trimRight { + let range = NSMakeRange(0, string.characters.count) + string = TextNode.trailingWhiteSpace.stringByReplacingMatches(in: string, options: [], range: range, withTemplate: "") + } + return string } } +extension TextNode.TrimBehavior: Equatable {} +public func == (lhs: TextNode.TrimBehavior, rhs: TextNode.TrimBehavior) -> Bool { + return (lhs.trimLeft == rhs.trimLeft) && (lhs.trimRight == rhs.trimRight) +} + public protocol Resolvable { func resolve(_ context: Context) throws -> Any? diff --git a/Sources/Parser.swift b/Sources/Parser.swift index 1a59edba..d612aeaa 100644 --- a/Sources/Parser.swift +++ b/Sources/Parser.swift @@ -19,6 +19,7 @@ public class TokenParser { fileprivate var tokens: [Token] fileprivate let environment: Environment + private var previousWhiteSpace: WhitespaceBehavior.Behavior? public init(tokens: [Token], environment: Environment) { self.tokens = tokens @@ -38,10 +39,14 @@ public class TokenParser { switch token { case .text(let text): - nodes.append(TextNode(text: text)) + let leadingTrim = (previousWhiteSpace ?? .unspecified) == .trim + let trailingTrim = (peekWhitespace() ?? .unspecified) == .trim + let behavior = TextNode.TrimBehavior(trimLeft: leadingTrim, trimRight: trailingTrim) + nodes.append(TextNode(text: text, tBehavior: behavior)) case .variable: nodes.append(VariableNode(variable: try compileFilter(token.contents))) case .block: + previousWhiteSpace = token.whitespace?.trailing if let parse_until = parse_until , parse_until(self, token) { prependToken(token) return nodes @@ -67,6 +72,10 @@ public class TokenParser { return nil } + func peekWhitespace() -> WhitespaceBehavior.Behavior? { + return tokens.first?.whitespace?.leading + } + public func prependToken(_ token:Token) { tokens.insert(token, at: 0) } diff --git a/Sources/Tokenizer.swift b/Sources/Tokenizer.swift index 0e4bf1e9..440e94e3 100644 --- a/Sources/Tokenizer.swift +++ b/Sources/Tokenizer.swift @@ -40,6 +40,23 @@ extension String { } } +public struct WhitespaceBehavior: Equatable { + public enum Behavior { + case unspecified + case trim + case keep + } + let leading: Behavior + let trailing: Behavior + static func defaultBehavior() -> WhitespaceBehavior { + return WhitespaceBehavior(leading: .unspecified, trailing: .unspecified) + } +} + +public func == (lhs: WhitespaceBehavior, rhs: WhitespaceBehavior) -> Bool { + return (lhs.leading == rhs.leading) && (lhs.trailing == rhs.trailing) +} + public enum Token : Equatable { /// A token representing a piece of text. @@ -52,25 +69,20 @@ public enum Token : Equatable { case comment(value: String) /// A token representing a template block. - case block(value: String) + case block(value: String, newline: WhitespaceBehavior) /// Returns the underlying value as an array seperated by spaces public func components() -> [String] { - switch self { - case .block(let value): - return value.smartSplit() - case .variable(let value): - return value.smartSplit() - case .text(let value): - return value.smartSplit() - case .comment(let value): - return value.smartSplit() - } + return contents.smartSplit() + } + + public static func mkBlock(_ value: String) -> Token { + return .block(value: value, newline: WhitespaceBehavior.defaultBehavior()) } public var contents: String { switch self { - case .block(let value): + case .block(let value, _): return value case .variable(let value): return value @@ -80,6 +92,12 @@ public enum Token : Equatable { return value } } + public var whitespace: WhitespaceBehavior? { + switch self { + case .variable, .comment, .text: return nil + case .block(_, let ws): return ws + } + } } @@ -89,8 +107,8 @@ public func == (lhs: Token, rhs: Token) -> Bool { return lhsValue == rhsValue case (.variable(let lhsValue), .variable(let rhsValue)): return lhsValue == rhsValue - case (.block(let lhsValue), .block(let rhsValue)): - return lhsValue == rhsValue + case (.block(let lhsValue, let lhsBehavior), .block(let rhsValue, let rhsBehavior)): + return (lhsValue == rhsValue) && (lhsBehavior == rhsBehavior) case (.comment(let lhsValue), .comment(let rhsValue)): return lhsValue == rhsValue default: diff --git a/Tests/StencilTests/ForNodeSpec.swift b/Tests/StencilTests/ForNodeSpec.swift index 8804f0bf..b29f4aae 100644 --- a/Tests/StencilTests/ForNodeSpec.swift +++ b/Tests/StencilTests/ForNodeSpec.swift @@ -155,7 +155,7 @@ func testForNode() { $0.it("handles invalid input") { let tokens: [Token] = [ - .block(value: "for i"), + Token.mkBlock("for i"), ] let parser = TokenParser(tokens: tokens, environment: Environment()) let error = TemplateSyntaxError("'for' statements should use the following 'for x in y where condition' `for i`.") diff --git a/Tests/StencilTests/IfNodeSpec.swift b/Tests/StencilTests/IfNodeSpec.swift index c77122f9..2e69ebfb 100644 --- a/Tests/StencilTests/IfNodeSpec.swift +++ b/Tests/StencilTests/IfNodeSpec.swift @@ -7,9 +7,9 @@ func testIfNode() { $0.describe("parsing") { $0.it("can parse an if block") { let tokens: [Token] = [ - .block(value: "if value"), + Token.mkBlock("if value"), .text(value: "true"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -25,11 +25,11 @@ func testIfNode() { $0.it("can parse an if with else block") { let tokens: [Token] = [ - .block(value: "if value"), + Token.mkBlock("if value"), .text(value: "true"), - .block(value: "else"), + Token.mkBlock("else"), .text(value: "false"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -50,13 +50,13 @@ func testIfNode() { $0.it("can parse an if with elif block") { let tokens: [Token] = [ - .block(value: "if value"), + Token.mkBlock("if value"), .text(value: "true"), - .block(value: "elif something"), + Token.mkBlock("elif something"), .text(value: "some"), - .block(value: "else"), + Token.mkBlock("else"), .text(value: "false"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -81,11 +81,11 @@ func testIfNode() { $0.it("can parse an if with elif block without else") { let tokens: [Token] = [ - .block(value: "if value"), + Token.mkBlock("if value"), .text(value: "true"), - .block(value: "elif something"), + Token.mkBlock("elif something"), .text(value: "some"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -106,15 +106,15 @@ func testIfNode() { $0.it("can parse an if with multiple elif block") { let tokens: [Token] = [ - .block(value: "if value"), + Token.mkBlock("if value"), .text(value: "true"), - .block(value: "elif something1"), + Token.mkBlock("elif something1"), .text(value: "some1"), - .block(value: "elif something2"), + Token.mkBlock("elif something2"), .text(value: "some2"), - .block(value: "else"), + Token.mkBlock("else"), .text(value: "false"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -144,9 +144,9 @@ func testIfNode() { $0.it("can parse an if with complex expression") { let tokens: [Token] = [ - .block(value: "if value == \"test\" and not name"), + Token.mkBlock("if value == \"test\" and not name"), .text(value: "true"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -156,11 +156,11 @@ func testIfNode() { $0.it("can parse an ifnot block") { let tokens: [Token] = [ - .block(value: "ifnot value"), + Token.mkBlock("ifnot value"), .text(value: "false"), - .block(value: "else"), + Token.mkBlock("else"), .text(value: "true"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -180,7 +180,7 @@ func testIfNode() { $0.it("throws an error when parsing an if block without an endif") { let tokens: [Token] = [ - .block(value: "if value"), + Token.mkBlock("if value"), ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -190,7 +190,7 @@ func testIfNode() { $0.it("throws an error when parsing an ifnot without an endif") { let tokens: [Token] = [ - .block(value: "ifnot value"), + Token.mkBlock("ifnot value"), ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -242,9 +242,9 @@ func testIfNode() { $0.it("supports variable filters in the if expression") { let tokens: [Token] = [ - .block(value: "if value|uppercase == \"TEST\""), + Token.mkBlock("if value|uppercase == \"TEST\""), .text(value: "true"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) @@ -256,9 +256,9 @@ func testIfNode() { $0.it("evaluates nil properties as false") { let tokens: [Token] = [ - .block(value: "if instance.value"), + Token.mkBlock("if instance.value"), .text(value: "true"), - .block(value: "endif") + Token.mkBlock("endif") ] let parser = TokenParser(tokens: tokens, environment: Environment()) diff --git a/Tests/StencilTests/IncludeSpec.swift b/Tests/StencilTests/IncludeSpec.swift index 02978965..f3d586c4 100644 --- a/Tests/StencilTests/IncludeSpec.swift +++ b/Tests/StencilTests/IncludeSpec.swift @@ -11,7 +11,7 @@ func testInclude() { $0.describe("parsing") { $0.it("throws an error when no template is given") { - let tokens: [Token] = [ .block(value: "include") ] + let tokens: [Token] = [ Token.mkBlock("include") ] let parser = TokenParser(tokens: tokens, environment: Environment()) let error = TemplateSyntaxError("'include' tag takes one argument, the template file to be included") @@ -19,7 +19,7 @@ func testInclude() { } $0.it("can parse a valid include block") { - let tokens: [Token] = [ .block(value: "include \"test.html\"") ] + let tokens: [Token] = [ Token.mkBlock("include \"test.html\"") ] let parser = TokenParser(tokens: tokens, environment: Environment()) let nodes = try parser.parse() diff --git a/Tests/StencilTests/LexerSpec.swift b/Tests/StencilTests/LexerSpec.swift index ac9eb502..786ce558 100644 --- a/Tests/StencilTests/LexerSpec.swift +++ b/Tests/StencilTests/LexerSpec.swift @@ -54,7 +54,17 @@ func testLexer() { try expect(tokens[0]) == Token.variable(value: "thing") try expect(tokens[1]) == Token.variable(value: "name") } + $0.it("can tokenize whitespace control characters") { + let fBlock = "if hello" + let sBlock = "ta da" + let lexer = Lexer(templateString: "{%+ \(fBlock) -%}{% \(sBlock) -%}") + let tokens = lexer.tokenize() + let newLineBehaviors = (WhitespaceBehavior(leading: .keep, trailing: .trim), WhitespaceBehavior(leading: .unspecified, trailing: .trim)) + try expect(tokens.count) == 2 + try expect(tokens[0]) == Token.block(value: fBlock, newline: newLineBehaviors.0) + try expect(tokens[1]) == Token.block(value: sBlock, newline: newLineBehaviors.1) + } $0.it("can tokenize an unclosed block") { let lexer = Lexer(templateString: "{%}") let _ = lexer.tokenize() diff --git a/Tests/StencilTests/NodeSpec.swift b/Tests/StencilTests/NodeSpec.swift index 431d225d..0c8d528c 100644 --- a/Tests/StencilTests/NodeSpec.swift +++ b/Tests/StencilTests/NodeSpec.swift @@ -22,6 +22,21 @@ func testNode() { let node = TextNode(text: "Hello World") try expect(try node.render(context)) == "Hello World" } + $0.it("Trims leading whitespace") { + let text = " \nSome text " + let node = TextNode(text: text, tBehavior: TextNode.TrimBehavior(trimLeft: true, trimRight: false)) + try expect(try node.render(context)) == "Some text " + } + $0.it("Trims trailing whitespace") { + let text = " \nSome text " + let node = TextNode(text: text, tBehavior: TextNode.TrimBehavior(trimLeft: false, trimRight: true)) + try expect(try node.render(context)) == " \nSome text" + } + $0.it("Trims all whitespace") { + let text = " \nSome text " + let node = TextNode(text: text, tBehavior: TextNode.TrimBehavior(trimLeft: true, trimRight: true)) + try expect(try node.render(context)) == "Some text" + } } $0.describe("VariableNode") { diff --git a/Tests/StencilTests/NowNodeSpec.swift b/Tests/StencilTests/NowNodeSpec.swift index 4adba361..60706041 100644 --- a/Tests/StencilTests/NowNodeSpec.swift +++ b/Tests/StencilTests/NowNodeSpec.swift @@ -8,7 +8,7 @@ func testNowNode() { describe("NowNode") { $0.describe("parsing") { $0.it("parses default format without any now arguments") { - let tokens: [Token] = [ .block(value: "now") ] + let tokens: [Token] = [ Token.mkBlock("now") ] let parser = TokenParser(tokens: tokens, environment: Environment()) let nodes = try parser.parse() @@ -18,7 +18,7 @@ func testNowNode() { } $0.it("parses now with a format") { - let tokens: [Token] = [ .block(value: "now \"HH:mm\"") ] + let tokens: [Token] = [ Token.mkBlock("now \"HH:mm\"") ] let parser = TokenParser(tokens: tokens, environment: Environment()) let nodes = try parser.parse() let node = nodes.first as? NowNode diff --git a/Tests/StencilTests/ParserSpec.swift b/Tests/StencilTests/ParserSpec.swift index b5c9bb29..cb2e4797 100644 --- a/Tests/StencilTests/ParserSpec.swift +++ b/Tests/StencilTests/ParserSpec.swift @@ -44,7 +44,7 @@ func testTokenParser() { } let parser = TokenParser(tokens: [ - .block(value: "known"), + Token.mkBlock("known"), ], environment: Environment(extensions: [simpleExtension])) let nodes = try parser.parse() @@ -53,10 +53,30 @@ func testTokenParser() { $0.it("errors when parsing an unknown tag") { let parser = TokenParser(tokens: [ - .block(value: "unknown"), + Token.mkBlock("unknown"), ], environment: Environment()) try expect(try parser.parse()).toThrow(TemplateSyntaxError("Unknown template tag 'unknown'")) } + + $0.it("Can trim whitespace") { + + let simpleExtension = Extension() + simpleExtension.registerSimpleTag("known") { _ in + return "" + } + + let parser = TokenParser(tokens: [ + Token.block(value: "known", newline: WhitespaceBehavior(leading: .unspecified, trailing: .trim)), + Token.text(value: " \nSome text "), + Token.block(value: "known", newline: WhitespaceBehavior(leading: .keep, trailing: .trim)) + ], environment: Environment(extensions: [simpleExtension])) + + let nodes = try parser.parse() + try expect(nodes.count) == 3 + let textNode = nodes[1] as? TextNode + try expect(textNode?.text) == " \nSome text " + try expect(textNode?.trimBehavior) == TextNode.TrimBehavior(trimLeft: true, trimRight: false) + } } } diff --git a/Tests/StencilTests/TemplateSpec.swift b/Tests/StencilTests/TemplateSpec.swift index ad03851e..f87d0187 100644 --- a/Tests/StencilTests/TemplateSpec.swift +++ b/Tests/StencilTests/TemplateSpec.swift @@ -15,5 +15,20 @@ func testTemplate() { let result = try template.render([ "name": "Kyle" ]) try expect(result) == "Hello World" } + $0.it("Respects whitespace control symbols in for tags") { + let template: Template = "{% for num in numbers -%}\n {{num}}\n{%- endfor %}" + let result = try template.render([ "numbers": Array(1...9) ]) + try expect(result) == "123456789" + } + $0.it("Respects whitespace control symbols in if tags") { + let template: Template = "{% if value -%}\n {{text}}\n{%- endif %}" + let result = try template.render([ "text": "hello", "value": true ]) + try expect(result) == "hello" + } + $0.it("Respects whitespace control symbols in ifnot tags") { + let template: Template = "{% ifnot value %}{% else -%}\n {{text}}\n{%- endif %}" + let result = try template.render([ "text": "hello", "value": true ]) + try expect(result) == "hello" + } } }