replace scanAll to avoid creating a slice of tokens in memory

normalizer.go

@@ -100,7 +100,11 @@ func (n *Normalizer) Normalize(input string, lexerOpts ...lexerOption) (normaliz
 	var lastToken Token // The last token that is not whitespace or comment
 	var groupablePlaceholder groupablePlaceholder
 
-	for _, token := range lexer.ScanAll() {
+	for {
+		token := lexer.Scan()
+		if token.Type == EOF {
+			break
+		}
 		n.collectMetadata(&token, &lastToken, statementMetadata)
 		n.normalizeSQL(&token, &lastToken, &normalizedSQLBuilder, &groupablePlaceholder)
 	}

obfuscate_and_normalize.go

@@ -21,7 +21,11 @@ func ObfuscateAndNormalize(input string, obfuscator *Obfuscator, normalizer *Nor
 	var lastToken Token // The last token that is not whitespace or comment
 	var groupablePlaceholder groupablePlaceholder
 
-	for _, token := range lexer.ScanAll() {
+	for {
+		token := lexer.Scan()
+		if token.Type == EOF {
+			break
+		}
 		token.Value = obfuscator.ObfuscateTokenValue(token, lexerOpts...)
 		normalizer.collectMetadata(&token, &lastToken, statementMetadata)
 		normalizer.normalizeSQL(&token, &lastToken, &normalizedSQLBuilder, &groupablePlaceholder)

obfuscator.go

@@ -74,7 +74,11 @@ func (o *Obfuscator) Obfuscate(input string, lexerOpts ...lexerOption) string {
 		input,
 		lexerOpts...,
 	)
-	for _, token := range lexer.ScanAll() {
+	for {
+		token := lexer.Scan()
+		if token.Type == EOF {
+			break
+		}
 		obfuscatedSQL.WriteString(o.ObfuscateTokenValue(token, lexerOpts...))
 	}