parser: don't cut off end-of-line comments

Previously, the parser (actually the scanner) removed comments that came
at the end of a statement from the raw sql that was returned along with
the parsed AST. This behavior is now removed.

Release note: None

pkg/sql/parser/parse.go

@@ -167,7 +167,6 @@ func (p *Parser) scanOneStmt() (sql string, tokens []sqlSymType, done bool) {
 			return p.scanner.In()[startPos:], tokens, true
 		}
 		preValID = lval.id
-		posBeforeScan := p.scanner.Pos()
 		tokens = append(tokens, sqlSymType{})
 		lval = &tokens[len(tokens)-1]
 		p.scanner.Scan(lval)
@@ -179,8 +178,13 @@ func (p *Parser) scanOneStmt() (sql string, tokens []sqlSymType, done bool) {
 			curFuncBodyCnt--
 		}
 		if lval.id == 0 || (curFuncBodyCnt == 0 && lval.id == ';') {
+			endPos := p.scanner.Pos()
+			if lval.id == ';' {
+				// Don't include the ending semicolon, if there is one, in the raw SQL.
+				endPos--
+			}
 			tokens = tokens[:len(tokens)-1]
-			return p.scanner.In()[startPos:posBeforeScan], tokens, (lval.id == 0)
+			return p.scanner.In()[startPos:endPos], tokens, (lval.id == 0)
 		}
 		lval.pos -= startPos
 	}

pkg/sql/parser/parse_internal_test.go

@@ -44,7 +44,7 @@ func TestScanOneStmt(t *testing.T) {
 		{
 			sql: `SELECT 1 /* comment */  ; /* comment */  ; /* comment */ `,
 			exp: []stmt{{
-				sql: `SELECT 1`,
+				sql: `SELECT 1 /* comment */  `,
 				tok: []int{SELECT, ICONST},
 			}},
 		},
@@ -73,11 +73,11 @@ func TestScanOneStmt(t *testing.T) {
 			sql: ` ; /* x */ SELECT 1  ; SET /* y */ ; ;  INSERT INTO table;  ;`,
 			exp: []stmt{
 				{
-					sql: `SELECT 1`,
+					sql: `SELECT 1  `,
 					tok: []int{SELECT, ICONST},
 				},
 				{
-					sql: `SET`,
+					sql: `SET /* y */ `,
 					tok: []int{SET},
 				},
 				{

pkg/sql/parser/parse_test.go

@@ -639,10 +639,12 @@ func TestParseSQL(t *testing.T) {
 		{in: ``, exp: nil},
 		{in: `SELECT 1`, exp: []string{`SELECT 1`}},
 		{in: `SELECT 1;`, exp: []string{`SELECT 1`}},
-		{in: `SELECT 1 /* comment */`, exp: []string{`SELECT 1`}},
+		// We currently chop off beginning-of-line comments.
+		{in: `/* comment */ SELECT 1`, exp: []string{`SELECT 1`}},
+		{in: `SELECT 1 /* comment */`, exp: []string{`SELECT 1 /* comment */`}},
 		{in: `SELECT 1;SELECT 2`, exp: []string{`SELECT 1`, `SELECT 2`}},
-		{in: `SELECT 1 /* comment */ ;SELECT 2`, exp: []string{`SELECT 1`, `SELECT 2`}},
-		{in: `SELECT 1 /* comment */ ; /* comment */ SELECT 2`, exp: []string{`SELECT 1`, `SELECT 2`}},
+		{in: `SELECT 1 /* comment */ ;SELECT 2`, exp: []string{`SELECT 1 /* comment */ `, `SELECT 2`}},
+		{in: `SELECT 1 /* comment */ ; SELECT 2`, exp: []string{`SELECT 1 /* comment */ `, `SELECT 2`}},
 	}
 	var p parser.Parser // Verify that the same parser can be reused.
 	for _, d := range testData {
@@ -656,7 +658,7 @@ func TestParseSQL(t *testing.T) {
 				res = append(res, stmts[i].SQL)
 			}
 			if !reflect.DeepEqual(res, d.exp) {
-				t.Errorf("expected \n%v\n, but found %v", res, d.exp)
+				t.Errorf("expected \n%v\n, but found %v", d.exp, res)
 			}
 		})
 	}