fix parsing of invalid document (#499)

lexer/lexer_test.go

@@ -2421,6 +2421,14 @@ a: |invalid`,
 			name: "invalid double-quoted",
 			src:  `a: "\"key\": \"value:\"`,
 		},
+		{
+			name: "invalid document folded",
+			src:  ">\n>",
+		},
+		{
+			name: "invalid document number",
+			src:  ">\n1",
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

parser/parser_test.go

@@ -1099,6 +1099,33 @@ b: - 2
 [1:8] ',' or ']' must be specified
 >  1 | foo: [$[should not be allowed]]
               ^
+`,
+		},
+		{
+			">\n>",
+			`
+[2:1] found invalid token
+   1 | >
+>  2 | >
+       ^
+`,
+		},
+		{
+			">\n1",
+			`
+[2:1] found invalid token
+   1 | >
+>  2 | 1
+       ^
+`,
+		},
+		{
+			"|\n1",
+			`
+[2:1] found invalid token
+   1 | |
+>  2 | 1
+       ^
 `,
 		},
 	}

scanner/scanner.go

@@ -837,6 +837,9 @@ func (s *Scanner) scanLiteralHeaderOption(ctx *Context) error {
 			case "", "+", "-",
 				"0", "1", "2", "3", "4", "5", "6", "7", "8", "9":
 				hasComment := len(opt) < orgOptLen
+				if s.column == 1 {
+					s.lastDelimColumn = 1
+				}
 				if header == '|' {
 					if hasComment {
 						commentLen := orgOptLen - len(opt)
@@ -961,6 +964,13 @@ func (s *Scanner) scan(ctx *Context) error {
 				if tk := ctx.lastToken(); tk != nil {
 					// If literal/folded content is empty, no string token is added.
 					// Therefore, add an empty string token.
+					// But if literal/folded token column is 1, it is invalid at down state.
+					if tk.Position.Column == 1 {
+						return ErrInvalidToken(
+							"could not find document",
+							token.Invalid(string(ctx.obuf), s.pos()),
+						)
+					}
 					if tk.Type != token.StringType {
 						ctx.addToken(token.String("", "", s.pos()))
 					}