From 83c5e1bc832680ea28c32764c738fb1a6407ed8e Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 8 Dec 2022 13:33:06 +1100
Subject: [PATCH] Include blank new lines in leading header preprocessing #1462

---
 acceptance_tests/leading-seperator.sh        | 58 ++++++++++++++++++++
 pkg/yqlib/decoder_yaml.go                    |  8 +++
 pkg/yqlib/doc/operators/comment-operators.md |  3 +
 pkg/yqlib/doc/operators/path.md              |  1 +
 pkg/yqlib/operator_anchors_aliases_test.go   |  3 +-
 pkg/yqlib/operator_comments_test.go          |  2 +-
 pkg/yqlib/operator_multiply_test.go          |  4 +-
 pkg/yqlib/operator_traverse_path_test.go     |  3 +-
 8 files changed, 75 insertions(+), 7 deletions(-)

diff --git a/acceptance_tests/leading-seperator.sh b/acceptance_tests/leading-seperator.sh
index 9419ecf5344..5aceacbe815 100755
--- a/acceptance_tests/leading-seperator.sh
+++ b/acceptance_tests/leading-seperator.sh
@@ -34,6 +34,64 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+
+testLeadingSeperatorWithNewlinesNewDoc() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+
+
+---
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+
+
+---
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorWithNewlinesMoreComments() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+
+---
+# great
+
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+
+---
+# great
+
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
 testLeadingSeperatorWithDirective() {
   cat >test.yml <<EOL
 %YAML 1.1
diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go
index b4591e61c88..a87233bcc8c 100644
--- a/pkg/yqlib/decoder_yaml.go
+++ b/pkg/yqlib/decoder_yaml.go
@@ -39,6 +39,14 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
 			return reader, sb.String(), nil
 		} else if err != nil {
 			return reader, sb.String(), err
+		} else if string(peekBytes[0]) == "\n" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
 		} else if string(peekBytes) == "---" {
 			_, err := reader.ReadString('\n')
 			sb.WriteString("$yqDocSeperator$\n")
diff --git a/pkg/yqlib/doc/operators/comment-operators.md b/pkg/yqlib/doc/operators/comment-operators.md
index 9d1b9da29d7..3ba9e65c163 100644
--- a/pkg/yqlib/doc/operators/comment-operators.md
+++ b/pkg/yqlib/doc/operators/comment-operators.md
@@ -247,6 +247,7 @@ Note the use of `...` to ensure key nodes are included.
 Given a sample.yml file of:
 ```yaml
 # hi
+
 a: cat # comment
 # great
 b: # key comment
@@ -265,6 +266,7 @@ b:
 Given a sample.yml file of:
 ```yaml
 # welcome!
+
 a: cat # meow
 # have a great day
 ```
@@ -293,6 +295,7 @@ yq '. | head_comment' sample.yml
 will output
 ```yaml
 welcome!
+
 ```
 
 ## Head comment with document split
diff --git a/pkg/yqlib/doc/operators/path.md b/pkg/yqlib/doc/operators/path.md
index 1e0076ac41a..3356e8678f8 100644
--- a/pkg/yqlib/doc/operators/path.md
+++ b/pkg/yqlib/doc/operators/path.md
@@ -127,6 +127,7 @@ Like pick but recursive. This uses `ireduce` to deeply set the selected paths in
 
 Given a sample.yml file of:
 ```yaml
+
 parentA: bob
 parentB:
   child1: i am child1
diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go
index 07a3fdfd458..7b52cc48e1e 100644
--- a/pkg/yqlib/operator_anchors_aliases_test.go
+++ b/pkg/yqlib/operator_anchors_aliases_test.go
@@ -12,8 +12,7 @@ var specDocument = `- &CENTER { x: 1, y: 2 }
 
 var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
 
-var simpleArrayRef = `
-item_value: &item_value
+var simpleArrayRef = `item_value: &item_value
   value: true
 
 thingOne:
diff --git a/pkg/yqlib/operator_comments_test.go b/pkg/yqlib/operator_comments_test.go
index e117e181644..ed7fef4835d 100644
--- a/pkg/yqlib/operator_comments_test.go
+++ b/pkg/yqlib/operator_comments_test.go
@@ -221,7 +221,7 @@ var commentOperatorScenarios = []expressionScenario{
 		document:              "# welcome!\n\na: cat # meow\n\n# have a great day",
 		expression:            `. | head_comment`,
 		expected: []string{
-			"D0, P[], (!!str)::welcome!\n",
+			"D0, P[], (!!str)::welcome!\n\n",
 		},
 	},
 	{
diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go
index 4c9a2060750..d775986ce02 100644
--- a/pkg/yqlib/operator_multiply_test.go
+++ b/pkg/yqlib/operator_multiply_test.go
@@ -108,7 +108,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document2:  docNoComments,
 		expression: `select(fi == 0) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+			"D0, P[], (!!map)::# here\n\na: apple\nb: banana\n",
 		},
 	},
 	{
@@ -126,7 +126,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document2:  docWithHeader,
 		expression: `select(fi == 0) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::# here\nb: banana\na: apple\n",
+			"D0, P[], (!!map)::# here\n\nb: banana\na: apple\n",
 		},
 	},
 	{
diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go
index 56b3cc9c498..405a523e5e4 100644
--- a/pkg/yqlib/operator_traverse_path_test.go
+++ b/pkg/yqlib/operator_traverse_path_test.go
@@ -4,8 +4,7 @@ import (
 	"testing"
 )
 
-var mergeDocSample = `
-foo: &foo
+var mergeDocSample = `foo: &foo
   a: foo_a
   thing: foo_thing
   c: foo_c