Merge pull request #613 from vmware-tanzu/fix-589

Sort files from plucked data values to preserve processing order

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16.0
+          go-version: 1.17.0
 
       - name: generate website/generate.go
         if: startsWith(github.ref, 'refs/tags/')
@@ -32,7 +32,7 @@ jobs:
         uses: goreleaser/goreleaser-action@5e15885530fb01d81d1f24e8a6f54ebbd0fed7eb
         if: startsWith(github.ref, 'refs/tags/')
         with:
-          version: 0.162.0
+          version: 0.181.1
           args: release --rm-dist --debug
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-all.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v1
       with:
-        go-version: "1.16"
+        go-version: "1.17"
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:

.github/workflows/test-windows.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v1
       with:
-        go-version: "1.16"
+        go-version: "1.17"
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
     - name: Run Tests

go.mod

@@ -1,6 +1,6 @@
 module github.com/k14s/ytt
 
-go 1.16
+go 1.17
 
 require (
 	github.com/aws/aws-lambda-go v1.26.0
@@ -14,3 +14,13 @@ require (
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
 )
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/kr/text v0.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

pkg/cmd/template/schema_test.go

@@ -219,7 +219,7 @@ rendered: #@ data.values
 `
 
 		filesToProcess := files.NewSortedFiles([]*files.File{
-			files.MustNewFileFromSource(files.NewBytesSource("schema1.yml", []byte(schemaYAML1))),
+			files.MustNewFileFromSource(files.NewBytesSource("values/schema1.yml", []byte(schemaYAML1))),
 			files.MustNewFileFromSource(files.NewBytesSource("schema2.yml", []byte(schemaYAML2))),
 			files.MustNewFileFromSource(files.NewBytesSource("dataValues.yml", []byte(dataValuesYAML))),
 			files.MustNewFileFromSource(files.NewBytesSource("template.yml", []byte(templateYAML))),

pkg/workspace/library_execution.go

@@ -54,6 +54,8 @@ func (ll *LibraryExecution) Schemas(schemaOverlays []*schema.DocumentSchemaEnvel
 		return nil, nil, err
 	}
 
+	SortFilesInLibrary(schemaFiles)
+
 	documentSchemas, err := collectSchemaDocs(schemaFiles, loader)
 	if err != nil {
 		return nil, nil, err

pkg/yamltemplate/filetests/ytt-library/url/url.yml

@@ -15,6 +15,7 @@ query_param_value:
 query_params:
   #! keys are sorted
   test1: #@ url.query_params_encode({"y":["2","3"],"x":["1"],"z":[""],"w":[]})
+  #! semicolons in query strings were deprecated in v0.38.0; see impl for details.
   test2: #@ url.query_params_decode("y=2&x=1&y=3;z")
   test3: #@ url.query_params_encode({"w":[]})
   test4: #@ url.query_params_decode("")

pkg/yttlibrary/url.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/url"
 	"sort"
+	"strings"
 
 	"github.com/k14s/starlark-go/starlark"
 	"github.com/k14s/starlark-go/starlarkstruct"
@@ -169,6 +170,7 @@ func (b urlModule) QueryParamsDecode(thread *starlark.Thread, f *starlark.Builti
 		return starlark.None, err
 	}
 
+	encodedVal = b.allowQuerySemicolons(encodedVal)
 	urlVals, err := url.ParseQuery(encodedVal)
 	if err != nil {
 		return starlark.None, err
@@ -187,6 +189,19 @@ func (b urlModule) QueryParamsDecode(thread *starlark.Thread, f *starlark.Builti
 	return core.NewGoValue(result).AsStarlarkValue(), nil
 }
 
+// allowQuerySemicolons restores pre-Go 1.17 handling of query parameters.
+// Doing so defers making a breaking change as we upgrade from Go 1.16 to 1.17.
+// We expect to pass along this breaking change to end-users at some point, but not at this time (circa v0.37.x).
+// Shamelessly stolen from https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/net/http/server.go;l=2892-2908
+// See also:
+// - https://golang.org/doc/go1.17#semicolons
+// - https://github.com/golang/go/issues/25192
+//   - particularly https://github.com/golang/go/issues/25192#issuecomment-789799446 which spells-out the vulnerability
+//     specifically.
+func (b urlModule) allowQuerySemicolons(encodedVal string) string {
+	return strings.ReplaceAll(encodedVal, ";", "&")
+}
+
 func (b urlModule) sortedKeys(vals url.Values) []string {
 	var result []string
 	for k := range vals {