[yaml]: Update YAML parser to yaml.v3

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
 	go.starlark.net v0.0.0-20190604130855-6ddc71c0ba77
 	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71
 )
 
 replace github.com/kylelemons/godebug => github.com/jmillikin-stripe/godebug v0.0.0-20180620173319-8279e1966bc1

go.sum

@@ -13,3 +13,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71 h1:Xe2gvTZUJpsvOWUnvmL/tmhVBZUmHSvLbMjRj6NUUKo=
+gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/go/skycfg/yaml.go

@@ -20,9 +20,10 @@ import (
 	"bytes"
 	"fmt"
 	"reflect"
+	"strings"
 
 	"go.starlark.net/starlark"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // YamlModule returns a Starlark module for YAML helpers.
@@ -44,24 +45,101 @@ func yamlMarshal() starlark.Callable {
 	return starlark.NewBuiltin("yaml.marshal", fnYamlMarshal)
 }
 
+// starlarkToYAMLNode parses v into *yaml.Node.
+func starlarkToYAMLNode(v starlark.Value) (*yaml.Node, error) {
+	switch v := v.(type) {
+	case starlark.NoneType:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Value: "null",
+		}, nil
+	case starlark.Bool:
+		return &yaml.Node{
+			Kind: yaml.ScalarNode,
+			// Both "True" and "true" are valid YAML but we'll use
+			// lowercase to stay consistent with older API (yaml.v2).
+			Value: strings.ToLower(v.String()),
+		}, nil
+	case starlark.Int:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Value: v.String(),
+		}, nil
+	case starlark.Float:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Value: fmt.Sprintf("%g", v),
+		}, nil
+	case starlark.String:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Value: string(v),
+		}, nil
+	case starlark.Indexable: // Tuple, List
+		var elems []*yaml.Node
+		for i, n := 0, starlark.Len(v); i < n; i++ {
+			nn, err := starlarkToYAMLNode(v.Index(i))
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert %d-th element of %v to YAML: %v", i, v, err)
+			}
+			elems = append(elems, nn)
+		}
+		return &yaml.Node{
+			Kind:    yaml.SequenceNode,
+			Value:   v.String(),
+			Content: elems,
+		}, nil
+	case *starlark.Dict:
+		var elems []*yaml.Node
+		for _, pair := range v.Items() {
+			key, err := starlarkToYAMLNode(pair[0])
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert key %v to YAML: %v", pair[0], err)
+			}
+			if key.Kind != yaml.ScalarNode {
+				return nil, fmt.Errorf("key `%v' is not scalar", key.Value)
+			}
+			elems = append(elems, key)
+
+			val, err := starlarkToYAMLNode(pair[1])
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert value %v to YAML: %v", pair[1], err)
+			}
+			elems = append(elems, val)
+		}
+		return &yaml.Node{
+			Kind:    yaml.MappingNode,
+			Value:   v.String(),
+			Content: elems,
+		}, nil
+	default:
+		return nil, fmt.Errorf("TypeError: value %s (type `%s') can't be converted to YAML", v.String(), v.Type())
+	}
+}
+
 func fnYamlMarshal(t *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
 	var v starlark.Value
 	if err := starlark.UnpackArgs(fn.Name(), args, kwargs, "value", &v); err != nil {
 		return nil, err
 	}
-	var buf bytes.Buffer
-	if err := writeJSON(&buf, v); err != nil {
-		return nil, err
-	}
-	var jsonObj interface{}
-	if err := yaml.Unmarshal(buf.Bytes(), &jsonObj); err != nil {
+
+	node, err := starlarkToYAMLNode(v)
+	if err != nil {
 		return nil, err
 	}
-	yamlBytes, err := yaml.Marshal(jsonObj)
-	if err != nil {
+
+	buf := bytes.Buffer{}
+	enc := yaml.NewEncoder(&buf)
+	enc.SetIndent(2)
+	if err := enc.Encode(&yaml.Node{
+		Kind:    yaml.DocumentNode,
+		Content: []*yaml.Node{node},
+	}); err != nil {
 		return nil, err
 	}
-	return starlark.String(yamlBytes), nil
+	enc.Close()
+
+	return starlark.String(buf.String()), nil
 }
 
 // yamlUnmarshal returns a Starlark function for unmarshaling yaml content to
@@ -77,71 +155,79 @@ func fnYamlUnmarshal(t *starlark.Thread, fn *starlark.Builtin, args starlark.Tup
 	if err := starlark.UnpackPositionalArgs(fn.Name(), args, nil, 1, &blob); err != nil {
 		return nil, err
 	}
-	var inflated interface{}
-	if err := yaml.Unmarshal([]byte(blob), &inflated); err != nil {
+	var doc yaml.Node
+	if err := yaml.Unmarshal([]byte(blob), &doc); err != nil {
 		return nil, err
 	}
-	return toStarlarkValue(inflated)
+	return toStarlarkValue(doc.Content[0])
 }
 
-// toStarlarkScalarValue converts a scalar [obj] value to its starlark Value
-func toStarlarkScalarValue(obj interface{}) (starlark.Value, bool){
+// toStarlarkScalarValue converts a scalar node value to corresponding
+// starlark.Value.
+func toStarlarkScalarValue(node *yaml.Node) (starlark.Value, error) {
+	var obj interface{}
+	if err := node.Decode(&obj); err != nil {
+		return nil, err
+	}
+
 	if obj == nil {
-		return starlark.None, true
+		return starlark.None, nil
 	}
-	rt := reflect.TypeOf(obj)
+
+	t := reflect.TypeOf(obj)
 	v := reflect.ValueOf(obj)
-	switch rt.Kind() {
+	switch t.Kind() {
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		return starlark.MakeInt64(v.Int()), true
+		return starlark.MakeInt64(v.Int()), nil
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		return starlark.MakeUint64(v.Uint()), true
+		return starlark.MakeUint64(v.Uint()), nil
 	case reflect.Bool:
-		return starlark.Bool(v.Bool()), true
+		return starlark.Bool(v.Bool()), nil
 	case reflect.Float32, reflect.Float64:
-		return starlark.Float(v.Float()), true
+		return starlark.Float(v.Float()), nil
 	case reflect.String:
-		return starlark.String(v.String()), true
+		return starlark.String(v.String()), nil
 	default:
-		return nil, false
+		return nil, fmt.Errorf("unsupported type: %v", t)
 	}
 }
 
-// toStarlarkValue is a DFS walk to translate the DAG from go to starlark
-func toStarlarkValue(obj interface{}) (starlark.Value, error) {
-	if objval, ok := toStarlarkScalarValue(obj); ok {
-		return objval, nil
-	}
-	rt := reflect.TypeOf(obj)
-	switch rt.Kind() {
-	case reflect.Map:
-		ret := &starlark.Dict{}
-		for k, v := range obj.(map[interface{}]interface{}) {
-			keyval, ok := toStarlarkScalarValue(k)
-			if !ok {
-				return nil, fmt.Errorf("%s (%v) is not a supported key type", rt.Kind(), obj)
+// toStarlarkValue is a DFS walk to translate the DAG from Go to Starlark.
+func toStarlarkValue(node *yaml.Node) (starlark.Value, error) {
+	switch node.Kind {
+	case yaml.ScalarNode:
+		return toStarlarkScalarValue(node)
+	case yaml.MappingNode:
+		out := &starlark.Dict{}
+		for ki, vi := 0, 1; vi < len(node.Content); ki, vi = ki+2, vi+2 {
+			k, v := node.Content[ki], node.Content[vi]
+
+			kv, err := toStarlarkScalarValue(k)
+			if err != nil {
+				return nil, fmt.Errorf("`%s' not a supported key type: %v", k.Value, err)
 			}
-			starval, err := toStarlarkValue(v)
+
+			vv, err := toStarlarkValue(v)
 			if err != nil {
 				return nil, err
 			}
-			if err = ret.SetKey(keyval, starval); err != nil {
+
+			if err = out.SetKey(kv, vv); err != nil {
 				return nil, err
 			}
 		}
-		return ret, nil
-	case reflect.Slice:
-		slice := obj.([]interface{})
-		starvals := make([]starlark.Value, len(slice))
-		for i, element := range slice {
-			v, err := toStarlarkValue(element)
+		return out, nil
+	case yaml.SequenceNode:
+		out := make([]starlark.Value, len(node.Content))
+		for i, e := range node.Content {
+			vv, err := toStarlarkValue(e)
 			if err != nil {
 				return nil, err
 			}
-			starvals[i] = v
+			out[i] = vv
 		}
-		return starlark.NewList(starvals), nil
+		return starlark.NewList(out), nil
 	default:
-		return nil, fmt.Errorf("%s (%v) is not a supported type", rt.Kind(), obj)
+		return nil, fmt.Errorf("`%s' not a supported value", node.Value)
 	}
 }

internal/go/skycfg/yaml_test.go

@@ -41,20 +41,20 @@ func TestSkyToYaml(t *testing.T) {
 `,
 		},
 		YamlTestCase{
-			skyExpr: `{"a": 5, 13: 2, "k": {"k2": "v"}}`,
+			skyExpr: `{13: 2, "k": {"k2": "v"}, "a": 5.11}`,
 			expOutput: `13: 2
-a: 5
 k:
   k2: v
+a: 5.11
 `,
 		},
 		YamlTestCase{
-			skyExpr: `[1, 2, 3, "abc", None, 15, True, False, {"k": "v"}]`,
+			skyExpr: `[1, 2, 3, None, "abc", 15, True, False, {"k": "v"}]`,
 			expOutput: `- 1
 - 2
 - 3
-- abc
 - null
+- abc
 - 15
 - true
 - false
@@ -198,8 +198,8 @@ func TestYamlToSky(t *testing.T) {
 		},
 		{
 			name: "negative Int64 key mapped to String",
-			key: starlark.MakeInt64(-2147483649),
-			want:  `"nInt64Key"`,
+			key:  starlark.MakeInt64(-2147483649),
+			want: `"nInt64Key"`,
 		},
 		{
 			name: "Uint key mapped to String",
@@ -230,9 +230,9 @@ func TestYamlToSky(t *testing.T) {
 			if testCase.want != got.String() {
 				t.Error(
 					"Bad return value from yaml.unmarshal",
-					"\nExpected:",
+					"Expected:",
 					testCase.want,
-					"\nGot:",
+					"Got:",
 					got,
 				)
 			}