diff --git a/cmd/clusterctl/client/cluster/cert_manager.go b/cmd/clusterctl/client/cluster/cert_manager.go index c5366c64dc33..c3679424fd10 100644 --- a/cmd/clusterctl/client/cluster/cert_manager.go +++ b/cmd/clusterctl/client/cluster/cert_manager.go @@ -27,6 +27,7 @@ import ( manifests "sigs.k8s.io/cluster-api/cmd/clusterctl/config" "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util" logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log" + utilyaml "sigs.k8s.io/cluster-api/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -180,7 +181,7 @@ func (cm *certManagerClient) getManifestObjs() ([]unstructured.Unstructured, err return nil, err } - objs, err := util.ToUnstructured(yaml) + objs, err := utilyaml.ToUnstructured(yaml) if err != nil { return nil, errors.Wrap(err, "failed to parse yaml for cert-manager manifest") } diff --git a/cmd/clusterctl/client/cluster/inventory.go b/cmd/clusterctl/client/cluster/inventory.go index 67f8663d970f..3b6a0ae7bf99 100644 --- a/cmd/clusterctl/client/cluster/inventory.go +++ b/cmd/clusterctl/client/cluster/inventory.go @@ -27,8 +27,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" "sigs.k8s.io/cluster-api/cmd/clusterctl/config" - "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util" logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log" + utilyaml "sigs.k8s.io/cluster-api/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -129,7 +129,7 @@ func (p *inventoryClient) EnsureCustomResourceDefinitions() error { } // Transform the yaml in a list of objects. - objs, err := util.ToUnstructured(yaml) + objs, err := utilyaml.ToUnstructured(yaml) if err != nil { return errors.Wrap(err, "failed to parse yaml for clusterctl inventory CRDs") } diff --git a/cmd/clusterctl/client/repository/components.go b/cmd/clusterctl/client/repository/components.go index 422fb66fff9c..5927aac2e391 100644 --- a/cmd/clusterctl/client/repository/components.go +++ b/cmd/clusterctl/client/repository/components.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/scheme" "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util" + utilyaml "sigs.k8s.io/cluster-api/util/yaml" ) const ( @@ -204,7 +205,7 @@ func NewComponents(provider config.Provider, configClient config.Client, rawyaml } // Transform the yaml in a list of objects, so following transformation can work on typed objects (instead of working on a string/slice of bytes) - objs, err := util.ToUnstructured(yaml) + objs, err := utilyaml.ToUnstructured(yaml) if err != nil { return nil, errors.Wrap(err, "failed to parse yaml") } diff --git a/cmd/clusterctl/client/repository/template.go b/cmd/clusterctl/client/repository/template.go index 08d08f49d265..c36c2cb647ff 100644 --- a/cmd/clusterctl/client/repository/template.go +++ b/cmd/clusterctl/client/repository/template.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util" + utilyaml "sigs.k8s.io/cluster-api/util/yaml" ) // Template wraps a YAML file that defines the cluster objects (Cluster, Machines etc.). @@ -86,7 +87,7 @@ func NewTemplate(rawYaml []byte, configVariablesClient config.VariablesClient, t } // Transform the yaml in a list of objects, so following transformation can work on typed objects (instead of working on a string/slice of bytes). - objs, err := util.ToUnstructured(yaml) + objs, err := utilyaml.ToUnstructured(yaml) if err != nil { return nil, errors.Wrap(err, "failed to parse yaml") } diff --git a/cmd/clusterctl/internal/util/yaml.go b/cmd/clusterctl/internal/util/yaml.go index 5db3731555e3..2892daf00303 100644 --- a/cmd/clusterctl/internal/util/yaml.go +++ b/cmd/clusterctl/internal/util/yaml.go @@ -17,14 +17,9 @@ limitations under the License. package util import ( - "bufio" "bytes" - "io" - "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - utilyaml "k8s.io/apimachinery/pkg/util/yaml" - "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/yaml" ) @@ -53,46 +48,6 @@ func JoinYaml(yamls ...[]byte) []byte { return r } -// ToUnstructured takes a YAML and converts it to a list of Unstructured objects -func ToUnstructured(rawyaml []byte) ([]unstructured.Unstructured, error) { - var ret []unstructured.Unstructured //nolint - - reader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(rawyaml))) - count := 1 - for { - // Read one YAML document at a time, until io.EOF is returned - b, err := reader.Read() - if err != nil { - if err == io.EOF { - break - } - return nil, errors.Wrapf(err, "failed to read yaml") - } - if len(b) == 0 { - break - } - - var m map[string]interface{} - if err := yaml.Unmarshal(b, &m); err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal the %s yaml document: %q", util.Ordinalize(count), string(b)) - } - - var u unstructured.Unstructured - u.SetUnstructuredContent(m) - - // Ignore empty objects. - // Empty objects are generated if there are weird things in manifest files like e.g. two --- in a row without a yaml doc in the middle - if u.Object == nil { - continue - } - - ret = append(ret, u) - count++ - } - - return ret, nil -} - // FromUnstructured takes a list of Unstructured objects and converts it into a YAML func FromUnstructured(objs []unstructured.Unstructured) ([]byte, error) { var ret [][]byte //nolint diff --git a/cmd/clusterctl/internal/util/yaml_test.go b/cmd/clusterctl/internal/util/yaml_test.go index f08026a1c014..8b929bb1f8ce 100644 --- a/cmd/clusterctl/internal/util/yaml_test.go +++ b/cmd/clusterctl/internal/util/yaml_test.go @@ -20,138 +20,23 @@ import ( "testing" . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) -func TestToUnstructured(t *testing.T) { - type args struct { - rawyaml []byte - } - tests := []struct { - name string - args args - wantObjsCount int - wantErr bool - err string - }{ - { - name: "single object", - args: args{ - rawyaml: []byte("apiVersion: v1\n" + - "kind: ConfigMap\n"), - }, - wantObjsCount: 1, - wantErr: false, - }, - { - name: "multiple objects are detected", - args: args{ - rawyaml: []byte("apiVersion: v1\n" + - "kind: ConfigMap\n" + - "---\n" + - "apiVersion: v1\n" + - "kind: Secret\n"), - }, - wantObjsCount: 2, - wantErr: false, - }, - { - name: "empty object are dropped", - args: args{ - rawyaml: []byte("---\n" + //empty objects before - "---\n" + - "---\n" + - "apiVersion: v1\n" + - "kind: ConfigMap\n" + - "---\n" + // empty objects in the middle - "---\n" + - "---\n" + - "apiVersion: v1\n" + - "kind: Secret\n" + - "---\n" + //empty objects after - "---\n" + - "---\n"), - }, - wantObjsCount: 2, - wantErr: false, - }, - { - name: "--- in the middle of objects are ignored", - args: args{ - []byte("apiVersion: v1\n" + - "kind: ConfigMap\n" + - "data: \n" + - " key: |\n" + - " ··Several lines of text,\n" + - " ··with some --- \n" + - " ---\n" + - " ··in the middle\n" + - "---\n" + - "apiVersion: v1\n" + - "kind: Secret\n"), - }, - wantObjsCount: 2, - wantErr: false, - }, - { - name: "returns error for invalid yaml", - args: args{ - rawyaml: []byte("apiVersion: v1\n" + - "kind: ConfigMap\n" + - "---\n" + - "apiVersion: v1\n" + - "foobar\n" + - "kind: Secret\n"), - }, - wantErr: true, - err: "failed to unmarshal the 2nd yaml document", - }, - { - name: "returns error for invalid yaml", - args: args{ - rawyaml: []byte("apiVersion: v1\n" + - "kind: ConfigMap\n" + - "---\n" + - "apiVersion: v1\n" + - "kind: Pod\n" + - "---\n" + - "apiVersion: v1\n" + - "kind: Deployment\n" + - "---\n" + - "apiVersion: v1\n" + - "foobar\n" + - "kind: ConfigMap\n"), - }, - wantErr: true, - err: "failed to unmarshal the 4th yaml document", - }, - { - name: "returns error for invalid yaml", - args: args{ - rawyaml: []byte("apiVersion: v1\n" + - "foobar\n" + - "kind: ConfigMap\n" + - "---\n" + - "apiVersion: v1\n" + - "kind: Secret\n"), - }, - wantErr: true, - err: "failed to unmarshal the 1st yaml document", +func TestFromUnstructured(t *testing.T) { + rawyaml := []byte("apiVersion: v1\n" + + "kind: ConfigMap") + + unstructuredObj := unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - got, err := ToUnstructured(tt.args.rawyaml) - if tt.wantErr { - g.Expect(err).To(HaveOccurred()) - if len(tt.err) != 0 { - g.Expect(err.Error()).To(ContainSubstring(tt.err)) - } - return - } - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(got).To(HaveLen(tt.wantObjsCount)) - }) - } + convertedyaml, err := FromUnstructured([]unstructured.Unstructured{unstructuredObj}) + g := NewWithT(t) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(string(rawyaml)).To(Equal(string(convertedyaml))) } diff --git a/util/yaml/yaml.go b/util/yaml/yaml.go index d897d1dd91b2..f19a45881981 100644 --- a/util/yaml/yaml.go +++ b/util/yaml/yaml.go @@ -28,9 +28,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/streaming" - "k8s.io/apimachinery/pkg/util/yaml" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes/scheme" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/yaml" ) func ExtractClusterReferences(out *ParseOutput, c *clusterv1.Cluster) (res []*unstructured.Unstructured) { @@ -151,7 +153,7 @@ func Parse(input ParseInput) (*ParseOutput, error) { } type yamlDecoder struct { - reader *yaml.YAMLReader + reader *utilyaml.YAMLReader decoder runtime.Decoder close func() error } @@ -179,8 +181,48 @@ func (d *yamlDecoder) Close() error { func NewYAMLDecoder(r io.ReadCloser) streaming.Decoder { return &yamlDecoder{ - reader: yaml.NewYAMLReader(bufio.NewReader(r)), + reader: utilyaml.NewYAMLReader(bufio.NewReader(r)), decoder: scheme.Codecs.UniversalDeserializer(), close: r.Close, } } + +// ToUnstructured takes a YAML and converts it to a list of Unstructured objects +func ToUnstructured(rawyaml []byte) ([]unstructured.Unstructured, error) { + var ret []unstructured.Unstructured //nolint + + reader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(rawyaml))) + count := 1 + for { + // Read one YAML document at a time, until io.EOF is returned + b, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + return nil, errors.Wrapf(err, "failed to read yaml") + } + if len(b) == 0 { + break + } + + var m map[string]interface{} + if err := yaml.Unmarshal(b, &m); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal the %s yaml document: %q", util.Ordinalize(count), string(b)) + } + + var u unstructured.Unstructured + u.SetUnstructuredContent(m) + + // Ignore empty objects. + // Empty objects are generated if there are weird things in manifest files like e.g. two --- in a row without a yaml doc in the middle + if u.Object == nil { + continue + } + + ret = append(ret, u) + count++ + } + + return ret, nil +} diff --git a/util/yaml/yaml_test.go b/util/yaml/yaml_test.go index 83cd979567c7..2103ce2e3693 100644 --- a/util/yaml/yaml_test.go +++ b/util/yaml/yaml_test.go @@ -314,3 +314,134 @@ func createTempFile(contents string) (filename string, reterr error) { _, _ = f.WriteString(contents) return f.Name(), nil } + +func TestToUnstructured(t *testing.T) { + type args struct { + rawyaml []byte + } + tests := []struct { + name string + args args + wantObjsCount int + wantErr bool + err string + }{ + { + name: "single object", + args: args{ + rawyaml: []byte("apiVersion: v1\n" + + "kind: ConfigMap\n"), + }, + wantObjsCount: 1, + wantErr: false, + }, + { + name: "multiple objects are detected", + args: args{ + rawyaml: []byte("apiVersion: v1\n" + + "kind: ConfigMap\n" + + "---\n" + + "apiVersion: v1\n" + + "kind: Secret\n"), + }, + wantObjsCount: 2, + wantErr: false, + }, + { + name: "empty object are dropped", + args: args{ + rawyaml: []byte("---\n" + //empty objects before + "---\n" + + "---\n" + + "apiVersion: v1\n" + + "kind: ConfigMap\n" + + "---\n" + // empty objects in the middle + "---\n" + + "---\n" + + "apiVersion: v1\n" + + "kind: Secret\n" + + "---\n" + //empty objects after + "---\n" + + "---\n"), + }, + wantObjsCount: 2, + wantErr: false, + }, + { + name: "--- in the middle of objects are ignored", + args: args{ + []byte("apiVersion: v1\n" + + "kind: ConfigMap\n" + + "data: \n" + + " key: |\n" + + " ··Several lines of text,\n" + + " ··with some --- \n" + + " ---\n" + + " ··in the middle\n" + + "---\n" + + "apiVersion: v1\n" + + "kind: Secret\n"), + }, + wantObjsCount: 2, + wantErr: false, + }, + { + name: "returns error for invalid yaml", + args: args{ + rawyaml: []byte("apiVersion: v1\n" + + "kind: ConfigMap\n" + + "---\n" + + "apiVersion: v1\n" + + "foobar\n" + + "kind: Secret\n"), + }, + wantErr: true, + err: "failed to unmarshal the 2nd yaml document", + }, + { + name: "returns error for invalid yaml", + args: args{ + rawyaml: []byte("apiVersion: v1\n" + + "kind: ConfigMap\n" + + "---\n" + + "apiVersion: v1\n" + + "kind: Pod\n" + + "---\n" + + "apiVersion: v1\n" + + "kind: Deployment\n" + + "---\n"), + }, + wantErr: true, + err: "failed to unmarshal the 4th yaml document", + }, + { + name: "returns error for invalid yaml", + args: args{ + rawyaml: []byte("apiVersion: v1\n" + + "foobar\n" + + "kind: ConfigMap\n" + + "---\n" + + "apiVersion: v1\n" + + "kind: Secret\n"), + }, + wantErr: true, + err: "failed to unmarshal the 1st yaml document", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := ToUnstructured(tt.args.rawyaml) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + if len(tt.err) != 0 { + g.Expect(err.Error()).To(ContainSubstring(tt.err)) + } + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(HaveLen(tt.wantObjsCount)) + }) + } +}