From 261bc07fcd1f6743cf451a9691d91606116cb39d Mon Sep 17 00:00:00 2001 From: Varsha Munishwar Date: Tue, 5 Apr 2022 17:03:42 -0700 Subject: [PATCH 1/2] store @schema/validation assertions on a node's meta Signed-off-by: Varsha Munishwar Signed-off-by: Garrett Cheadle --- pkg/assertions/yamlmeta.go | 20 +- pkg/cmd/template/assert_validate_test.go | 335 +++++++++++++++++++- pkg/schema/annotations.go | 71 +++++ pkg/schema/assign.go | 15 + pkg/schema/schema.go | 34 +- pkg/schema/type.go | 52 +++ pkg/schema/yamlmeta.go | 6 +- pkg/workspace/data_values_pre_processing.go | 2 + 8 files changed, 523 insertions(+), 12 deletions(-) diff --git a/pkg/assertions/yamlmeta.go b/pkg/assertions/yamlmeta.go index 41d248da..704de0f1 100644 --- a/pkg/assertions/yamlmeta.go +++ b/pkg/assertions/yamlmeta.go @@ -3,12 +3,22 @@ package assertions -import "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" +import ( + "fmt" + + "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" +) + +const validations = "validations" // AddValidations appends validation Rules to node's validations metadata, later retrieved via GetValidations(). func AddValidations(node yamlmeta.Node, rules []Rule) { - metas := node.GetMeta("validations") - if currRules, ok := metas.([]Rule); ok { + metas := node.GetMeta(validations) + if metas != nil { + currRules, ok := metas.([]Rule) + if !ok { + panic(fmt.Sprintf("Unexpected validations in meta : %s %v", node.GetPosition().AsCompactString(), metas)) + } rules = append(currRules, rules...) } SetValidations(node, rules) @@ -16,12 +26,12 @@ func AddValidations(node yamlmeta.Node, rules []Rule) { // SetValidations attaches validation Rules to node's metadata, later retrieved via GetValidations(). func SetValidations(node yamlmeta.Node, rules []Rule) { - node.SetMeta("validations", rules) + node.SetMeta(validations, rules) } // GetValidations retrieves validation Rules from node metadata, set previously via SetValidations(). func GetValidations(node yamlmeta.Node) []Rule { - metas := node.GetMeta("validations") + metas := node.GetMeta(validations) if rules, ok := metas.([]Rule); ok { return rules } diff --git a/pkg/cmd/template/assert_validate_test.go b/pkg/cmd/template/assert_validate_test.go index 38eeb7c4..dc37b997 100644 --- a/pkg/cmd/template/assert_validate_test.go +++ b/pkg/cmd/template/assert_validate_test.go @@ -4,10 +4,10 @@ package template_test import ( + "testing" + cmdtpl "github.com/vmware-tanzu/carvel-ytt/pkg/cmd/template" "github.com/vmware-tanzu/carvel-ytt/pkg/files" - - "testing" ) func TestAssertValidateOnDataValuesSucceeds(t *testing.T) { @@ -371,3 +371,334 @@ values: #@ data.values assertFails(t, filesToProcess, expectedErr, opts) }) } + +func TestSchemaValidationSucceeds(t *testing.T) { + t.Run("when validations pass using --data-values-inspect", func(t *testing.T) { + opts := cmdtpl.NewOptions() + opts.DataValuesFlags.Inspect = true + + schemaYAML := `#@ load("@ytt:assert", "assert") +#@data/values-schema +#@schema/validation ("a non empty data values", lambda v: True if v else assert.fail("data values was empty")) +--- +#@schema/validation ("a map with more than 3 elements", lambda v: True if len(v) > 3 else assert.fail("length of map was less than 3")) +my_map: + #@schema/validation ("a non-empty string", lambda v: True if len(v) > 0 else assert.fail("length of string was 0")) + string: server.example.com + #@schema/validation ("an int over 9000", lambda v: True if v > 9000 else assert.fail("int was less than 9000")) + int: 54321 + #@schema/validation ("a float less than pi", lambda v: True if v < 3.1415 else assert.fail("float was more than 3.1415")) + float: 2.3 + #@schema/validation ("bool evaluating to true", lambda v: v) + bool: true + #@schema/validation ("a null value", lambda v: True if v == None else assert.fail("value was not null")) + #@schema/nullable + nil: "" + #@schema/validation ("an array with 1 or more items", lambda v: True if len(v) >= 1 else assert.fail("array was empty")) + #@schema/default ['abc'] + my_array: + #@schema/validation ("a non-empty string", lambda v: True if len(v) > 0 else assert.fail("length of string was 0")) + - abc +` + + expected := `my_map: + string: server.example.com + int: 54321 + float: 2.3 + bool: true + nil: null + my_array: + - abc +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("schema.yml", []byte(schemaYAML))), + }) + + assertSucceedsDocSet(t, filesToProcess, expected, opts) + }) + t.Run("when validations pass on an array item using --data-values-inspect", func(t *testing.T) { + opts := cmdtpl.NewOptions() + opts.DataValuesFlags.Inspect = true + schemaYAML := `#@ load("@ytt:assert", "assert") +#@data/values-schema +--- +my_array: +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +- 5 +` + dvYAML := `#@data/values +--- +my_array: +- 5 +- 6 +- 7 +` + + expected := `my_array: +- 5 +- 6 +- 7 +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("schema.yml", []byte(schemaYAML))), + files.MustNewFileFromSource(files.NewBytesSource("dv.yml", []byte(dvYAML))), + }) + + assertSucceedsDocSet(t, filesToProcess, expected, opts) + }) + t.Run("when validations pass with other optional schema annotations", func(t *testing.T) { + opts := cmdtpl.NewOptions() + opts.DataValuesFlags.Inspect = true + schemaYAML := `#@ load("@ytt:assert", "assert") +#@data/values-schema +--- +#@schema/default [5] +my_array: +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +- 6 + +#@schema/type any= True +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +my_map: 5 + +#@schema/nullable +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +my_other_map: 5 +` + dvYAML := `#@data/values +--- +my_other_map: 6 +` + + expected := `my_array: +- 5 +my_map: 5 +my_other_map: 6 +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("schema.yml", []byte(schemaYAML))), + files.MustNewFileFromSource(files.NewBytesSource("dv.yml", []byte(dvYAML))), + }) + + assertSucceedsDocSet(t, filesToProcess, expected, opts) + }) + t.Run("when validations on library values pass", func(t *testing.T) { + opts := cmdtpl.NewOptions() + configYAML := ` +#@ load("@ytt:template", "template") +#@ load("@ytt:library", "library") +--- + +#@ def additional_vals(): +int: 10 +#@ end + +#@ lib = library.get("lib") +#@ lib2 = lib.with_data_values_schema(additional_vals()) +--- #@ template.replace(lib.eval()) +--- #@ template.replace(lib2.eval()) +` + + libSchemaYAML := `#@ load("@ytt:assert", "assert") + +#@data/values-schema +--- +#@schema/validation ("an integer over 1", lambda v: True if v > 1 else assert.fail("value was less than 1")) +int: 2 +` + + libConfigYAML := ` +#@ load("@ytt:data", "data") +--- +values: #@ data.values +` + + expected := `values: + int: 2 +--- +values: + int: 10 +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("config.yml", []byte(configYAML))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/schema.yml", []byte(libSchemaYAML))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/config.yml", []byte(libConfigYAML))), + }) + + assertSucceedsDocSet(t, filesToProcess, expected, opts) + }) +} + +func TestSchemaValidationFails(t *testing.T) { + t.Run("when validations fail", func(t *testing.T) { + opts := cmdtpl.NewOptions() + schemaYAML := `#@ load("@ytt:assert", "assert") + +#@data/values-schema +--- + #@schema/validation ("a non-empty string", lambda v: True if len(v) > 0 else assert.fail("length of string was 0")) + string: "" + #@schema/validation ("an int over 9000", lambda v: True if v > 9000 else assert.fail("int was less than 9000")) + int: 5432 + #@schema/validation ("a float less than pi", lambda v: True if v < 3.1415 else assert.fail("float was more than 3.1415")) + float: 21.3 + #@schema/validation ("bool evaluating to true", lambda v: v) + bool: false + #@schema/validation ("a null value", lambda v: True if v == None else assert.fail("value was not null")) + nil: "" + #@schema/validation ("an array with 1 or more items", lambda v: True if len(v) >= 1 else assert.fail("array was empty")) + my_array: + #@schema/validation ("a non-empty string", lambda v: True if len(v) > 0 else assert.fail("length of string was 0")) + - abc +` + + expectedErr := `One or more data values were invalid: +- "string" (schema.yml:6) requires "a non-empty string"; assert.fail: fail: length of string was 0 (by schema.yml:5) +- "int" (schema.yml:8) requires "an int over 9000"; assert.fail: fail: int was less than 9000 (by schema.yml:7) +- "float" (schema.yml:10) requires "a float less than pi"; assert.fail: fail: float was more than 3.1415 (by schema.yml:9) +- "bool" (schema.yml:12) requires "bool evaluating to true" (by schema.yml:11) +- "nil" (schema.yml:14) requires "a null value"; assert.fail: fail: value was not null (by schema.yml:13) +- "my_array" (schema.yml:16) requires "an array with 1 or more items"; assert.fail: fail: array was empty (by schema.yml:15)` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("schema.yml", []byte(schemaYAML))), + }) + + assertFails(t, filesToProcess, expectedErr, opts) + }) + t.Run("when validations fail on an array item with data values overlays", func(t *testing.T) { + opts := cmdtpl.NewOptions() + + schemaYAML := `#@ load("@ytt:assert", "assert") +#@data/values-schema +--- +my_array: +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +- 5 +` + dvYAML := `#@data/values +--- +my_array: +- 5 +- 6 +- 7 +- 1 +- 2 +- 3 +` + + expectedErr := `One or more data values were invalid: +- array item (dv.yml:7) requires "an integer larger than 4"; assert.fail: fail: values less than 5 (by schema.yml:5) +- array item (dv.yml:8) requires "an integer larger than 4"; assert.fail: fail: values less than 5 (by schema.yml:5) +- array item (dv.yml:9) requires "an integer larger than 4"; assert.fail: fail: values less than 5 (by schema.yml:5)` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("schema.yml", []byte(schemaYAML))), + files.MustNewFileFromSource(files.NewBytesSource("dv.yml", []byte(dvYAML))), + }) + + assertFails(t, filesToProcess, expectedErr, opts) + }) + t.Run("when validations on a document fail", func(t *testing.T) { + opts := cmdtpl.NewOptions() + schemaYAML := `#@ load("@ytt:assert", "assert") +#@data/values-schema +#@schema/validation ("need more than 1 data value", lambda v: True if len(v) > 1 else assert.fail("less than 1 data values present")) +--- +my_map: "abc" +` + + expectedErr := `One or more data values were invalid: +- document (schema.yml:4) requires "need more than 1 data value"; assert.fail: fail: less than 1 data values present (by schema.yml:3) +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("schema.yml", []byte(schemaYAML))), + }) + + assertFails(t, filesToProcess, expectedErr, opts) + }) + t.Run("when validations fail with other optional schema annotations", func(t *testing.T) { + opts := cmdtpl.NewOptions() + opts.DataValuesFlags.Inspect = true + schemaYAML := `#@ load("@ytt:assert", "assert") +#@data/values-schema +--- +#@schema/default [3] +my_array: +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +- 6 + +#@schema/type any= True +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +my_map: 3 + +#@schema/nullable +#@schema/validation ("an integer larger than 4", lambda v: True if v > 4 else assert.fail("values less than 5")) +my_other_map: 5 +` + dvYAML := `#@data/values +--- +my_other_map: 3 +` + + expectedErr := `One or more data values were invalid: +- array item (schema.yml:5) requires "an integer larger than 4"; assert.fail: fail: values less than 5 (by schema.yml:6) +- "my_map" (schema.yml:11) requires "an integer larger than 4"; assert.fail: fail: values less than 5 (by schema.yml:10) +- "my_other_map" (dv.yml:3) requires "an integer larger than 4"; assert.fail: fail: values less than 5 (by schema.yml:14) +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("schema.yml", []byte(schemaYAML))), + files.MustNewFileFromSource(files.NewBytesSource("dv.yml", []byte(dvYAML))), + }) + + assertFails(t, filesToProcess, expectedErr, opts) + }) + t.Run("when validations on library schema fail", func(t *testing.T) { + opts := cmdtpl.NewOptions() + configYAML := ` +#@ load("@ytt:template", "template") +#@ load("@ytt:library", "library") +--- + +#@ lib = library.get("lib") +--- #@ template.replace(lib.eval()) +` + + libSchemaYAML := `#@ load("@ytt:assert", "assert") + +#@data/values-schema +--- +#@schema/validation ("an integer over 1", lambda v: True if v > 1 else assert.fail("value was less than 1")) +int: 1 +` + + libConfigYAML := ` +#@ load("@ytt:data", "data") +--- +values: #@ data.values +` + + expectedErr := `- library.eval: Evaluating library 'lib': One or more data values were invalid: + in + config.yml:7 | --- #@ template.replace(lib.eval()) + + reason: + - "int" (_ytt_lib/lib/schema.yml:6) requires "an integer over 1"; assert.fail: fail: value was less than 1 (by _ytt_lib/lib/schema.yml:5) +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("config.yml", []byte(configYAML))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/schema.yml", []byte(libSchemaYAML))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/config.yml", []byte(libConfigYAML))), + }) + + assertFails(t, filesToProcess, expectedErr, opts) + }) + +} diff --git a/pkg/schema/annotations.go b/pkg/schema/annotations.go index ead5cca3..a904d05f 100644 --- a/pkg/schema/annotations.go +++ b/pkg/schema/annotations.go @@ -8,12 +8,14 @@ import ( "strings" "github.com/k14s/starlark-go/starlark" + "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/filepos" "github.com/vmware-tanzu/carvel-ytt/pkg/template" "github.com/vmware-tanzu/carvel-ytt/pkg/template/core" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) +// Declare @schema/... annotation names const ( AnnotationNullable template.AnnotationName = "schema/nullable" AnnotationType template.AnnotationName = "schema/type" @@ -23,6 +25,7 @@ const ( AnnotationExamples template.AnnotationName = "schema/examples" AnnotationDeprecated template.AnnotationName = "schema/deprecated" TypeAnnotationKwargAny string = "any" + AnnotationValidation template.AnnotationName = "schema/validation" ) type Annotation interface { @@ -71,6 +74,12 @@ type ExampleAnnotation struct { pos *filepos.Position } +// ValidationAnnotation is a wrapper for validations provided via @schema/validation annotation +type ValidationAnnotation struct { + rules []assertions.Rule + pos *filepos.Position +} + // Example contains a yaml example and its description type Example struct { description string @@ -402,6 +411,41 @@ func NewExampleAnnotation(ann template.NodeAnnotation, pos *filepos.Position) (* return &ExampleAnnotation{examples, ann.Position}, nil } +// NewValidationAnnotation checks the argument provided via @schema/validation annotation, and returns wrapper for the rules defined +func NewValidationAnnotation(ann template.NodeAnnotation, pos *filepos.Position) (*ValidationAnnotation, error) { + var rules []assertions.Rule + + if len(ann.Kwargs) != 0 { + return nil, fmt.Errorf("Invalid @%s annotation - expected @%s to have 2-tuple as argument(s), but found keyword argument (by %s)", AnnotationValidation, AnnotationValidation, ann.Position.AsCompactString()) + } + if len(ann.Args) == 0 { + return nil, fmt.Errorf("Invalid @%s annotation - expected @%s to have 2-tuple as argument(s), but found no arguments (by %s)", AnnotationValidation, AnnotationValidation, ann.Position.AsCompactString()) + } + for _, arg := range ann.Args { + ruleTuple, ok := arg.(starlark.Tuple) + if !ok { + return nil, fmt.Errorf("Invalid @%s annotation - expected @%s to have 2-tuple as argument(s), but found: %s (by %s)", AnnotationValidation, AnnotationValidation, arg.String(), ann.Position.AsCompactString()) + } + if len(ruleTuple) != 2 { + return nil, fmt.Errorf("Invalid @%s annotation - expected @%s 2-tuple, but found tuple with length %v (by %s)", AnnotationValidation, AnnotationValidation, len(ruleTuple), ann.Position.AsCompactString()) + } + message, ok := ruleTuple[0].(starlark.String) + if !ok { + return nil, fmt.Errorf("Invalid @%s annotation - expected first item in the 2-tuple to be a string describing a valid value, but was %s (at %s)", AnnotationValidation, ruleTuple[0].Type(), ann.Position.AsCompactString()) + } + lambda, ok := ruleTuple[1].(starlark.Callable) + if !ok { + return nil, fmt.Errorf("Invalid @%s annotation - expected second item in the 2-tuple to be an assertion function, but was %s (at %s)", AnnotationValidation, ruleTuple[1].Type(), ann.Position.AsCompactString()) + } + rules = append(rules, assertions.Rule{ + Msg: message.GoString(), + Assertion: lambda, + Position: ann.Position, + }) + } + return &ValidationAnnotation{rules, ann.Position}, nil +} + // NewTypeFromAnn returns type information given by annotation. func (t *TypeAnnotation) NewTypeFromAnn() (Type, error) { if t.any { @@ -444,6 +488,11 @@ func (t *TitleAnnotation) NewTypeFromAnn() (Type, error) { return nil, nil } +// NewTypeFromAnn returns type information given by annotation. +func (v *ValidationAnnotation) NewTypeFromAnn() (Type, error) { + return nil, nil +} + // GetPosition returns position of the source comment used to create this annotation. func (n *NullableAnnotation) GetPosition() *filepos.Position { return n.pos @@ -479,6 +528,16 @@ func (t *TitleAnnotation) GetPosition() *filepos.Position { return t.pos } +// GetPosition returns position of the source comment used to create this annotation. +func (v *ValidationAnnotation) GetPosition() *filepos.Position { + return nil +} + +// GetRules gets the validation rules from @schema/validation annotation +func (v *ValidationAnnotation) GetRules() []assertions.Rule { + return v.rules +} + func (t *TypeAnnotation) IsAny() bool { return t.any } @@ -612,6 +671,18 @@ func processOptionalAnnotation(node yamlmeta.Node, optionalAnnotation template.A return nil, nil } +func processValidationAnnotation(node yamlmeta.Node) (*ValidationAnnotation, error) { + nodeAnnotations := template.NewAnnotations(node) + if nodeAnnotations.Has(AnnotationValidation) { + validationAnn, err := NewValidationAnnotation(nodeAnnotations[AnnotationValidation], node.GetPosition()) + if err != nil { + return nil, err + } + return validationAnn, nil + } + return nil, nil +} + func getTypeFromAnnotations(anns []Annotation, pos *filepos.Position) (Type, error) { annsCopy := append([]Annotation{}, anns...) diff --git a/pkg/schema/assign.go b/pkg/schema/assign.go index de9708fd..50f5d975 100644 --- a/pkg/schema/assign.go +++ b/pkg/schema/assign.go @@ -6,6 +6,7 @@ package schema import ( "fmt" + "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -159,3 +160,17 @@ func (n NullType) AssignTypeTo(node yamlmeta.Node) TypeCheck { chk.Violations = append(chk.Violations, childCheck.Violations...) return chk } + +// AssignSchemaValidations implements the visitor interface to set validations from schema validation rules +type AssignSchemaValidations struct{} + +// Visit Extracts the validations from Node's Type and sets them in Node's meta +// This visitor returns nil if node has no assigned type or when the execution is completed +func (AssignSchemaValidations) Visit(node yamlmeta.Node) error { + if schemaType := GetType(node); schemaType != nil { + if rules := schemaType.GetValidations(); rules != nil { + assertions.SetValidations(node, rules) + } + } + return nil +} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index ff5d8c98..134e4d20 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -6,6 +6,7 @@ package schema import ( "fmt" + "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/filepos" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -22,9 +23,14 @@ func NewDocumentType(doc *yamlmeta.Document) (*DocumentType, error) { return nil, err } + validations, err := getValidations(doc) + if err != nil { + return nil, err + } + typeOfValue.SetDefaultValue(defaultValue) - return &DocumentType{Source: doc, Position: doc.Position, ValueType: typeOfValue, defaultValue: defaultValue}, nil + return &DocumentType{Source: doc, Position: doc.Position, ValueType: typeOfValue, defaultValue: defaultValue, validations: validations}, nil } func NewMapType(m *yamlmeta.Map) (*MapType, error) { @@ -52,9 +58,14 @@ func NewMapItemType(item *yamlmeta.MapItem) (*MapItemType, error) { return nil, err } + validations, err := getValidations(item) + if err != nil { + return nil, err + } + typeOfValue.SetDefaultValue(defaultValue) - return &MapItemType{Key: item.Key, ValueType: typeOfValue, defaultValue: defaultValue, Position: item.Position}, nil + return &MapItemType{Key: item.Key, ValueType: typeOfValue, defaultValue: defaultValue, Position: item.Position, validations: validations}, nil } func NewArrayType(a *yamlmeta.Array) (*ArrayType, error) { @@ -68,6 +79,7 @@ func NewArrayType(a *yamlmeta.Array) (*ArrayType, error) { } arrayItemType, err := NewArrayItemType(a.Items[0]) + if err != nil { return nil, err } @@ -86,9 +98,14 @@ func NewArrayItemType(item *yamlmeta.ArrayItem) (*ArrayItemType, error) { return nil, err } + validations, err := getValidations(item) + if err != nil { + return nil, err + } + typeOfValue.SetDefaultValue(defaultValue) - return &ArrayItemType{ValueType: typeOfValue, defaultValue: defaultValue, Position: item.GetPosition()}, nil + return &ArrayItemType{ValueType: typeOfValue, defaultValue: defaultValue, Position: item.GetPosition(), validations: validations}, nil } func getType(node yamlmeta.Node) (Type, error) { @@ -145,6 +162,17 @@ func getValue(node yamlmeta.Node, t Type) (interface{}, error) { return t.GetDefaultValue(), nil } +func getValidations(node yamlmeta.Node) ([]assertions.Rule, error) { + validationAnn, err := processValidationAnnotation(node) + if err != nil { + return nil, err + } + if validationAnn != nil { + return validationAnn.GetRules(), nil + } + return nil, nil +} + // getValueFromAnn extracts the value from the annotation and validates its type func getValueFromAnn(defaultAnn *DefaultAnnotation, t Type) (interface{}, error) { var typeCheck TypeCheck diff --git a/pkg/schema/type.go b/pkg/schema/type.go index d35bf23b..6407a843 100644 --- a/pkg/schema/type.go +++ b/pkg/schema/type.go @@ -6,6 +6,7 @@ package schema import ( "fmt" + "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/filepos" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -28,6 +29,7 @@ type Type interface { SetExamples([]Example) IsDeprecated() (bool, string) SetDeprecated(bool, string) + GetValidations() []assertions.Rule String() string } @@ -45,40 +47,50 @@ type DocumentType struct { ValueType Type // typically one of: MapType, ArrayType, ScalarType Position *filepos.Position defaultValue interface{} + validations []assertions.Rule } + type MapType struct { Items []*MapItemType Position *filepos.Position documentation documentation } + type MapItemType struct { Key interface{} // usually a string ValueType Type Position *filepos.Position defaultValue interface{} + validations []assertions.Rule } + type ArrayType struct { ItemsType Type Position *filepos.Position defaultValue interface{} documentation documentation } + type ArrayItemType struct { ValueType Type Position *filepos.Position defaultValue interface{} + validations []assertions.Rule } + type ScalarType struct { ValueType interface{} Position *filepos.Position defaultValue interface{} documentation documentation } + type AnyType struct { defaultValue interface{} Position *filepos.Position documentation documentation } + type NullType struct { ValueType Type Position *filepos.Position @@ -564,6 +576,46 @@ func (n *NullType) SetDeprecated(deprecated bool, notice string) { n.documentation.deprecated = deprecated } +// GetValidations provides validations from @schema/validation for a node +func (t *DocumentType) GetValidations() []assertions.Rule { + return t.validations +} + +// GetValidations provides validations from @schema/validation for a node +func (m *MapType) GetValidations() []assertions.Rule { + return nil +} + +// GetValidations provides validations from @schema/validation for a node +func (t *MapItemType) GetValidations() []assertions.Rule { + return t.validations +} + +// GetValidations provides validations from @schema/validation for a node +func (a *ArrayType) GetValidations() []assertions.Rule { + return nil +} + +// GetValidations provides validations from @schema/validation for a node +func (a *ArrayItemType) GetValidations() []assertions.Rule { + return a.validations +} + +// GetValidations provides validations from @schema/validation for a node +func (s *ScalarType) GetValidations() []assertions.Rule { + return nil +} + +// GetValidations provides validations from @schema/validation for a node +func (a AnyType) GetValidations() []assertions.Rule { + return nil +} + +// GetValidations provides validations from @schema/validation for a node +func (n NullType) GetValidations() []assertions.Rule { + return nil +} + // String produces a user-friendly name of the expected type. func (t *DocumentType) String() string { return yamlmeta.TypeName(&yamlmeta.Document{}) diff --git a/pkg/schema/yamlmeta.go b/pkg/schema/yamlmeta.go index fc3174c1..929bb04a 100644 --- a/pkg/schema/yamlmeta.go +++ b/pkg/schema/yamlmeta.go @@ -5,9 +5,11 @@ package schema import "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" +const schemaType = "schema/type" + // GetType retrieves schema metadata from `n`, set previously via SetType(). func GetType(n yamlmeta.Node) Type { - t := n.GetMeta("schema/type") + t := n.GetMeta(schemaType) if t == nil { return nil } @@ -16,5 +18,5 @@ func GetType(n yamlmeta.Node) Type { // SetType attaches schema metadata to `n`, later retrieved via GetType(). func SetType(n yamlmeta.Node, t Type) { - n.SetMeta("schema/type", t) + n.SetMeta(schemaType, t) } diff --git a/pkg/workspace/data_values_pre_processing.go b/pkg/workspace/data_values_pre_processing.go index 750b1196..030852ed 100644 --- a/pkg/workspace/data_values_pre_processing.go +++ b/pkg/workspace/data_values_pre_processing.go @@ -68,6 +68,8 @@ func (pp DataValuesPreProcessing) apply(files []*FileInLibrary) (*datavalues.Env if len(typeCheck.Violations) > 0 { return nil, nil, schema.NewSchemaError("One or more data values were invalid", typeCheck.Violations...) } + // Walk updates node's validations meta from Node's assigned type + yamlmeta.Walk(dvsDoc, schema.AssignSchemaValidations{}) } if dvsDoc == nil { From df80e9e5e2bdb0ec5fe07cc1b4a53a67a001eec1 Mon Sep 17 00:00:00 2001 From: Garrett Cheadle Date: Mon, 9 May 2022 14:28:39 -0700 Subject: [PATCH 2/2] refactor package and method names and return direct cast Signed-off-by: Varsha Munishwar --- pkg/assertions/yamlmeta.go | 39 --------------------- pkg/schema/annotations.go | 10 +++--- pkg/schema/assign.go | 4 +-- pkg/schema/schema.go | 4 +-- pkg/schema/type.go | 26 +++++++------- pkg/{assertions => validations}/assert.go | 8 ++--- pkg/{assertions => validations}/validate.go | 2 +- pkg/validations/yamlmeta.go | 33 +++++++++++++++++ pkg/workspace/library_execution.go | 4 +-- 9 files changed, 62 insertions(+), 68 deletions(-) delete mode 100644 pkg/assertions/yamlmeta.go rename pkg/{assertions => validations}/assert.go (97%) rename pkg/{assertions => validations}/validate.go (99%) create mode 100644 pkg/validations/yamlmeta.go diff --git a/pkg/assertions/yamlmeta.go b/pkg/assertions/yamlmeta.go deleted file mode 100644 index 704de0f1..00000000 --- a/pkg/assertions/yamlmeta.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2022 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package assertions - -import ( - "fmt" - - "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" -) - -const validations = "validations" - -// AddValidations appends validation Rules to node's validations metadata, later retrieved via GetValidations(). -func AddValidations(node yamlmeta.Node, rules []Rule) { - metas := node.GetMeta(validations) - if metas != nil { - currRules, ok := metas.([]Rule) - if !ok { - panic(fmt.Sprintf("Unexpected validations in meta : %s %v", node.GetPosition().AsCompactString(), metas)) - } - rules = append(currRules, rules...) - } - SetValidations(node, rules) -} - -// SetValidations attaches validation Rules to node's metadata, later retrieved via GetValidations(). -func SetValidations(node yamlmeta.Node, rules []Rule) { - node.SetMeta(validations, rules) -} - -// GetValidations retrieves validation Rules from node metadata, set previously via SetValidations(). -func GetValidations(node yamlmeta.Node) []Rule { - metas := node.GetMeta(validations) - if rules, ok := metas.([]Rule); ok { - return rules - } - return nil -} diff --git a/pkg/schema/annotations.go b/pkg/schema/annotations.go index a904d05f..0e0a52c6 100644 --- a/pkg/schema/annotations.go +++ b/pkg/schema/annotations.go @@ -8,10 +8,10 @@ import ( "strings" "github.com/k14s/starlark-go/starlark" - "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/filepos" "github.com/vmware-tanzu/carvel-ytt/pkg/template" "github.com/vmware-tanzu/carvel-ytt/pkg/template/core" + "github.com/vmware-tanzu/carvel-ytt/pkg/validations" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -76,7 +76,7 @@ type ExampleAnnotation struct { // ValidationAnnotation is a wrapper for validations provided via @schema/validation annotation type ValidationAnnotation struct { - rules []assertions.Rule + rules []validations.Rule pos *filepos.Position } @@ -413,7 +413,7 @@ func NewExampleAnnotation(ann template.NodeAnnotation, pos *filepos.Position) (* // NewValidationAnnotation checks the argument provided via @schema/validation annotation, and returns wrapper for the rules defined func NewValidationAnnotation(ann template.NodeAnnotation, pos *filepos.Position) (*ValidationAnnotation, error) { - var rules []assertions.Rule + var rules []validations.Rule if len(ann.Kwargs) != 0 { return nil, fmt.Errorf("Invalid @%s annotation - expected @%s to have 2-tuple as argument(s), but found keyword argument (by %s)", AnnotationValidation, AnnotationValidation, ann.Position.AsCompactString()) @@ -437,7 +437,7 @@ func NewValidationAnnotation(ann template.NodeAnnotation, pos *filepos.Position) if !ok { return nil, fmt.Errorf("Invalid @%s annotation - expected second item in the 2-tuple to be an assertion function, but was %s (at %s)", AnnotationValidation, ruleTuple[1].Type(), ann.Position.AsCompactString()) } - rules = append(rules, assertions.Rule{ + rules = append(rules, validations.Rule{ Msg: message.GoString(), Assertion: lambda, Position: ann.Position, @@ -534,7 +534,7 @@ func (v *ValidationAnnotation) GetPosition() *filepos.Position { } // GetRules gets the validation rules from @schema/validation annotation -func (v *ValidationAnnotation) GetRules() []assertions.Rule { +func (v *ValidationAnnotation) GetRules() []validations.Rule { return v.rules } diff --git a/pkg/schema/assign.go b/pkg/schema/assign.go index 50f5d975..d305894d 100644 --- a/pkg/schema/assign.go +++ b/pkg/schema/assign.go @@ -6,7 +6,7 @@ package schema import ( "fmt" - "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" + "github.com/vmware-tanzu/carvel-ytt/pkg/validations" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -169,7 +169,7 @@ type AssignSchemaValidations struct{} func (AssignSchemaValidations) Visit(node yamlmeta.Node) error { if schemaType := GetType(node); schemaType != nil { if rules := schemaType.GetValidations(); rules != nil { - assertions.SetValidations(node, rules) + validations.SetRules(node, rules) } } return nil diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 134e4d20..ed379709 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -6,8 +6,8 @@ package schema import ( "fmt" - "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/filepos" + "github.com/vmware-tanzu/carvel-ytt/pkg/validations" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -162,7 +162,7 @@ func getValue(node yamlmeta.Node, t Type) (interface{}, error) { return t.GetDefaultValue(), nil } -func getValidations(node yamlmeta.Node) ([]assertions.Rule, error) { +func getValidations(node yamlmeta.Node) ([]validations.Rule, error) { validationAnn, err := processValidationAnnotation(node) if err != nil { return nil, err diff --git a/pkg/schema/type.go b/pkg/schema/type.go index 6407a843..58ed5ff3 100644 --- a/pkg/schema/type.go +++ b/pkg/schema/type.go @@ -6,8 +6,8 @@ package schema import ( "fmt" - "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/filepos" + "github.com/vmware-tanzu/carvel-ytt/pkg/validations" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -29,7 +29,7 @@ type Type interface { SetExamples([]Example) IsDeprecated() (bool, string) SetDeprecated(bool, string) - GetValidations() []assertions.Rule + GetValidations() []validations.Rule String() string } @@ -47,7 +47,7 @@ type DocumentType struct { ValueType Type // typically one of: MapType, ArrayType, ScalarType Position *filepos.Position defaultValue interface{} - validations []assertions.Rule + validations []validations.Rule } type MapType struct { @@ -61,7 +61,7 @@ type MapItemType struct { ValueType Type Position *filepos.Position defaultValue interface{} - validations []assertions.Rule + validations []validations.Rule } type ArrayType struct { @@ -75,7 +75,7 @@ type ArrayItemType struct { ValueType Type Position *filepos.Position defaultValue interface{} - validations []assertions.Rule + validations []validations.Rule } type ScalarType struct { @@ -577,42 +577,42 @@ func (n *NullType) SetDeprecated(deprecated bool, notice string) { } // GetValidations provides validations from @schema/validation for a node -func (t *DocumentType) GetValidations() []assertions.Rule { +func (t *DocumentType) GetValidations() []validations.Rule { return t.validations } // GetValidations provides validations from @schema/validation for a node -func (m *MapType) GetValidations() []assertions.Rule { +func (m *MapType) GetValidations() []validations.Rule { return nil } // GetValidations provides validations from @schema/validation for a node -func (t *MapItemType) GetValidations() []assertions.Rule { +func (t *MapItemType) GetValidations() []validations.Rule { return t.validations } // GetValidations provides validations from @schema/validation for a node -func (a *ArrayType) GetValidations() []assertions.Rule { +func (a *ArrayType) GetValidations() []validations.Rule { return nil } // GetValidations provides validations from @schema/validation for a node -func (a *ArrayItemType) GetValidations() []assertions.Rule { +func (a *ArrayItemType) GetValidations() []validations.Rule { return a.validations } // GetValidations provides validations from @schema/validation for a node -func (s *ScalarType) GetValidations() []assertions.Rule { +func (s *ScalarType) GetValidations() []validations.Rule { return nil } // GetValidations provides validations from @schema/validation for a node -func (a AnyType) GetValidations() []assertions.Rule { +func (a AnyType) GetValidations() []validations.Rule { return nil } // GetValidations provides validations from @schema/validation for a node -func (n NullType) GetValidations() []assertions.Rule { +func (n NullType) GetValidations() []validations.Rule { return nil } diff --git a/pkg/assertions/assert.go b/pkg/validations/assert.go similarity index 97% rename from pkg/assertions/assert.go rename to pkg/validations/assert.go index 8147eb30..57d301d8 100644 --- a/pkg/assertions/assert.go +++ b/pkg/validations/assert.go @@ -1,7 +1,7 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -package assertions +package validations import ( "fmt" @@ -84,7 +84,7 @@ func (a *convertAssertAnnsToValidations) Visit(node yamlmeta.Node) error { return syntaxErr } // store rules in node's validations meta without overriding any existing rules - AddValidations(node, rules) + AddRules(node, rules) } return nil @@ -134,13 +134,13 @@ func newValidationRunner(threadName string) *validationRunner { return &validationRunner{thread: &starlark.Thread{Name: threadName}, chk: AssertCheck{[]error{}}} } -// Visit if `node` is has validations in its meta. +// Visit if `node` has validations in its meta. // Runs the validation Rules, any violations from running the assertions are collected. // // This visitor stores error(violations) in the validationRunner and returns nil. func (a *validationRunner) Visit(node yamlmeta.Node) error { // get rules in node's meta - rules := GetValidations(node) + rules := GetRules(node) if rules == nil { return nil } diff --git a/pkg/assertions/validate.go b/pkg/validations/validate.go similarity index 99% rename from pkg/assertions/validate.go rename to pkg/validations/validate.go index 5b6f730a..75b10e37 100644 --- a/pkg/assertions/validate.go +++ b/pkg/validations/validate.go @@ -1,7 +1,7 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -package assertions +package validations import ( "fmt" diff --git a/pkg/validations/yamlmeta.go b/pkg/validations/yamlmeta.go new file mode 100644 index 00000000..226aa6a3 --- /dev/null +++ b/pkg/validations/yamlmeta.go @@ -0,0 +1,33 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package validations + +import ( + "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" +) + +const validations = "validations" + +// AddRules appends validation Rules to node's validations metadata, later retrieved via GetRules(). +func AddRules(node yamlmeta.Node, rules []Rule) { + metas := node.GetMeta(validations) + if metas != nil { + rules = append(metas.([]Rule), rules...) + } + SetRules(node, rules) +} + +// SetRules attaches validation Rules to node's metadata, later retrieved via GetRules(). +func SetRules(node yamlmeta.Node, rules []Rule) { + node.SetMeta(validations, rules) +} + +// GetRules retrieves validation Rules from node metadata, set previously via SetRules(). +func GetRules(node yamlmeta.Node) []Rule { + metas := node.GetMeta(validations) + if metas == nil { + return nil + } + return metas.([]Rule) +} diff --git a/pkg/workspace/library_execution.go b/pkg/workspace/library_execution.go index 539bedf5..16c56c07 100644 --- a/pkg/workspace/library_execution.go +++ b/pkg/workspace/library_execution.go @@ -8,10 +8,10 @@ import ( "strings" "github.com/k14s/starlark-go/starlark" - "github.com/vmware-tanzu/carvel-ytt/pkg/assertions" "github.com/vmware-tanzu/carvel-ytt/pkg/cmd/ui" "github.com/vmware-tanzu/carvel-ytt/pkg/files" "github.com/vmware-tanzu/carvel-ytt/pkg/template" + "github.com/vmware-tanzu/carvel-ytt/pkg/validations" "github.com/vmware-tanzu/carvel-ytt/pkg/workspace/datavalues" "github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta" ) @@ -103,7 +103,7 @@ func (ll *LibraryExecution) Values(valuesOverlays []*datavalues.Envelope, schema // Returns an error if the arguments to an @assert/validate are invalid, // otherwise, checks the AssertCheck for violations, and returns nil if there are no violations. func (ll *LibraryExecution) validateValues(values *datavalues.Envelope) error { - assertCheck, err := assertions.ProcessAndRunValidations(values.Doc, "assert-data-values") + assertCheck, err := validations.ProcessAndRunValidations(values.Doc, "assert-data-values") if err != nil { return err }