diff --git a/ast/annotations.go b/ast/annotations.go index 981bdbbfa3..68f9f645cf 100644 --- a/ast/annotations.go +++ b/ast/annotations.go @@ -68,10 +68,10 @@ type ( } AnnotationsRef struct { - Location *Location `json:"location"` - Path Ref `json:"path"` + Location *Location `json:"location"` // The location of the node the annotations are applied to + Path Ref `json:"path"` // The path of the node the annotations are applied to Annotations *Annotations `json:"annotations,omitempty"` - node Node + node Node // The node the annotations are applied to } ) @@ -90,10 +90,22 @@ func (a *Annotations) SetLoc(l *Location) { a.Location = l } -// Compare returns an integer indicating if s is less than, equal to, or greater +// Compare returns an integer indicating if a is less than, equal to, or greater // than other. func (a *Annotations) Compare(other *Annotations) int { + if a == nil && other == nil { + return 0 + } + + if a == nil { + return -1 + } + + if other == nil { + return 1 + } + if cmp := scopeCompare(a.Scope, other.Scope); cmp != 0 { return cmp } @@ -141,6 +153,15 @@ func (a *Annotations) GetTargetPath() Ref { } } +func NewAnnotationsRef(a *Annotations) *AnnotationsRef { + return &AnnotationsRef{ + Location: a.node.Loc(), + Path: a.GetTargetPath(), + Annotations: a, + node: a.node, + } +} + func (ar *AnnotationsRef) GetPackage() *Package { switch n := ar.node.(type) { case *Package: @@ -287,6 +308,147 @@ func (a *Annotations) Copy(node Node) *Annotations { return &cpy } +// toObject constructs an AST Object from a. +func (a *Annotations) toObject() (*Object, *Error) { + obj := NewObject() + + if a == nil { + return &obj, nil + } + + if len(a.Scope) > 0 { + obj.Insert(StringTerm("scope"), StringTerm(a.Scope)) + } + + if len(a.Title) > 0 { + obj.Insert(StringTerm("title"), StringTerm(a.Title)) + } + + if len(a.Description) > 0 { + obj.Insert(StringTerm("description"), StringTerm(a.Description)) + } + + if len(a.Organizations) > 0 { + orgs := make([]*Term, 0, len(a.Organizations)) + for _, org := range a.Organizations { + orgs = append(orgs, StringTerm(org)) + } + obj.Insert(StringTerm("organizations"), ArrayTerm(orgs...)) + } + + if len(a.RelatedResources) > 0 { + rrs := make([]*Term, 0, len(a.RelatedResources)) + for _, rr := range a.RelatedResources { + rrObj := NewObject(Item(StringTerm("ref"), StringTerm(rr.Ref.String()))) + if len(rr.Description) > 0 { + rrObj.Insert(StringTerm("description"), StringTerm(rr.Description)) + } + rrs = append(rrs, NewTerm(rrObj)) + } + obj.Insert(StringTerm("related_resources"), ArrayTerm(rrs...)) + } + + if len(a.Authors) > 0 { + as := make([]*Term, 0, len(a.Authors)) + for _, author := range a.Authors { + aObj := NewObject() + if len(author.Name) > 0 { + aObj.Insert(StringTerm("name"), StringTerm(author.Name)) + } + if len(author.Email) > 0 { + aObj.Insert(StringTerm("email"), StringTerm(author.Email)) + } + as = append(as, NewTerm(aObj)) + } + obj.Insert(StringTerm("authors"), ArrayTerm(as...)) + } + + if len(a.Schemas) > 0 { + ss := make([]*Term, 0, len(a.Schemas)) + for _, s := range a.Schemas { + sObj := NewObject() + if len(s.Path) > 0 { + sObj.Insert(StringTerm("path"), NewTerm(s.Path.toArray())) + } + if len(s.Schema) > 0 { + sObj.Insert(StringTerm("schema"), NewTerm(s.Schema.toArray())) + } + if s.Definition != nil { + def, err := InterfaceToValue(s.Definition) + if err != nil { + return nil, NewError(CompileErr, a.Location, "invalid definition in schema annotation: %s", err.Error()) + } + sObj.Insert(StringTerm("definition"), NewTerm(def)) + } + ss = append(ss, NewTerm(sObj)) + } + obj.Insert(StringTerm("schemas"), ArrayTerm(ss...)) + } + + if len(a.Custom) > 0 { + c, err := InterfaceToValue(a.Custom) + if err != nil { + return nil, NewError(CompileErr, a.Location, "invalid custom annotation %s", err.Error()) + } + obj.Insert(StringTerm("custom"), NewTerm(c)) + } + + return &obj, nil +} + +func attachAnnotationsNodes(mod *Module) Errors { + var errs Errors + + // Find first non-annotation statement following each annotation and attach + // the annotation to that statement. + for _, a := range mod.Annotations { + for _, stmt := range mod.stmts { + _, ok := stmt.(*Annotations) + if !ok { + if stmt.Loc().Row > a.Location.Row { + a.node = stmt + break + } + } + } + + if a.Scope == "" { + switch a.node.(type) { + case *Rule: + a.Scope = annotationScopeRule + case *Package: + a.Scope = annotationScopePackage + case *Import: + a.Scope = annotationScopeImport + } + } + + if err := validateAnnotationScopeAttachment(a); err != nil { + errs = append(errs, err) + } + } + + return errs +} + +func validateAnnotationScopeAttachment(a *Annotations) *Error { + + switch a.Scope { + case annotationScopeRule, annotationScopeDocument: + if _, ok := a.node.(*Rule); ok { + return nil + } + return newScopeAttachmentErr(a, "rule") + case annotationScopePackage, annotationScopeSubpackages: + if _, ok := a.node.(*Package); ok { + return nil + } + return newScopeAttachmentErr(a, "package") + } + + return NewError(ParseErr, a.Loc(), "invalid annotation scope '%v'", a.Scope) +} + // Copy returns a deep copy of a. func (a *AuthorAnnotation) Copy() *AuthorAnnotation { cpy := *a @@ -481,32 +643,22 @@ func (as *AnnotationSet) Flatten() []*AnnotationsRef { refs = as.byPath.flatten(refs) - for p, a := range as.byPackage { - refs = append(refs, &AnnotationsRef{ - Location: p.Location, - Path: p.Path, - Annotations: a, - node: p, - }) + for _, a := range as.byPackage { + refs = append(refs, NewAnnotationsRef(a)) } - for r, as := range as.byRule { + for _, as := range as.byRule { for _, a := range as { - refs = append(refs, &AnnotationsRef{ - Location: r.Location, - Path: r.Path(), - Annotations: a, - node: r, - }) + refs = append(refs, NewAnnotationsRef(a)) } } - // Sort by path, then location, for stable output + // Sort by path, then annotation location, for stable output sort.SliceStable(refs, func(i, j int) bool { if refs[i].Path.Compare(refs[j].Path) < 0 { return true } - if refs[i].Location.Compare(refs[j].Location) < 0 { + if refs[i].Annotations.Location.Compare(refs[j].Annotations.Location) < 0 { return true } return false @@ -515,6 +667,59 @@ func (as *AnnotationSet) Flatten() []*AnnotationsRef { return refs } +// Chain returns the chain of annotations leading up to the given rule. +// The returned slice is ordered as follows +// 0. Entries for the given rule, ordered from the METADATA block declared immediately above the rule, to the block declared farthest away (always at least one entry) +// 1. The 'document' scope entry, if any +// 2. The 'package' scope entry, if any +// 3. Entries for the 'subpackages' scope, if any; ordered from the closest package path to the fartest. E.g.: 'do.re.mi', 'do.re', 'do' +// The returned slice is guaranteed to always contain at least one entry, corresponding to the given rule. +func (as *AnnotationSet) Chain(rule *Rule) []*AnnotationsRef { + var refs []*AnnotationsRef + + ruleAnnots := as.GetRuleScope(rule) + + if len(ruleAnnots) >= 1 { + for _, a := range ruleAnnots { + refs = append(refs, NewAnnotationsRef(a)) + } + } else { + // Make sure there is always a leading entry representing the passed rule, even if it has no annotations + refs = append(refs, &AnnotationsRef{ + Location: rule.Location, + Path: rule.Path(), + node: rule, + }) + } + + if len(refs) > 1 { + // Sort by annotation location; chain must start with annotations declared closest to rule, then going outward + sort.SliceStable(refs, func(i, j int) bool { + return refs[i].Annotations.Location.Compare(refs[j].Annotations.Location) > 0 + }) + } + + docAnnots := as.GetDocumentScope(rule.Path()) + if docAnnots != nil { + refs = append(refs, NewAnnotationsRef(docAnnots)) + } + + pkg := rule.Module.Package + pkgAnnots := as.GetPackageScope(pkg) + if pkgAnnots != nil { + refs = append(refs, NewAnnotationsRef(pkgAnnots)) + } + + subPkgAnnots := as.GetSubpackagesScope(pkg.Path) + // We need to reverse the order, as subPkgAnnots ordering will start at the root, + // whereas we want to end at the root. + for i := len(subPkgAnnots) - 1; i >= 0; i-- { + refs = append(refs, NewAnnotationsRef(subPkgAnnots[i])) + } + + return refs +} + func newAnnotationTree() *annotationTreeNode { return &annotationTreeNode{ Value: nil, @@ -550,6 +755,7 @@ func (t *annotationTreeNode) get(path Ref) *annotationTreeNode { return node } +// ancestors returns a slice of annotations in ascending order, starting with the root of ref; e.g.: 'root', 'root.foo', 'root.foo.bar'. func (t *annotationTreeNode) ancestors(path Ref) (result []*Annotations) { node := t for _, k := range path { @@ -570,12 +776,7 @@ func (t *annotationTreeNode) ancestors(path Ref) (result []*Annotations) { func (t *annotationTreeNode) flatten(refs []*AnnotationsRef) []*AnnotationsRef { if a := t.Value; a != nil { - refs = append(refs, &AnnotationsRef{ - Location: a.Location, - Path: a.GetTargetPath(), - Annotations: a, - node: a.node, - }) + refs = append(refs, NewAnnotationsRef(a)) } for _, c := range t.Children { refs = c.flatten(refs) diff --git a/ast/annotations_test.go b/ast/annotations_test.go index 01e8686436..322cfc00fd 100644 --- a/ast/annotations_test.go +++ b/ast/annotations_test.go @@ -10,25 +10,28 @@ import ( "testing" ) +// Test of example code in docs/content/annotations.md func ExampleAnnotationSet_Flatten() { - modules := map[string]string{ - "foo.rego": `# METADATA + modules := [][]string{ + { + "foo.rego", `# METADATA # scope: subpackages # organizations: # - Acme Corp. -package foo`, - "mod": `# METADATA +package foo`}, + { + "mod", `# METADATA # description: A couple of useful rules package foo.bar # METADATA # title: My Rule P -p := 7`, +p := 7`}, } parsed := make([]*Module, 0, len(modules)) - for f, module := range modules { - pm, err := ParseModuleWithOpts(f, module, ParserOptions{ProcessAnnotation: true}) + for _, entry := range modules { + pm, err := ParseModuleWithOpts(entry[0], entry[1], ParserOptions{ProcessAnnotation: true}) if err != nil { panic(err) } @@ -49,12 +52,61 @@ p := 7`, } // Output: - // data.foo at foo.rego:1 has annotations {"scope":"subpackages","organizations":["Acme Corp."]} + // data.foo at foo.rego:5 has annotations {"scope":"subpackages","organizations":["Acme Corp."]} // data.foo.bar at mod:3 has annotations {"scope":"package","description":"A couple of useful rules"} // data.foo.bar.p at mod:7 has annotations {"scope":"rule","title":"My Rule P"} } -func TestAnnotationSetFlatten(t *testing.T) { +// Test of example code in docs/content/annotations.md +func ExampleAnnotationSet_Chain() { + modules := [][]string{ + { + "foo.rego", `# METADATA +# scope: subpackages +# organizations: +# - Acme Corp. +package foo`}, + { + "mod", `# METADATA +# description: A couple of useful rules +package foo.bar + +# METADATA +# title: My Rule P +p := 7`}, + } + + parsed := make([]*Module, 0, len(modules)) + for _, entry := range modules { + pm, err := ParseModuleWithOpts(entry[0], entry[1], ParserOptions{ProcessAnnotation: true}) + if err != nil { + panic(err) + } + parsed = append(parsed, pm) + } + + as, err := BuildAnnotationSet(parsed) + if err != nil { + panic(err) + } + + rule := parsed[1].Rules[0] + + flattened := as.Chain(rule) + for _, entry := range flattened { + fmt.Printf("%v at %v has annotations %v\n", + entry.Path, + entry.Location, + entry.Annotations) + } + + // Output: + // data.foo.bar.p at mod:7 has annotations {"scope":"rule","title":"My Rule P"} + // data.foo.bar at mod:3 has annotations {"scope":"package","description":"A couple of useful rules"} + // data.foo at foo.rego:5 has annotations {"scope":"subpackages","organizations":["Acme Corp."]} +} + +func TestAnnotationSet_Flatten(t *testing.T) { tests := []struct { note string modules map[string]string @@ -144,7 +196,7 @@ p = 1`, }, { Path: MustParseRef("data.test.p"), - Location: &Location{File: "module", Row: 16}, + Location: &Location{File: "module", Row: 44}, Annotations: &Annotations{ Scope: "document", Title: "doc", @@ -229,7 +281,7 @@ package root2`, expected: []AnnotationsRef{ { Path: MustParseRef("data.root"), - Location: &Location{File: "root", Row: 1}, + Location: &Location{File: "root", Row: 4}, Annotations: &Annotations{ Scope: "subpackages", Title: "ROOT", @@ -237,7 +289,7 @@ package root2`, }, { Path: MustParseRef("data.root.bar"), - Location: &Location{File: "root.bar", Row: 1}, + Location: &Location{File: "root.bar", Row: 4}, Annotations: &Annotations{ Scope: "subpackages", Title: "BAR", @@ -253,7 +305,7 @@ package root2`, }, { Path: MustParseRef("data.root.foo"), - Location: &Location{File: "root.foo", Row: 1}, + Location: &Location{File: "root.foo", Row: 4}, Annotations: &Annotations{ Scope: "subpackages", Title: "FOO", @@ -269,7 +321,7 @@ package root2`, }, { Path: MustParseRef("data.root2"), - Location: &Location{File: "root2", Row: 1}, + Location: &Location{File: "root2", Row: 4}, Annotations: &Annotations{ Scope: "subpackages", Title: "ROOT2", @@ -399,7 +451,7 @@ p[v] {v = 2}`, i, a.Location, expected.Location) } if expected.Annotations.Compare(a.Annotations) != 0 { - t.Fatalf("annotations of AnnotationRef at %d\n%v\n doesn't match expected\n%v", + t.Fatalf("annotations of AnnotationRef at %d\n%v\ndoesn't match expected\n%v", i, a.Annotations, expected.Annotations) } } @@ -407,6 +459,568 @@ p[v] {v = 2}`, } } +func TestAnnotationSet_Chain(t *testing.T) { + tests := []struct { + note string + modules map[string]string + moduleToAnalyze string + ruleOnLineToAnalyze int + expected []AnnotationsRef + }{ + { + note: "simple module (all annotation types)", + modules: map[string]string{ + "module": `# METADATA +# title: pkg +# description: pkg +# organizations: +# - pkg +# related_resources: +# - https://pkg +# authors: +# - pkg +# schemas: +# - input.foo: {"type": "boolean"} +# custom: +# pkg: pkg +package test + +# METADATA +# scope: document +# title: doc +# description: doc +# organizations: +# - doc +# related_resources: +# - https://doc +# authors: +# - doc +# schemas: +# - input.bar: {"type": "integer"} +# custom: +# doc: doc + +# METADATA +# title: rule +# description: rule +# organizations: +# - rule +# related_resources: +# - https://rule +# authors: +# - rule +# schemas: +# - input.baz: {"type": "string"} +# custom: +# rule: rule +p = 1`, + }, + moduleToAnalyze: "module", + ruleOnLineToAnalyze: 44, + expected: []AnnotationsRef{ + { // Rule annotation is always first + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 44}, + Annotations: &Annotations{ + Scope: "rule", + Title: "rule", + Description: "rule", + Organizations: []string{"rule"}, + RelatedResources: []*RelatedResourceAnnotation{ + { + Ref: mustParseURL("https://rule"), + }, + }, + Authors: []*AuthorAnnotation{ + { + Name: "rule", + }, + }, + Schemas: []*SchemaAnnotation{ + schemaAnnotationFromMap("input.baz", map[string]interface{}{ + "type": "string", + }), + }, + Custom: map[string]interface{}{ + "rule": "rule", + }, + }, + }, + { + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 44}, + Annotations: &Annotations{ + Scope: "document", + Title: "doc", + Description: "doc", + Organizations: []string{"doc"}, + RelatedResources: []*RelatedResourceAnnotation{ + { + Ref: mustParseURL("https://doc"), + }, + }, + Authors: []*AuthorAnnotation{ + { + Name: "doc", + }, + }, + Schemas: []*SchemaAnnotation{ + schemaAnnotationFromMap("input.bar", map[string]interface{}{ + "type": "integer", + }), + }, + Custom: map[string]interface{}{ + "doc": "doc", + }, + }, + }, + { + Path: MustParseRef("data.test"), + Location: &Location{File: "module", Row: 14}, + Annotations: &Annotations{ + Scope: "package", + Title: "pkg", + Description: "pkg", + Organizations: []string{"pkg"}, + RelatedResources: []*RelatedResourceAnnotation{ + { + Ref: mustParseURL("https://pkg"), + }, + }, + Authors: []*AuthorAnnotation{ + { + Name: "pkg", + }, + }, + Schemas: []*SchemaAnnotation{ + schemaAnnotationFromMap("input.foo", map[string]interface{}{ + "type": "boolean", + }), + }, + Custom: map[string]interface{}{ + "pkg": "pkg", + }, + }, + }, + }, + }, + { + note: "no annotations on rule", + modules: map[string]string{ + "module": `# METADATA +# title: pkg +# description: pkg +package test + +# METADATA +# scope: document +# title: doc +# description: doc + +p = 1`, + }, + moduleToAnalyze: "module", + ruleOnLineToAnalyze: 11, + expected: []AnnotationsRef{ + { // Rule entry is always first, even if no annotations are present + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 11}, + Annotations: nil, + }, + { + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 11}, + Annotations: &Annotations{ + Scope: "document", + Title: "doc", + Description: "doc", + }, + }, + + { + Path: MustParseRef("data.test"), + Location: &Location{File: "module", Row: 4}, + Annotations: &Annotations{ + Scope: "package", + Title: "pkg", + Description: "pkg", + }, + }, + }, + }, + { + note: "multiple subpackages", + modules: map[string]string{ + "root": `# METADATA +# scope: subpackages +# title: ROOT +package root`, + "root.foo": `# METADATA +# title: FOO +# scope: subpackages +package root.foo`, + "root.foo.bar": `# METADATA +# scope: subpackages +# description: subpackages scope applied to rule in other module +# title: BAR-sub + +# METADATA +# title: BAR-other +# description: This metadata is on the path of the queried rule, but shouldn't show up in the result as it's in a different module. +package root.foo.bar + +# METADATA +# scope: document +# description: document scope applied to rule in other module +# title: P-doc +p = 1`, + "rule": `# METADATA +# title: BAR +package root.foo.bar + +# METADATA +# title: P +p = 1`, + }, + moduleToAnalyze: "rule", + ruleOnLineToAnalyze: 7, + expected: []AnnotationsRef{ + { + Path: MustParseRef("data.root.foo.bar.p"), + Location: &Location{File: "rule", Row: 7}, + Annotations: &Annotations{ + Scope: "rule", + Title: "P", + }, + }, + { + Path: MustParseRef("data.root.foo.bar.p"), + Location: &Location{File: "root.foo.bar", Row: 15}, + Annotations: &Annotations{ + Scope: "document", + Title: "P-doc", + Description: "document scope applied to rule in other module", + }, + }, + { + Path: MustParseRef("data.root.foo.bar"), + Location: &Location{File: "rule", Row: 3}, + Annotations: &Annotations{ + Scope: "package", + Title: "BAR", + }, + }, + { + Path: MustParseRef("data.root.foo.bar"), + Location: &Location{File: "root.foo.bar", Row: 9}, + Annotations: &Annotations{ + Scope: "subpackages", + Title: "BAR-sub", + Description: "subpackages scope applied to rule in other module", + }, + }, + { + Path: MustParseRef("data.root.foo"), + Location: &Location{File: "root.foo", Row: 4}, + Annotations: &Annotations{ + Scope: "subpackages", + Title: "FOO", + }, + }, + { + Path: MustParseRef("data.root"), + Location: &Location{File: "root", Row: 4}, + Annotations: &Annotations{ + Scope: "subpackages", + Title: "ROOT", + }, + }, + }, + }, + { + note: "multiple metadata blocks for single rule (order)", + modules: map[string]string{ + "module": `package test + +# METADATA +# title: One + +# METADATA +# title: Two + +# METADATA +# title: Three + +# METADATA +# title: Four +p = true`, + }, + moduleToAnalyze: "module", + ruleOnLineToAnalyze: 14, + expected: []AnnotationsRef{ // Rule annotations order is expected to start closest to the rule, moving out + { + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 14}, + Annotations: &Annotations{ + Scope: "rule", + Title: "Four", + }, + }, + { + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 14}, + Annotations: &Annotations{ + Scope: "rule", + Title: "Three", + }, + }, + { + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 14}, + Annotations: &Annotations{ + Scope: "rule", + Title: "Two", + }, + }, + { + Path: MustParseRef("data.test.p"), + Location: &Location{File: "module", Row: 14}, + Annotations: &Annotations{ + Scope: "rule", + Title: "One", + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + compiler := MustCompileModulesWithOpts(tc.modules, + CompileOpts{ParserOptions: ParserOptions{ProcessAnnotation: true}}) + + as := compiler.GetAnnotationSet() + if as == nil { + t.Fatalf("Expected compiled AnnotationSet, got nil") + } + + m := compiler.Modules[tc.moduleToAnalyze] + if m == nil { + t.Fatalf("no such module: %s", tc.moduleToAnalyze) + } + + var rule *Rule + for _, r := range m.Rules { + if r.Location.Row == tc.ruleOnLineToAnalyze { + rule = r + break + } + } + if rule == nil { + t.Fatalf("no rule found on line %d in module '%s'", + tc.ruleOnLineToAnalyze, tc.moduleToAnalyze) + } + + chain := as.Chain(rule) + + if len(chain) != len(tc.expected) { + t.Fatalf("chained AnnotationSet\n%v\n\ndoesn't match expected\n\n%v", + toJSON(chain), toJSON(tc.expected)) + } + + for i, expected := range tc.expected { + a := chain[i] + if !expected.Path.Equal(a.Path) { + t.Fatalf("path of AnnotationRef at %d '%v' doesn't match expected '%v'", + i, a.Path, expected.Path) + } + if expected.Location.File != a.Location.File || expected.Location.Row != a.Location.Row { + t.Fatalf("location of AnnotationRef at %d '%v' doesn't match expected '%v'", + i, a.Location, expected.Location) + } + if expected.Annotations.Compare(a.Annotations) != 0 { + t.Fatalf("annotations of AnnotationRef at %d\n%v\n\ndoesn't match expected\n\n%v", + i, a.Annotations, expected.Annotations) + } + } + }) + } +} + +func TestAnnotations_toObject(t *testing.T) { + annotations := Annotations{ + Scope: annotationScopeRule, + Title: "A title", + Description: "A description", + Organizations: []string{ + "Acme Corp.", + "Tyrell Corp.", + }, + RelatedResources: []*RelatedResourceAnnotation{ + { + Ref: mustParseURL("https://example.com"), + Description: "An example", + }, + { + Ref: mustParseURL("https://another.example.com"), + }, + }, + Authors: []*AuthorAnnotation{ + { + Name: "John Doe", + Email: "john@example.com", + }, + { + Name: "Jane Doe", + }, + { + Email: "jeff@example.com", + }, + }, + Schemas: []*SchemaAnnotation{ + { + Path: MustParseRef("input.foo"), + Schema: MustParseRef("schema.a"), + }, + schemaAnnotationFromMap("input.bar", map[string]interface{}{ + "type": "boolean", + }), + }, + Custom: map[string]interface{}{ + "number": 42, + "float": 2.2, + "string": "foo bar baz", + "bool": true, + "list": []interface{}{ + "a", "b", + }, + "list_of_lists": []interface{}{ + []interface{}{ + "a", "b", + }, + []interface{}{ + "b", "c", + }, + }, + "list_of_maps": []interface{}{ + map[string]interface{}{ + "one": 1, + "two": 2, + }, + map[string]interface{}{ + "two": 2, + "three": 3, + }, + }, + "map": map[string]interface{}{ + "nested_number": 1, + "nested_map": map[string]interface{}{ + "do": "re", + "mi": "fa", + }, + "nested_list": []interface{}{ + 1, 2, 3, + }, + }, + }, + } + + expected := NewObject( + Item(StringTerm("scope"), StringTerm(annotationScopeRule)), + Item(StringTerm("title"), StringTerm("A title")), + Item(StringTerm("description"), StringTerm("A description")), + Item(StringTerm("organizations"), ArrayTerm( + StringTerm("Acme Corp."), + StringTerm("Tyrell Corp."), + )), + Item(StringTerm("related_resources"), ArrayTerm( + ObjectTerm( + Item(StringTerm("ref"), StringTerm("https://example.com")), + Item(StringTerm("description"), StringTerm("An example")), + ), + ObjectTerm( + Item(StringTerm("ref"), StringTerm("https://another.example.com")), + ), + )), + Item(StringTerm("authors"), ArrayTerm( + ObjectTerm( + Item(StringTerm("name"), StringTerm("John Doe")), + Item(StringTerm("email"), StringTerm("john@example.com")), + ), + ObjectTerm( + Item(StringTerm("name"), StringTerm("Jane Doe")), + ), + ObjectTerm( + Item(StringTerm("email"), StringTerm("jeff@example.com")), + ), + )), + Item(StringTerm("schemas"), ArrayTerm( + ObjectTerm( + Item(StringTerm("path"), ArrayTerm(StringTerm("input"), StringTerm("foo"))), + Item(StringTerm("schema"), ArrayTerm(StringTerm("schema"), StringTerm("a"))), + ), + ObjectTerm( + Item(StringTerm("path"), ArrayTerm(StringTerm("input"), StringTerm("bar"))), + Item(StringTerm("definition"), ObjectTerm( + Item(StringTerm("type"), StringTerm("boolean")), + )), + ), + )), + Item(StringTerm("custom"), ObjectTerm( + Item(StringTerm("number"), NumberTerm("42")), + Item(StringTerm("float"), NumberTerm("2.2")), + Item(StringTerm("string"), StringTerm("foo bar baz")), + Item(StringTerm("bool"), BooleanTerm(true)), + Item(StringTerm("list"), ArrayTerm( + StringTerm("a"), + StringTerm("b"), + )), + Item(StringTerm("list_of_lists"), ArrayTerm( + ArrayTerm( + StringTerm("a"), + StringTerm("b"), + ), + ArrayTerm( + StringTerm("b"), + StringTerm("c"), + ), + )), + Item(StringTerm("list_of_maps"), ArrayTerm( + ObjectTerm( + Item(StringTerm("one"), NumberTerm("1")), + Item(StringTerm("two"), NumberTerm("2")), + ), + ObjectTerm( + Item(StringTerm("two"), NumberTerm("2")), + Item(StringTerm("three"), NumberTerm("3")), + ), + )), + Item(StringTerm("map"), ObjectTerm( + Item(StringTerm("nested_number"), NumberTerm("1")), + Item(StringTerm("nested_map"), ObjectTerm( + Item(StringTerm("do"), StringTerm("re")), + Item(StringTerm("mi"), StringTerm("fa")), + )), + Item(StringTerm("nested_list"), ArrayTerm( + NumberTerm("1"), + NumberTerm("2"), + NumberTerm("3"), + )), + )), + )), + ) + + obj, err := annotations.toObject() + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + + if Compare(*obj, expected) != 0 { + t.Fatalf("object generated from annotations\n\n%v\n\ndoesn't match expected\n\n%v", + *obj, expected) + } +} + func toJSON(v interface{}) string { b, _ := json.Marshal(v) return string(b) diff --git a/ast/builtins.go b/ast/builtins.go index cbfe54783a..4ce91b0978 100644 --- a/ast/builtins.go +++ b/ast/builtins.go @@ -232,6 +232,8 @@ var DefaultBuiltins = [...]*Builtin{ // Rego RegoParseModule, + RegoMetadataChain, + RegoMetadataRule, // OPA OPARuntime, @@ -2161,6 +2163,24 @@ var RegoParseModule = &Builtin{ ), } +// RegoMetadataChain returns the chain of metadata for the active rule +var RegoMetadataChain = &Builtin{ + Name: "rego.metadata.chain", + Decl: types.NewFunction( + types.Args(), + types.NewArray(nil, types.A), + ), +} + +// RegoMetadataRule returns the metadata for the active rule +var RegoMetadataRule = &Builtin{ + Name: "rego.metadata.rule", + Decl: types.NewFunction( + types.Args(), + types.A, + ), +} + /** * OPA */ diff --git a/ast/compile.go b/ast/compile.go index 6c738d0e10..4f23549fa4 100644 --- a/ast/compile.go +++ b/ast/compile.go @@ -258,12 +258,13 @@ func NewCompiler() *Compiler { metricName string f func() }{ - {"CheckDuplicateImports", "compile_stage_check_duplicate_imports", c.checkDuplicateImports}, - {"CheckKeywordOverrides", "compile_stage_check_keyword_overrides", c.checkKeywordOverrides}, // Reference resolution should run first as it may be used to lazily // load additional modules. If any stages run before resolution, they // need to be re-run after resolution. {"ResolveRefs", "compile_stage_resolve_refs", c.resolveAllRefs}, + {"CheckKeywordOverrides", "compile_stage_check_keyword_overrides", c.checkKeywordOverrides}, + {"CheckDuplicateImports", "compile_stage_check_duplicate_imports", c.checkDuplicateImports}, + {"RemoveImports", "compile_stage_remove_imports", c.removeImports}, {"SetModuleTree", "compile_stage_set_module_tree", c.setModuleTree}, {"SetRuleTree", "compile_stage_set_rule_tree", c.setRuleTree}, // The local variable generator must be initialized after references are @@ -274,6 +275,9 @@ func NewCompiler() *Compiler { {"CheckVoidCalls", "compile_stage_check_void_calls", c.checkVoidCalls}, {"RewritePrintCalls", "compile_stage_rewrite_print_calls", c.rewritePrintCalls}, {"RewriteExprTerms", "compile_stage_rewrite_expr_terms", c.rewriteExprTerms}, + {"ParseMetadataBlocks", "compile_stage_parse_metadata_blocks", c.parseMetadataBlocks}, + {"SetAnnotationSet", "compile_stage_set_annotationset", c.setAnnotationSet}, + {"RewriteRegoMetadataCalls", "compile_stage_rewrite_rego_metadata_calls", c.rewriteRegoMetadataCalls}, {"SetGraph", "compile_stage_set_graph", c.setGraph}, {"RewriteComprehensionTerms", "compile_stage_rewrite_comprehension_terms", c.rewriteComprehensionTerms}, {"RewriteRefsInHead", "compile_stage_rewrite_refs_in_head", c.rewriteRefsInHead}, @@ -285,7 +289,6 @@ func NewCompiler() *Compiler { {"RewriteEquals", "compile_stage_rewrite_equals", c.rewriteEquals}, {"RewriteDynamicTerms", "compile_stage_rewrite_dynamic_terms", c.rewriteDynamicTerms}, {"CheckRecursion", "compile_stage_check_recursion", c.checkRecursion}, - {"SetAnnotationSet", "compile_stage_set_annotationset", c.setAnnotationSet}, {"CheckTypes", "compile_stage_check_types", c.checkTypes}, // must be run after CheckRecursion {"CheckUnsafeBuiltins", "compile_state_check_unsafe_builtins", c.checkUnsafeBuiltins}, {"CheckDeprecatedBuiltins", "compile_state_check_deprecated_builtins", c.checkDeprecatedBuiltins}, @@ -1425,9 +1428,6 @@ func (c *Compiler) resolveAllRefs() { } } } - - // Once imports have been resolved, they are no longer needed. - mod.Imports = nil } if c.moduleLoader != nil { @@ -1452,6 +1452,12 @@ func (c *Compiler) resolveAllRefs() { } } +func (c *Compiler) removeImports() { + for name := range c.Modules { + c.Modules[name].Imports = nil + } +} + func (c *Compiler) initLocalVarGen() { c.localvargen = newLocalVarGeneratorForModuleSet(c.sorted, c.Modules) } @@ -1728,6 +1734,212 @@ func (c *Compiler) rewriteDynamicTerms() { } } +func (c *Compiler) parseMetadataBlocks() { + // Only parse annotations if rego.metadata built-ins are called + regoMetadataCalled := false + for _, name := range c.sorted { + mod := c.Modules[name] + WalkExprs(mod, func(expr *Expr) bool { + if isRegoMetadataChainCall(expr) || isRegoMetadataRuleCall(expr) { + regoMetadataCalled = true + } + return regoMetadataCalled + }) + + if regoMetadataCalled { + break + } + } + + if regoMetadataCalled { + // NOTE: Possible optimization: only parse annotations for modules on the path of rego.metadata-calling module + for _, name := range c.sorted { + mod := c.Modules[name] + + if len(mod.Annotations) == 0 { + var errs Errors + mod.Annotations, errs = parseAnnotations(mod.Comments) + errs = append(errs, attachAnnotationsNodes(mod)...) + for _, err := range errs { + c.err(err) + } + } + } + } +} + +func (c *Compiler) rewriteRegoMetadataCalls() { + eqFactory := newEqualityFactory(c.localvargen) + + for _, name := range c.sorted { + mod := c.Modules[name] + + WalkRules(mod, func(rule *Rule) bool { + var chainCalled bool + var ruleCalled bool + + WalkExprs(rule, func(expr *Expr) bool { + if isRegoMetadataChainCall(expr) { + chainCalled = true + } else if isRegoMetadataRuleCall(expr) { + ruleCalled = true + } + return chainCalled && ruleCalled + }) + + if chainCalled || ruleCalled { + body := make(Body, 0, len(rule.Body)+2) + + var metadataChainVar Var + if chainCalled { + // Create and inject metadata chain for rule + + chain, err := createMetadataChain(c.annotationSet.Chain(rule)) + if err != nil { + c.err(err) + return false + } + + eq := eqFactory.Generate(chain) + metadataChainVar = eq.Operands()[0].Value.(Var) + body.Append(eq) + } + + var metadataRuleVar Var + if ruleCalled { + // Create and inject metadata for rule + + var metadataRuleTerm *Term + + a := getPrimaryRuleAnnotations(c.annotationSet, rule) + if a != nil { + annotObj, err := a.toObject() + if err != nil { + c.err(err) + return false + } + metadataRuleTerm = NewTerm(*annotObj) + } else { + // If rule has no annotations, assign an empty object + metadataRuleTerm = ObjectTerm() + } + + eq := eqFactory.Generate(metadataRuleTerm) + metadataRuleVar = eq.Operands()[0].Value.(Var) + body.Append(eq) + } + + for _, expr := range rule.Body { + body.Append(expr) + } + rule.Body = body + + vis := func(b Body) bool { + for _, err := range rewriteRegoMetadataCalls(metadataChainVar, metadataRuleVar, b, &c.RewrittenVars) { + c.err(err) + } + return false + } + WalkBodies(rule.Head, vis) + WalkBodies(rule.Body, vis) + } + + return false + }) + } +} + +func getPrimaryRuleAnnotations(as *AnnotationSet, rule *Rule) *Annotations { + annots := as.GetRuleScope(rule) + + if len(annots) == 0 { + return nil + } + + // Sort by annotation location; chain must start with annotations declared closest to rule, then going outward + sort.SliceStable(annots, func(i, j int) bool { + return annots[i].Location.Compare(annots[j].Location) > 0 + }) + + return annots[0] +} + +func rewriteRegoMetadataCalls(metadataChainVar Var, metadataRuleVar Var, body Body, rewrittenVars *map[Var]Var) Errors { + var errs Errors + + WalkClosures(body, func(x interface{}) bool { + switch x := x.(type) { + case *ArrayComprehension: + errs = rewriteRegoMetadataCalls(metadataChainVar, metadataRuleVar, x.Body, rewrittenVars) + case *SetComprehension: + errs = rewriteRegoMetadataCalls(metadataChainVar, metadataRuleVar, x.Body, rewrittenVars) + case *ObjectComprehension: + errs = rewriteRegoMetadataCalls(metadataChainVar, metadataRuleVar, x.Body, rewrittenVars) + case *Every: + errs = rewriteRegoMetadataCalls(metadataChainVar, metadataRuleVar, x.Body, rewrittenVars) + } + return true + }) + + for i := range body { + expr := body[i] + var metadataVar Var + + if isRegoMetadataChainCall(expr) { + metadataVar = metadataChainVar + } else if isRegoMetadataRuleCall(expr) { + metadataVar = metadataRuleVar + } else { + continue + } + + // NOTE(johanfylling): An alternative strategy would be to walk the body and replace all operands[0] + // usages with *metadataChainVar + operands := expr.Operands() + if len(operands) > 0 { // There is an output var to rewrite + rewrittenVar := operands[0] + newExpr := Equality.Expr(rewrittenVar, NewTerm(metadataVar)) + newExpr.Generated = true + newExpr.Location = expr.Location + body.Set(newExpr, i) + } else { // No output var, just rewrite expr to metadataVar + body.Set(NewExpr(NewTerm(metadataVar)), i) + } + } + + return errs +} + +func isRegoMetadataChainCall(x *Expr) bool { + return x.IsCall() && x.Operator().Equal(RegoMetadataChain.Ref()) +} + +func isRegoMetadataRuleCall(x *Expr) bool { + return x.IsCall() && x.Operator().Equal(RegoMetadataRule.Ref()) +} + +func createMetadataChain(chain []*AnnotationsRef) (*Term, *Error) { + + metaArray := NewArray() + for _, link := range chain { + p := link.Path.toArray(). + Slice(1, -1) // Dropping leading 'data' element of path + obj := NewObject( + Item(StringTerm("path"), NewTerm(p)), + ) + if link.Annotations != nil { + annotObj, err := link.Annotations.toObject() + if err != nil { + return nil, err + } + obj.Insert(StringTerm("annotations"), NewTerm(*annotObj)) + } + metaArray = metaArray.Append(NewTerm(obj)) + } + + return NewTerm(metaArray), nil +} + func (c *Compiler) rewriteLocalVars() { for _, name := range c.sorted { diff --git a/ast/compile_test.go b/ast/compile_test.go index f56c65a28d..19bf7317a1 100644 --- a/ast/compile_test.go +++ b/ast/compile_test.go @@ -1738,6 +1738,9 @@ func TestCompilerCheckDuplicateImports(t *testing.T) { import input.foo import data.foo import data.bar.foo + + p := noconflict + q := foo `, expectedErrors: Errors{ &Error{ @@ -1755,6 +1758,9 @@ func TestCompilerCheckDuplicateImports(t *testing.T) { import input.noconflict import input.foo import input.bar as foo + + p := noconflict + q := foo `, expectedErrors: Errors{ &Error{ @@ -2335,6 +2341,274 @@ elsekw { assertRulesEqual(t, rule5, expected5) } +func TestCompilerRewriteRegoMetadataCalls(t *testing.T) { + tests := []struct { + note string + module string + exp string + }{ + { + note: "rego.metadata called, no metadata", + module: `package test + +p { + rego.metadata.chain()[0].path == ["test", "p"] + rego.metadata.rule() == {} +}`, + exp: `package test + +p = true { + __local2__ = [{"path": ["test", "p"]}] + __local3__ = {} + __local0__ = __local2__ + equal(__local0__[0].path, ["test", "p"]) + __local1__ = __local3__ + equal(__local1__, {}) +}`, + }, + { + note: "rego.metadata called, no output var, no metadata", + module: `package test + +p { + rego.metadata.chain() + rego.metadata.rule() +}`, + exp: `package test + +p = true { + __local0__ = [{"path": ["test", "p"]}] + __local1__ = {} + __local0__ + __local1__ +}`, + }, + { + note: "rego.metadata called, with metadata", + module: `# METADATA +# description: A test package +package test + +# METADATA +# title: My P Rule +p { + rego.metadata.chain()[0].title == "My P Rule" + rego.metadata.chain()[1].description == "A test package" +} + +# METADATA +# title: My Other P Rule +p { + rego.metadata.rule().title == "My Other P Rule" +}`, + exp: `# METADATA +# {"scope":"package","description":"A test package"} +package test + +# METADATA +# {"scope":"rule","title":"My P Rule"} +p = true { + __local3__ = [ + {"annotations": {"scope": "rule", "title": "My P Rule"}, "path": ["test", "p"]}, + {"annotations": {"description": "A test package", "scope": "package"}, "path": ["test"]} + ] + __local0__ = __local3__ + equal(__local0__[0].title, "My P Rule") + __local1__ = __local3__ + equal(__local1__[1].description, "A test package") +} + +# METADATA +# {"scope":"rule","title":"My Other P Rule"} +p = true { + __local4__ = {"scope": "rule", "title": "My Other P Rule"} + __local2__ = __local4__ + equal(__local2__.title, "My Other P Rule") +}`, + }, + { + note: "rego.metadata referenced multiple times", + module: `# METADATA +# description: TEST +package test + +p { + rego.metadata.chain()[0].path == ["test", "p"] + rego.metadata.chain()[1].path == ["test"] +}`, + exp: `# METADATA +# {"scope":"package","description":"TEST"} +package test + +p = true { + __local2__ = [ + {"path": ["test", "p"]}, + {"annotations": {"description": "TEST", "scope": "package"}, "path": ["test"]} + ] + __local0__ = __local2__ + equal(__local0__[0].path, ["test", "p"]) + __local1__ = __local2__ + equal(__local1__[1].path, ["test"]) }`, + }, + { + note: "rego.metadata return value", + module: `package test + +p := rego.metadata.chain()`, + exp: `package test + +p := __local0__ { + __local1__ = [{"path": ["test", "p"]}] + true + __local0__ = __local1__ +}`, + }, + { + note: "rego.metadata argument in function call", + module: `package test + +p { + q(rego.metadata.chain()) +} + +q(s) { + s == ["test", "p"] +}`, + exp: `package test + +p = true { + __local2__ = [{"path": ["test", "p"]}] + __local1__ = __local2__ + data.test.q(__local1__) +} + +q(__local0__) = true { + equal(__local0__, ["test", "p"]) +}`, + }, + { + note: "rego.metadata used in array comprehension", + module: `package test + +p = [x | x := rego.metadata.chain()]`, + exp: `package test + +p = [__local0__ | __local1__ = __local2__; __local0__ = __local1__] { + __local2__ = [{"path": ["test", "p"]}] + true +}`, + }, + { + note: "rego.metadata used in nested array comprehension", + module: `package test + +p { + y := [x | x := rego.metadata.chain()] + y[0].path == ["test", "p"] +}`, + exp: `package test + +p = true { + __local3__ = [{"path": ["test", "p"]}]; + __local1__ = [__local0__ | __local2__ = __local3__; __local0__ = __local2__]; + equal(__local1__[0].path, ["test", "p"]) +}`, + }, + { + note: "rego.metadata used in set comprehension", + module: `package test + +p = {x | x := rego.metadata.chain()}`, + exp: `package test + +p = {__local0__ | __local1__ = __local2__; __local0__ = __local1__} { + __local2__ = [{"path": ["test", "p"]}] + true +}`, + }, + { + note: "rego.metadata used in nested set comprehension", + module: `package test + +p { + y := {x | x := rego.metadata.chain()} + y[0].path == ["test", "p"] +}`, + exp: `package test + +p = true { + __local3__ = [{"path": ["test", "p"]}] + __local1__ = {__local0__ | __local2__ = __local3__; __local0__ = __local2__} + equal(__local1__[0].path, ["test", "p"]) +}`, + }, + { + note: "rego.metadata used in object comprehension", + module: `package test + +p = {i: x | x := rego.metadata.chain()[i]}`, + exp: `package test + +p = {i: __local0__ | __local1__ = __local2__; __local0__ = __local1__[i]} { + __local2__ = [{"path": ["test", "p"]}] + true +}`, + }, + { + note: "rego.metadata used in nested object comprehension", + module: `package test + +p { + y := {i: x | x := rego.metadata.chain()[i]} + y[0].path == ["test", "p"] +}`, + exp: `package test + +p = true { + __local3__ = [{"path": ["test", "p"]}] + __local1__ = {i: __local0__ | __local2__ = __local3__; __local0__ = __local2__[i]} + equal(__local1__[0].path, ["test", "p"]) +}`, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + c := NewCompiler() + c.Modules = map[string]*Module{ + "test.rego": MustParseModule(tc.module), + } + compileStages(c, c.rewriteRegoMetadataCalls) + assertNotFailed(t, c) + + result := c.Modules["test.rego"] + exp := MustParseModuleWithOpts(tc.exp, ParserOptions{ProcessAnnotation: true}) + + if result.Compare(exp) != 0 { + t.Fatalf("\nExpected:\n\n%v\n\nGot:\n\n%v", exp, result) + } + }) + } +} + +func TestCompilerOverridingSelfCalls(t *testing.T) { + c := NewCompiler() + c.Modules = map[string]*Module{ + "self.rego": MustParseModule(`package self.metadata + +chain(x) = "foo" +rule := "bar"`), + "test.rego": MustParseModule(`package test +import data.self + +p := self.metadata.chain(42) +q := self.metadata.rule`), + } + + compileStages(c, nil) + assertNotFailed(t, c) +} + func TestCompilerRewriteLocalAssignments(t *testing.T) { tests := []struct { diff --git a/ast/parser.go b/ast/parser.go index 1f9e54c20b..4d1f4b46e9 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -352,34 +352,54 @@ func (p *Parser) Parse() ([]Statement, []*Comment, Errors) { func (p *Parser) parseAnnotations(stmts []Statement) []Statement { + annotStmts, errs := parseAnnotations(p.s.comments) + for _, err := range errs { + p.error(err.Location, err.Message) + } + + for _, annotStmt := range annotStmts { + stmts = append(stmts, annotStmt) + } + + return stmts +} + +func parseAnnotations(comments []*Comment) ([]*Annotations, Errors) { + var hint = []byte("METADATA") var curr *metadataParser var blocks []*metadataParser - for i := 0; i < len(p.s.comments); i++ { + for i := 0; i < len(comments); i++ { if curr != nil { - if p.s.comments[i].Location.Row == p.s.comments[i-1].Location.Row+1 && p.s.comments[i].Location.Col == 1 { - curr.Append(p.s.comments[i]) + if comments[i].Location.Row == comments[i-1].Location.Row+1 && comments[i].Location.Col == 1 { + curr.Append(comments[i]) continue } curr = nil } - if bytes.HasPrefix(bytes.TrimSpace(p.s.comments[i].Text), hint) { - curr = newMetadataParser(p.s.comments[i].Location) + if bytes.HasPrefix(bytes.TrimSpace(comments[i].Text), hint) { + curr = newMetadataParser(comments[i].Location) blocks = append(blocks, curr) } } + var stmts []*Annotations + var errs Errors for _, b := range blocks { a, err := b.Parse() if err != nil { - p.error(b.loc, err.Error()) + errs = append(errs, &Error{ + Code: ParseErr, + Message: err.Error(), + Location: b.loc, + }) } else { stmts = append(stmts, a) } } - return stmts + return stmts, errs } func (p *Parser) parsePackage() *Package { @@ -2269,7 +2289,7 @@ func (p *Parser) futureImport(imp *Import, allowedFutureKeywords map[string]toke } if imp.Alias != "" { - p.errorf(imp.Path.Location, "future keyword imports cannot be aliased") + p.errorf(imp.Path.Location, "`future` imports cannot be aliased") return } diff --git a/ast/parser_ext.go b/ast/parser_ext.go index a1f2fbb959..c302defb7b 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -606,6 +606,7 @@ func parseModule(filename string, stmts []Statement, comments []*Comment) (*Modu mod := &Module{ Package: _package, + stmts: stmts, } // The comments slice only holds comments that were not their own statements. @@ -645,34 +646,7 @@ func parseModule(filename string, stmts []Statement, comments []*Comment) (*Modu return nil, errs } - // Find first non-annotation statement following each annotation and attach - // the annotation to that statement. - for _, a := range mod.Annotations { - for _, stmt := range stmts { - _, ok := stmt.(*Annotations) - if !ok { - if stmt.Loc().Row > a.Location.Row { - a.node = stmt - break - } - } - } - - if a.Scope == "" { - switch a.node.(type) { - case *Rule: - a.Scope = annotationScopeRule - case *Package: - a.Scope = annotationScopePackage - case *Import: - a.Scope = annotationScopeImport - } - } - - if err := validateAnnotationScopeAttachment(a); err != nil { - errs = append(errs, err) - } - } + errs = append(errs, attachAnnotationsNodes(mod)...) if len(errs) > 0 { return nil, errs @@ -681,24 +655,6 @@ func parseModule(filename string, stmts []Statement, comments []*Comment) (*Modu return mod, nil } -func validateAnnotationScopeAttachment(a *Annotations) *Error { - - switch a.Scope { - case annotationScopeRule, annotationScopeDocument: - if _, ok := a.node.(*Rule); ok { - return nil - } - return newScopeAttachmentErr(a, "rule") - case annotationScopePackage, annotationScopeSubpackages: - if _, ok := a.node.(*Package); ok { - return nil - } - return newScopeAttachmentErr(a, "package") - } - - return NewError(ParseErr, a.Loc(), "invalid annotation scope '%v'", a.Scope) -} - func newScopeAttachmentErr(a *Annotations, want string) *Error { var have string if a.node != nil { diff --git a/ast/parser_test.go b/ast/parser_test.go index bba75ae77e..96a17af7e6 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -1201,8 +1201,8 @@ func TestFutureImports(t *testing.T) { assertParseErrorContains(t, "future", "import future", "invalid import, must be `future.keywords`") assertParseErrorContains(t, "future.a", "import future.a", "invalid import, must be `future.keywords`") assertParseErrorContains(t, "unknown keyword", "import future.keywords.xyz", "unexpected keyword, must be one of [every in]") - assertParseErrorContains(t, "all keyword import + alias", "import future.keywords as xyz", "future keyword imports cannot be aliased") - assertParseErrorContains(t, "keyword import + alias", "import future.keywords.in as xyz", "future keyword imports cannot be aliased") + assertParseErrorContains(t, "all keyword import + alias", "import future.keywords as xyz", "`future` imports cannot be aliased") + assertParseErrorContains(t, "keyword import + alias", "import future.keywords.in as xyz", "`future` imports cannot be aliased") assertParseImport(t, "import kw with kw in options", "import future.keywords.in", &Import{Path: RefTerm(VarTerm("future"), StringTerm("keywords"), StringTerm("in"))}, @@ -3915,7 +3915,7 @@ func assertParseOneTermNegated(t *testing.T, msg string, input string, correct * assertParseOneExprNegated(t, msg, input, &Expr{Terms: correct}) } -func assertParseRule(t *testing.T, msg string, input string, correct *Rule) { +func assertParseRule(t *testing.T, msg string, input string, correct *Rule, opts ...ParserOptions) { t.Helper() assertParseOne(t, msg, input, func(parsed interface{}) { t.Helper() @@ -3923,5 +3923,6 @@ func assertParseRule(t *testing.T, msg string, input string, correct *Rule) { if !rule.Equal(correct) { t.Errorf("Error on test \"%s\": rules not equal: %v (parsed), %v (correct)", msg, rule, correct) } - }) + }, + opts...) } diff --git a/ast/policy.go b/ast/policy.go index fdb3d897ed..b9f4adb1cd 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -144,6 +144,7 @@ type ( Annotations []*Annotations `json:"annotations,omitempty"` Rules []*Rule `json:"rules,omitempty"` Comments []*Comment `json:"comments,omitempty"` + stmts []Statement } // Comment contains the raw text from the comment in the definition. @@ -260,11 +261,7 @@ func (mod *Module) Copy() *Module { cpy := *mod cpy.Rules = make([]*Rule, len(mod.Rules)) - var nodes map[Node]Node - - if len(mod.Annotations) > 0 { - nodes = make(map[Node]Node) - } + nodes := make(map[Node]Node) for i := range mod.Rules { cpy.Rules[i] = mod.Rules[i].Copy() @@ -297,6 +294,11 @@ func (mod *Module) Copy() *Module { cpy.Comments[i] = mod.Comments[i].Copy() } + cpy.stmts = make([]Statement, len(mod.stmts)) + for i := range mod.stmts { + cpy.stmts[i] = nodes[mod.stmts[i]] + } + return &cpy } diff --git a/ast/term.go b/ast/term.go index a6db2e8d9b..90bc94b8bd 100644 --- a/ast/term.go +++ b/ast/term.go @@ -1085,6 +1085,18 @@ func (ref Ref) OutputVars() VarSet { return vis.Vars() } +func (ref Ref) toArray() *Array { + a := NewArray() + for _, term := range ref { + if _, ok := term.Value.(String); ok { + a = a.Append(term) + } else { + a = a.Append(StringTerm(term.Value.String())) + } + } + return a +} + // QueryIterator defines the interface for querying AST documents with references. type QueryIterator func(map[Var]Value, Value) error diff --git a/capabilities.json b/capabilities.json index d5edb2daa7..2f9e6e6409 100644 --- a/capabilities.json +++ b/capabilities.json @@ -2856,6 +2856,27 @@ "type": "function" } }, + { + "name": "rego.metadata.chain", + "decl": { + "result": { + "dynamic": { + "type": "any" + }, + "type": "array" + }, + "type": "function" + } + }, + { + "name": "rego.metadata.rule", + "decl": { + "result": { + "type": "any" + }, + "type": "function" + } + }, { "name": "rego.parse_module", "decl": { diff --git a/cmd/inspect_test.go b/cmd/inspect_test.go index 141b73c2d2..eefe206f37 100644 --- a/cmd/inspect_test.go +++ b/cmd/inspect_test.go @@ -312,7 +312,7 @@ doc-descr Package: test Rule: p -Location: %[1]s/x.rego:18 +Location: %[1]s/x.rego:50 Organizations: doc-org diff --git a/docs/content/annotations.md b/docs/content/annotations.md index 87941aed8b..9940447de5 100644 --- a/docs/content/annotations.md +++ b/docs/content/annotations.md @@ -6,7 +6,7 @@ weight: 18 The package and individual rules in a module can be annotated with a rich set of metadata. -```rego +```live:rego/metadata:query:read_only # METADATA # title: My rule # description: A rule that determines if x is allowed. @@ -50,149 +50,35 @@ a metadata block determines how that metadata block will be applied. If the immediately follows the annotation. The `scope` values that are currently supported are: -* `rule` - applies to the individual rule statement. Default, when metadata block precedes rule. -* `document` - applies to all of the rules with the same name in the same package -* `package` - applies to all of the rules in the package, Default, when metadata block precedes package. -* `subpackages` - applies to all of the rules in the package and all subpackages (recursively) +* `rule` - applies to the individual rule statement (within the same file). Default, when metadata block precedes rule. +* `document` - applies to all of the rules with the same name in the same package (across multiple files) +* `package` - applies to all of the rules in the package (within the same file). Default, when metadata block precedes package. +* `subpackages` - applies to all of the rules in the package and all subpackages (recursively, across multiple files) -In case of overlap, schema annotations override each other as follows: +Since the `document` scope annotation applies to all rules with the same name in the same package +and the `subpackages` scope annotation applies to all packages with a matching path, metadata blocks with +these scopes are applied over all files with applicable package- and rule paths. +As there is no ordering across files in the same package, the `document` and `subpackages` scope annotations +can only be specified **once** per path. +The `document` scope annotation can be applied to any rule in the set (i.e., ordering does not matter.) -``` -rule overrides document -document overrides package -package overrides subpackages -``` - -The following sections explain how the different scopes work. - -#### Rule and Document Scopes - -``` -# METADATA -# scope: rule -# schemas: -# - input: schema.input -# - data.acl: schema["acl-schema"] -allow { - access := data.acl["alice"] - access[_] == input.operation -} - -allow { - access := data.acl["bob"] - access[_] == input.operation -} -``` - -In the example above, the second rule does not include an annotation, so type -checking of the second rule would not take schemas into account. To enable type -checking on the second (or other rules in the same file) we could specify the -annotation multiple times: - -``` -# METADATA -# scope: rule -# schemas: -# - input: schema.input -# - data.acl: schema["acl-schema"] -allow { - access := data.acl["alice"] - access[_] == input.operation -} - -# METADATA -# scope: rule -# schemas: -# - input: schema.input -# - data.acl: schema["acl-schema"] -allow { - access := data.acl["bob"] - access[_] == input.operation -} -``` - -This is obviously redundant and error-prone. To avoid this problem, we can -define the metadata block once on a rule with scope `document`: +#### Example -``` +```live:rego/metadata/scope:query:read_only # METADATA # scope: document -# schemas: -# - input: schema.input -# - data.acl: schema["acl-schema"] -allow { - access := data.acl["alice"] - access[_] == input.operation -} - -allow { - access := data.acl["bob"] - access[_] == input.operation -} -``` - -In this example, the metadata with `document` scope has the same affect as the -two `rule` scoped metadata blocks in the previous example. - -Since the `document` scope annotation applies to all rules with the same name in -the same package (which can span multiple files) and there is no ordering across -files in the same package, `document` scope annotations can only be specified -**once** per rule set. The `document` scope annotation can be applied to any -rule in the set (i.e., ordering does not matter.) +# description: A set of rules that determines if x is allowed. -#### Package and Subpackage Scopes - -Annotations can be defined at the `package` level and are then applied to all rules -within the package: - -``` # METADATA -# scope: package -# schemas: -# - input: schema.input -# - data.acl: schema["acl-schema"] -package example - -allow { - access := data.acl["alice"] - access[_] == input.operation -} - +# title: Allow Ones allow { - access := data.acl["bob"] - access[_] == input.operation + x == 1 } -``` -`package` scoped schema annotations are useful when all rules in the same -package operate on the same input structure. In some cases, when policies are -organized into many sub-packages, it is useful to declare schemas recursively -for them using the `subpackages` scope. For example: - -``` -# METADTA -# scope: subpackages -# schemas: -# - input: schema.input -package kubernetes.admission -``` - -This snippet would declare the top-level schema for `input` for the -`kubernetes.admission` package as well as all subpackages. If admission control -rules were defined inside packages like `kubernetes.admission.workloads.pods`, -they would be able to pick up that one schema declaration. - -#### Example - -```rego # METADATA -# scope: document -# schemas: -# - input: schema.input -# - data.acl: schema["acl-schema"] +# title: Allow Twos allow { - access := data.acl["alice"] - access[_] == input.operation + x == 2 } ``` @@ -202,7 +88,7 @@ The `title` annotation is a string value giving a human-readable name to the ann #### Example -```rego +```live:rego/metadata/title:query:read_only # METADATA # title: Allow Ones allow { @@ -222,7 +108,7 @@ The `description` annotation is a string value describing the annotation target, #### Example -```rego +```live:rego/metadata/description:query:read_only # METADATA # description: | # The 'allow' rule... @@ -249,7 +135,7 @@ When a *related-resource* entry is presented as a string, it needs to be a valid #### Examples -```rego +```live:rego/metadata/related_resources1:query:read_only # METADATA # related_resources: # - ref: https://example.com @@ -261,7 +147,7 @@ allow { } ``` -```rego +```live:rego/metadata/related_resources2:query:read_only # METADATA # organizations: # - https://example.com/foo @@ -292,7 +178,7 @@ Optionally, the last word may represent an email, if enclosed with `<>`. #### Examples -```rego +```live:rego/metadata/authors1:query:read_only # METADATA # authors: # - name: John Doe @@ -304,7 +190,7 @@ allow { } ``` -```rego +```live:rego/metadata/authors2:query:read_only # METADATA # authors: # - John Doe @@ -321,7 +207,7 @@ The `organizations` annotation is a list of string values representing the organ #### Example -```rego +```live:rego/metadata/organizations:query:read_only # METADATA # organizations: # - Acme Corp. @@ -339,7 +225,7 @@ In-depth information on this topic can be found [here](../schemas#schema-annotat #### Example -```rego +```live:rego/metadata/schemas:query:read_only # METADATA # schemas: # - input: schema.input @@ -356,7 +242,7 @@ The `custom` annotation is a mapping of user-defined data, mapping string keys t #### Example -```rego +```live:rego/metadata/custom:query:read_only # METADATA # custom: # my_int: 42 @@ -385,8 +271,8 @@ opa inspect -a ### Go API -The `ast.AnnotationSet` is a collection of all `ast.Annotations` declared in a set of modules. -An `ast.AnnotationSet` can be created from a slice of compiled modules: +The ``ast.AnnotationSet`` is a collection of all ``ast.Annotations`` declared in a set of modules. +An ``ast.AnnotationSet`` can be created from a slice of compiled modules: ```go var modules []*ast.Module @@ -397,7 +283,7 @@ if err != nil { } ``` -or can be retrieved from an `ast.Compiler` instance: +or can be retrieved from an ``ast.Compiler`` instance: ```go var modules []*ast.Module @@ -407,7 +293,7 @@ compiler.Compile(modules) as := compiler.GetAnnotationSet() ``` -The `ast.AnnotationSet` can be flattened into a slice of `ast.AnnotationsRef`, which is a complete, sorted list of all +The ``ast.AnnotationSet`` can be flattened into a slice of ``ast.AnnotationsRef``, which is a complete, sorted list of all annotations, grouped by the path and location of their targeted package or -rule. ```go @@ -418,4 +304,59 @@ for _, entry := range flattened { entry.Location, entry.Annotations) } + +// Output: +// data.foo at foo.rego:5 has annotations {"scope":"subpackages","organizations":["Acme Corp."]} +// data.foo.bar at mod:3 has annotations {"scope":"package","description":"A couple of useful rules"} +// data.foo.bar.p at mod:7 has annotations {"scope":"rule","title":"My Rule P"} +// +// For modules: +// # METADATA +// # scope: subpackages +// # organizations: +// # - Acme Corp. +// package foo +// --- +// # METADATA +// # description: A couple of useful rules +// package foo.bar +// +// # METADATA +// # title: My Rule P +// p := 7 +``` + +Given an ``ast.Rule``, the ``ast.AnnotationSet`` can return the chain of annotations declared for that rule, and its path ancestry. +The returned slice is ordered starting with the annotations for the rule, going outward to the farthest node with declared annotations +in the rule's path ancestry. + +```go +var rule *ast.Rule +... +chain := ast.Chain(rule) +for _, link := range chain { + fmt.Printf("link at %v has annotations %v\n", + link.Path, + link.Annotations) +} + +// Output: +// data.foo.bar.p at mod:7 has annotations {"scope":"rule","title":"My Rule P"} +// data.foo.bar at mod:3 has annotations {"scope":"package","description":"A couple of useful rules"} +// data.foo at foo.rego:5 has annotations {"scope":"subpackages","organizations":["Acme Corp."]} +// +// For modules: +// # METADATA +// # scope: subpackages +// # organizations: +// # - Acme Corp. +// package foo +// --- +// # METADATA +// # description: A couple of useful rules +// package foo.bar +// +// # METADATA +// # title: My Rule P +// p := 7 ``` \ No newline at end of file diff --git a/docs/content/policy-reference.md b/docs/content/policy-reference.md index 98aec9ad90..1e34932642 100644 --- a/docs/content/policy-reference.md +++ b/docs/content/policy-reference.md @@ -1096,6 +1096,80 @@ net.cidr_contains_matches({["1.1.0.0/16", "foo"], "1.1.2.0/24"}, {"x": "1.1.1.12 | Built-in | Description | Wasm Support | | ------- |-------------|---------------| | ``output := rego.parse_module(filename, string)`` | ``rego.parse_module`` parses the input ``string`` as a Rego module and returns the AST as a JSON object ``output``. | ``SDK-dependent`` | +| ``output := rego.metadata.chain()`` | Each entry in the ``output`` array represents a node in the path ancestry (chain) of the active rule that also has declared [annotations](../annotations).``output`` is ordered starting at the active rule, going outward to the most distant node in its package ancestry. A chain entry is a JSON document with two members: ``path``, an array representing the path of the node; and ``annotations``, a JSON document containing the annotations declared for the node. The first entry in the chain always points to the active rule, even if it has no declared annotations (in which case the ``annotations`` member is not present). | ✅ | +| ``output := rego.metadata.rule()`` | Returns a JSON object ``output`` containing the set of [annotations](../annotations) declared for the active rule and using the `rule` [scope](../annotations#scope). If no annotations are declared, an empty object is returned. | ✅ | + +#### Metadata Merge strategies + +When multiple [annotations](../annotations) are declared along the path ancestry (chain) for a rule, how any given annotation should be selected, inherited or merged depends on the semantics of the annotation, the context of the rule, and the preferences of the developer. +OPA doesn't presume what merge strategy is appropriate; instead, this lies in the hands of the developer. The following example demonstrates how some string and list type annotations in a metadata chain can be merged into a single metadata object. + +```live:rego/metadata:query:read_only +# METADATA +# title: My Example Package +# description: A set of rules illustrating how metadata annotations can be merged. +# authors: +# - John Doe +# organizations: +# - Acme Corp. +package example + +import future.keywords.in + +# METADATA +# scope: document +# description: A rule that merges metadata annotations in various ways. + +# METADATA +# title: My Allow Rule +# authors: +# - Jane Doe +allow { + meta := merge(rego.metadata.chain()) + meta.title == "My Allow Rule" # 'title' pulled from 'rule' scope + meta.description == "A rule that merges metadata annotations in various ways." # 'description' pulled from 'document' scope + meta.authors == { # 'authors' joined from 'package' and 'rule' scopes + {"email": "jane@example.com", "name": "Jane Doe"}, + {"email": "john@example.com", "name": "John Doe"} + } + meta.organizations == {"Acme Corp."} # 'organizations' pulled from 'package' scope +} + +allow { + meta := merge(rego.metadata.chain()) + meta.title == null # No 'title' present in 'rule' or 'document' scopes + meta.description == "A rule that merges metadata annotations in various ways." # 'description' pulled from 'document' scope + meta.authors == { # 'authors' pulled from 'package' scope + {"email": "john@example.com", "name": "John Doe"} + } + meta.organizations == {"Acme Corp."} # 'organizations' pulled from 'package' scope +} + +merge(chain) = meta { + ruleAndDoc := ["rule", "document"] + meta := { + "title": override_annot(chain, "title", ruleAndDoc), # looks for 'title' in 'rule' scope, then 'document' scope + "description": override_annot(chain, "description", ruleAndDoc), # looks for 'description' in 'rule' scope, then 'document' scope + "related_resources": override_annot(chain, "related_resources", ruleAndDoc), # looks for 'related_resources' in 'rule' scope, then 'document' scope + "authors": merge_annot(chain, "authors"), # merges all 'authors' across all scopes + "organizations": merge_annot(chain, "organizations"), # merges all 'organizations' across all scopes + } +} + +override_annot(chain, name, scopes) = val { + val := [v | + link := chain[_] + link.annotations.scope in scopes + v := link.annotations[name] + ][0] +} else = null + +merge_annot(chain, name) = val { + val := {v | + v := chain[_].annotations[name][_] + } +} else = null +``` ### OPA diff --git a/docs/content/schemas.md b/docs/content/schemas.md index 43b87635ef..a8424f1d55 100644 --- a/docs/content/schemas.md +++ b/docs/content/schemas.md @@ -207,6 +207,111 @@ On a different note, schema annotations can also be added to policy files part o The *scope* of the `schema` annotation can be controlled through the [scope](../annotations#scope) annotation +In case of overlap, schema annotations override each other as follows: + +``` +rule overrides document +document overrides package +package overrides subpackages +``` + +The following sections explain how the different scopes affect `schema` annotation +overriding for type checking. + +#### Rule and Document Scopes + +In the example above, the second rule does not include an annotation so type +checking of the second rule would not take schemas into account. To enable type +checking on the second (or other rules in the same file) we could specify the +annotation multiple times: + +``` +# METADATA +# scope: rule +# schemas: +# - input: schema.input +# - data.acl: schema["acl-schema"] +allow { + access := data.acl["alice"] + access[_] == input.operation +} + +# METADATA +# scope: rule +# schemas: +# - input: schema.input +# - data.acl: schema["acl-schema"] +allow { + access := data.acl["bob"] + access[_] == input.operation +} +``` + +This is obviously redundant and error-prone. To avoid this problem, we can +define the annotation once on a rule with scope `document`: + +``` +# METADATA +# scope: document +# schemas: +# - input: schema.input +# - data.acl: schema["acl-schema"] +allow { + access := data.acl["alice"] + access[_] == input.operation +} + +allow { + access := data.acl["bob"] + access[_] == input.operation +} +``` + +In this example, the annotation with `document` scope has the same affect as the +two `rule` scoped annotations in the previous example. + +#### Package and Subpackage Scopes + +Annotations can be defined at the `package` level and then applied to all rules +within the package: + +``` +# METADATA +# scope: package +# schemas: +# - input: schema.input +# - data.acl: schema["acl-schema"] +package example + +allow { + access := data.acl["alice"] + access[_] == input.operation +} + +allow { + access := data.acl["bob"] + access[_] == input.operation +} +``` + +`package` scoped schema annotations are useful when all rules in the same +package operate on the same input structure. In some cases, when policies are +organized into many sub-packages, it is useful to declare schemas recursively +for them using the `subpackages` scope. For example: + +``` +# METADTA +# scope: subpackages +# schemas: +# - input: schema.input +package kubernetes.admission +``` + +This snippet would declare the top-level schema for `input` for the +`kubernetes.admission` package as well as all subpackages. If admission control +rules were defined inside packages like `kubernetes.admission.workloads.pods`, +they would be able to pick up that one schema declaration. + ### Overriding JSON Schemas are often incomplete specifications of the format of data. For example, a Kubernetes Admission Review resource has a field `object` which can contain any other Kubernetes resource. A schema for Admission Review has a generic type `object` for that field that has no further specification. To allow more precise type checking in such cases, we support overriding existing schemas.