From 60494fe8d17333486a8adc940fc50759772e58e8 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Sat, 20 Oct 2018 17:04:16 +0300 Subject: [PATCH 1/2] schema: split schema package into multiple files --- schema/loader.go | 487 ++++++++++++++++++++++++++ schema/loader_test.go | 289 ++++++++++++++++ schema/namespaces.go | 57 ++++ schema/namespaces_test.go | 60 ++++ schema/schema.go | 695 +------------------------------------- schema/schema_test.go | 590 +------------------------------- schema/writer.go | 172 ++++++++++ schema/writer_test.go | 260 ++++++++++++++ 8 files changed, 1341 insertions(+), 1269 deletions(-) create mode 100644 schema/loader.go create mode 100644 schema/loader_test.go create mode 100644 schema/namespaces.go create mode 100644 schema/namespaces_test.go create mode 100644 schema/writer.go create mode 100644 schema/writer_test.go diff --git a/schema/loader.go b/schema/loader.go new file mode 100644 index 000000000..e4c76ac89 --- /dev/null +++ b/schema/loader.go @@ -0,0 +1,487 @@ +package schema + +import ( + "context" + "errors" + "fmt" + "reflect" + + "github.com/cayleygraph/cayley/graph/path" + "github.com/cayleygraph/cayley/quad" + + "github.com/cayleygraph/cayley/graph" + "github.com/cayleygraph/cayley/graph/iterator" +) + +var ( + errNotFound = errors.New("not found") + errRequiredFieldIsMissing = errors.New("required field is missing") +) + +// Optimize flags controls an optimization step performed before queries. +var Optimize = true + +// IsNotFound check if error is related to a missing object (either because of wrong ID or because of type constrains). +func IsNotFound(err error) bool { + return err == errNotFound || err == errRequiredFieldIsMissing +} + +// LoadTo will load a sub-graph of objects starting from ids (or from any nodes, if empty) +// to a destination Go object. Destination can be a struct, slice or channel. +// +// Mapping to quads is done via Go struct tag "quad" or "json" as a fallback. +// +// A simplest mapping is an "@id" tag which saves node ID (subject of a quad) into tagged field. +// +// type Node struct{ +// ID quad.IRI `json:"@id"` // or `quad:"@id"` +// } +// +// Field with an "@id" tag is omitted, but in case of Go->quads mapping new ID will be generated +// using GenerateID callback, which can be changed to provide a custom mappings. +// +// All other tags are interpreted as a predicate name for a specific field: +// +// type Person struct{ +// ID quad.IRI `json:"@id"` +// Name string `json:"name"` +// } +// p := Person{"bob","Bob"} +// // is equivalent to triple: +// // "Bob" +// +// Predicate IRIs in RDF can have a long namespaces, but they can be written in short +// form. They will be expanded automatically if namespace prefix is registered within +// QuadStore or globally via "voc" package. +// There is also a special predicate name "@type" which is mapped to "rdf:type" IRI. +// +// voc.RegisterPrefix("ex:", "http://example.org/") +// type Person struct{ +// ID quad.IRI `json:"@id"` +// Type quad.IRI `json:"@type"` +// Name string `json:"ex:name"` // will be expanded to http://example.org/name +// } +// p := Person{"bob",quad.IRI("Person"),"Bob"} +// // is equivalent to triples: +// // +// // "Bob" +// +// Predicate link direction can be reversed with a special tag syntax (not available for "json" tag): +// +// type Person struct{ +// ID quad.IRI `json:"@id"` +// Name string `json:"name"` // same as `quad:"name"` or `quad:"name > *"` +// Parents []quad.IRI `quad:"isParentOf < *"` +// } +// p := Person{"bob","Bob",[]quad.IRI{"alice","fred"}} +// // is equivalent to triples: +// // "Bob" +// // +// // +// +// All fields in structs are interpreted as required (except slices), thus struct will not be +// loaded if one of fields is missing. An "optional" tag can be specified to relax this requirement. +// Also, "required" can be specified for slices to alter default value. +// +// type Person struct{ +// ID quad.IRI `json:"@id"` +// Name string `json:"name"` // required field +// ThirdName string `quad:"thirdName,optional"` // can be empty +// FollowedBy []quad.IRI `quad:"follows"` +// } +func (c *Config) LoadTo(ctx context.Context, qs graph.QuadStore, dst interface{}, ids ...quad.Value) error { + return c.LoadToDepth(ctx, qs, dst, -1, ids...) +} + +// LoadToDepth is the same as LoadTo, but stops at a specified depth. +// Negative value means unlimited depth, and zero means top level only. +func (c *Config) LoadToDepth(ctx context.Context, qs graph.QuadStore, dst interface{}, depth int, ids ...quad.Value) error { + if dst == nil { + return fmt.Errorf("nil destination object") + } + var it graph.Iterator + if len(ids) != 0 { + fixed := iterator.NewFixed() + for _, id := range ids { + fixed.Add(qs.ValueOf(id)) + } + it = fixed + } + var rv reflect.Value + if v, ok := dst.(reflect.Value); ok { + rv = v + } else { + rv = reflect.ValueOf(dst) + } + return c.LoadIteratorToDepth(ctx, qs, rv, depth, it) +} + +// LoadPathTo is the same as LoadTo, but starts loading objects from a given path. +func (c *Config) LoadPathTo(ctx context.Context, qs graph.QuadStore, dst interface{}, p *path.Path) error { + return c.LoadIteratorTo(ctx, qs, reflect.ValueOf(dst), p.BuildIterator()) +} + +// LoadIteratorTo is a lower level version of LoadTo. +// +// It expects an iterator of nodes to be passed explicitly and +// destination value to be obtained via reflect package manually. +// +// Nodes iterator can be nil, All iterator will be used in this case. +func (c *Config) LoadIteratorTo(ctx context.Context, qs graph.QuadStore, dst reflect.Value, list graph.Iterator) error { + return c.LoadIteratorToDepth(ctx, qs, dst, -1, list) +} + +// LoadIteratorToDepth is the same as LoadIteratorTo, but stops at a specified depth. +// Negative value means unlimited depth, and zero means top level only. +func (c *Config) LoadIteratorToDepth(ctx context.Context, qs graph.QuadStore, dst reflect.Value, depth int, list graph.Iterator) error { + if depth >= 0 { + // 0 depth means "current level only" for user, but it's easier to make depth=0 a stop condition + depth++ + } + l := c.newLoader(qs) + return l.loadIteratorToDepth(ctx, dst, depth, list) +} + +type loader struct { + c *Config + qs graph.QuadStore + + pathForType map[reflect.Type]*path.Path + pathForTypeRoot map[reflect.Type]*path.Path +} + +func (c *Config) newLoader(qs graph.QuadStore) *loader { + return &loader{ + c: c, + qs: qs, + + pathForType: make(map[reflect.Type]*path.Path), + pathForTypeRoot: make(map[reflect.Type]*path.Path), + } +} + +func (l *loader) makePathForType(rt reflect.Type, tagPref string, rootOnly bool) (*path.Path, error) { + for rt.Kind() == reflect.Ptr { + rt = rt.Elem() + } + if rt.Kind() != reflect.Struct { + return nil, fmt.Errorf("expected struct, got %v", rt) + } + if tagPref == "" { + m := l.pathForType + if rootOnly { + m = l.pathForTypeRoot + } + if p, ok := m[rt]; ok { + return p, nil + } + } + + p := path.StartMorphism() + + if iri := getTypeIRI(rt); iri != quad.IRI("") { + p = p.Has(l.c.iri(iriType), iri) + } + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + if f.Anonymous { + pa, err := l.makePathForType(f.Type, tagPref+f.Name+".", rootOnly) + if err != nil { + return nil, err + } + p = p.Follow(pa) + continue + } + name := f.Name + rule, err := l.c.fieldRule(f) + if err != nil { + return nil, err + } else if rule == nil { // skip + continue + } + ft := f.Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + if err = checkFieldType(ft); err != nil { + return nil, err + } + switch rule := rule.(type) { + case idRule: + p = p.Tag(tagPref + name) + case constraintRule: + var nodes []quad.Value + if rule.Val != "" { + nodes = []quad.Value{rule.Val} + } + if rule.Rev { + p = p.HasReverse(rule.Pred, nodes...) + } else { + p = p.Has(rule.Pred, nodes...) + } + case saveRule: + tag := tagPref + name + if rule.Opt { + if !rootOnly { + if rule.Rev { + p = p.SaveOptionalReverse(rule.Pred, tag) + } else { + p = p.SaveOptional(rule.Pred, tag) + } + } + } else if rootOnly { // do not save field, enforce constraint only + if rule.Rev { + p = p.HasReverse(rule.Pred) + } else { + p = p.Has(rule.Pred) + } + } else { + if rule.Rev { + p = p.SaveReverse(rule.Pred, tag) + } else { + p = p.Save(rule.Pred, tag) + } + } + } + } + if tagPref != "" { + return p, nil + } + m := l.pathForType + if rootOnly { + m = l.pathForTypeRoot + } + m[rt] = p + return p, nil +} + +func (l *loader) loadToValue(ctx context.Context, dst reflect.Value, depth int, m map[string][]graph.Value, tagPref string) error { + if ctx == nil { + ctx = context.TODO() + } + for dst.Kind() == reflect.Ptr { + dst = dst.Elem() + } + rt := dst.Type() + if rt.Kind() != reflect.Struct { + return fmt.Errorf("expected struct, got %v", rt) + } + var fields fieldRules + if v := ctx.Value(fieldsCtxKey{}); v != nil { + fields = v.(fieldRules) + } else { + nfields, err := l.c.rulesFor(rt) + if err != nil { + return err + } + fields = nfields + } + if depth != 0 { // do not check required fields if depth limit is reached + for name, field := range fields { + if r, ok := field.(saveRule); ok && !r.Opt { + if vals := m[name]; len(vals) == 0 { + return errRequiredFieldIsMissing + } + } + } + } + for i := 0; i < rt.NumField(); i++ { + select { + case <-ctx.Done(): + return context.Canceled + default: + } + f := rt.Field(i) + name := f.Name + if err := checkFieldType(f.Type); err != nil { + return err + } + df := dst.Field(i) + if f.Anonymous { + if err := l.loadToValue(ctx, df, depth, m, tagPref+name+"."); err != nil { + return fmt.Errorf("load anonymous field %s failed: %v", f.Name, err) + } + continue + } + rules := fields[tagPref+name] + if rules == nil { + continue + } + arr, ok := m[tagPref+name] + if !ok || len(arr) == 0 { + continue + } + ft := f.Type + native := isNative(ft) + for ft.Kind() == reflect.Ptr || ft.Kind() == reflect.Slice { + native = native || isNative(ft) + ft = ft.Elem() + } + recursive := !native && ft.Kind() == reflect.Struct + for _, fv := range arr { + var sv reflect.Value + if recursive { + sv = reflect.New(ft).Elem() + sit := iterator.NewFixed() + sit.Add(fv) + err := l.loadIteratorToDepth(ctx, sv, depth-1, sit) + if err == errRequiredFieldIsMissing { + continue + } else if err != nil { + return err + } + } else { + fv := l.qs.NameOf(fv) + if fv == nil { + continue + } + sv = reflect.ValueOf(fv) + } + if err := DefaultConverter.SetValue(df, sv); err != nil { + return fmt.Errorf("field %s: %v", f.Name, err) + } + } + } + return nil +} + +func (l *loader) iteratorForType(root graph.Iterator, rt reflect.Type, rootOnly bool) (graph.Iterator, error) { + p, err := l.makePathForType(rt, "", rootOnly) + if err != nil { + return nil, err + } + return l.iteratorFromPath(root, p) +} + +func (l *loader) loadIteratorToDepth(ctx context.Context, dst reflect.Value, depth int, list graph.Iterator) error { + if ctx == nil { + ctx = context.TODO() + } + if dst.Kind() == reflect.Ptr { + dst = dst.Elem() + } + et := dst.Type() + slice, chanl := false, false + if dst.Kind() == reflect.Slice { + et = et.Elem() + slice = true + } else if dst.Kind() == reflect.Chan { + et = et.Elem() + chanl = true + defer dst.Close() + } + fields, err := l.c.rulesFor(et) + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + rootOnly := depth == 0 + it, err := l.iteratorForType(list, et, rootOnly) + if err != nil { + return err + } + defer it.Close() + + ctx = context.WithValue(ctx, fieldsCtxKey{}, fields) + for it.Next(ctx) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + mp := make(map[string]graph.Value) + it.TagResults(mp) + if len(mp) == 0 { + continue + } + cur := dst + if slice || chanl { + cur = reflect.New(et) + } + mo := make(map[string][]graph.Value, len(mp)) + for k, v := range mp { + mo[k] = []graph.Value{v} + } + for it.NextPath(ctx) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + mp = make(map[string]graph.Value) + it.TagResults(mp) + if len(mp) == 0 { + continue + } + // TODO(dennwc): replace with something more efficient + for k, v := range mp { + if sl, ok := mo[k]; !ok { + mo[k] = []graph.Value{v} + } else if len(sl) == 1 { + if !keysEqual(sl[0], v) { + mo[k] = append(sl, v) + } + } else { + found := false + for _, sv := range sl { + if keysEqual(sv, v) { + found = true + break + } + } + if !found { + mo[k] = append(sl, v) + } + } + } + } + err := l.loadToValue(ctx, cur, depth, mo, "") + if err == errRequiredFieldIsMissing { + if !slice && !chanl { + return err + } + continue + } else if err != nil { + return err + } + if slice { + dst.Set(reflect.Append(dst, cur.Elem())) + } else if chanl { + dst.Send(cur.Elem()) + } else { + return nil + } + } + if err := it.Err(); err != nil { + return err + } + if slice || chanl { + return nil + } + if list != nil && list.Type() != graph.All { + // distinguish between missing object and type constraints + list.Reset() + and := iterator.NewAnd(l.qs, list, l.qs.NodesAllIterator()) + defer and.Close() + if and.Next(ctx) { + return errRequiredFieldIsMissing + } + } + return errNotFound +} + +func (l *loader) iteratorFromPath(root graph.Iterator, p *path.Path) (graph.Iterator, error) { + it := p.BuildIteratorOn(l.qs) + if root != nil { + it = iterator.NewAnd(l.qs, root, it) + } + if Optimize { + it, _ = it.Optimize() + it, _ = l.qs.OptimizeIterator(it) + } + return it, nil +} diff --git a/schema/loader_test.go b/schema/loader_test.go new file mode 100644 index 000000000..728f63d30 --- /dev/null +++ b/schema/loader_test.go @@ -0,0 +1,289 @@ +package schema_test + +import ( + "reflect" + "testing" + + "github.com/cayleygraph/cayley/graph" + "github.com/cayleygraph/cayley/graph/iterator" + "github.com/cayleygraph/cayley/graph/memstore" + "github.com/cayleygraph/cayley/quad" + "github.com/cayleygraph/cayley/schema" +) + +func TestLoadIteratorTo(t *testing.T) { + sch := schema.NewConfig() + for i, c := range testFillValueCases { + t.Run(c.name, func(t *testing.T) { + qs := memstore.New(c.quads...) + out := reflect.New(reflect.TypeOf(c.expect)) + var it graph.Iterator + if c.from != nil { + fixed := iterator.NewFixed() + for _, id := range c.from { + fixed.Add(qs.ValueOf(id)) + } + it = fixed + } + depth := c.depth + if depth == 0 { + depth = -1 + } + if err := sch.LoadIteratorToDepth(nil, qs, out, depth, it); err != nil { + t.Errorf("case %d failed: %v", i+1, err) + return + } + got := out.Elem().Interface() + if s, ok := got.(interface { + Sort() + }); ok { + s.Sort() + } + if s, ok := c.expect.(interface { + Sort() + }); ok { + s.Sort() + } + if !reflect.DeepEqual(got, c.expect) { + t.Errorf("case %d failed: objects are different\n%#v\n%#v", + i+1, out.Elem().Interface(), c.expect, + ) + } + }) + } +} + +var testFillValueCases = []struct { + name string + expect interface{} + quads []quad.Quad + depth int + from []quad.Value +}{ + { + name: "complex object", + expect: struct { + rdfType struct{} `quad:"rdf:type > some:Type"` + ID quad.IRI `quad:"@id"` + Name string `quad:"name"` + Values []string `quad:"values"` + Items []item `quad:"items"` + Sub *item `quad:"sub"` + Val int `quad:"val"` + }{ + ID: "1234", + Name: "some item", + Values: []string{"val1", "val2"}, + Items: []item{ + {ID: "sub1", Name: "Sub 1"}, + {ID: "sub2", Name: "Sub 2"}, + }, + Sub: &item{ID: "sub3", Name: "Sub 3"}, + Val: 123, + }, + quads: []quad.Quad{ + {iri("1234"), typeIRI, iri("some:Type"), nil}, + {iri("1234"), iri("name"), quad.String("some item"), nil}, + {iri("1234"), iri("values"), quad.String("val1"), nil}, + {iri("1234"), iri("values"), quad.String("val2"), nil}, + {iri("sub1"), typeIRI, iri("some:item"), nil}, + {iri("sub1"), iri("name"), quad.String("Sub 1"), nil}, + {iri("1234"), iri("items"), iri("sub1"), nil}, + {iri("sub2"), typeIRI, iri("some:item"), nil}, + {iri("sub2"), iri("name"), quad.String("Sub 2"), nil}, + {iri("1234"), iri("items"), iri("sub2"), nil}, + {iri("sub3"), typeIRI, iri("some:item"), nil}, + {iri("sub3"), iri("name"), quad.String("Sub 3"), nil}, + {iri("1234"), iri("sub"), iri("sub3"), nil}, + {iri("1234"), iri("val"), quad.Int(123), nil}, + }, + }, + { + name: "complex object (id value)", + expect: struct { + rdfType struct{} `quad:"rdf:type > some:Type"` + ID quad.Value `quad:"@id"` + Name string `quad:"name"` + Values []string `quad:"values"` + Items []item `quad:"items"` + }{ + ID: quad.BNode("1234"), + Name: "some item", + Values: []string{"val1", "val2"}, + Items: []item{ + {ID: "sub1", Name: "Sub 1"}, + {ID: "sub2", Name: "Sub 2"}, + }, + }, + quads: []quad.Quad{ + {quad.BNode("1234"), typeIRI, iri("some:Type"), nil}, + {quad.BNode("1234"), iri("name"), quad.String("some item"), nil}, + {quad.BNode("1234"), iri("values"), quad.String("val1"), nil}, + {quad.BNode("1234"), iri("values"), quad.String("val2"), nil}, + {iri("sub1"), typeIRI, iri("some:item"), nil}, + {iri("sub1"), iri("name"), quad.String("Sub 1"), nil}, + {quad.BNode("1234"), iri("items"), iri("sub1"), nil}, + {iri("sub2"), typeIRI, iri("some:item"), nil}, + {iri("sub2"), iri("name"), quad.String("Sub 2"), nil}, + {quad.BNode("1234"), iri("items"), iri("sub2"), nil}, + }, + }, + { + name: "embedded object", + expect: struct { + rdfType struct{} `quad:"rdf:type > some:Type"` + item2 + ID quad.IRI `quad:"@id"` + Values []string `quad:"values"` + }{ + item2: item2{Name: "Sub 1", Spec: "special"}, + ID: "1234", + Values: []string{"val1", "val2"}, + }, + quads: []quad.Quad{ + {iri("1234"), typeIRI, iri("some:Type"), nil}, + {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, + {iri("1234"), iri("spec"), quad.String("special"), nil}, + {iri("1234"), iri("values"), quad.String("val1"), nil}, + {iri("1234"), iri("values"), quad.String("val2"), nil}, + }, + }, + { + name: "type shorthand", + expect: struct { + rdfType struct{} `quad:"@type > some:Type"` + item2 + ID quad.IRI `quad:"@id"` + Values []string `quad:"values"` + }{ + item2: item2{Name: "Sub 1", Spec: "special"}, + ID: "1234", + Values: []string{"val1", "val2"}, + }, + quads: []quad.Quad{ + {iri("1234"), typeIRI, iri("some:Type"), nil}, + {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, + {iri("1234"), iri("spec"), quad.String("special"), nil}, + {iri("1234"), iri("values"), quad.String("val1"), nil}, + {iri("1234"), iri("values"), quad.String("val2"), nil}, + }, + }, + { + name: "tree", + expect: treeItem{ + ID: iri("n1"), + Name: "Node 1", + Children: []treeItem{ + { + ID: iri("n2"), + Name: "Node 2", + }, + { + ID: iri("n3"), + Name: "Node 3", + Children: []treeItem{ + { + ID: iri("n4"), + Name: "Node 4", + }, + }, + }, + }, + }, + quads: treeQuads, + from: []quad.Value{iri("n1")}, + }, + { + name: "tree with depth limit 1", + expect: treeItem{ + ID: iri("n1"), + Name: "Node 1", + Children: []treeItem{ + { + ID: iri("n2"), + Name: "Node 2", + }, + { + ID: iri("n3"), + Name: "Node 3", + Children: []treeItem{ + { + ID: iri("n4"), + }, + }, + }, + }, + }, + depth: 1, + quads: treeQuads, + from: []quad.Value{iri("n1")}, + }, + { + name: "tree with depth limit 2", + expect: treeItemOpt{ + ID: iri("n1"), + Name: "Node 1", + Children: []treeItemOpt{ + { + ID: iri("n2"), + Name: "Node 2", + }, + { + ID: iri("n3"), + Name: "Node 3", + Children: []treeItemOpt{ + { + ID: iri("n4"), + Name: "Node 4", + }, + }, + }, + }, + }, + depth: 2, + quads: treeQuads, + from: []quad.Value{iri("n1")}, + }, + { + name: "tree with required children", + expect: treeItemReq{ + ID: iri("n1"), + Name: "Node 1", + Children: []treeItemReq{ + { + ID: iri("n3"), + Name: "Node 3", + // TODO(dennwc): a strange behavior: this field is required, but it's empty for current object, + // because all it's children are missing the same field. Leaving this as-is for now because + // it's weird to set Children field as required in a tree. + Children: nil, + }, + }, + }, + quads: treeQuads, + from: []quad.Value{iri("n1")}, + }, + { + name: "simple object", + expect: subObject{ + genObject: genObject{ + ID: "1234", + Name: "Obj", + }, + Num: 3, + }, + quads: []quad.Quad{ + {iri("1234"), iri("name"), quad.String("Obj"), nil}, + {iri("1234"), iri("num"), quad.Int(3), nil}, + }, + }, + { + name: "coords", + expect: Coords{Lat: 12.3, Lng: 34.5}, + quads: []quad.Quad{ + {iri("c1"), typeIRI, iri("ex:Coords"), nil}, + {iri("c1"), iri("ex:lat"), quad.Float(12.3), nil}, + {iri("c1"), iri("ex:lng"), quad.Float(34.5), nil}, + }, + }, +} diff --git a/schema/namespaces.go b/schema/namespaces.go new file mode 100644 index 000000000..0bb1d7777 --- /dev/null +++ b/schema/namespaces.go @@ -0,0 +1,57 @@ +package schema + +import ( + "context" + "fmt" + "reflect" + + "github.com/cayleygraph/cayley/graph" + "github.com/cayleygraph/cayley/quad" + "github.com/cayleygraph/cayley/voc" +) + +type namespace struct { + _ struct{} `quad:"@type > cayley:namespace"` + Full quad.IRI `quad:"@id"` + Prefix quad.IRI `quad:"cayley:prefix"` +} + +// WriteNamespaces will writes namespaces list into graph. +func (c *Config) WriteNamespaces(w quad.Writer, n *voc.Namespaces) error { + rules, err := c.rulesFor(reflect.TypeOf(namespace{})) + if err != nil { + return fmt.Errorf("can't load rules: %v", err) + } + wr := c.newWriter(w) + for _, ns := range n.List() { + obj := namespace{ + Full: quad.IRI(ns.Full), + Prefix: quad.IRI(ns.Prefix), + } + rv := reflect.ValueOf(obj) + if err = wr.writeValueAs(obj.Full, rv, "", rules); err != nil { + return err + } + } + return nil +} + +// LoadNamespaces will load namespaces stored in graph to a specified list. +// If destination list is empty, global namespace registry will be used. +func (c *Config) LoadNamespaces(ctx context.Context, qs graph.QuadStore, dest *voc.Namespaces) error { + var list []namespace + if err := c.LoadTo(ctx, qs, &list); err != nil { + return err + } + register := dest.Register + if dest == nil { + register = voc.Register + } + for _, ns := range list { + register(voc.Namespace{ + Prefix: string(ns.Prefix), + Full: string(ns.Full), + }) + } + return nil +} diff --git a/schema/namespaces_test.go b/schema/namespaces_test.go new file mode 100644 index 000000000..ad39f3493 --- /dev/null +++ b/schema/namespaces_test.go @@ -0,0 +1,60 @@ +package schema_test + +import ( + "context" + "reflect" + "sort" + "testing" + + "github.com/cayleygraph/cayley/graph" + "github.com/cayleygraph/cayley/graph/memstore" + "github.com/cayleygraph/cayley/quad" + "github.com/cayleygraph/cayley/schema" + "github.com/cayleygraph/cayley/voc" +) + +func TestSaveNamespaces(t *testing.T) { + sch := schema.NewConfig() + save := []voc.Namespace{ + {Full: "http://example.org/", Prefix: "ex:"}, + {Full: "http://cayley.io/", Prefix: "c:"}, + } + var ns voc.Namespaces + for _, n := range save { + ns.Register(n) + } + qs := memstore.New() + err := sch.WriteNamespaces(qs, &ns) + if err != nil { + t.Fatal(err) + } + var ns2 voc.Namespaces + err = sch.LoadNamespaces(context.TODO(), qs, &ns2) + if err != nil { + t.Fatal(err) + } + got := ns2.List() + sort.Sort(voc.ByFullName(save)) + sort.Sort(voc.ByFullName(got)) + if !reflect.DeepEqual(save, got) { + t.Fatalf("wrong namespaces returned: got: %v, expect: %v", got, save) + } + qr := graph.NewQuadStoreReader(qs) + q, err := quad.ReadAll(qr) + qr.Close() + if err != nil { + t.Fatal(err) + } + expect := []quad.Quad{ + quad.MakeIRI("http://cayley.io/", "cayley:prefix", "c:", ""), + quad.MakeIRI("http://cayley.io/", "rdf:type", "cayley:namespace", ""), + + quad.MakeIRI("http://example.org/", "cayley:prefix", "ex:", ""), + quad.MakeIRI("http://example.org/", "rdf:type", "cayley:namespace", ""), + } + sort.Sort(quad.ByQuadString(expect)) + sort.Sort(quad.ByQuadString(q)) + if !reflect.DeepEqual(expect, q) { + t.Fatalf("wrong quads returned: got: %v, expect: %v", q, expect) + } +} diff --git a/schema/schema.go b/schema/schema.go index 244feba8e..9ff9d5ca9 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -5,21 +5,19 @@ package schema import ( - "context" - "errors" "fmt" "reflect" "strings" "sync" "github.com/cayleygraph/cayley/graph" - "github.com/cayleygraph/cayley/graph/iterator" "github.com/cayleygraph/cayley/graph/path" "github.com/cayleygraph/cayley/quad" - "github.com/cayleygraph/cayley/voc" "github.com/cayleygraph/cayley/voc/rdf" ) +var reflQuadValue = reflect.TypeOf((*quad.Value)(nil)).Elem() + type ErrReqFieldNotSet struct { Field string } @@ -58,10 +56,6 @@ type Config struct { // Label will be added to all quads written. Does not affect queries. Label quad.Value - pathForTypeMu sync.RWMutex - pathForType map[reflect.Type]*path.Path - pathForTypeRoot map[reflect.Type]*path.Path - rulesForTypeMu sync.RWMutex rulesForType map[reflect.Type]fieldRules } @@ -215,35 +209,19 @@ func checkFieldType(ftp reflect.Type) error { return nil } -// Optimize flags controls an optimization step performed before queries. -var Optimize = true - -func iteratorFromPath(qs graph.QuadStore, root graph.Iterator, p *path.Path) (graph.Iterator, error) { - it := p.BuildIteratorOn(qs) - if root != nil { - it = iterator.NewAnd(qs, root, it) - } - if Optimize { - it, _ = it.Optimize() - it, _ = qs.OptimizeIterator(it) - } - return it, nil -} - -func (c *Config) iteratorForType(qs graph.QuadStore, root graph.Iterator, rt reflect.Type, rootOnly bool) (graph.Iterator, error) { - p, err := c.makePathForType(rt, "", rootOnly) - if err != nil { - return nil, err - } - return iteratorFromPath(qs, root, p) -} - var ( typesMu sync.RWMutex typeToIRI = make(map[reflect.Type]quad.IRI) iriToType = make(map[quad.IRI]reflect.Type) ) +func getTypeIRI(rt reflect.Type) quad.IRI { + typesMu.RLock() + iri := typeToIRI[rt] + typesMu.RUnlock() + return iri +} + // RegisterType associates an IRI with a given Go type. // // All queries and writes will require or add a type triple. @@ -278,122 +256,10 @@ func RegisterType(iri quad.IRI, obj interface{}) { iriToType[full] = rt } -func (c *Config) makePathForType(rt reflect.Type, tagPref string, rootOnly bool) (*path.Path, error) { - for rt.Kind() == reflect.Ptr { - rt = rt.Elem() - } - if rt.Kind() != reflect.Struct { - return nil, fmt.Errorf("expected struct, got %v", rt) - } - if tagPref != "" { - c.pathForTypeMu.RLock() - m := c.pathForType - if rootOnly { - m = c.pathForTypeRoot - } - p, ok := m[rt] - c.pathForTypeMu.RUnlock() - if ok { - return p, nil - } - } - - p := path.StartMorphism() - typesMu.RLock() - iri := typeToIRI[rt] - typesMu.RUnlock() - if iri != quad.IRI("") { - p = p.Has(c.iri(iriType), iri) - } - for i := 0; i < rt.NumField(); i++ { - f := rt.Field(i) - if f.Anonymous { - pa, err := c.makePathForType(f.Type, tagPref+f.Name+".", rootOnly) - if err != nil { - return nil, err - } - p = p.Follow(pa) - continue - } - name := f.Name - rule, err := c.fieldRule(f) - if err != nil { - return nil, err - } else if rule == nil { // skip - continue - } - ft := f.Type - if ft.Kind() == reflect.Ptr { - ft = ft.Elem() - } - if err = checkFieldType(ft); err != nil { - return nil, err - } - switch rule := rule.(type) { - case idRule: - p = p.Tag(tagPref + name) - case constraintRule: - var nodes []quad.Value - if rule.Val != "" { - nodes = []quad.Value{rule.Val} - } - if rule.Rev { - p = p.HasReverse(rule.Pred, nodes...) - } else { - p = p.Has(rule.Pred, nodes...) - } - case saveRule: - tag := tagPref + name - if rule.Opt { - if !rootOnly { - if rule.Rev { - p = p.SaveOptionalReverse(rule.Pred, tag) - } else { - p = p.SaveOptional(rule.Pred, tag) - } - } - } else if rootOnly { // do not save field, enforce constraint only - if rule.Rev { - p = p.HasReverse(rule.Pred) - } else { - p = p.Has(rule.Pred) - } - } else { - if rule.Rev { - p = p.SaveReverse(rule.Pred, tag) - } else { - p = p.Save(rule.Pred, tag) - } - } - } - } - if tagPref == "" { - return p, nil - } - - c.pathForTypeMu.Lock() - defer c.pathForTypeMu.Unlock() - var m map[reflect.Type]*path.Path - if rootOnly { - m = c.pathForTypeRoot - } else { - m = c.pathForType - } - if m == nil { - m = make(map[reflect.Type]*path.Path) - if rootOnly { - c.pathForTypeRoot = m - } else { - c.pathForType = m - } - } - m[rt] = p - return p, nil -} - // PathForType builds a path (morphism) for a given Go type. func (c *Config) PathForType(rt reflect.Type) (*path.Path, error) { - return c.makePathForType(rt, "", false) + l := c.newLoader(nil) + return l.makePathForType(rt, "", false) } func anonFieldType(fld reflect.StructField) (reflect.Type, bool) { @@ -506,106 +372,6 @@ func init() { }) } -// IsNotFound check if error is related to a missing object (either because of wrong ID or because of type constrains). -func IsNotFound(err error) bool { - return err == errNotFound || err == errRequiredFieldIsMissing -} - -var ( - errNotFound = errors.New("not found") - errRequiredFieldIsMissing = errors.New("required field is missing") -) - -func (c *Config) loadToValue(ctx context.Context, qs graph.QuadStore, dst reflect.Value, depth int, m map[string][]graph.Value, tagPref string) error { - if ctx == nil { - ctx = context.TODO() - } - for dst.Kind() == reflect.Ptr { - dst = dst.Elem() - } - rt := dst.Type() - if rt.Kind() != reflect.Struct { - return fmt.Errorf("expected struct, got %v", rt) - } - var fields fieldRules - if v := ctx.Value(fieldsCtxKey{}); v != nil { - fields = v.(fieldRules) - } else { - nfields, err := c.rulesFor(rt) - if err != nil { - return err - } - fields = nfields - } - if depth != 0 { // do not check required fields if depth limit is reached - for name, field := range fields { - if r, ok := field.(saveRule); ok && !r.Opt { - if vals := m[name]; len(vals) == 0 { - return errRequiredFieldIsMissing - } - } - } - } - for i := 0; i < rt.NumField(); i++ { - select { - case <-ctx.Done(): - return context.Canceled - default: - } - f := rt.Field(i) - name := f.Name - if err := checkFieldType(f.Type); err != nil { - return err - } - df := dst.Field(i) - if f.Anonymous { - if err := c.loadToValue(ctx, qs, df, depth, m, tagPref+name+"."); err != nil { - return fmt.Errorf("load anonymous field %s failed: %v", f.Name, err) - } - continue - } - rules := fields[tagPref+name] - if rules == nil { - continue - } - arr, ok := m[tagPref+name] - if !ok || len(arr) == 0 { - continue - } - ft := f.Type - native := isNative(ft) - for ft.Kind() == reflect.Ptr || ft.Kind() == reflect.Slice { - native = native || isNative(ft) - ft = ft.Elem() - } - recursive := !native && ft.Kind() == reflect.Struct - for _, fv := range arr { - var sv reflect.Value - if recursive { - sv = reflect.New(ft).Elem() - sit := iterator.NewFixed() - sit.Add(fv) - err := c.loadIteratorToDepth(ctx, qs, sv, depth-1, sit) - if err == errRequiredFieldIsMissing { - continue - } else if err != nil { - return err - } - } else { - fv := qs.NameOf(fv) - if fv == nil { - continue - } - sv = reflect.ValueOf(fv) - } - if err := DefaultConverter.SetValue(df, sv); err != nil { - return fmt.Errorf("field %s: %v", f.Name, err) - } - } - } - return nil -} - func isNative(rt reflect.Type) bool { // TODO(dennwc): replace _, ok := quad.AsValue(reflect.Zero(rt).Interface()) return ok @@ -626,332 +392,10 @@ func keysEqual(v1, v2 graph.Value) bool { return v1 == v2 } -// LoadTo will load a sub-graph of objects starting from ids (or from any nodes, if empty) -// to a destination Go object. Destination can be a struct, slice or channel. -// -// Mapping to quads is done via Go struct tag "quad" or "json" as a fallback. -// -// A simplest mapping is an "@id" tag which saves node ID (subject of a quad) into tagged field. -// -// type Node struct{ -// ID quad.IRI `json:"@id"` // or `quad:"@id"` -// } -// -// Field with an "@id" tag is omitted, but in case of Go->quads mapping new ID will be generated -// using GenerateID callback, which can be changed to provide a custom mappings. -// -// All other tags are interpreted as a predicate name for a specific field: -// -// type Person struct{ -// ID quad.IRI `json:"@id"` -// Name string `json:"name"` -// } -// p := Person{"bob","Bob"} -// // is equivalent to triple: -// // "Bob" -// -// Predicate IRIs in RDF can have a long namespaces, but they can be written in short -// form. They will be expanded automatically if namespace prefix is registered within -// QuadStore or globally via "voc" package. -// There is also a special predicate name "@type" which is mapped to "rdf:type" IRI. -// -// voc.RegisterPrefix("ex:", "http://example.org/") -// type Person struct{ -// ID quad.IRI `json:"@id"` -// Type quad.IRI `json:"@type"` -// Name string `json:"ex:name"` // will be expanded to http://example.org/name -// } -// p := Person{"bob",quad.IRI("Person"),"Bob"} -// // is equivalent to triples: -// // -// // "Bob" -// -// Predicate link direction can be reversed with a special tag syntax (not available for "json" tag): -// -// type Person struct{ -// ID quad.IRI `json:"@id"` -// Name string `json:"name"` // same as `quad:"name"` or `quad:"name > *"` -// Parents []quad.IRI `quad:"isParentOf < *"` -// } -// p := Person{"bob","Bob",[]quad.IRI{"alice","fred"}} -// // is equivalent to triples: -// // "Bob" -// // -// // -// -// All fields in structs are interpreted as required (except slices), thus struct will not be -// loaded if one of fields is missing. An "optional" tag can be specified to relax this requirement. -// Also, "required" can be specified for slices to alter default value. -// -// type Person struct{ -// ID quad.IRI `json:"@id"` -// Name string `json:"name"` // required field -// ThirdName string `quad:"thirdName,optional"` // can be empty -// FollowedBy []quad.IRI `quad:"follows"` -// } -func (c *Config) LoadTo(ctx context.Context, qs graph.QuadStore, dst interface{}, ids ...quad.Value) error { - return c.LoadToDepth(ctx, qs, dst, -1, ids...) -} - -// LoadToDepth is the same as LoadTo, but stops at a specified depth. -// Negative value means unlimited depth, and zero means top level only. -func (c *Config) LoadToDepth(ctx context.Context, qs graph.QuadStore, dst interface{}, depth int, ids ...quad.Value) error { - if dst == nil { - return fmt.Errorf("nil destination object") - } - var it graph.Iterator - if len(ids) != 0 { - fixed := iterator.NewFixed() - for _, id := range ids { - fixed.Add(qs.ValueOf(id)) - } - it = fixed - } - var rv reflect.Value - if v, ok := dst.(reflect.Value); ok { - rv = v - } else { - rv = reflect.ValueOf(dst) - } - return c.LoadIteratorToDepth(ctx, qs, rv, depth, it) -} - -// LoadPathTo is the same as LoadTo, but starts loading objects from a given path. -func (c *Config) LoadPathTo(ctx context.Context, qs graph.QuadStore, dst interface{}, p *path.Path) error { - return c.LoadIteratorTo(ctx, qs, reflect.ValueOf(dst), p.BuildIterator()) -} - -// LoadIteratorTo is a lower level version of LoadTo. -// -// It expects an iterator of nodes to be passed explicitly and -// destination value to be obtained via reflect package manually. -// -// Nodes iterator can be nil, All iterator will be used in this case. -func (c *Config) LoadIteratorTo(ctx context.Context, qs graph.QuadStore, dst reflect.Value, list graph.Iterator) error { - return c.LoadIteratorToDepth(ctx, qs, dst, -1, list) -} - -// LoadIteratorToDepth is the same as LoadIteratorTo, but stops at a specified depth. -// Negative value means unlimited depth, and zero means top level only. -func (c *Config) LoadIteratorToDepth(ctx context.Context, qs graph.QuadStore, dst reflect.Value, depth int, list graph.Iterator) error { - if depth >= 0 { - // 0 depth means "current level only" for user, but it's easier to make depth=0 a stop condition - depth++ - } - return c.loadIteratorToDepth(ctx, qs, dst, depth, list) -} - -func (c *Config) loadIteratorToDepth(ctx context.Context, qs graph.QuadStore, dst reflect.Value, depth int, list graph.Iterator) error { - if ctx == nil { - ctx = context.Background() - } - if dst.Kind() == reflect.Ptr { - dst = dst.Elem() - } - et := dst.Type() - slice, chanl := false, false - if dst.Kind() == reflect.Slice { - et = et.Elem() - slice = true - } else if dst.Kind() == reflect.Chan { - et = et.Elem() - chanl = true - defer dst.Close() - } - fields, err := c.rulesFor(et) - if err != nil { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - rootOnly := depth == 0 - it, err := c.iteratorForType(qs, list, et, rootOnly) - if err != nil { - return err - } - defer it.Close() - - ctx = context.WithValue(ctx, fieldsCtxKey{}, fields) - for it.Next(ctx) { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - mp := make(map[string]graph.Value) - it.TagResults(mp) - if len(mp) == 0 { - continue - } - cur := dst - if slice || chanl { - cur = reflect.New(et) - } - mo := make(map[string][]graph.Value, len(mp)) - for k, v := range mp { - mo[k] = []graph.Value{v} - } - for it.NextPath(ctx) { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - mp = make(map[string]graph.Value) - it.TagResults(mp) - if len(mp) == 0 { - continue - } - // TODO(dennwc): replace with more efficient - for k, v := range mp { - if sl, ok := mo[k]; !ok { - mo[k] = []graph.Value{v} - } else if len(sl) == 1 { - if !keysEqual(sl[0], v) { - mo[k] = append(sl, v) - } - } else { - found := false - for _, sv := range sl { - if keysEqual(sv, v) { - found = true - break - } - } - if !found { - mo[k] = append(sl, v) - } - } - } - } - err := c.loadToValue(ctx, qs, cur, depth, mo, "") - if err == errRequiredFieldIsMissing { - if !slice && !chanl { - return err - } - continue - } else if err != nil { - return err - } - if slice { - dst.Set(reflect.Append(dst, cur.Elem())) - } else if chanl { - dst.Send(cur.Elem()) - } else { - return nil - } - } - if err := it.Err(); err != nil { - return err - } - if slice || chanl { - return nil - } - if list != nil && list.Type() != graph.All { - // distinguish between missing object and type constraints - list.Reset() - and := iterator.NewAnd(qs, list, qs.NodesAllIterator()) - defer and.Close() - if and.Next(ctx) { - return errRequiredFieldIsMissing - } - } - return errNotFound -} - func isZero(rv reflect.Value) bool { return rv.Interface() == reflect.Zero(rv.Type()).Interface() // TODO(dennwc): rewrite } -func (c *Config) writeQuad(w quad.Writer, s, p, o quad.Value, rev bool) error { - if rev { - s, o = o, s - } - return w.WriteQuad(quad.Quad{Subject: s, Predicate: p, Object: o, Label: c.Label}) -} - -// writeOneValReflect writes a set of quads corresponding to a value. It may omit writing quads if value is zero. -func (c *Config) writeOneValReflect(w quad.Writer, id quad.Value, pred quad.Value, rv reflect.Value, rev bool, seen map[uintptr]quad.Value) error { - if isZero(rv) { - return nil - } - // write field value and get an ID - sid, err := c.writeAsQuads(w, rv, seen) - if err != nil { - return err - } - // write a quad pointing to this value - return c.writeQuad(w, id, pred, sid, rev) -} - -func (c *Config) writeTypeInfo(w quad.Writer, id quad.Value, rt reflect.Type) error { - typesMu.RLock() - iri := typeToIRI[rt] - typesMu.RUnlock() - if iri == quad.IRI("") { - return nil - } - return c.writeQuad(w, id, c.iri(iriType), c.iri(iri), false) -} - -func (c *Config) writeValueAs(w quad.Writer, id quad.Value, rv reflect.Value, pref string, rules fieldRules, seen map[uintptr]quad.Value) error { - switch kind := rv.Kind(); kind { - case reflect.Ptr, reflect.Map: - ptr := rv.Pointer() - if _, ok := seen[ptr]; ok { - return nil - } - seen[ptr] = id - if kind == reflect.Ptr { - rv = rv.Elem() - } - } - rt := rv.Type() - if err := c.writeTypeInfo(w, id, rt); err != nil { - return err - } - for i := 0; i < rt.NumField(); i++ { - f := rt.Field(i) - if f.Anonymous { - if err := c.writeValueAs(w, id, rv.Field(i), pref+f.Name+".", rules, seen); err != nil { - return err - } - continue - } - switch r := rules[pref+f.Name].(type) { - case constraintRule: - s, o := id, quad.Value(r.Val) - if r.Rev { - s, o = o, s - } - if err := w.WriteQuad(quad.Quad{Subject: s, Predicate: r.Pred, Object: o, Label: c.Label}); err != nil { - return err - } - case saveRule: - if f.Type.Kind() == reflect.Slice { - sl := rv.Field(i) - for j := 0; j < sl.Len(); j++ { - if err := c.writeOneValReflect(w, id, r.Pred, sl.Index(j), r.Rev, seen); err != nil { - return err - } - } - } else { - fv := rv.Field(i) - if !r.Opt && isZero(fv) { - return ErrReqFieldNotSet{Field: f.Name} - } - if err := c.writeOneValReflect(w, id, r.Pred, fv, r.Rev, seen); err != nil { - return err - } - } - } - } - return nil -} - func (c *Config) idFor(rules fieldRules, rt reflect.Type, rv reflect.Value, pref string) (id quad.Value, err error) { hasAnon := false for i := 0; i < rt.NumField(); i++ { @@ -988,120 +432,3 @@ func (c *Config) idFor(rules fieldRules, rt reflect.Type, rv reflect.Value, pref } return } - -// WriteAsQuads writes a single value in form of quads into specified quad writer. -// -// It returns an identifier of the object in the output sub-graph. If an object has -// an annotated ID field, it's value will be converted to quad.Value and returned. -// Otherwise, a new BNode will be generated using GenerateID function. -// -// See LoadTo for a list of quads mapping rules. -func (c *Config) WriteAsQuads(w quad.Writer, o interface{}) (quad.Value, error) { - return c.writeAsQuads(w, reflect.ValueOf(o), make(map[uintptr]quad.Value)) -} - -var reflQuadValue = reflect.TypeOf((*quad.Value)(nil)).Elem() - -func (c *Config) writeAsQuads(w quad.Writer, rv reflect.Value, seen map[uintptr]quad.Value) (quad.Value, error) { - rt := rv.Type() - // if node is a primitive - return directly - if rt.Implements(reflQuadValue) { - return rv.Interface().(quad.Value), nil - } - prv := rv - kind := rt.Kind() - // check if we've seen this node already - switch kind { - case reflect.Ptr, reflect.Map: - ptr := prv.Pointer() - if sid, ok := seen[ptr]; ok { - return sid, nil - } - if kind == reflect.Ptr { - rv = rv.Elem() - rt = rv.Type() - kind = rt.Kind() - } - } - // check if it's a type that quads package supports - // note, that it may be a struct such as time.Time - if val, ok := quad.AsValue(rv.Interface()); ok { - return val, nil - } - // TODO(dennwc): support maps - if kind != reflect.Struct { - return nil, fmt.Errorf("unsupported type: %v", rt) - } - // get conversion rules for this struct type - rules, err := c.rulesFor(rt) - if err != nil { - return nil, fmt.Errorf("can't load rules: %v", err) - } - if len(rules) == 0 { - return nil, fmt.Errorf("no rules for struct: %v", rt) - } - // get an ID from the struct value - id, err := c.idFor(rules, rt, rv, "") - if err != nil { - return nil, err - } - if id == nil { - id = c.genID(prv.Interface()) - } - // save a node ID to avoid loops - switch prv.Kind() { - case reflect.Ptr, reflect.Map: - ptr := prv.Pointer() - seen[ptr] = id - } - if err = c.writeValueAs(w, id, rv, "", rules, seen); err != nil { - return nil, err - } - return id, nil -} - -type namespace struct { - _ struct{} `quad:"@type > cayley:namespace"` - Full quad.IRI `quad:"@id"` - Prefix quad.IRI `quad:"cayley:prefix"` -} - -// WriteNamespaces will writes namespaces list into graph. -func (c *Config) WriteNamespaces(w quad.Writer, n *voc.Namespaces) error { - rules, err := c.rulesFor(reflect.TypeOf(namespace{})) - if err != nil { - return fmt.Errorf("can't load rules: %v", err) - } - seen := make(map[uintptr]quad.Value) - for _, ns := range n.List() { - obj := namespace{ - Full: quad.IRI(ns.Full), - Prefix: quad.IRI(ns.Prefix), - } - rv := reflect.ValueOf(obj) - if err = c.writeValueAs(w, obj.Full, rv, "", rules, seen); err != nil { - return err - } - } - return nil -} - -// LoadNamespaces will load namespaces stored in graph to a specified list. -// If destination list is empty, global namespace registry will be used. -func (c *Config) LoadNamespaces(ctx context.Context, qs graph.QuadStore, dest *voc.Namespaces) error { - var list []namespace - if err := c.LoadTo(ctx, qs, &list); err != nil { - return err - } - register := dest.Register - if dest == nil { - register = voc.Register - } - for _, ns := range list { - register(voc.Namespace{ - Prefix: string(ns.Prefix), - Full: string(ns.Full), - }) - } - return nil -} diff --git a/schema/schema_test.go b/schema/schema_test.go index 2b3d221b7..78b189379 100644 --- a/schema/schema_test.go +++ b/schema/schema_test.go @@ -1,20 +1,19 @@ package schema_test import ( - "context" - "reflect" "sort" - "testing" - "github.com/cayleygraph/cayley/graph" - "github.com/cayleygraph/cayley/graph/iterator" - "github.com/cayleygraph/cayley/graph/memstore" "github.com/cayleygraph/cayley/quad" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/cayley/voc" "github.com/cayleygraph/cayley/voc/rdf" ) +func init() { + voc.RegisterPrefix("ex:", "http://example.org/") + schema.RegisterType(quad.IRI("ex:Coords"), Coords{}) +} + type item struct { rdfType struct{} `quad:"rdf:type > some:item"` ID quad.IRI `quad:"@id"` @@ -99,11 +98,6 @@ type subSubObject struct { Num2 int `quad:"num2"` } -func init() { - voc.RegisterPrefix("ex:", "http://example.org/") - schema.RegisterType(quad.IRI("ex:Coords"), Coords{}) -} - type Coords struct { Lat float64 `json:"ex:lat"` Lng float64 `json:"ex:lng"` @@ -119,257 +113,6 @@ func iri(s string) quad.IRI { return quad.IRI(s) } const typeIRI = quad.IRI(rdf.Type) -var testWriteValueCases = []struct { - name string - obj interface{} - id quad.Value - expect []quad.Quad - err error -}{ - { - "complex object", - struct { - rdfType struct{} `quad:"rdf:type > some:Type"` - ID quad.IRI `quad:"@id"` - Name string `quad:"name"` - Values []string `quad:"values"` - Items []item `quad:"items"` - Sub *item `quad:"sub"` - }{ - ID: "1234", - Name: "some item", - Values: []string{"val1", "val2"}, - Items: []item{ - {ID: "sub1", Name: "Sub 1"}, - {ID: "sub2", Name: "Sub 2"}, - }, - Sub: &item{ID: "sub3", Name: "Sub 3"}, - }, - iri("1234"), - []quad.Quad{ - {iri("1234"), typeIRI, iri("some:Type"), nil}, - {iri("1234"), iri("name"), quad.String(`some item`), nil}, - {iri("1234"), iri("values"), quad.String(`val1`), nil}, - {iri("1234"), iri("values"), quad.String(`val2`), nil}, - - {iri("sub1"), typeIRI, iri("some:item"), nil}, - {iri("sub1"), iri("name"), quad.String(`Sub 1`), nil}, - {iri("1234"), iri("items"), iri("sub1"), nil}, - - {iri("sub2"), typeIRI, iri("some:item"), nil}, - {iri("sub2"), iri("name"), quad.String(`Sub 2`), nil}, - {iri("1234"), iri("items"), iri("sub2"), nil}, - - {iri("sub3"), typeIRI, iri("some:item"), nil}, - {iri("sub3"), iri("name"), quad.String(`Sub 3`), nil}, - {iri("1234"), iri("sub"), iri("sub3"), nil}, - }, - nil, - }, - { - "complex object (embedded)", - struct { - rdfType struct{} `quad:"rdf:type > some:Type"` - item2 - ID quad.IRI `quad:"@id"` - Values []string `quad:"values"` - }{ - item2: item2{Name: "Sub 1", Spec: "special"}, - ID: "1234", - Values: []string{"val1", "val2"}, - }, - iri("1234"), - []quad.Quad{ - {iri("1234"), typeIRI, iri("some:Type"), nil}, - {iri("1234"), iri("name"), quad.String(`Sub 1`), nil}, - {iri("1234"), iri("spec"), quad.String(`special`), nil}, - {iri("1234"), iri("values"), quad.String(`val1`), nil}, - {iri("1234"), iri("values"), quad.String(`val2`), nil}, - }, - nil, - }, - { - "type shorthand", - struct { - rdfType struct{} `quad:"@type > some:Type"` - item2 - ID quad.IRI `quad:"@id"` - Values []string `quad:"values"` - }{ - item2: item2{Name: "Sub 1", Spec: "special"}, - ID: "1234", - Values: []string{"val1", "val2"}, - }, - iri("1234"), - []quad.Quad{ - {iri("1234"), typeIRI, iri("some:Type"), nil}, - {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, - {iri("1234"), iri("spec"), quad.String("special"), nil}, - {iri("1234"), iri("values"), quad.String("val1"), nil}, - {iri("1234"), iri("values"), quad.String("val2"), nil}, - }, - nil, - }, - { - "json tags", - struct { - rdfType struct{} `quad:"@type > some:Type"` - item2 - ID quad.IRI `json:"@id"` - Values []string `json:"values,omitempty"` - }{ - item2: item2{Name: "Sub 1", Spec: "special"}, - ID: "1234", - Values: []string{"val1", "val2"}, - }, - iri("1234"), - []quad.Quad{ - {iri("1234"), typeIRI, iri("some:Type"), nil}, - {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, - {iri("1234"), iri("spec"), quad.String("special"), nil}, - {iri("1234"), iri("values"), quad.String("val1"), nil}, - {iri("1234"), iri("values"), quad.String("val2"), nil}, - }, - nil, - }, - { - "simple object", - subObject{ - genObject: genObject{ - ID: "1234", - Name: "Obj", - }, - Num: 3, - }, - iri("1234"), - []quad.Quad{ - {iri("1234"), iri("name"), quad.String("Obj"), nil}, - {iri("1234"), iri("num"), quad.Int(3), nil}, - }, - nil, - }, - { - "simple object (embedded multiple levels)", - subSubObject{ - subObject: subObject{ - genObject: genObject{ - ID: "1234", - Name: "Obj", - }, - Num: 3, - }, - Num2: 4, - }, - iri("1234"), - []quad.Quad{ - {iri("1234"), iri("name"), quad.String("Obj"), nil}, - {iri("1234"), iri("num"), quad.Int(3), nil}, - {iri("1234"), iri("num2"), quad.Int(4), nil}, - }, - nil, - }, - { - "required field not set", - item2{Name: "partial"}, - nil, nil, - schema.ErrReqFieldNotSet{Field: "Spec"}, - }, - { - "single tree node", - treeItemOpt{ - ID: iri("n1"), - Name: "Node 1", - }, - iri("n1"), - []quad.Quad{ - {iri("n1"), iri("name"), quad.String("Node 1"), nil}, - }, - nil, - }, - { - "coords", - Coords{Lat: 12.3, Lng: 34.5}, - nil, - []quad.Quad{ - {nil, typeIRI, iri("ex:Coords"), nil}, - {nil, iri("ex:lat"), quad.Float(12.3), nil}, - {nil, iri("ex:lng"), quad.Float(34.5), nil}, - }, - nil, - }, - { - "self loop", - func() *NodeLoop { - a := &NodeLoop{ID: iri("A"), Name: "Node A"} - a.Next = a - return a - }(), - iri("A"), - []quad.Quad{ - {iri("A"), iri("name"), quad.String("Node A"), nil}, - {iri("A"), iri("next"), iri("A"), nil}, - }, - nil, - }, - { - "pointer chain", - func() *NodeLoop { - a := &NodeLoop{ID: iri("A"), Name: "Node A"} - b := &NodeLoop{ID: iri("B"), Name: "Node B"} - c := &NodeLoop{ID: iri("C"), Name: "Node C"} - - a.Next = b - b.Next = c - c.Next = a - return a - }(), - iri("A"), - []quad.Quad{ - {iri("A"), iri("name"), quad.String("Node A"), nil}, - {iri("B"), iri("name"), quad.String("Node B"), nil}, - {iri("C"), iri("name"), quad.String("Node C"), nil}, - {iri("C"), iri("next"), iri("A"), nil}, - {iri("B"), iri("next"), iri("C"), nil}, - {iri("A"), iri("next"), iri("B"), nil}, - }, - nil, - }, -} - -type quadSlice []quad.Quad - -func (s *quadSlice) WriteQuad(q quad.Quad) error { - *s = append(*s, q) - return nil -} - -func TestWriteAsQuads(t *testing.T) { - sch := schema.NewConfig() - for _, c := range testWriteValueCases { - t.Run(c.name, func(t *testing.T) { - var out quadSlice - id, err := sch.WriteAsQuads(&out, c.obj) - if err != c.err { - t.Errorf("unexpected error: %v (expected: %v)", err, c.err) - } else if c.err != nil { - return // case with expected error; omit other checks - } - if c.id == nil { - for i := range out { - if c.expect[i].Subject == nil { - c.expect[i].Subject = id - } - } - } else if id != c.id { - t.Errorf("ids are different: %v vs %v", id, c.id) - } - if !reflect.DeepEqual([]quad.Quad(out), c.expect) { - t.Errorf("quad sets are different\n%#v\n%#v", []quad.Quad(out), c.expect) - } - }) - } -} - var treeQuads = []quad.Quad{ {iri("n1"), iri("name"), quad.String("Node 1"), nil}, {iri("n2"), iri("name"), quad.String("Node 2"), nil}, @@ -382,326 +125,3 @@ var treeQuads = []quad.Quad{ {iri("n3"), iri("child"), iri("n4"), nil}, } - -var testFillValueCases = []struct { - name string - expect interface{} - quads []quad.Quad - depth int - from []quad.Value -}{ - { - name: "complex object", - expect: struct { - rdfType struct{} `quad:"rdf:type > some:Type"` - ID quad.IRI `quad:"@id"` - Name string `quad:"name"` - Values []string `quad:"values"` - Items []item `quad:"items"` - Sub *item `quad:"sub"` - Val int `quad:"val"` - }{ - ID: "1234", - Name: "some item", - Values: []string{"val1", "val2"}, - Items: []item{ - {ID: "sub1", Name: "Sub 1"}, - {ID: "sub2", Name: "Sub 2"}, - }, - Sub: &item{ID: "sub3", Name: "Sub 3"}, - Val: 123, - }, - quads: []quad.Quad{ - {iri("1234"), typeIRI, iri("some:Type"), nil}, - {iri("1234"), iri("name"), quad.String("some item"), nil}, - {iri("1234"), iri("values"), quad.String("val1"), nil}, - {iri("1234"), iri("values"), quad.String("val2"), nil}, - {iri("sub1"), typeIRI, iri("some:item"), nil}, - {iri("sub1"), iri("name"), quad.String("Sub 1"), nil}, - {iri("1234"), iri("items"), iri("sub1"), nil}, - {iri("sub2"), typeIRI, iri("some:item"), nil}, - {iri("sub2"), iri("name"), quad.String("Sub 2"), nil}, - {iri("1234"), iri("items"), iri("sub2"), nil}, - {iri("sub3"), typeIRI, iri("some:item"), nil}, - {iri("sub3"), iri("name"), quad.String("Sub 3"), nil}, - {iri("1234"), iri("sub"), iri("sub3"), nil}, - {iri("1234"), iri("val"), quad.Int(123), nil}, - }, - }, - { - name: "complex object (id value)", - expect: struct { - rdfType struct{} `quad:"rdf:type > some:Type"` - ID quad.Value `quad:"@id"` - Name string `quad:"name"` - Values []string `quad:"values"` - Items []item `quad:"items"` - }{ - ID: quad.BNode("1234"), - Name: "some item", - Values: []string{"val1", "val2"}, - Items: []item{ - {ID: "sub1", Name: "Sub 1"}, - {ID: "sub2", Name: "Sub 2"}, - }, - }, - quads: []quad.Quad{ - {quad.BNode("1234"), typeIRI, iri("some:Type"), nil}, - {quad.BNode("1234"), iri("name"), quad.String("some item"), nil}, - {quad.BNode("1234"), iri("values"), quad.String("val1"), nil}, - {quad.BNode("1234"), iri("values"), quad.String("val2"), nil}, - {iri("sub1"), typeIRI, iri("some:item"), nil}, - {iri("sub1"), iri("name"), quad.String("Sub 1"), nil}, - {quad.BNode("1234"), iri("items"), iri("sub1"), nil}, - {iri("sub2"), typeIRI, iri("some:item"), nil}, - {iri("sub2"), iri("name"), quad.String("Sub 2"), nil}, - {quad.BNode("1234"), iri("items"), iri("sub2"), nil}, - }, - }, - { - name: "embedded object", - expect: struct { - rdfType struct{} `quad:"rdf:type > some:Type"` - item2 - ID quad.IRI `quad:"@id"` - Values []string `quad:"values"` - }{ - item2: item2{Name: "Sub 1", Spec: "special"}, - ID: "1234", - Values: []string{"val1", "val2"}, - }, - quads: []quad.Quad{ - {iri("1234"), typeIRI, iri("some:Type"), nil}, - {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, - {iri("1234"), iri("spec"), quad.String("special"), nil}, - {iri("1234"), iri("values"), quad.String("val1"), nil}, - {iri("1234"), iri("values"), quad.String("val2"), nil}, - }, - }, - { - name: "type shorthand", - expect: struct { - rdfType struct{} `quad:"@type > some:Type"` - item2 - ID quad.IRI `quad:"@id"` - Values []string `quad:"values"` - }{ - item2: item2{Name: "Sub 1", Spec: "special"}, - ID: "1234", - Values: []string{"val1", "val2"}, - }, - quads: []quad.Quad{ - {iri("1234"), typeIRI, iri("some:Type"), nil}, - {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, - {iri("1234"), iri("spec"), quad.String("special"), nil}, - {iri("1234"), iri("values"), quad.String("val1"), nil}, - {iri("1234"), iri("values"), quad.String("val2"), nil}, - }, - }, - { - name: "tree", - expect: treeItem{ - ID: iri("n1"), - Name: "Node 1", - Children: []treeItem{ - { - ID: iri("n2"), - Name: "Node 2", - }, - { - ID: iri("n3"), - Name: "Node 3", - Children: []treeItem{ - { - ID: iri("n4"), - Name: "Node 4", - }, - }, - }, - }, - }, - quads: treeQuads, - from: []quad.Value{iri("n1")}, - }, - { - name: "tree with depth limit 1", - expect: treeItem{ - ID: iri("n1"), - Name: "Node 1", - Children: []treeItem{ - { - ID: iri("n2"), - Name: "Node 2", - }, - { - ID: iri("n3"), - Name: "Node 3", - Children: []treeItem{ - { - ID: iri("n4"), - }, - }, - }, - }, - }, - depth: 1, - quads: treeQuads, - from: []quad.Value{iri("n1")}, - }, - { - name: "tree with depth limit 2", - expect: treeItemOpt{ - ID: iri("n1"), - Name: "Node 1", - Children: []treeItemOpt{ - { - ID: iri("n2"), - Name: "Node 2", - }, - { - ID: iri("n3"), - Name: "Node 3", - Children: []treeItemOpt{ - { - ID: iri("n4"), - Name: "Node 4", - }, - }, - }, - }, - }, - depth: 2, - quads: treeQuads, - from: []quad.Value{iri("n1")}, - }, - { - name: "tree with required children", - expect: treeItemReq{ - ID: iri("n1"), - Name: "Node 1", - Children: []treeItemReq{ - { - ID: iri("n3"), - Name: "Node 3", - // TODO(dennwc): a strange behavior: this field is required, but it's empty for current object, - // because all it's children are missing the same field. Leaving this as-is for now because - // it's weird to set Children field as required in a tree. - Children: nil, - }, - }, - }, - quads: treeQuads, - from: []quad.Value{iri("n1")}, - }, - { - name: "simple object", - expect: subObject{ - genObject: genObject{ - ID: "1234", - Name: "Obj", - }, - Num: 3, - }, - quads: []quad.Quad{ - {iri("1234"), iri("name"), quad.String("Obj"), nil}, - {iri("1234"), iri("num"), quad.Int(3), nil}, - }, - }, - { - name: "coords", - expect: Coords{Lat: 12.3, Lng: 34.5}, - quads: []quad.Quad{ - {iri("c1"), typeIRI, iri("ex:Coords"), nil}, - {iri("c1"), iri("ex:lat"), quad.Float(12.3), nil}, - {iri("c1"), iri("ex:lng"), quad.Float(34.5), nil}, - }, - }, -} - -func TestLoadIteratorTo(t *testing.T) { - sch := schema.NewConfig() - for i, c := range testFillValueCases { - t.Run(c.name, func(t *testing.T) { - qs := memstore.New(c.quads...) - out := reflect.New(reflect.TypeOf(c.expect)) - var it graph.Iterator - if c.from != nil { - fixed := iterator.NewFixed() - for _, id := range c.from { - fixed.Add(qs.ValueOf(id)) - } - it = fixed - } - depth := c.depth - if depth == 0 { - depth = -1 - } - if err := sch.LoadIteratorToDepth(nil, qs, out, depth, it); err != nil { - t.Errorf("case %d failed: %v", i+1, err) - return - } - got := out.Elem().Interface() - if s, ok := got.(interface { - Sort() - }); ok { - s.Sort() - } - if s, ok := c.expect.(interface { - Sort() - }); ok { - s.Sort() - } - if !reflect.DeepEqual(got, c.expect) { - t.Errorf("case %d failed: objects are different\n%#v\n%#v", - i+1, out.Elem().Interface(), c.expect, - ) - } - }) - } -} - -func TestSaveNamespaces(t *testing.T) { - sch := schema.NewConfig() - save := []voc.Namespace{ - {Full: "http://example.org/", Prefix: "ex:"}, - {Full: "http://cayley.io/", Prefix: "c:"}, - } - var ns voc.Namespaces - for _, n := range save { - ns.Register(n) - } - qs := memstore.New() - err := sch.WriteNamespaces(qs, &ns) - if err != nil { - t.Fatal(err) - } - var ns2 voc.Namespaces - err = sch.LoadNamespaces(context.TODO(), qs, &ns2) - if err != nil { - t.Fatal(err) - } - got := ns2.List() - sort.Sort(voc.ByFullName(save)) - sort.Sort(voc.ByFullName(got)) - if !reflect.DeepEqual(save, got) { - t.Fatalf("wrong namespaces returned: got: %v, expect: %v", got, save) - } - qr := graph.NewQuadStoreReader(qs) - q, err := quad.ReadAll(qr) - qr.Close() - if err != nil { - t.Fatal(err) - } - expect := []quad.Quad{ - quad.MakeIRI("http://cayley.io/", "cayley:prefix", "c:", ""), - quad.MakeIRI("http://cayley.io/", "rdf:type", "cayley:namespace", ""), - - quad.MakeIRI("http://example.org/", "cayley:prefix", "ex:", ""), - quad.MakeIRI("http://example.org/", "rdf:type", "cayley:namespace", ""), - } - sort.Sort(quad.ByQuadString(expect)) - sort.Sort(quad.ByQuadString(q)) - if !reflect.DeepEqual(expect, q) { - t.Fatalf("wrong quads returned: got: %v, expect: %v", q, expect) - } -} diff --git a/schema/writer.go b/schema/writer.go new file mode 100644 index 000000000..057922f60 --- /dev/null +++ b/schema/writer.go @@ -0,0 +1,172 @@ +package schema + +import ( + "fmt" + "reflect" + + "github.com/cayleygraph/cayley/quad" +) + +// WriteAsQuads writes a single value in form of quads into specified quad writer. +// +// It returns an identifier of the object in the output sub-graph. If an object has +// an annotated ID field, it's value will be converted to quad.Value and returned. +// Otherwise, a new BNode will be generated using GenerateID function. +// +// See LoadTo for a list of quads mapping rules. +func (c *Config) WriteAsQuads(w quad.Writer, o interface{}) (quad.Value, error) { + wr := c.newWriter(w) + return wr.writeAsQuads(reflect.ValueOf(o)) +} + +type writer struct { + c *Config + w quad.Writer + seen map[uintptr]quad.Value +} + +func (c *Config) newWriter(w quad.Writer) *writer { + return &writer{c: c, w: w, seen: make(map[uintptr]quad.Value)} +} + +func (w *writer) writeQuad(s, p, o quad.Value, rev bool) error { + if rev { + s, o = o, s + } + return w.w.WriteQuad(quad.Quad{Subject: s, Predicate: p, Object: o, Label: w.c.Label}) +} + +// writeOneValReflect writes a set of quads corresponding to a value. It may omit writing quads if value is zero. +func (w *writer) writeOneValReflect(id quad.Value, pred quad.Value, rv reflect.Value, rev bool) error { + if isZero(rv) { + return nil + } + // write field value and get an ID + sid, err := w.writeAsQuads(rv) + if err != nil { + return err + } + // write a quad pointing to this value + return w.writeQuad(id, pred, sid, rev) +} + +func (w *writer) writeTypeInfo(id quad.Value, rt reflect.Type) error { + iri := getTypeIRI(rt) + if iri == quad.IRI("") { + return nil + } + return w.writeQuad(id, w.c.iri(iriType), w.c.iri(iri), false) +} + +func (w *writer) writeValueAs(id quad.Value, rv reflect.Value, pref string, rules fieldRules) error { + switch kind := rv.Kind(); kind { + case reflect.Ptr, reflect.Map: + ptr := rv.Pointer() + if _, ok := w.seen[ptr]; ok { + return nil + } + w.seen[ptr] = id + if kind == reflect.Ptr { + rv = rv.Elem() + } + } + rt := rv.Type() + if err := w.writeTypeInfo(id, rt); err != nil { + return err + } + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + if f.Anonymous { + if err := w.writeValueAs(id, rv.Field(i), pref+f.Name+".", rules); err != nil { + return err + } + continue + } + switch r := rules[pref+f.Name].(type) { + case constraintRule: + s, o := id, quad.Value(r.Val) + if r.Rev { + s, o = o, s + } + if err := w.writeQuad(s, r.Pred, o, false); err != nil { + return err + } + case saveRule: + if f.Type.Kind() == reflect.Slice { + sl := rv.Field(i) + for j := 0; j < sl.Len(); j++ { + if err := w.writeOneValReflect(id, r.Pred, sl.Index(j), r.Rev); err != nil { + return err + } + } + } else { + fv := rv.Field(i) + if !r.Opt && isZero(fv) { + return ErrReqFieldNotSet{Field: f.Name} + } + if err := w.writeOneValReflect(id, r.Pred, fv, r.Rev); err != nil { + return err + } + } + } + } + return nil +} + +func (w *writer) writeAsQuads(rv reflect.Value) (quad.Value, error) { + rt := rv.Type() + // if node is a primitive - return directly + if rt.Implements(reflQuadValue) { + return rv.Interface().(quad.Value), nil + } + prv := rv + kind := rt.Kind() + // check if we've seen this node already + switch kind { + case reflect.Ptr, reflect.Map: + ptr := prv.Pointer() + if sid, ok := w.seen[ptr]; ok { + return sid, nil + } + if kind == reflect.Ptr { + rv = rv.Elem() + rt = rv.Type() + kind = rt.Kind() + } + } + // check if it's a type that quads package supports + // note, that it may be a struct such as time.Time + if val, ok := quad.AsValue(rv.Interface()); ok { + return val, nil + } + // TODO(dennwc): support maps + if kind != reflect.Struct { + return nil, fmt.Errorf("unsupported type: %v", rt) + } + // get conversion rules for this struct type + rules, err := w.c.rulesFor(rt) + if err != nil { + return nil, fmt.Errorf("can't load rules: %v", err) + } + if len(rules) == 0 { + return nil, fmt.Errorf("no rules for struct: %v", rt) + } + // get an ID from the struct value + id, err := w.c.idFor(rules, rt, rv, "") + if err != nil { + return nil, err + } + if id == nil { + id = w.c.genID(prv.Interface()) + } + // save a node ID to avoid loops + switch prv.Kind() { + case reflect.Ptr, reflect.Map: + ptr := prv.Pointer() + w.seen[ptr] = id + } + if err = w.writeValueAs(id, rv, "", rules); err != nil { + return nil, err + } + return id, nil +} diff --git a/schema/writer_test.go b/schema/writer_test.go new file mode 100644 index 000000000..2b872f2e7 --- /dev/null +++ b/schema/writer_test.go @@ -0,0 +1,260 @@ +package schema_test + +import ( + "reflect" + "testing" + + "github.com/cayleygraph/cayley/quad" + "github.com/cayleygraph/cayley/schema" +) + +type quadSlice []quad.Quad + +func (s *quadSlice) WriteQuad(q quad.Quad) error { + *s = append(*s, q) + return nil +} + +func TestWriteAsQuads(t *testing.T) { + sch := schema.NewConfig() + for _, c := range testWriteValueCases { + t.Run(c.name, func(t *testing.T) { + var out quadSlice + id, err := sch.WriteAsQuads(&out, c.obj) + if err != c.err { + t.Errorf("unexpected error: %v (expected: %v)", err, c.err) + } else if c.err != nil { + return // case with expected error; omit other checks + } + if c.id == nil { + for i := range out { + if c.expect[i].Subject == nil { + c.expect[i].Subject = id + } + } + } else if id != c.id { + t.Errorf("ids are different: %v vs %v", id, c.id) + } + if !reflect.DeepEqual([]quad.Quad(out), c.expect) { + t.Errorf("quad sets are different\n%#v\n%#v", []quad.Quad(out), c.expect) + } + }) + } +} + +var testWriteValueCases = []struct { + name string + obj interface{} + id quad.Value + expect []quad.Quad + err error +}{ + { + "complex object", + struct { + rdfType struct{} `quad:"rdf:type > some:Type"` + ID quad.IRI `quad:"@id"` + Name string `quad:"name"` + Values []string `quad:"values"` + Items []item `quad:"items"` + Sub *item `quad:"sub"` + }{ + ID: "1234", + Name: "some item", + Values: []string{"val1", "val2"}, + Items: []item{ + {ID: "sub1", Name: "Sub 1"}, + {ID: "sub2", Name: "Sub 2"}, + }, + Sub: &item{ID: "sub3", Name: "Sub 3"}, + }, + iri("1234"), + []quad.Quad{ + {iri("1234"), typeIRI, iri("some:Type"), nil}, + {iri("1234"), iri("name"), quad.String(`some item`), nil}, + {iri("1234"), iri("values"), quad.String(`val1`), nil}, + {iri("1234"), iri("values"), quad.String(`val2`), nil}, + + {iri("sub1"), typeIRI, iri("some:item"), nil}, + {iri("sub1"), iri("name"), quad.String(`Sub 1`), nil}, + {iri("1234"), iri("items"), iri("sub1"), nil}, + + {iri("sub2"), typeIRI, iri("some:item"), nil}, + {iri("sub2"), iri("name"), quad.String(`Sub 2`), nil}, + {iri("1234"), iri("items"), iri("sub2"), nil}, + + {iri("sub3"), typeIRI, iri("some:item"), nil}, + {iri("sub3"), iri("name"), quad.String(`Sub 3`), nil}, + {iri("1234"), iri("sub"), iri("sub3"), nil}, + }, + nil, + }, + { + "complex object (embedded)", + struct { + rdfType struct{} `quad:"rdf:type > some:Type"` + item2 + ID quad.IRI `quad:"@id"` + Values []string `quad:"values"` + }{ + item2: item2{Name: "Sub 1", Spec: "special"}, + ID: "1234", + Values: []string{"val1", "val2"}, + }, + iri("1234"), + []quad.Quad{ + {iri("1234"), typeIRI, iri("some:Type"), nil}, + {iri("1234"), iri("name"), quad.String(`Sub 1`), nil}, + {iri("1234"), iri("spec"), quad.String(`special`), nil}, + {iri("1234"), iri("values"), quad.String(`val1`), nil}, + {iri("1234"), iri("values"), quad.String(`val2`), nil}, + }, + nil, + }, + { + "type shorthand", + struct { + rdfType struct{} `quad:"@type > some:Type"` + item2 + ID quad.IRI `quad:"@id"` + Values []string `quad:"values"` + }{ + item2: item2{Name: "Sub 1", Spec: "special"}, + ID: "1234", + Values: []string{"val1", "val2"}, + }, + iri("1234"), + []quad.Quad{ + {iri("1234"), typeIRI, iri("some:Type"), nil}, + {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, + {iri("1234"), iri("spec"), quad.String("special"), nil}, + {iri("1234"), iri("values"), quad.String("val1"), nil}, + {iri("1234"), iri("values"), quad.String("val2"), nil}, + }, + nil, + }, + { + "json tags", + struct { + rdfType struct{} `quad:"@type > some:Type"` + item2 + ID quad.IRI `json:"@id"` + Values []string `json:"values,omitempty"` + }{ + item2: item2{Name: "Sub 1", Spec: "special"}, + ID: "1234", + Values: []string{"val1", "val2"}, + }, + iri("1234"), + []quad.Quad{ + {iri("1234"), typeIRI, iri("some:Type"), nil}, + {iri("1234"), iri("name"), quad.String("Sub 1"), nil}, + {iri("1234"), iri("spec"), quad.String("special"), nil}, + {iri("1234"), iri("values"), quad.String("val1"), nil}, + {iri("1234"), iri("values"), quad.String("val2"), nil}, + }, + nil, + }, + { + "simple object", + subObject{ + genObject: genObject{ + ID: "1234", + Name: "Obj", + }, + Num: 3, + }, + iri("1234"), + []quad.Quad{ + {iri("1234"), iri("name"), quad.String("Obj"), nil}, + {iri("1234"), iri("num"), quad.Int(3), nil}, + }, + nil, + }, + { + "simple object (embedded multiple levels)", + subSubObject{ + subObject: subObject{ + genObject: genObject{ + ID: "1234", + Name: "Obj", + }, + Num: 3, + }, + Num2: 4, + }, + iri("1234"), + []quad.Quad{ + {iri("1234"), iri("name"), quad.String("Obj"), nil}, + {iri("1234"), iri("num"), quad.Int(3), nil}, + {iri("1234"), iri("num2"), quad.Int(4), nil}, + }, + nil, + }, + { + "required field not set", + item2{Name: "partial"}, + nil, nil, + schema.ErrReqFieldNotSet{Field: "Spec"}, + }, + { + "single tree node", + treeItemOpt{ + ID: iri("n1"), + Name: "Node 1", + }, + iri("n1"), + []quad.Quad{ + {iri("n1"), iri("name"), quad.String("Node 1"), nil}, + }, + nil, + }, + { + "coords", + Coords{Lat: 12.3, Lng: 34.5}, + nil, + []quad.Quad{ + {nil, typeIRI, iri("ex:Coords"), nil}, + {nil, iri("ex:lat"), quad.Float(12.3), nil}, + {nil, iri("ex:lng"), quad.Float(34.5), nil}, + }, + nil, + }, + { + "self loop", + func() *NodeLoop { + a := &NodeLoop{ID: iri("A"), Name: "Node A"} + a.Next = a + return a + }(), + iri("A"), + []quad.Quad{ + {iri("A"), iri("name"), quad.String("Node A"), nil}, + {iri("A"), iri("next"), iri("A"), nil}, + }, + nil, + }, + { + "pointer chain", + func() *NodeLoop { + a := &NodeLoop{ID: iri("A"), Name: "Node A"} + b := &NodeLoop{ID: iri("B"), Name: "Node B"} + c := &NodeLoop{ID: iri("C"), Name: "Node C"} + + a.Next = b + b.Next = c + c.Next = a + return a + }(), + iri("A"), + []quad.Quad{ + {iri("A"), iri("name"), quad.String("Node A"), nil}, + {iri("B"), iri("name"), quad.String("Node B"), nil}, + {iri("C"), iri("name"), quad.String("Node C"), nil}, + {iri("C"), iri("next"), iri("A"), nil}, + {iri("B"), iri("next"), iri("C"), nil}, + {iri("A"), iri("next"), iri("B"), nil}, + }, + nil, + }, +} From 2656e31be26bc3008f191b06e714b387d5855beb Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Sat, 20 Oct 2018 22:09:59 +0300 Subject: [PATCH 2/2] schema: support loading objects with loops --- schema/loader.go | 104 ++++++++++++++++++++++++++++-------------- schema/loader_test.go | 77 +++++++++++++++++++++++++++++-- 2 files changed, 145 insertions(+), 36 deletions(-) diff --git a/schema/loader.go b/schema/loader.go index e4c76ac89..03b3d3e49 100644 --- a/schema/loader.go +++ b/schema/loader.go @@ -148,6 +148,8 @@ type loader struct { pathForType map[reflect.Type]*path.Path pathForTypeRoot map[reflect.Type]*path.Path + + seen map[quad.Value]reflect.Value } func (c *Config) newLoader(qs graph.QuadStore) *loader { @@ -157,6 +159,8 @@ func (c *Config) newLoader(qs graph.QuadStore) *loader { pathForType: make(map[reflect.Type]*path.Path), pathForTypeRoot: make(map[reflect.Type]*path.Path), + + seen: make(map[quad.Value]reflect.Value), } } @@ -288,7 +292,7 @@ func (l *loader) loadToValue(ctx context.Context, dst reflect.Value, depth int, for i := 0; i < rt.NumField(); i++ { select { case <-ctx.Done(): - return context.Canceled + return ctx.Err() default: } f := rt.Field(i) @@ -313,18 +317,32 @@ func (l *loader) loadToValue(ctx context.Context, dst reflect.Value, depth int, } ft := f.Type native := isNative(ft) + ptr := ft.Kind() == reflect.Ptr for ft.Kind() == reflect.Ptr || ft.Kind() == reflect.Slice { - native = native || isNative(ft) ft = ft.Elem() + native = native || isNative(ft) + switch ft.Kind() { + case reflect.Ptr: + ptr = true + case reflect.Slice: + ptr = false + } } recursive := !native && ft.Kind() == reflect.Struct for _, fv := range arr { var sv reflect.Value if recursive { + if ptr { + fv := l.qs.NameOf(fv) + var ok bool + sv, ok = l.seen[fv] + if ok && sv.Type().AssignableTo(f.Type) { + df.Set(sv) + continue + } + } sv = reflect.New(ft).Elem() - sit := iterator.NewFixed() - sit.Add(fv) - err := l.loadIteratorToDepth(ctx, sv, depth-1, sit) + err := l.loadIteratorToDepth(ctx, sv, depth-1, iterator.NewFixed(fv)) if err == errRequiredFieldIsMissing { continue } else if err != nil { @@ -353,6 +371,19 @@ func (l *loader) iteratorForType(root graph.Iterator, rt reflect.Type, rootOnly return l.iteratorFromPath(root, p) } +func mergeMap(dst map[string][]graph.Value, m map[string]graph.Value) { +loop: + for k, v := range m { + sl := dst[k] + for _, sv := range sl { + if keysEqual(sv, v) { + continue loop + } + } + dst[k] = append(sl, v) + } +} + func (l *loader) loadIteratorToDepth(ctx context.Context, dst reflect.Value, depth int, list graph.Iterator) error { if ctx == nil { ctx = context.TODO() @@ -374,11 +405,20 @@ func (l *loader) loadIteratorToDepth(ctx context.Context, dst reflect.Value, dep if err != nil { return err } - select { - case <-ctx.Done(): + + ctxDone := func() bool { + select { + case <-ctx.Done(): + return true + default: + } + return false + } + + if ctxDone() { return ctx.Err() - default: } + rootOnly := depth == 0 it, err := l.iteratorForType(list, et, rootOnly) if err != nil { @@ -388,10 +428,22 @@ func (l *loader) loadIteratorToDepth(ctx context.Context, dst reflect.Value, dep ctx = context.WithValue(ctx, fieldsCtxKey{}, fields) for it.Next(ctx) { - select { - case <-ctx.Done(): + if ctxDone() { return ctx.Err() - default: + } + id := l.qs.NameOf(it.Result()) + if id != nil { + if sv, ok := l.seen[id]; ok { + if slice { + dst.Set(reflect.Append(dst, sv.Elem())) + } else if chanl { + dst.Send(sv.Elem()) + } else { + dst.Set(sv) + return nil + } + continue + } } mp := make(map[string]graph.Value) it.TagResults(mp) @@ -407,10 +459,8 @@ func (l *loader) loadIteratorToDepth(ctx context.Context, dst reflect.Value, dep mo[k] = []graph.Value{v} } for it.NextPath(ctx) { - select { - case <-ctx.Done(): + if ctxDone() { return ctx.Err() - default: } mp = make(map[string]graph.Value) it.TagResults(mp) @@ -418,26 +468,14 @@ func (l *loader) loadIteratorToDepth(ctx context.Context, dst reflect.Value, dep continue } // TODO(dennwc): replace with something more efficient - for k, v := range mp { - if sl, ok := mo[k]; !ok { - mo[k] = []graph.Value{v} - } else if len(sl) == 1 { - if !keysEqual(sl[0], v) { - mo[k] = append(sl, v) - } - } else { - found := false - for _, sv := range sl { - if keysEqual(sv, v) { - found = true - break - } - } - if !found { - mo[k] = append(sl, v) - } - } + mergeMap(mo, mp) + } + if id != nil { + sv := cur + if sv.Kind() != reflect.Ptr && sv.CanAddr() { + sv = sv.Addr() } + l.seen[id] = sv } err := l.loadToValue(ctx, cur, depth, mo, "") if err == errRequiredFieldIsMissing { diff --git a/schema/loader_test.go b/schema/loader_test.go index 728f63d30..c89481a80 100644 --- a/schema/loader_test.go +++ b/schema/loader_test.go @@ -11,12 +11,78 @@ import ( "github.com/cayleygraph/cayley/schema" ) +func TestLoadLoop(t *testing.T) { + sch := schema.NewConfig() + + a := &NodeLoop{ID: iri("A"), Name: "Node A"} + a.Next = a + + qs := memstore.New([]quad.Quad{ + {a.ID, iri("name"), quad.String(a.Name), nil}, + {a.ID, iri("next"), a.ID, nil}, + }...) + + b := &NodeLoop{} + if err := sch.LoadIteratorTo(nil, qs, reflect.ValueOf(b), nil); err != nil { + t.Error(err) + return + } + if a.ID != b.ID || a.Name != b.Name { + t.Fatalf("%#v vs %#v", a, b) + } + if b != b.Next { + t.Fatalf("loop is broken: %p vs %p", b, b.Next) + } + + a = &NodeLoop{ID: iri("A"), Name: "Node A"} + b = &NodeLoop{ID: iri("B"), Name: "Node B"} + c := &NodeLoop{ID: iri("C"), Name: "Node C"} + a.Next = b + b.Next = c + c.Next = a + + qs = memstore.New([]quad.Quad{ + {a.ID, iri("name"), quad.String(a.Name), nil}, + {b.ID, iri("name"), quad.String(b.Name), nil}, + {c.ID, iri("name"), quad.String(c.Name), nil}, + {a.ID, iri("next"), b.ID, nil}, + {b.ID, iri("next"), c.ID, nil}, + {c.ID, iri("next"), a.ID, nil}, + }...) + + a1 := &NodeLoop{} + if err := sch.LoadIteratorTo(nil, qs, reflect.ValueOf(a1), nil); err != nil { + t.Error(err) + return + } + if a.ID != a1.ID || a.Name != a1.Name { + t.Fatalf("%#v vs %#v", a, b) + } + b1 := a1.Next + c1 := b1.Next + if b.ID != b1.ID || b.Name != b1.Name { + t.Fatalf("%#v vs %#v", a, b) + } + if c.ID != c1.ID || c.Name != c1.Name { + t.Fatalf("%#v vs %#v", a, b) + } + if a1 != c1.Next { + t.Fatalf("loop is broken: %p vs %p", a1, c1.Next) + } +} + func TestLoadIteratorTo(t *testing.T) { sch := schema.NewConfig() for i, c := range testFillValueCases { t.Run(c.name, func(t *testing.T) { qs := memstore.New(c.quads...) - out := reflect.New(reflect.TypeOf(c.expect)) + rt := reflect.TypeOf(c.expect) + var out reflect.Value + if rt.Kind() == reflect.Ptr { + out = reflect.New(rt.Elem()) + } else { + out = reflect.New(rt) + } var it graph.Iterator if c.from != nil { fixed := iterator.NewFixed() @@ -33,7 +99,12 @@ func TestLoadIteratorTo(t *testing.T) { t.Errorf("case %d failed: %v", i+1, err) return } - got := out.Elem().Interface() + var got interface{} + if rt.Kind() == reflect.Ptr { + got = out.Interface() + } else { + got = out.Elem().Interface() + } if s, ok := got.(interface { Sort() }); ok { @@ -46,7 +117,7 @@ func TestLoadIteratorTo(t *testing.T) { } if !reflect.DeepEqual(got, c.expect) { t.Errorf("case %d failed: objects are different\n%#v\n%#v", - i+1, out.Elem().Interface(), c.expect, + i+1, got, c.expect, ) } })