Skip to content
This repository has been archived by the owner on Dec 13, 2022. It is now read-only.

Commit

Permalink
feat(facets): Remove Facets from Outserv (#3)
Browse files Browse the repository at this point in the history 
We don't need Facets in GraphQL. Their storage and processing is expensive. And GraphQL does not require them. Remove this feature.

This PR does not fix up the tests. It also keeps types/facets directory, which should be removed once the tests are fixed up.
  • Loading branch information
@manishrjain
manishrjain authored Feb 13, 2022
1 parent 08bfe44 commit 959b06b
Show file tree
Hide file tree
Showing 30 changed files with 612 additions and 3,186 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,9 @@ test:
./test.sh

image:
@GOOS=linux $(MAKE) dgraph
@GOOS=linux $(MAKE) outserv
@mkdir -p linux
@mv ./dgraph/dgraph ./linux/dgraph
@mv ./outserv/outserv ./linux/outserv
@docker build -f contrib/Dockerfile -t dgraph/dgraph:$(subst /,-,${BUILD_BRANCH}) .
@rm -r linux

Expand Down
188 changes: 4 additions & 184 deletions chunker/json_parser.go
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,5 @@
/*
* Copyright 2018 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Portions Copyright 2018 Dgraph Labs, Inc. are available under the Apache 2.0 license.
// Portions Copyright 2022 Outcaste, Inc. are available under the Smart License.

package chunker

Expand All @@ -30,7 +17,6 @@ import (
"github.com/outcaste-io/dgo/v210/protos/api"
"github.com/outcaste-io/outserv/protos/pb"
"github.com/outcaste-io/outserv/types"
"github.com/outcaste-io/outserv/types/facets"
"github.com/outcaste-io/outserv/x"
"github.com/pkg/errors"
geom "github.com/twpayne/go-geom"
Expand All @@ -47,146 +33,6 @@ func stripSpaces(str string) string {
}, str)
}

// handleBasicFacetsType parses a facetVal to string/float64/bool/datetime type.
func handleBasicFacetsType(key string, facetVal interface{}) (*api.Facet, error) {
var jsonValue interface{}
var valueType api.Facet_ValType
switch v := facetVal.(type) {
case string:
if t, err := types.ParseTime(v); err == nil {
valueType = api.Facet_DATETIME
jsonValue = t
} else {
facet, err := facets.FacetFor(key, strconv.Quote(v))
if err != nil {
return nil, err
}

// FacetFor function already converts the value to binary so there is no need
// for the conversion again after the switch block.
return facet, nil
}
case json.Number:
number := facetVal.(json.Number)
if strings.Contains(number.String(), ".") {
jsonFloat, err := number.Float64()
if err != nil {
return nil, err
}
jsonValue = jsonFloat
valueType = api.Facet_FLOAT
} else {
jsonInt, err := number.Int64()
if err != nil {
return nil, err
}
jsonValue = jsonInt
valueType = api.Facet_INT
}
// these int64/float64 cases are needed for the FastParseJSON simdjson
// parser, which doesn't use json.Number
case int64:
jsonValue = v
valueType = api.Facet_INT
case float64:
jsonValue = v
valueType = api.Facet_FLOAT
case bool:
jsonValue = v
valueType = api.Facet_BOOL
default:
return nil, errors.Errorf("facet value can only be string/number/bool.")
}

// Convert facet val interface{} to binary.
binaryValueFacet, err := facets.ToBinary(key, jsonValue, valueType)
if err != nil {
return nil, err
}

return binaryValueFacet, nil
}

// parseMapFacets parses facets which are of map type. Facets for scalar list predicates are
// specified in map format. For example below predicate nickname and kind facet associated with it.
// Here nickname "bob" doesn't have any facet associated with it.
// {
// "nickname": ["alice", "bob", "josh"],
// "nickname|kind": {
// "0": "friends",
// "2": "official"
// }
// }
// Parsed response would a slice of maps[int]*api.Facet, one map for each facet.
// Map key would be the index of scalar value for respective facets.
func parseMapFacets(m map[string]interface{}, prefix string) ([]map[int]*api.Facet, error) {
// This happens at root.
if prefix == "" {
return nil, nil
}

var mapSlice []map[int]*api.Facet
for fname, facetVal := range m {
if facetVal == nil {
continue
}
if !strings.HasPrefix(fname, prefix) {
continue
}

fm, ok := facetVal.(map[string]interface{})
if !ok {
return nil, errors.Errorf("facets format should be of type map for "+
"scalarlist predicates, found: %v for facet: %v", facetVal, fname)
}

idxMap := make(map[int]*api.Facet, len(fm))
for sidx, val := range fm {
key := fname[len(prefix):]
facet, err := handleBasicFacetsType(key, val)
if err != nil {
return nil, errors.Wrapf(err, "facet: %s, index: %s", fname, sidx)
}
idx, err := strconv.Atoi(sidx)
if err != nil {
return nil, errors.Wrapf(err, "facet: %s, index: %s", fname, sidx)
}
idxMap[idx] = facet
}
mapSlice = append(mapSlice, idxMap)
}

return mapSlice, nil
}

// parseScalarFacets parses facets which should be of type string/json.Number/bool.
// It returns []*api.Facet, one *api.Facet for each facet.
func parseScalarFacets(m map[string]interface{}, prefix string) ([]*api.Facet, error) {
// This happens at root.
if prefix == "" {
return nil, nil
}

var facetsForPred []*api.Facet
for fname, facetVal := range m {
if facetVal == nil {
continue
}
if !strings.HasPrefix(fname, prefix) {
continue
}

key := fname[len(prefix):]
facet, err := handleBasicFacetsType(key, facetVal)
if err != nil {
return nil, errors.Wrapf(err, "facet: %s", fname)
}
facetsForPred = append(facetsForPred, facet)
}

return facetsForPred, nil
}

// This is the response for a map[string]interface{} i.e. a struct.
type mapResponse struct {
uid string // uid retrieved or allocated for the node.
Expand Down Expand Up @@ -511,15 +357,6 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred
Namespace: namespace,
}

prefix := pred + x.FacetDelimeter
if _, ok := v.([]interface{}); !ok {
fts, err := parseScalarFacets(mr.rawFacets, prefix)
if err != nil {
return mr, err
}
nq.Facets = fts
}

// Here we split predicate and lang directive (ex: "name@en"), if needed. With JSON
// mutations that's the only way to send language for a value.
nq.Predicate, nq.Lang = x.PredicateLang(nq.Predicate)
Expand Down Expand Up @@ -566,18 +403,12 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred
case []interface{}:
buf.PushPredHint(pred, pb.Metadata_LIST)

// NOTE: facetsMapSlice should be empty unless this is a scalar list
var facetsMapSlice []map[int]*api.Facet
for idx, item := range v {
if idx == 0 {
// determine if this is a scalar list
switch item.(type) {
case string, float64, json.Number, int64:
var err error
facetsMapSlice, err = parseMapFacets(mr.rawFacets, prefix)
if err != nil {
return mr, err
}
// Used to parse facet here.
default:
// not a scalar list, continue
}
Expand Down Expand Up @@ -615,13 +446,6 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred
// 1: *api.Facet
// }
// ]
var fts []*api.Facet
for _, fm := range facetsMapSlice {
if ft, ok := fm[idx]; ok {
fts = append(fts, ft)
}
}
nq.Facets = fts
buf.Push(&nq)
case map[string]interface{}:
// map[string]interface{} can mean geojson or a connecting entity.
Expand All @@ -639,7 +463,6 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred
return mr, err
}
nq.ObjectId = cr.uid
nq.Facets = cr.fcts
buf.Push(&nq)
default:
return mr,
Expand All @@ -651,10 +474,7 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred
}
}

fts, err := parseScalarFacets(mr.rawFacets, parentPred+x.FacetDelimeter)
mr.fcts = fts

return mr, err
return mr, nil
}

const (
Expand Down
27 changes: 2 additions & 25 deletions chunker/rdf_parser.go
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,5 @@
/*
* Copyright 2015-2018 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Portions Copyright 2015-2018 Dgraph Labs, Inc. are available under the Apache 2.0 license.
// Portions Copyright 2022 Outcaste, Inc. are available under the Smart License.

package chunker

Expand All @@ -26,7 +13,6 @@ import (
"github.com/outcaste-io/outserv/lex"
"github.com/outcaste-io/outserv/protos/pb"
"github.com/outcaste-io/outserv/types"
"github.com/outcaste-io/outserv/types/facets"
"github.com/outcaste-io/outserv/x"
"github.com/pkg/errors"
)
Expand Down Expand Up @@ -293,15 +279,6 @@ func parseFacetsRDF(it *lex.ItemIterator, rnq *api.NQuad) error {
return errors.Errorf("Unexpected end of facets.")
}
item = it.Item()
facetVal := ""
if item.Typ == itemText {
facetVal = item.Val
}
facet, err := facets.FacetFor(facetKey, facetVal)
if err != nil {
return err
}
rnq.Facets = append(rnq.Facets, facet)

// empty value case..
if item.Typ == itemRightRound {
Expand Down
21 changes: 4 additions & 17 deletions compose/compose.go
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,5 @@
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Portions Copyright 2019 Dgraph Labs, Inc. are available under the Apache 2.0 license.
// Portions Copyright 2022 Outcaste, Inc. are available under the Smart License.

package main

Expand Down Expand Up @@ -224,9 +211,9 @@ func initService(basename string, idx, grpcPort int) service {
// no data volume
}

svc.Command = "dgraph"
svc.Command = "outserv"
if opts.LocalBin {
svc.Command = "/gobin/dgraph"
svc.Command = "/gobin/outserv"
}
if opts.UserOwnership {
user, err := user.Current()
Expand Down
6 changes: 3 additions & 3 deletions contrib/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
# This gets built as part of release.sh. Must be run from /tmp/build, with the linux binaries
# already built and placed there.

FROM ubuntu:20.04
LABEL maintainer="Dgraph Labs <[email protected]>"
FROM ubuntu:21.10
# LABEL maintainer="Dgraph Labs <[email protected]>"

RUN apt-get update && \
apt-get install -y --no-install-recommends \
Expand All @@ -29,4 +29,4 @@ RUN mkdir /dgraph
WORKDIR /dgraph

ENV GODEBUG=madvdontneed=1
CMD ["dgraph"] # Shows the dgraph version and commands available.
CMD ["outserv"] # Shows the dgraph version and commands available.
Loading

0 comments on commit 959b06b

Please sign in to comment.