diff --git a/AUTHORS b/AUTHORS index 4835189a4..ef938bd27 100644 --- a/AUTHORS +++ b/AUTHORS @@ -11,6 +11,7 @@ Alexander Peters Andrew Dunham Bram Leenders +Brendan Ball Denys Smirnov Derek Liang Google Inc. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 1a0c9c9a4..432848197 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -11,6 +11,7 @@ Alexander Peters Andrew Dunham Barak Michener Bram Leenders +Brendan Ball Connor Newton Denys Smirnov Derek Liang diff --git a/README.md b/README.md index 2df4a9cea..ead5ed3e2 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Its goal is to be a part of the developer's toolbox where [Linked Data](http://l * Plays well with multiple backend stores: * KVs: [Bolt](https://github.com/boltdb/bolt), [LevelDB](https://github.com/google/leveldb) * NoSQL: [MongoDB](https://www.mongodb.org), [ElasticSearch](https://www.elastic.co/products/elasticsearch), [CouchDB](http://couchdb.apache.org/)/[PouchDB](https://pouchdb.com/) - * SQL: [PostgreSQL](http://www.postgresql.org), [CockroachDB](https://www.cockroachlabs.com), [MySQL](https://www.mysql.com) + * SQL: [PostgreSQL](http://www.postgresql.org), [CockroachDB](https://www.cockroachlabs.com), [MySQL](https://www.mysql.com), [SQLite](https://www.sqlite.org) * In-memory, ephemeral * Modular design; easy to extend with new languages and backends * Good test coverage diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 diff --git a/docs/Configuration.md b/docs/Configuration.md index 4c016525f..e67c7e1aa 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -45,6 +45,7 @@ All command line flags take precedence over the configuration file. * `postgres`: Stores the graph data and indices in a [PostgreSQL](https://www.postgresql.org) instance. * `cockroach`: Stores the graph data and indices in a [CockroachDB](https://www.cockroachlabs.com/product/cockroachdb/) cluster. * `mysql`: Stores the graph data and indices in a [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/) instance. + * `sqlite`: Stores the graph data and indices in a [SQLite](https://www.sqlite.org) database. #### **`store.address`** @@ -62,6 +63,7 @@ All command line flags take precedence over the configuration file. * `couch`: `http://user:pass@host:port/dbname` of the desired CouchDB server. * `postgres`,`cockroach`: `postgres://[username:password@]host[:port]/database-name?sslmode=disable` of the PostgreSQL database and credentials. Sslmode is optional. More option available on [pq](https://godoc.org/github.com/lib/pq) page. * `mysql`: `[username:password@]tcp(host[:3306])/database-name` of the MqSQL database and credentials. More option available on [driver](https://github.com/go-sql-driver/mysql#dsn-data-source-name) page. + * `sqlite`: `filepath` of the SQLite database. More options available on [driver](https://github.com/mattn/go-sqlite3#connection-string) page. #### **`store.read_only`** diff --git a/go.mod b/go.mod index 2fe0ec0e4..cef86736a 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/lib/pq v0.0.0-20170324204654-2704adc878c2 github.com/linkeddata/gojsonld v0.0.0-20170418210642-4f5db6791326 github.com/magiconair/properties v0.0.0-20170321093039-51463bfca257 // indirect + github.com/mattn/go-sqlite3 v1.10.0 github.com/mitchellh/mapstructure v0.0.0-20170422000251-cc8532a8e9a5 // indirect github.com/onsi/ginkgo v1.7.0 // indirect github.com/onsi/gomega v1.4.3 // indirect diff --git a/go.sum b/go.sum index 5cf4ff274..8e92c7593 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/linkeddata/gojsonld v0.0.0-20170418210642-4f5db6791326 h1:YP3lfXXYiQV github.com/linkeddata/gojsonld v0.0.0-20170418210642-4f5db6791326/go.mod h1:nfqkuSNlsk1bvti/oa7TThx4KmRMBmSxf3okHI9wp3E= github.com/magiconair/properties v0.0.0-20170321093039-51463bfca257 h1:nqPtTOaJx2zFKWinLXKQYQRfxXTZN3GoJ602Uo65VGY= github.com/magiconair/properties v0.0.0-20170321093039-51463bfca257/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mitchellh/mapstructure v0.0.0-20170422000251-cc8532a8e9a5 h1:AjVRQelY0a5wq4bUgs08/+iblph90u9H58uzaH7Cw3g= github.com/mitchellh/mapstructure v0.0.0-20170422000251-cc8532a8e9a5/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/graph/all/all.go b/graph/all/all.go index fb496fc01..3e01d48c9 100644 --- a/graph/all/all.go +++ b/graph/all/all.go @@ -12,4 +12,5 @@ import ( _ "github.com/cayleygraph/cayley/graph/sql/cockroach" _ "github.com/cayleygraph/cayley/graph/sql/mysql" _ "github.com/cayleygraph/cayley/graph/sql/postgres" + _ "github.com/cayleygraph/cayley/graph/sql/sqlite" ) diff --git a/graph/sql/cockroach/cockroach_test.go b/graph/sql/cockroach/cockroach_test.go index 2a657836d..358b94c61 100644 --- a/graph/sql/cockroach/cockroach_test.go +++ b/graph/sql/cockroach/cockroach_test.go @@ -61,6 +61,7 @@ func makeCockroach(t testing.TB) (string, graph.Options, func()) { var conf = &sqltest.Config{ TimeRound: true, + TimeInMcs: true, } func TestCockroach(t *testing.T) { diff --git a/graph/sql/postgres/postgres_test.go b/graph/sql/postgres/postgres_test.go index d75d57ca2..dfe315367 100644 --- a/graph/sql/postgres/postgres_test.go +++ b/graph/sql/postgres/postgres_test.go @@ -35,6 +35,7 @@ func makePostgres(t testing.TB) (string, graph.Options, func()) { var conf = &sqltest.Config{ TimeRound: true, + TimeInMcs: true, } func TestPostgres(t *testing.T) { diff --git a/graph/sql/sqlite/sqlite.go b/graph/sql/sqlite/sqlite.go new file mode 100644 index 000000000..e5bc3c2ee --- /dev/null +++ b/graph/sql/sqlite/sqlite.go @@ -0,0 +1,151 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "regexp" + "strings" + + "github.com/cayleygraph/cayley/clog" + "github.com/cayleygraph/cayley/graph" + graphlog "github.com/cayleygraph/cayley/graph/log" + csql "github.com/cayleygraph/cayley/graph/sql" + "github.com/cayleygraph/cayley/quad" + sqlite3 "github.com/mattn/go-sqlite3" +) + +const Type = "sqlite" + +var QueryDialect = csql.QueryDialect{ + RegexpOp: "REGEXP", + FieldQuote: func(name string) string { + return "`" + name + "`" + }, + Placeholder: func(n int) string { return "?" }, +} + +func init() { + regex := func(re, s string) (bool, error) { + return regexp.MatchString(re, s) + } + sql.Register("sqlite3-regexp", + &sqlite3.SQLiteDriver{ + ConnectHook: func(conn *sqlite3.SQLiteConn) error { + return conn.RegisterFunc("regexp", regex, true) + }, + }) + csql.Register(Type, csql.Registration{ + Driver: "sqlite3-regexp", + HashType: fmt.Sprintf(`BINARY(%d)`, quad.HashSize), + BytesType: `BLOB`, + HorizonType: `INTEGER`, + TimeType: `DATETIME`, + QueryDialect: QueryDialect, + NoOffsetWithoutLimit: true, + NoForeignKeys: true, + Error: func(err error) error { + return err + }, + Estimated: nil, + RunTx: runTxSqlite, + }) +} + +func runTxSqlite(tx *sql.Tx, nodes []graphlog.NodeUpdate, quads []graphlog.QuadUpdate, opts graph.IgnoreOpts) error { + // update node ref counts and insert nodes + var ( + // prepared statements for each value type + insertValue = make(map[csql.ValueType]*sql.Stmt) + updateValue *sql.Stmt + ) + for _, n := range nodes { + if n.RefInc >= 0 { + nodeKey, values, err := csql.NodeValues(csql.NodeHash{n.Hash}, n.Val) + if err != nil { + return err + } + values = append([]interface{}{n.RefInc}, values...) + values = append(values, n.RefInc) // one more time for UPDATE + stmt, ok := insertValue[nodeKey] + if !ok { + var ph = make([]string, len(values)-1) // excluding last increment + for i := range ph { + ph[i] = "?" + } + stmt, err = tx.Prepare(`INSERT INTO nodes(refs, hash, ` + + strings.Join(nodeKey.Columns(), ", ") + + `) VALUES (` + strings.Join(ph, ", ") + + `) ON CONFLICT(hash) DO UPDATE SET refs = refs + ?;`) + if err != nil { + return err + } + insertValue[nodeKey] = stmt + } + _, err = stmt.Exec(values...) + err = convInsertError(err) + if err != nil { + clog.Errorf("couldn't exec INSERT statement: %v", err) + return err + } + } else { + panic("unexpected node update") + } + } + for _, s := range insertValue { + s.Close() + } + if s := updateValue; s != nil { + s.Close() + } + insertValue = nil + updateValue = nil + + // now we can deal with quads + ignore := "" + if opts.IgnoreDup { + ignore = " OR IGNORE" + } + + var ( + insertQuad *sql.Stmt + err error + ) + for _, d := range quads { + dirs := make([]interface{}, 0, len(quad.Directions)) + for _, h := range d.Quad.Dirs() { + dirs = append(dirs, csql.NodeHash{h}.SQLValue()) + } + if !d.Del { + if insertQuad == nil { + insertQuad, err = tx.Prepare(`INSERT` + ignore + ` INTO quads(subject_hash, predicate_hash, object_hash, label_hash, ts) VALUES (?, ?, ?, ?, datetime());`) + if err != nil { + return err + } + insertValue = make(map[csql.ValueType]*sql.Stmt) + } + _, err := insertQuad.Exec(dirs...) + err = convInsertError(err) + if err != nil { + if _, ok := err.(*graph.DeltaError); !ok { + clog.Errorf("couldn't exec INSERT statement: %v", err) + } + return err + } + } else { + panic("unexpected quad delete") + } + } + return nil +} + +func convInsertError(err error) error { + if err == nil { + return nil + } + if e, ok := err.(sqlite3.Error); ok { + if e.Code == sqlite3.ErrConstraint { + return &graph.DeltaError{Err: graph.ErrQuadExists} + } + } + return err +} diff --git a/graph/sql/sqlite/sqlite_test.go b/graph/sql/sqlite/sqlite_test.go new file mode 100644 index 000000000..22da8025c --- /dev/null +++ b/graph/sql/sqlite/sqlite_test.go @@ -0,0 +1,34 @@ +package sqlite + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/cayleygraph/cayley/graph" + "github.com/cayleygraph/cayley/graph/sql/sqltest" +) + +func makeSqlite(t testing.TB) (string, graph.Options, func()) { + tmpFile, err := ioutil.TempFile("", fmt.Sprintf("cayley_test_%s*", Type)) + if err != nil { + t.Fatalf("Could not create working directory: %v", err) + } + return fmt.Sprintf("file:%s?_loc=UTC", tmpFile.Name()), nil, func() { + os.RemoveAll(tmpFile.Name()) + } +} + +var conf = &sqltest.Config{ + TimeRound: true, + TimeInMcs: false, +} + +func TestSqlite(t *testing.T) { + sqltest.TestAll(t, Type, makeSqlite, conf) +} + +func BenchmarkSqlite(t *testing.B) { + sqltest.BenchmarkAll(t, Type, makeSqlite, conf) +} diff --git a/graph/sql/sqltest/sqltest.go b/graph/sql/sqltest/sqltest.go index fd2db3aff..632454ebb 100644 --- a/graph/sql/sqltest/sqltest.go +++ b/graph/sql/sqltest/sqltest.go @@ -14,12 +14,13 @@ import ( type Config struct { TimeRound bool + TimeInMcs bool } func (c Config) quadStore() *graphtest.Config { return &graphtest.Config{ NoPrimitives: true, - TimeInMcs: true, + TimeInMcs: c.TimeInMcs, TimeRound: c.TimeRound, OptimizesComparison: true, } @@ -27,7 +28,7 @@ func (c Config) quadStore() *graphtest.Config { func TestAll(t *testing.T, typ string, fnc DatabaseFunc, c *Config) { if c == nil { - c = &Config{} + c = &Config{TimeInMcs: true} } create := makeDatabaseFunc(typ, fnc) t.Run("qs", func(t *testing.T) {