From 294cf61193a00b94c6d22086b250cad67e4a9df1 Mon Sep 17 00:00:00 2001 From: Ana Maria Franco Date: Fri, 12 Aug 2022 22:07:30 -0500 Subject: [PATCH] feat: Adding sqlite to mobile Signed-off-by: Ana Maria Franco --- component/storage/mysql/go.mod | 25 +- component/storage/sqlite/errors.go | 29 + component/storage/sqlite/go.mod | 24 + component/storage/sqlite/go.sum | 32 + component/storage/sqlite/store.go | 755 ++++++++++++++++++ component/storage/sqlite/store_test.go | 383 +++++++++ .../storage/sqlite/testdata/nosqlite.txt | 1 + 7 files changed, 1244 insertions(+), 5 deletions(-) create mode 100644 component/storage/sqlite/errors.go create mode 100644 component/storage/sqlite/go.mod create mode 100644 component/storage/sqlite/go.sum create mode 100644 component/storage/sqlite/store.go create mode 100644 component/storage/sqlite/store_test.go create mode 100644 component/storage/sqlite/testdata/nosqlite.txt diff --git a/component/storage/mysql/go.mod b/component/storage/mysql/go.mod index c72aa532..4864cddb 100644 --- a/component/storage/mysql/go.mod +++ b/component/storage/mysql/go.mod @@ -6,22 +6,37 @@ module github.com/hyperledger/aries-framework-go-ext/component/storage/mysql go 1.17 require ( - github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect github.com/cenkalti/backoff/v4 v4.1.0 - github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe // indirect github.com/go-sql-driver/mysql v1.5.0 - github.com/google/go-cmp v0.5.4 // indirect github.com/google/uuid v1.2.0 github.com/hyperledger/aries-framework-go/spi v0.0.0-20220330140627-07042d78580c github.com/hyperledger/aries-framework-go/test/component v0.0.0-20220330140627-07042d78580c + github.com/ory/dockertest/v3 v3.6.3 + github.com/stretchr/testify v1.7.0 +) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/cenkalti/backoff/v3 v3.0.0 // indirect + github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/google/go-cmp v0.5.4 // indirect github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.9.0 // indirect + github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/ory/dockertest/v3 v3.6.3 + github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/opencontainers/runc v1.0.0-rc9 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect - github.com/stretchr/testify v1.7.0 golang.org/x/net v0.0.0-20210421230115-4e50805a0758 // indirect + golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/component/storage/sqlite/errors.go b/component/storage/sqlite/errors.go new file mode 100644 index 00000000..35383f25 --- /dev/null +++ b/component/storage/sqlite/errors.go @@ -0,0 +1,29 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sqlite + +import ( + "errors" +) + +const ( + // Error messages we return. + failureWhileOpeningSQLiteConnectionErrMsg = "failure while opening SQLite connection using path %s: %w" + failureWhileClosingSQLiteConnection = "failure while closing SQLite DB connection: %w" + failureWhilePingingSQLiteErrMsg = "failure while pinging SQLite at path %s : %w" + failureWhileCreatingTableErrMsg = "failure while creating table %s: %w" + failureWhileExecutingInsertStatementErrMsg = "failure while executing insert statement on table %s: %w" + failureWhileQueryingRowErrMsg = "failure while querying row: %w" + failureWhileExecutingBatchStatementErrMsg = "failure while executing batch upsert on table %s: %w" + // Error messages returned from MySQL that we directly check for. + valueNotFoundErrMsgFromSQlite = "no rows" +) + +var ( + errBlankDBPath = errors.New("DB Path for new SQLite DB provider can't be blank") + errBlankStoreName = errors.New("store name is required") +) diff --git a/component/storage/sqlite/go.mod b/component/storage/sqlite/go.mod new file mode 100644 index 00000000..e4570f71 --- /dev/null +++ b/component/storage/sqlite/go.mod @@ -0,0 +1,24 @@ +// Copyright SecureKey Technologies Inc. All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +module github.com/hyperledger/aries-framework-go-ext/component/storage/sqlite + +go 1.17 + +require ( + github.com/google/uuid v1.3.0 + github.com/hyperledger/aries-framework-go/cmd/aries-agent-mobile v0.0.0-20220811152045-03f747c09617 + github.com/hyperledger/aries-framework-go/spi v0.0.0-20220606124520-53422361c38c + github.com/hyperledger/aries-framework-go/test/component v0.0.0-20220428211718-66cc046674a1 + github.com/mattn/go-sqlite3 v1.14.14 + github.com/stretchr/testify v1.7.2 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/component/storage/sqlite/go.sum b/component/storage/sqlite/go.sum new file mode 100644 index 00000000..398570ab --- /dev/null +++ b/component/storage/sqlite/go.sum @@ -0,0 +1,32 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hyperledger/aries-framework-go v0.1.8-0.20220322085443-50e8f9bd208b h1:q+Plvnq7/n3IwTO6IOhBXn//txifpeo/OErvM1ivh8s= +github.com/hyperledger/aries-framework-go/cmd/aries-agent-mobile v0.0.0-20220811152045-03f747c09617 h1:3QNyjpHGcKgdWF7WX6AFqfPeUSr14leIbqzq6qI9wWA= +github.com/hyperledger/aries-framework-go/cmd/aries-agent-mobile v0.0.0-20220811152045-03f747c09617/go.mod h1:NE+udomUMekXew+ytQa3G163zLaRJKWjXloSpG+/lV0= +github.com/hyperledger/aries-framework-go/component/storageutil v0.0.0-20220322085443-50e8f9bd208b h1:IoD7+sHQRLMouwHjhrOj5vhg+rPt/aKl4P+WiBZVHVk= +github.com/hyperledger/aries-framework-go/spi v0.0.0-20220606124520-53422361c38c h1:4JSde5+80U+W8IxwNt/Dd/3PPEIGYTI3RbjFXOi4o1g= +github.com/hyperledger/aries-framework-go/spi v0.0.0-20220606124520-53422361c38c/go.mod h1:4bD5c5fj5K7rkQurVa/8I8+TfNcI4bxIBzaUNcxTOTg= +github.com/hyperledger/aries-framework-go/test/component v0.0.0-20220428211718-66cc046674a1 h1:vxZ0DlFNLjgxMdBESLZu895AsI1JWL2SJerphwIn8Po= +github.com/hyperledger/aries-framework-go/test/component v0.0.0-20220428211718-66cc046674a1/go.mod h1:lykx3N+GX+sAWSxO2Ycc4Dz+ynV9b0Fv4NdP+ms4Alc= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/component/storage/sqlite/store.go b/component/storage/sqlite/store.go new file mode 100644 index 00000000..26ba6f9a --- /dev/null +++ b/component/storage/sqlite/store.go @@ -0,0 +1,755 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Created using https://github.com/hyperledger/aries-framework-go-ext/tree/main/component/storage/mysql as reference + +package sqlite + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + + _ "github.com/mattn/go-sqlite3" //nolint:gci // required for SQLite + + "github.com/hyperledger/aries-framework-go/spi/storage" +) + +const ( + tagMapKey = "TagMap" + storeConfigKey = "StoreConfig" +) + +const ( + expressionTagNameOnlyLength = 1 + expressionTagNameAndValueLength = 2 +) + +const ( + invalidQueryExpressionFormat = `"%s" is not in a valid expression format. ` + + "it must be in the following format: TagName:TagValue" + invalidTagName = `"%s" is an invalid tag name since it contains one or more ':' characters` + invalidTagValue = `"%s" is an invalid tag value since it contains one or more ':' characters` +) + +// TODO: Fully implement all methods. + +// ErrKeyRequired is returned when key is mandatory. +var ErrKeyRequired = errors.New("key is mandatory") + +type closer func(storeName string) + +type tagMapping map[string]map[string]struct{} // map[TagName](Set of database Keys) + +type dbEntry struct { + Value []byte `json:"value,omitempty"` + Tags []storage.Tag `json:"tags,omitempty"` +} + +// Provider represents a SQLite DB implementation of the storage.Provider interface. +type Provider struct { + dbPath string + db *sql.DB + dbs map[string]*store + dbPrefix string + lock sync.RWMutex +} + +// Option configures the SQLite provider. +type Option func(opts *Provider) + +// WithDBPrefix option is for adding prefix to db name. +func WithDBPrefix(dbPrefix string) Option { + return func(opts *Provider) { + opts.dbPrefix = dbPrefix + } +} + +// NewProvider instantiates Provider. +// Example DB Path ./test.db +// This provider's CreateStore(name) implementation creates stores that are backed by a table without a schema. +// The fully qualified name of the table is thus `name`. +// Use of `Batch()` has additional considerations - please read the docs for Batch() accordingly.. +func NewProvider(dbPath string, opts ...Option) (*Provider, error) { + if dbPath == "" { + return nil, errBlankDBPath + } + + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return nil, fmt.Errorf(failureWhileOpeningSQLiteConnectionErrMsg, dbPath, err) + } + + err = db.Ping() + if err != nil { + return nil, fmt.Errorf(failureWhilePingingSQLiteErrMsg, dbPath, err) + } + + p := &Provider{ + dbPath: dbPath, + db: db, + dbs: map[string]*store{}, + } + + for _, opt := range opts { + opt(p) + } + + return p, nil +} + +// OpenStore opens a store with the given name and returns a handle. +// If the store has never been opened before, then it is created. +// Store names are not case-sensitive. If name is blank, then an error will be returned. +// SQLite doesn't support the "-" character in the table names so they are replaced with "_" +// WARNING: This method will create a database and table based on the given name. Those database calls may be +// vulnerable to an SQL injection attack. Be very careful if you use a user-provided string in the store name! +func (p *Provider) OpenStore(name string) (storage.Store, error) { + if name == "" { + return nil, errBlankStoreName + } + + name = strings.ReplaceAll(strings.ToLower(name), "-", "_") + + if p.dbPrefix != "" { + name = p.dbPrefix + "_" + name + } + + p.lock.Lock() + defer p.lock.Unlock() + + // Check cache first + cachedStore, existsInCache := p.dbs[name] + if existsInCache { + return cachedStore, nil + } + + createTableStmt := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s "+ + "('key' VARCHAR(255) NOT NULL PRIMARY KEY, 'value' MEDIUMBLOB)", name) + + // creating key-value table inside the database + _, err := p.db.Exec(createTableStmt) + if err != nil { + return nil, fmt.Errorf(failureWhileCreatingTableErrMsg, name, err) + } + + // Opening new DB connection + storeDB, err := sql.Open("sqlite3", p.dbPath) + if err != nil { + return nil, fmt.Errorf(failureWhileOpeningSQLiteConnectionErrMsg, p.dbPath, err) + } + + store := &store{ + db: storeDB, + name: name, + tableName: name, + close: p.removeStore, + } + + p.dbs[name] = store + + return store, nil +} + +// SetStoreConfig sets the configuration on a store. This must be done before storing any data in order to make use +// of the Query method. +// TODO: Use proper SQlite indexing instead of the "Tag Map". +func (p *Provider) SetStoreConfig(name string, config storage.StoreConfiguration) error { + for _, tagName := range config.TagNames { + if strings.Contains(tagName, ":") { + return fmt.Errorf(invalidTagName, tagName) + } + } + + name = strings.ReplaceAll(strings.ToLower(name), "-", "_") + + if p.dbPrefix != "" { + name = p.dbPrefix + "_" + name + } + + openStore, ok := p.dbs[name] + if !ok { + return storage.ErrStoreNotFound + } + + configBytes, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal store configuration: %w", err) + } + + err = openStore.Put(storeConfigKey, configBytes) + if err != nil { + return fmt.Errorf("failed to put store store configuration: %w", err) + } + + return nil +} + +// GetStoreConfig returns the store's configuration. +// TODO: Check for underlying database's existence instead of looking at in-memory stores. +func (p *Provider) GetStoreConfig(name string) (storage.StoreConfiguration, error) { + name = strings.ReplaceAll(strings.ToLower(name), "-", "_") + + if p.dbPrefix != "" { + name = p.dbPrefix + "_" + name + } + + openStore, ok := p.dbs[name] + if !ok { + return storage.StoreConfiguration{}, storage.ErrStoreNotFound + } + + storeConfigBytes, err := openStore.Get(storeConfigKey) + if err != nil { + return storage.StoreConfiguration{}, + fmt.Errorf(`failed to get store configuration for "%s": %w`, name, err) + } + + var storeConfig storage.StoreConfiguration + + err = json.Unmarshal(storeConfigBytes, &storeConfig) + if err != nil { + return storage.StoreConfiguration{}, fmt.Errorf("failed to unmarshal store configuration: %w", err) + } + + return storeConfig, nil +} + +// GetOpenStores is currently not implemented. +func (p *Provider) GetOpenStores() []storage.Store { + panic("not implemented") +} + +// Close closes all stores created under this store provider. +func (p *Provider) Close() error { + p.lock.RLock() + + openStoresSnapshot := make([]*store, len(p.dbs)) + + var counter int + + for _, openStore := range p.dbs { + openStoresSnapshot[counter] = openStore + counter++ + } + p.lock.RUnlock() + + for _, openStore := range openStoresSnapshot { + err := openStore.Close() + if err != nil { + return fmt.Errorf(`failed to close open store with name "%s": %w`, openStore.name, err) + } + } + + return p.db.Close() +} + +func (p *Provider) removeStore(name string) { + p.lock.Lock() + defer p.lock.Unlock() + + _, ok := p.dbs[name] + if ok { + delete(p.dbs, name) + } +} + +type store struct { + db *sql.DB + name string + tableName string + close closer + lock sync.RWMutex +} + +func (s *store) Put(key string, value []byte, tags ...storage.Tag) error { + errInputValidation := validatePutInput(key, value, tags) + if errInputValidation != nil { + return errInputValidation + } + + var newDBEntry dbEntry + newDBEntry.Value = value + + if len(tags) > 0 { + newDBEntry.Tags = tags + + err := s.updateTagMap(key, tags) + if err != nil { + return fmt.Errorf("failed to update tag map: %w", err) + } + } + + entryBytes, err := json.Marshal(newDBEntry) + if err != nil { + return fmt.Errorf("failed to marshal new DB entry: %w", err) + } + + // create upsert query to insert the record, checking whether the key is already mapped to a value in the store. + insertStmt := "INSERT OR REPLACE INTO " + s.tableName + " VALUES (?, ?)" + // executing the prepared insert statement + _, err = s.db.Exec(insertStmt, key, entryBytes, entryBytes) + if err != nil { + return fmt.Errorf(failureWhileExecutingInsertStatementErrMsg, s.tableName, err) + } + + return nil +} + +func (s *store) Get(k string) ([]byte, error) { + retrievedDBEntry, err := s.getDBEntry(k) + if err != nil { + return nil, fmt.Errorf("failed to get DB entry: %w", err) + } + + return retrievedDBEntry.Value, nil +} + +func (s *store) GetTags(key string) ([]storage.Tag, error) { + retrievedDBEntry, err := s.getDBEntry(key) + if err != nil { + return nil, fmt.Errorf("failed to get DB entry: %w", err) + } + + return retrievedDBEntry.Tags, nil +} + +func (s *store) GetBulk(...string) ([][]byte, error) { + return nil, errors.New("not implemented") +} + +// Mobile providers doesn't currently support any of the current query options. +// pageSize will simply be ignored since it only relates to performance and not the actual end result. +func (s *store) Query(expression string, options ...storage.QueryOption) (storage.Iterator, error) { + err := checkForUnsupportedQueryOptions(options) + if err != nil { + return nil, err + } + + if expression == "" { + return nil, fmt.Errorf(invalidQueryExpressionFormat, expression) + } + + expressionSplit := strings.Split(expression, ":") + + var expressionTagName string + + var expressionTagValue string + + switch len(expressionSplit) { + case expressionTagNameOnlyLength: + expressionTagName = expressionSplit[0] + case expressionTagNameAndValueLength: + expressionTagName = expressionSplit[0] + expressionTagValue = expressionSplit[1] + default: + return nil, fmt.Errorf(invalidQueryExpressionFormat, expression) + } + + matchingDatabaseKeys, err := s.getDatabaseKeysMatchingQuery(expressionTagName, expressionTagValue) + if err != nil { + return nil, fmt.Errorf("failed to get database keys matching query: %w", err) + } + + return &iterator{keys: matchingDatabaseKeys, store: s}, nil +} + +// Delete will delete record with k key. +func (s *store) Delete(k string) error { + if k == "" { + return ErrKeyRequired + } + + // delete query to delete the record by key + _, err := s.db.Exec("DELETE FROM "+s.tableName+" WHERE `key`= ?", k) + if err != nil { + return fmt.Errorf(storage.ErrDataNotFound.Error(), err) + } + + err = s.removeFromTagMap(k) + if err != nil { + return fmt.Errorf("failed to remove key from tag map: %w", err) + } + + return nil +} + +// Batch performs batch upserts and deletions preserving the batch's ordering. +// Batch is a no-op if the batch is empty. +// Batch needs both `interpolateParams` and `multiStatements` enabled in the dataSourceName +// - see the following guide: https://github.com/go-sql-driver/mysql#parameters. +// All operations in the batch are executed in a single multi-statement query due to the ordering +// requirements. This means we cannot optimize INSERT statements as per +// https://dev.mysql.com/doc/refman/8.0/en/insert-optimization.html. +// Executing a single multi-statement query requires O(N) space for the query string and an additional +// slice with O(2*N) space holding the values for the query. Callers should take care of this additional +// memory usage by limiting the size of the batch. +func (s *store) Batch(batch []storage.Operation) error { // nolint:gocyclo + if len(batch) == 0 { + return errors.New("batch requires at least one operation") + } + + var ( + query string + values []interface{} + ) + + for i := range batch { + b := batch[i] + + if b.Key == "" { + return errors.New("key cannot be empty") + } + + err := s.bulkAppendToQuery(b, &query, &values) + if err != nil { + return fmt.Errorf(failureWhileExecutingBatchStatementErrMsg, s.tableName, err) + } + + if len(b.Value) > 0 { + err = s.updateTagMap(b.Key, b.Tags) + if err != nil && !errors.Is(err, storage.ErrDataNotFound) { + return fmt.Errorf("failed to update tag map: %w", err) + } + } else { + err = s.removeFromTagMap(b.Key) + if err != nil && !errors.Is(err, storage.ErrDataNotFound) { + return fmt.Errorf("failed to remove key from tag map: %w", err) + } + } + } + + _, err := s.db.Exec(query, values...) + if err != nil { + return fmt.Errorf(failureWhileExecutingBatchStatementErrMsg, s.tableName, err) + } + + return nil +} + +func (s *store) bulkAppendToQuery(b storage.Operation, query *string, values *[]interface{}) error { + if len(b.Value) > 0 { + value, err := json.Marshal(dbEntry{ + Value: b.Value, + Tags: b.Tags, + }) + if err != nil { + return fmt.Errorf("failed to marshal dbEntry: %w", err) + } + + *query += fmt.Sprintf( + "INSERT OR REPLACE INTO %s (`key`, `value`) VALUES (?, ?);\n", + s.tableName, + ) + + *values = append(*values, b.Key, value) + } else { + *query += fmt.Sprintf("DELETE FROM %s WHERE `KEY` = ?;\n", s.tableName) + *values = append(*values, b.Key) + } + + return nil +} + +// SQL store doesn't queue values, so there's never anything to flush. +func (s *store) Flush() error { + return nil +} + +func (s *store) Close() error { + s.close(s.name) + + err := s.db.Close() + if err != nil { + return fmt.Errorf(failureWhileClosingSQLiteConnection, err) + } + + return nil +} + +func (s *store) updateTagMap(key string, tags []storage.Tag) error { + s.lock.Lock() + defer s.lock.Unlock() + + tagMap, err := s.getTagMap(true) + if err != nil { + return fmt.Errorf("failed to get tag map: %w", err) + } + + for _, tag := range tags { + if tagMap[tag.Name] == nil { + tagMap[tag.Name] = make(map[string]struct{}) + } + + tagMap[tag.Name][key] = struct{}{} + } + + tagMapBytes, err := json.Marshal(tagMap) + if err != nil { + return fmt.Errorf("failed to marshal updated tag map: %w", err) + } + + err = s.Put(tagMapKey, tagMapBytes) + if err != nil { + return fmt.Errorf("failed to put updated tag map back into the store: %w", err) + } + + return nil +} + +func (s *store) getTagMap(createIfDoesNotExist bool) (tagMapping, error) { + tagMapBytes, err := s.Get(tagMapKey) + if err != nil { + if createIfDoesNotExist && errors.Is(err, storage.ErrDataNotFound) { + // Create the tag map if it has never been created before. + err = s.Put(tagMapKey, []byte("{}")) + if err != nil { + return nil, fmt.Errorf(`failed to create tag map for "%s": %w`, s.name, err) + } + + tagMapBytes = []byte("{}") + } else { + return nil, fmt.Errorf("failed to get data: %w", err) + } + } + + var tagMap tagMapping + + err = json.Unmarshal(tagMapBytes, &tagMap) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal tag map bytes: %w", err) + } + + return tagMap, nil +} + +func (s *store) getDBEntry(key string) (dbEntry, error) { + if key == "" { + return dbEntry{}, ErrKeyRequired + } + + var retrievedDBEntryBytes []byte + + // select query to fetch the record by key + err := s.db.QueryRow("SELECT `value` FROM "+s.tableName+" "+ + " WHERE `key` = ?", key).Scan(&retrievedDBEntryBytes) + if err != nil { + if strings.Contains(err.Error(), valueNotFoundErrMsgFromSQlite) { + return dbEntry{}, storage.ErrDataNotFound + } + + return dbEntry{}, fmt.Errorf(failureWhileQueryingRowErrMsg, err) + } + + var retrievedDBEntry dbEntry + + err = json.Unmarshal(retrievedDBEntryBytes, &retrievedDBEntry) + if err != nil { + return dbEntry{}, fmt.Errorf("failed to unmarshaled retrieved DB entry: %w", err) + } + + return retrievedDBEntry, nil +} + +func (s *store) removeFromTagMap(keyToRemove string) error { + s.lock.Lock() + defer s.lock.Unlock() + + tagMap, err := s.getTagMap(false) + if err != nil { + // If there's no tag map, then this means that tags have never been used. This means that there's no tag map + // to update. This isn't a problem. + if errors.Is(err, storage.ErrDataNotFound) { + return nil + } + + return fmt.Errorf("failed to get tag map: %w", err) + } + + for _, tagNameToKeys := range tagMap { + delete(tagNameToKeys, keyToRemove) + } + + tagMapBytes, err := json.Marshal(tagMap) + if err != nil { + return fmt.Errorf("failed to marshal updated tag map: %w", err) + } + + err = s.Put(tagMapKey, tagMapBytes) + if err != nil { + return fmt.Errorf("failed to put updated tag map back into the store: %w", err) + } + + return nil +} + +func (s *store) getDatabaseKeysMatchingQuery(expressionTagName, expressionTagValue string) ([]string, error) { + tagMap, err := s.getTagMap(false) + if err != nil { + // If there's no tag map, then this means that tags have never been used, and therefore no matching results. + if errors.Is(err, storage.ErrDataNotFound) { + return nil, nil + } + + return nil, fmt.Errorf("failed to get tag map: %w", err) + } + + if expressionTagValue == "" { + return getDatabaseKeysMatchingTagName(tagMap, expressionTagName), nil + } + + matchingDatabaseKeys, err := s.getDatabaseKeysMatchingTagNameAndValue(tagMap, expressionTagName, expressionTagValue) + if err != nil { + return nil, fmt.Errorf("failed to get database keys matching tag name and value: %w", err) + } + + return matchingDatabaseKeys, nil +} + +func (s *store) getDatabaseKeysMatchingTagNameAndValue(tagMap tagMapping, + expressionTagName, expressionTagValue string) ([]string, error) { + var matchingDatabaseKeys []string + + for tagName, databaseKeysSet := range tagMap { + if tagName == expressionTagName { + for databaseKey := range databaseKeysSet { + tags, err := s.GetTags(databaseKey) + if err != nil { + return nil, fmt.Errorf("failed to get tags: %w", err) + } + + for _, tag := range tags { + if tag.Name == expressionTagName && tag.Value == expressionTagValue { + matchingDatabaseKeys = append(matchingDatabaseKeys, databaseKey) + + break + } + } + } + + break + } + } + + return matchingDatabaseKeys, nil +} + +type iterator struct { + keys []string + currentIndex int + currentKey string + store *store +} + +func (i *iterator) Next() (bool, error) { + if len(i.keys) == i.currentIndex || len(i.keys) == 0 { + if len(i.keys) == i.currentIndex || len(i.keys) == 0 { + return false, nil + } + } + + i.currentKey = i.keys[i.currentIndex] + + i.currentIndex++ + + return true, nil +} + +func (i *iterator) Key() (string, error) { + return i.currentKey, nil +} + +func (i *iterator) Value() ([]byte, error) { + value, err := i.store.Get(i.currentKey) + if err != nil { + return nil, fmt.Errorf("failed to get value from store: %w", err) + } + + return value, nil +} + +func (i *iterator) Tags() ([]storage.Tag, error) { + tags, err := i.store.GetTags(i.currentKey) + if err != nil { + return nil, fmt.Errorf("failed to get tags from store: %w", err) + } + + return tags, nil +} + +func (i *iterator) TotalItems() (int, error) { + return len(i.keys), nil +} + +func (i *iterator) Close() error { + return nil +} + +func validatePutInput(key string, value []byte, tags []storage.Tag) error { + if key == "" { + return errors.New("key cannot be empty") + } + + if value == nil { + return errors.New("value cannot be nil") + } + + for _, tag := range tags { + if strings.Contains(tag.Name, ":") { + return fmt.Errorf(invalidTagName, tag.Name) + } + + if strings.Contains(tag.Value, ":") { + return fmt.Errorf(invalidTagValue, tag.Value) + } + } + + return nil +} + +func checkForUnsupportedQueryOptions(options []storage.QueryOption) error { + querySettings := getQueryOptions(options) + + if querySettings.InitialPageNum != 0 { + return errors.New("mySQL provider does not currently support " + + "setting the initial page number of query results") + } + + if querySettings.SortOptions != nil { + return errors.New("mySQL provider does not currently support custom sort options for query results") + } + + return nil +} + +func getQueryOptions(options []storage.QueryOption) storage.QueryOptions { + var queryOptions storage.QueryOptions + + for _, option := range options { + option(&queryOptions) + } + + return queryOptions +} + +func getDatabaseKeysMatchingTagName(tagMap tagMapping, expressionTagName string) []string { + var matchingDatabaseKeys []string + + for tagName, databaseKeysSet := range tagMap { + if tagName == expressionTagName { + for databaseKey := range databaseKeysSet { + matchingDatabaseKeys = append(matchingDatabaseKeys, databaseKey) + } + + break + } + } + + return matchingDatabaseKeys +} diff --git a/component/storage/sqlite/store_test.go b/component/storage/sqlite/store_test.go new file mode 100644 index 00000000..a15cf734 --- /dev/null +++ b/component/storage/sqlite/store_test.go @@ -0,0 +1,383 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sqlite_test + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/hyperledger/aries-framework-go-ext/component/storage/sqlite" + "github.com/hyperledger/aries-framework-go/spi/storage" + commontest "github.com/hyperledger/aries-framework-go/test/component/storage" +) + +func setupSQLiteDB(t testing.TB) string { + file, err := ioutil.TempFile("./testdata", "test-*.db") + if err != nil { + t.Fatalf("Failed to create sqlite file: %s", err) + } + + dbFolderPath, err := filepath.Abs("./testdata") + if err != nil { + t.Fatalf("Failed to get absolute path of sqlite directory: %s", err) + } + + dbPath := filepath.Join(dbFolderPath, filepath.Base(file.Name())) + + t.Cleanup(func() { + err := file.Close() + if err != nil { + t.Fatalf("Failed to close sqlite file: %s", err) + } + err = os.Remove(dbPath) + if err != nil { + t.Fatalf("Failed to clear sqlite file: %s", err) + } + }) + + return dbPath +} + +func TestSQLDBStore(t *testing.T) { + t.Run("Test SQL open store", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + _, err = provider.OpenStore("") + require.Error(t, err) + require.Equal(t, err.Error(), "store name is required") + + err = provider.Close() + require.NoError(t, err) + }) + t.Run("Test sql db store failures", func(t *testing.T) { + provider, err := sqlite.NewProvider("") + require.Error(t, err) + require.Contains(t, err.Error(), "DB Path for new SQLite DB provider can't be blank") + require.Nil(t, provider) + + _, err = sqlite.NewProvider("./testdata/nosqlite.txt") + require.Error(t, err) + require.Contains(t, err.Error(), "failure while pinging SQLite") + }) + t.Run("Test sqlDB multi store close by name", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path, sqlite.WithDBPrefix("prefixdb")) + require.NoError(t, err) + + const commonKey = "did:example:1" + data := []byte("value1") + + storeNames := []string{"store_1", "store_2", "store_3", "store_4", "store_5"} + storesToClose := []string{"store_1", "store_3", "store_5"} + + for _, name := range storeNames { + store, e := provider.OpenStore(name) + require.NoError(t, e) + + e = store.Put(commonKey, data) + require.NoError(t, e) + } + + for _, name := range storeNames { + store, e := provider.OpenStore(name) + require.NoError(t, e) + require.NotNil(t, store) + + dataRead, e := store.Get(commonKey) + require.NoError(t, e) + require.Equal(t, data, dataRead) + } + + for _, name := range storesToClose { + store, e := provider.OpenStore(name) + require.NoError(t, e) + require.NotNil(t, store) + + err = store.Close() + require.NoError(t, err) + } + + err = provider.Close() + require.NoError(t, err) + + // try close all again + err = provider.Close() + require.NoError(t, err) + }) + t.Run("Flush", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + store, err := provider.OpenStore("storename") + require.NoError(t, err) + + err = store.Flush() + require.NoError(t, err) + + err = provider.Close() + require.NoError(t, err) + }) +} + +func TestProvider_GetStoreConfig(t *testing.T) { + t.Run("Fail to get store configuration", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + storeName := randomStoreName() + + _, err = provider.OpenStore(storeName) + require.NoError(t, err) + + config, err := provider.GetStoreConfig(storeName) + require.EqualError(t, err, + fmt.Sprintf(`failed to get store configuration for "%s": `+ + `failed to get DB entry: data not found`, strings.ReplaceAll(storeName, "-", "_"))) + require.Empty(t, config) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestStore_Put(t *testing.T) { + t.Run("Fail to update tag map since the DB connection was closed", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + storeName := randomStoreName() + testStore, err := provider.OpenStore(storeName) + require.NoError(t, err) + + err = testStore.Close() + require.NoError(t, err) + + err = testStore.Put("key", []byte("value")) + require.EqualError(t, err, fmt.Sprintf("failure while executing insert statement on table %s: "+ + "sql: database is closed", strings.ReplaceAll(storeName, "-", "_"))) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) + t.Run("Fail to unmarshal tag map bytes", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStore, err := provider.OpenStore(randomStoreName()) + require.NoError(t, err) + + err = testStore.Put("TagMap", []byte("Not a proper tag map")) + require.NoError(t, err) + + err = testStore.Put("key", []byte("value"), storage.Tag{}) + require.EqualError(t, err, "failed to update tag map: failed to get tag map: "+ + "failed to unmarshal tag map bytes: invalid character 'N' looking for beginning of value") + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestSqlDBStore_Query(t *testing.T) { + t.Run("Fail to get tag map since the DB connection was closed", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStore, err := provider.OpenStore(randomStoreName()) + require.NoError(t, err) + + err = testStore.Close() + require.NoError(t, err) + + itr, err := testStore.Query("expression") + require.EqualError(t, err, "failed to get database keys matching query: failed to get tag map: "+ + "failed to get data: failed to get DB entry: failure while querying row: sql: database is closed") + require.Nil(t, itr) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestIterator(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStoreName := randomStoreName() + + testStore, err := provider.OpenStore(testStoreName) + require.NoError(t, err) + + storeConfig := storage.StoreConfiguration{TagNames: []string{}} + err = provider.SetStoreConfig(testStoreName, storeConfig) + require.NoError(t, err) + + itr, err := testStore.Query("expression") + require.NoError(t, err) + + t.Run("Fail to get value from store", func(t *testing.T) { + value, errValue := itr.Value() + require.EqualError(t, errValue, "failed to get value from store: failed to get DB entry: key is mandatory") + require.Nil(t, value) + }) + t.Run("Fail to get tags from store", func(t *testing.T) { + tags, errGetTags := itr.Tags() + require.EqualError(t, errGetTags, "failed to get tags from store: failed to get DB entry: key is mandatory") + require.Nil(t, tags) + }) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) +} + +func TestSqlDBStore_Common(t *testing.T) { + t.Run("Without prefix", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + commontest.TestProviderOpenStoreSetGetConfig(t, provider) + commontest.TestPutGet(t, provider) + commontest.TestStoreGetTags(t, provider) + commontest.TestStoreQuery(t, provider) + commontest.TestStoreDelete(t, provider) + commontest.TestStoreBatch(t, provider) + commontest.TestStoreClose(t, provider) + commontest.TestProviderClose(t, provider) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) + t.Run("With prefix", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path, sqlite.WithDBPrefix("dbprefix_")) + require.NoError(t, err) + + commontest.TestProviderOpenStoreSetGetConfig(t, provider) + commontest.TestPutGet(t, provider) + commontest.TestStoreGetTags(t, provider) + commontest.TestStoreQuery(t, provider) + commontest.TestStoreDelete(t, provider) + commontest.TestStoreBatch(t, provider) + commontest.TestStoreClose(t, provider) + commontest.TestProviderClose(t, provider) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestEnsureTagMapIsOnlyCreatedWhenNeeded(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + // We defer creating the tag map entry until we actually have to. This saves on space if a client does not need + // to use tags + querying. The only thing that should cause the tag map entry to be created is if a Put is done + // with tags. + + testStore, err := provider.OpenStore("TestStore") + require.NoError(t, err) + + storeConfig := storage.StoreConfiguration{TagNames: []string{"TagName1"}} + err = provider.SetStoreConfig("TestStore", storeConfig) + require.NoError(t, err) + + value, err := testStore.Get("TagMap") + require.True(t, errors.Is(err, storage.ErrDataNotFound), "unexpected error or no error") + require.Nil(t, value) + + err = testStore.Put("Key", []byte("value")) + require.NoError(t, err) + + value, err = testStore.Get("TagMap") + require.True(t, errors.Is(err, storage.ErrDataNotFound)) + require.Nil(t, value) + + err = testStore.Delete("Key") + require.NoError(t, err) + + value, err = testStore.Get("TagMap") + require.True(t, errors.Is(err, storage.ErrDataNotFound), "unexpected error or no error") + require.Nil(t, value) + + tag := []storage.Tag{{Name: "TagName1"}} + err = testStore.Put("Key", []byte("value"), tag...) + require.NoError(t, err) + + value, err = testStore.Get("TagMap") + require.NoError(t, err) + require.Equal(t, `{"TagName1":{"Key":{}}}`, string(value)) + + t.Cleanup(func() { + err = provider.Close() + require.NoError(t, err) + }) +} + +func TestStoreLargeData(t *testing.T) { + path := setupSQLiteDB(t) + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStore, err := provider.OpenStore(randomStoreName()) + require.NoError(t, err) + + // Store 1 MiB worth of data. + err = testStore.Put("key", make([]byte, 1000000)) + require.NoError(t, err) + + t.Cleanup(func() { + err = provider.Close() + require.NoError(t, err) + }) +} + +func randomStoreName() string { + return "store-" + uuid.New().String() +} diff --git a/component/storage/sqlite/testdata/nosqlite.txt b/component/storage/sqlite/testdata/nosqlite.txt new file mode 100644 index 00000000..25965255 --- /dev/null +++ b/component/storage/sqlite/testdata/nosqlite.txt @@ -0,0 +1 @@ +file not valid \ No newline at end of file