diff --git a/cmd/aries-agent-mobile/go.mod b/cmd/aries-agent-mobile/go.mod index 8892e7e5df..a59144d535 100644 --- a/cmd/aries-agent-mobile/go.mod +++ b/cmd/aries-agent-mobile/go.mod @@ -17,6 +17,8 @@ require ( nhooyr.io/websocket v1.8.3 ) +require github.com/kr/pretty v0.2.1 // indirect + require ( github.com/PaesslerAG/gval v1.1.0 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect @@ -24,7 +26,7 @@ require ( github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 // indirect github.com/btcsuite/btcd v0.22.0-beta // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect - github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -36,9 +38,10 @@ require ( github.com/kawamuray/jsonpath v0.0.0-20201211160320-7483bafabd7e // indirect github.com/kilic/bls12-381 v0.1.1-0.20210503002446-7b7597926c69 // indirect github.com/klauspost/compress v1.10.0 // indirect + github.com/mattn/go-sqlite3 v1.14.14 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/sha256-simd v0.1.1 // indirect - github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.0.4 // indirect github.com/multiformats/go-base36 v0.1.0 // indirect diff --git a/cmd/aries-agent-mobile/go.sum b/cmd/aries-agent-mobile/go.sum index aad427cd6c..915ef2e820 100644 --- a/cmd/aries-agent-mobile/go.sum +++ b/cmd/aries-agent-mobile/go.sum @@ -64,8 +64,9 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -209,13 +210,16 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.10.0 h1:92XGj1AcYzA6UrVdd4qIIBrT8OroryvRvdmg/IfmC7Y= github.com/klauspost/compress v1.10.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -227,8 +231,9 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= diff --git a/cmd/aries-agent-mobile/pkg/api/storage.go b/cmd/aries-agent-mobile/pkg/api/storage.go index 75272cd540..db9dc4db5d 100644 --- a/cmd/aries-agent-mobile/pkg/api/storage.go +++ b/cmd/aries-agent-mobile/pkg/api/storage.go @@ -112,6 +112,13 @@ type Iterator interface { // aries-framework-go/spi/storage/Tag. Tags() ([]byte, error) + // TotalItems returns a count of the number of entries (key + value + tags triplets) matched by the query + // that generated this Iterator. This count is not affected by the page settings used (i.e. the count is of all + // results as if you queried starting from the first page and with an unlimited page size). + // Depending on the storage implementation, you may need to ensure that the TagName used in the query is in the + // Store's StoreConfiguration before trying to call this method (or it may be optional, but recommended). + TotalItems() (int, error) + // Close closes this iterator object, freeing resources. Close() error } diff --git a/cmd/aries-agent-mobile/pkg/component/storage/sqlite/errors.go b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/errors.go new file mode 100644 index 0000000000..35383f2550 --- /dev/null +++ b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/errors.go @@ -0,0 +1,29 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sqlite + +import ( + "errors" +) + +const ( + // Error messages we return. + failureWhileOpeningSQLiteConnectionErrMsg = "failure while opening SQLite connection using path %s: %w" + failureWhileClosingSQLiteConnection = "failure while closing SQLite DB connection: %w" + failureWhilePingingSQLiteErrMsg = "failure while pinging SQLite at path %s : %w" + failureWhileCreatingTableErrMsg = "failure while creating table %s: %w" + failureWhileExecutingInsertStatementErrMsg = "failure while executing insert statement on table %s: %w" + failureWhileQueryingRowErrMsg = "failure while querying row: %w" + failureWhileExecutingBatchStatementErrMsg = "failure while executing batch upsert on table %s: %w" + // Error messages returned from MySQL that we directly check for. + valueNotFoundErrMsgFromSQlite = "no rows" +) + +var ( + errBlankDBPath = errors.New("DB Path for new SQLite DB provider can't be blank") + errBlankStoreName = errors.New("store name is required") +) diff --git a/cmd/aries-agent-mobile/pkg/component/storage/sqlite/store.go b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/store.go new file mode 100644 index 0000000000..5c277eccc8 --- /dev/null +++ b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/store.go @@ -0,0 +1,781 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Created using https://github.com/hyperledger/aries-framework-go-ext/tree/main/component/storage/mysql as reference.. + +package sqlite + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + + _ "github.com/mattn/go-sqlite3" //nolint:gci // required for SQLite + + "github.com/hyperledger/aries-framework-go/cmd/aries-agent-mobile/pkg/api" + "github.com/hyperledger/aries-framework-go/spi/storage" +) + +const ( + tagMapKey = "TagMap" + storeConfigKey = "StoreConfig" +) + +const ( + expressionTagNameOnlyLength = 1 + expressionTagNameAndValueLength = 2 +) + +const ( + invalidQueryExpressionFormat = `"%s" is not in a valid expression format. ` + + "it must be in the following format: TagName:TagValue" + invalidTagName = `"%s" is an invalid tag name since it contains one or more ':' characters` + invalidTagValue = `"%s" is an invalid tag value since it contains one or more ':' characters` +) + +// TODO: Fully implement all methods. + +// ErrKeyRequired is returned when key is mandatory. +var ErrKeyRequired = errors.New("key is mandatory") + +type closer func(storeName string) + +type tagMapping map[string]map[string]struct{} // map[TagName](Set of database Keys) + +type dbEntry struct { + Value []byte `json:"value,omitempty"` + Tags []storage.Tag `json:"tags,omitempty"` +} + +// Provider represents a SQLite DB implementation of the storage.Provider interface. +type Provider struct { + dbPath string + db *sql.DB + dbs map[string]*store + dbPrefix string + lock sync.RWMutex +} + +// Option configures the SQLite provider. +type Option func(opts *Provider) + +// WithDBPrefix option is for adding prefix to db name. +func WithDBPrefix(dbPrefix string) Option { + return func(opts *Provider) { + opts.dbPrefix = dbPrefix + } +} + +// GetProvider is used to be able to get the SQLite provider and expose it in Android. +func GetProvider(dbPath string) api.Provider { + provider, err := NewProvider(dbPath) + if err != nil { + panic(fmt.Errorf(failureWhileOpeningSQLiteConnectionErrMsg, dbPath, err)) + } + + return provider +} + +// NewProvider instantiates Provider. +// Example DB Path ./test.db +// This provider's CreateStore(name) implementation creates stores that are backed by a table without a schema. +// The fully qualified name of the table is thus `name`. +// Use of `Batch()` has additional considerations - please read the docs for Batch() accordingly. +func NewProvider(dbPath string, opts ...Option) (*Provider, error) { + if dbPath == "" { + return nil, errBlankDBPath + } + + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return nil, fmt.Errorf(failureWhileOpeningSQLiteConnectionErrMsg, dbPath, err) + } + + err = db.Ping() + if err != nil { + return nil, fmt.Errorf(failureWhilePingingSQLiteErrMsg, dbPath, err) + } + + p := &Provider{ + dbPath: dbPath, + db: db, + dbs: map[string]*store{}, + } + + for _, opt := range opts { + opt(p) + } + + return p, nil +} + +// OpenStore opens a store with the given name and returns a handle. +// If the store has never been opened before, then it is created. +// Store names are not case-sensitive. If name is blank, then an error will be returned. +// SQLite doesn't support the "-" character in the table names so they are replaced with "_" +// WARNING: This method will create a database and table based on the given name. Those database calls may be +// vulnerable to an SQL injection attack. Be very careful if you use a user-provided string in the store name! +func (p *Provider) OpenStore(name string) (api.Store, error) { + if name == "" { + return nil, errBlankStoreName + } + + name = strings.ReplaceAll(strings.ToLower(name), "-", "_") + + if p.dbPrefix != "" { + name = p.dbPrefix + "_" + name + } + + p.lock.Lock() + defer p.lock.Unlock() + + // Check cache first + cachedStore, existsInCache := p.dbs[name] + if existsInCache { + return cachedStore, nil + } + + createTableStmt := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s "+ + "('key' VARCHAR(255) NOT NULL PRIMARY KEY, 'value' MEDIUMBLOB)", name) + + // creating key-value table inside the database + _, err := p.db.Exec(createTableStmt) + if err != nil { + return nil, fmt.Errorf(failureWhileCreatingTableErrMsg, name, err) + } + + // Opening new DB connection + storeDB, err := sql.Open("sqlite3", p.dbPath) + if err != nil { + return nil, fmt.Errorf(failureWhileOpeningSQLiteConnectionErrMsg, p.dbPath, err) + } + + store := &store{ + db: storeDB, + name: name, + tableName: name, + close: p.removeStore, + } + + p.dbs[name] = store + + return store, nil +} + +// SetStoreConfig sets the configuration on a store. This must be done before storing any data in order to make use +// of the Query method. +// TODO: Use proper SQlite indexing instead of the "Tag Map". +func (p *Provider) SetStoreConfig(name string, storeConfigBytes []byte) error { + config := storage.StoreConfiguration{} + + err := json.Unmarshal(storeConfigBytes, &config) + if err != nil { + return fmt.Errorf("failed to unmarshal store configuration: %w", err) + } + + for _, tagName := range config.TagNames { + if strings.Contains(tagName, ":") { + return fmt.Errorf(invalidTagName, tagName) + } + } + + name = strings.ReplaceAll(strings.ToLower(name), "-", "_") + + if p.dbPrefix != "" { + name = p.dbPrefix + "_" + name + } + + openStore, ok := p.dbs[name] + if !ok { + return storage.ErrStoreNotFound + } + + configBytes, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal store configuration: %w", err) + } + + err = openStore.Put(storeConfigKey, configBytes, []byte{}) + if err != nil { + return fmt.Errorf("failed to put store store configuration: %w", err) + } + + return nil +} + +// GetStoreConfig returns the store's configuration. +// TODO: Check for underlying database's existence instead of looking at in-memory stores. +func (p *Provider) GetStoreConfig(name string) ([]byte, error) { + name = strings.ReplaceAll(strings.ToLower(name), "-", "_") + + if p.dbPrefix != "" { + name = p.dbPrefix + "_" + name + } + + openStore, ok := p.dbs[name] + if !ok { + return []byte{}, storage.ErrStoreNotFound + } + + storeConfigBytes, err := openStore.Get(storeConfigKey) + if err != nil { + return []byte{}, + fmt.Errorf(`failed to get store configuration for "%s": %w`, name, err) + } + + return storeConfigBytes, nil +} + +// GetOpenStores is currently not implemented. +func (p *Provider) GetOpenStores() []storage.Store { + panic("not implemented") +} + +// Close closes all stores created under this store provider. +func (p *Provider) Close() error { + p.lock.RLock() + + openStoresSnapshot := make([]*store, len(p.dbs)) + + var counter int + + for _, openStore := range p.dbs { + openStoresSnapshot[counter] = openStore + counter++ + } + p.lock.RUnlock() + + for _, openStore := range openStoresSnapshot { + err := openStore.Close() + if err != nil { + return fmt.Errorf(`failed to close open store with name "%s": %w`, openStore.name, err) + } + } + + return p.db.Close() +} + +func (p *Provider) removeStore(name string) { + p.lock.Lock() + defer p.lock.Unlock() + + _, ok := p.dbs[name] + if ok { + delete(p.dbs, name) + } +} + +type store struct { + db *sql.DB + name string + tableName string + close closer + lock sync.RWMutex +} + +func (s *store) Put(key string, value, tagsBytes []byte) error { + tags := []storage.Tag{} + + if len(tagsBytes) > 0 { + err := json.Unmarshal(tagsBytes, &tags) + if err != nil { + return fmt.Errorf("failed to unmarshal tags: %w", err) + } + } + + errInputValidation := validatePutInput(key, value, tags) + if errInputValidation != nil { + return errInputValidation + } + + var newDBEntry dbEntry + newDBEntry.Value = value + + if len(tags) > 0 { + newDBEntry.Tags = tags + + err := s.updateTagMap(key, tags) + if err != nil { + return fmt.Errorf("failed to update tag map: %w", err) + } + } + + entryBytes, err := json.Marshal(newDBEntry) + if err != nil { + return fmt.Errorf("failed to marshal new DB entry: %w", err) + } + + // create upsert query to insert the record, checking whether the key is already mapped to a value in the store. + insertStmt := "INSERT OR REPLACE INTO " + s.tableName + " VALUES (?, ?)" + // executing the prepared insert statement + _, err = s.db.Exec(insertStmt, key, entryBytes, entryBytes) + if err != nil { + return fmt.Errorf(failureWhileExecutingInsertStatementErrMsg, s.tableName, err) + } + + return nil +} + +func (s *store) Get(k string) ([]byte, error) { + retrievedDBEntry, err := s.getDBEntry(k) + if err != nil { + return nil, fmt.Errorf("failed to get DB entry: %w", err) + } + + return retrievedDBEntry.Value, nil +} + +func (s *store) GetTags(key string) ([]byte, error) { + retrievedDBEntry, err := s.getDBEntry(key) + if err != nil { + return nil, fmt.Errorf("failed to get DB entry: %w", err) + } + + tagsBytes, err := json.Marshal(retrievedDBEntry.Tags) + if err != nil { + return []byte{}, fmt.Errorf("failed to marshal tags: %w", err) + } + + return tagsBytes, nil +} + +func (s *store) GetBulk(keys []byte) ([]byte, error) { + return nil, errors.New("not implemented") +} + +// Mobile providers doesn't currently support any of the current query options. +// pageSize will simply be ignored since it only relates to performance and not the actual end result. +func (s *store) Query(expression string, pageSize int) (api.Iterator, error) { + if expression == "" { + return nil, fmt.Errorf(invalidQueryExpressionFormat, expression) + } + + expressionSplit := strings.Split(expression, ":") + + var expressionTagName string + + var expressionTagValue string + + switch len(expressionSplit) { + case expressionTagNameOnlyLength: + expressionTagName = expressionSplit[0] + case expressionTagNameAndValueLength: + expressionTagName = expressionSplit[0] + expressionTagValue = expressionSplit[1] + default: + return nil, fmt.Errorf(invalidQueryExpressionFormat, expression) + } + + matchingDatabaseKeys, err := s.getDatabaseKeysMatchingQuery(expressionTagName, expressionTagValue) + if err != nil { + return nil, fmt.Errorf("failed to get database keys matching query: %w", err) + } + + return &iterator{keys: matchingDatabaseKeys, store: s}, nil +} + +// Delete will delete record with k key. +func (s *store) Delete(k string) error { + if k == "" { + return ErrKeyRequired + } + + // delete query to delete the record by key + _, err := s.db.Exec("DELETE FROM "+s.tableName+" WHERE `key`= ?", k) + if err != nil { + return fmt.Errorf(storage.ErrDataNotFound.Error(), err) + } + + err = s.removeFromTagMap(k) + if err != nil { + return fmt.Errorf("failed to remove key from tag map: %w", err) + } + + return nil +} + +// Batch performs batch upserts and deletions preserving the batch's ordering. +// Batch is a no-op if the batch is empty. +// Batch needs both `interpolateParams` and `multiStatements` enabled in the dataSourceName +// - see the following guide: https://github.com/go-sql-driver/mysql#parameters. +// All operations in the batch are executed in a single multi-statement query due to the ordering +// requirements. This means we cannot optimize INSERT statements as per +// https://dev.mysql.com/doc/refman/8.0/en/insert-optimization.html. +// Executing a single multi-statement query requires O(N) space for the query string and an additional +// slice with O(2*N) space holding the values for the query. Callers should take care of this additional +// memory usage by limiting the size of the batch. +func (s *store) Batch(operations []byte) error { // nolint:gocyclo + var batch []storage.Operation + + err := json.Unmarshal(operations, &batch) + if err != nil { + return fmt.Errorf("failed to unmarshal batch operations: %w", err) + } + + if len(batch) == 0 { + return errors.New("batch requires at least one operation") + } + + var ( + query string + values []interface{} + ) + + for i := range batch { + b := batch[i] + + if b.Key == "" { + return errors.New("key cannot be empty") + } + + err = s.bulkAppendToQuery(b, &query, &values) + if err != nil { + return fmt.Errorf(failureWhileExecutingBatchStatementErrMsg, s.tableName, err) + } + + if len(b.Value) > 0 { + err = s.updateTagMap(b.Key, b.Tags) + if err != nil && !errors.Is(err, storage.ErrDataNotFound) { + return fmt.Errorf("failed to update tag map: %w", err) + } + } else { + err = s.removeFromTagMap(b.Key) + if err != nil && !errors.Is(err, storage.ErrDataNotFound) { + return fmt.Errorf("failed to remove key from tag map: %w", err) + } + } + } + + _, err = s.db.Exec(query, values...) + if err != nil { + return fmt.Errorf(failureWhileExecutingBatchStatementErrMsg, s.tableName, err) + } + + return nil +} + +func (s *store) bulkAppendToQuery(b storage.Operation, query *string, values *[]interface{}) error { + if len(b.Value) > 0 { + value, err := json.Marshal(dbEntry{ + Value: b.Value, + Tags: b.Tags, + }) + if err != nil { + return fmt.Errorf("failed to marshal dbEntry: %w", err) + } + + *query += fmt.Sprintf( + "INSERT OR REPLACE INTO %s (`key`, `value`) VALUES (?, ?);\n", + s.tableName, + ) + + *values = append(*values, b.Key, value) + } else { + *query += fmt.Sprintf("DELETE FROM %s WHERE `KEY` = ?;\n", s.tableName) + *values = append(*values, b.Key) + } + + return nil +} + +// SQL store doesn't queue values, so there's never anything to flush. +func (s *store) Flush() error { + return nil +} + +func (s *store) Close() error { + s.close(s.name) + + err := s.db.Close() + if err != nil { + return fmt.Errorf(failureWhileClosingSQLiteConnection, err) + } + + return nil +} + +func (s *store) updateTagMap(key string, tags []storage.Tag) error { + s.lock.Lock() + defer s.lock.Unlock() + + tagMap, err := s.getTagMap(true) + if err != nil { + return fmt.Errorf("failed to get tag map: %w", err) + } + + for _, tag := range tags { + if tagMap[tag.Name] == nil { + tagMap[tag.Name] = make(map[string]struct{}) + } + + tagMap[tag.Name][key] = struct{}{} + } + + tagMapBytes, err := json.Marshal(tagMap) + if err != nil { + return fmt.Errorf("failed to marshal updated tag map: %w", err) + } + + err = s.Put(tagMapKey, tagMapBytes, []byte{}) + if err != nil { + return fmt.Errorf("failed to put updated tag map back into the store: %w", err) + } + + return nil +} + +func (s *store) getTagMap(createIfDoesNotExist bool) (tagMapping, error) { + tagMapBytes, err := s.Get(tagMapKey) + if err != nil { + if createIfDoesNotExist && errors.Is(err, storage.ErrDataNotFound) { + // Create the tag map if it has never been created before. + err = s.Put(tagMapKey, []byte("{}"), []byte{}) + if err != nil { + return nil, fmt.Errorf(`failed to create tag map for "%s": %w`, s.name, err) + } + + tagMapBytes = []byte("{}") + } else { + return nil, fmt.Errorf("failed to get data: %w", err) + } + } + + var tagMap tagMapping + + err = json.Unmarshal(tagMapBytes, &tagMap) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal tag map bytes: %w", err) + } + + return tagMap, nil +} + +func (s *store) getDBEntry(key string) (dbEntry, error) { + if key == "" { + return dbEntry{}, ErrKeyRequired + } + + var retrievedDBEntryBytes []byte + + // select query to fetch the record by key + err := s.db.QueryRow("SELECT `value` FROM "+s.tableName+" "+ + " WHERE `key` = ?", key).Scan(&retrievedDBEntryBytes) + if err != nil { + if strings.Contains(err.Error(), valueNotFoundErrMsgFromSQlite) { + return dbEntry{}, storage.ErrDataNotFound + } + + return dbEntry{}, fmt.Errorf(failureWhileQueryingRowErrMsg, err) + } + + var retrievedDBEntry dbEntry + + err = json.Unmarshal(retrievedDBEntryBytes, &retrievedDBEntry) + if err != nil { + return dbEntry{}, fmt.Errorf("failed to unmarshaled retrieved DB entry: %w", err) + } + + return retrievedDBEntry, nil +} + +func (s *store) removeFromTagMap(keyToRemove string) error { + s.lock.Lock() + defer s.lock.Unlock() + + tagMap, err := s.getTagMap(false) + if err != nil { + // If there's no tag map, then this means that tags have never been used. This means that there's no tag map + // to update. This isn't a problem. + if errors.Is(err, storage.ErrDataNotFound) { + return nil + } + + return fmt.Errorf("failed to get tag map: %w", err) + } + + for _, tagNameToKeys := range tagMap { + delete(tagNameToKeys, keyToRemove) + } + + tagMapBytes, err := json.Marshal(tagMap) + if err != nil { + return fmt.Errorf("failed to marshal updated tag map: %w", err) + } + + err = s.Put(tagMapKey, tagMapBytes, []byte{}) + if err != nil { + return fmt.Errorf("failed to put updated tag map back into the store: %w", err) + } + + return nil +} + +func (s *store) getDatabaseKeysMatchingQuery(expressionTagName, expressionTagValue string) ([]string, error) { + tagMap, err := s.getTagMap(false) + if err != nil { + // If there's no tag map, then this means that tags have never been used, and therefore no matching results. + if errors.Is(err, storage.ErrDataNotFound) { + return nil, nil + } + + return nil, fmt.Errorf("failed to get tag map: %w", err) + } + + if expressionTagValue == "" { + return getDatabaseKeysMatchingTagName(tagMap, expressionTagName), nil + } + + matchingDatabaseKeys, err := s.getDatabaseKeysMatchingTagNameAndValue(tagMap, expressionTagName, expressionTagValue) + if err != nil { + return nil, fmt.Errorf("failed to get database keys matching tag name and value: %w", err) + } + + return matchingDatabaseKeys, nil +} + +func (s *store) getDatabaseKeysMatchingTagNameAndValue(tagMap tagMapping, expressionTagName, expressionTagValue string, +) ([]string, error) { + var matchingDatabaseKeys []string + + for tagName, databaseKeysSet := range tagMap { + if tagName == expressionTagName { + tags, err := s.getTagsMatchingTagNameAndValue(databaseKeysSet, expressionTagName, expressionTagValue) + if err != nil { + return nil, fmt.Errorf("failed to get tags: %w", err) + } + + matchingDatabaseKeys = append(matchingDatabaseKeys, tags...) + + break + } + } + + return matchingDatabaseKeys, nil +} + +func (s *store) getTagsMatchingTagNameAndValue(databaseKeysSet map[string]struct{}, expressionTagName, + expressionTagValue string, +) ([]string, error) { + var matchingDatabaseKeys []string + + for databaseKey := range databaseKeysSet { + tagsBytes, err := s.GetTags(databaseKey) + if err != nil { + return nil, fmt.Errorf("failed to get tags: %w", err) + } + + tags := []storage.Tag{} + + if len(tagsBytes) > 0 { + err = json.Unmarshal(tagsBytes, &tags) + if err != nil { + return []string{}, fmt.Errorf("failed to unmarshal saved tags: %w", err) + } + } + + for _, tag := range tags { + if tag.Name == expressionTagName && tag.Value == expressionTagValue { + matchingDatabaseKeys = append(matchingDatabaseKeys, databaseKey) + + break + } + } + } + + return matchingDatabaseKeys, nil +} + +type iterator struct { + keys []string + currentIndex int + currentKey string + store *store +} + +func (i *iterator) Next() (bool, error) { + if len(i.keys) == i.currentIndex || len(i.keys) == 0 { + if len(i.keys) == i.currentIndex || len(i.keys) == 0 { + return false, nil + } + } + + i.currentKey = i.keys[i.currentIndex] + + i.currentIndex++ + + return true, nil +} + +func (i *iterator) Key() (string, error) { + return i.currentKey, nil +} + +func (i *iterator) Value() ([]byte, error) { + value, err := i.store.Get(i.currentKey) + if err != nil { + return nil, fmt.Errorf("failed to get value from store: %w", err) + } + + return value, nil +} + +func (i *iterator) Tags() ([]byte, error) { + tags, err := i.store.GetTags(i.currentKey) + if err != nil { + return nil, fmt.Errorf("failed to get tags from store: %w", err) + } + + return tags, nil +} + +func (i *iterator) TotalItems() (int, error) { + return len(i.keys), nil +} + +func (i *iterator) Close() error { + return nil +} + +func validatePutInput(key string, value []byte, tags []storage.Tag) error { + if key == "" { + return errors.New("key cannot be empty") + } + + if value == nil { + return errors.New("value cannot be nil") + } + + for _, tag := range tags { + if strings.Contains(tag.Name, ":") { + return fmt.Errorf(invalidTagName, tag.Name) + } + + if strings.Contains(tag.Value, ":") { + return fmt.Errorf(invalidTagValue, tag.Value) + } + } + + return nil +} + +func getDatabaseKeysMatchingTagName(tagMap tagMapping, expressionTagName string) []string { + var matchingDatabaseKeys []string + + for tagName, databaseKeysSet := range tagMap { + if tagName == expressionTagName { + for databaseKey := range databaseKeysSet { + matchingDatabaseKeys = append(matchingDatabaseKeys, databaseKey) + } + + break + } + } + + return matchingDatabaseKeys +} diff --git a/cmd/aries-agent-mobile/pkg/component/storage/sqlite/store_test.go b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/store_test.go new file mode 100644 index 0000000000..b640d9a413 --- /dev/null +++ b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/store_test.go @@ -0,0 +1,400 @@ +/* +Copyright SecureKey Technologies Inc. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sqlite_test + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/hyperledger/aries-framework-go/cmd/aries-agent-mobile/pkg/component/storage/sqlite" + "github.com/hyperledger/aries-framework-go/cmd/aries-agent-mobile/pkg/wrappers/storage" + basestorage "github.com/hyperledger/aries-framework-go/spi/storage" + commontest "github.com/hyperledger/aries-framework-go/test/component/storage" +) + +func setupSQLiteDB(t testing.TB) string { + file, err := ioutil.TempFile("./testdata", "test-*.db") + if err != nil { + t.Fatalf("Failed to create sqlite file: %s", err) + } + + dbFolderPath, err := filepath.Abs("./testdata") + if err != nil { + t.Fatalf("Failed to get absolute path of sqlite directory: %s", err) + } + + dbPath := filepath.Join(dbFolderPath, filepath.Base(file.Name())) + + t.Cleanup(func() { + err := file.Close() + if err != nil { + t.Fatalf("Failed to close sqlite file: %s", err) + } + err = os.Remove(dbPath) + if err != nil { + t.Fatalf("Failed to clear sqlite file: %s", err) + } + }) + + return dbPath +} + +func TestSQLDBStore(t *testing.T) { + t.Run("Test SQL open store", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + _, err = provider.OpenStore("") + require.Error(t, err) + require.Equal(t, err.Error(), "store name is required") + + err = provider.Close() + require.NoError(t, err) + }) + t.Run("Test sql db store failures", func(t *testing.T) { + provider, err := sqlite.NewProvider("") + require.Error(t, err) + require.Contains(t, err.Error(), "DB Path for new SQLite DB provider can't be blank") + require.Nil(t, provider) + + _, err = sqlite.NewProvider("./testdata/nosqlite.txt") + require.Error(t, err) + require.Contains(t, err.Error(), "failure while pinging SQLite") + }) + t.Run("Test sqlDB multi store close by name", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path, sqlite.WithDBPrefix("prefixdb")) + require.NoError(t, err) + + const commonKey = "did:example:1" + data := []byte("value1") + + storeNames := []string{"store_1", "store_2", "store_3", "store_4", "store_5"} + storesToClose := []string{"store_1", "store_3", "store_5"} + + for _, name := range storeNames { + store, e := provider.OpenStore(name) + require.NoError(t, e) + + e = store.Put(commonKey, data, []byte{}) + require.NoError(t, e) + } + + for _, name := range storeNames { + store, e := provider.OpenStore(name) + require.NoError(t, e) + require.NotNil(t, store) + + dataRead, e := store.Get(commonKey) + require.NoError(t, e) + require.Equal(t, data, dataRead) + } + + for _, name := range storesToClose { + store, e := provider.OpenStore(name) + require.NoError(t, e) + require.NotNil(t, store) + + err = store.Close() + require.NoError(t, err) + } + + err = provider.Close() + require.NoError(t, err) + + // try close all again + err = provider.Close() + require.NoError(t, err) + }) + t.Run("Flush", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + store, err := provider.OpenStore("storename") + require.NoError(t, err) + + err = store.Flush() + require.NoError(t, err) + + err = provider.Close() + require.NoError(t, err) + }) +} + +func TestProvider_GetStoreConfig(t *testing.T) { + t.Run("Fail to get store configuration", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + storeName := randomStoreName() + + _, err = provider.OpenStore(storeName) + require.NoError(t, err) + + config, err := provider.GetStoreConfig(storeName) + require.EqualError(t, err, + fmt.Sprintf(`failed to get store configuration for "%s": `+ + `failed to get DB entry: data not found`, strings.ReplaceAll(storeName, "-", "_"))) + require.Empty(t, config) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestStore_Put(t *testing.T) { + t.Run("Fail to update tag map since the DB connection was closed", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + storeName := randomStoreName() + testStore, err := provider.OpenStore(storeName) + require.NoError(t, err) + + err = testStore.Close() + require.NoError(t, err) + + err = testStore.Put("key", []byte("value"), []byte{}) + require.EqualError(t, err, fmt.Sprintf("failure while executing insert statement on table %s: "+ + "sql: database is closed", strings.ReplaceAll(storeName, "-", "_"))) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) + t.Run("Fail to unmarshal tag map bytes", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStore, err := provider.OpenStore(randomStoreName()) + require.NoError(t, err) + + err = testStore.Put("TagMap", []byte("Not a proper tag map"), []byte{}) + require.NoError(t, err) + + tag := basestorage.Tag{} + tagBytes, err := json.Marshal(tag) + require.NoError(t, err) + + err = testStore.Put("key", []byte("value"), tagBytes) + require.EqualError(t, err, "failed to unmarshal tags: json: "+ + "cannot unmarshal object into Go value of type []storage.Tag") + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestSqlDBStore_Query(t *testing.T) { + t.Run("Fail to get tag map since the DB connection was closed", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStore, err := provider.OpenStore(randomStoreName()) + require.NoError(t, err) + + err = testStore.Close() + require.NoError(t, err) + + itr, err := testStore.Query("expression", 0) + require.EqualError(t, err, "failed to get database keys matching query: failed to get tag map: "+ + "failed to get data: failed to get DB entry: failure while querying row: sql: database is closed") + require.Nil(t, itr) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestSqlDBIterator(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStoreName := randomStoreName() + + testStore, err := provider.OpenStore(testStoreName) + require.NoError(t, err) + + storeConfig := basestorage.StoreConfiguration{TagNames: []string{}} + storeConfigBytes, err := json.Marshal(storeConfig) + require.NoError(t, err) + + err = provider.SetStoreConfig(testStoreName, storeConfigBytes) + require.NoError(t, err) + + itr, err := testStore.Query("expression", 0) + require.NoError(t, err) + + t.Run("Fail to get value from store", func(t *testing.T) { + value, errValue := itr.Value() + require.EqualError(t, errValue, "failed to get value from store: failed to get DB entry: key is mandatory") + require.Nil(t, value) + }) + t.Run("Fail to get tags from store", func(t *testing.T) { + tags, errGetTags := itr.Tags() + require.EqualError(t, errGetTags, "failed to get tags from store: failed to get DB entry: key is mandatory") + require.Nil(t, tags) + }) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) +} + +func TestSqlDBStore_Common(t *testing.T) { + t.Run("Without prefix", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + wrapperProvider := storage.New(provider) + commontest.TestProviderOpenStoreSetGetConfig(t, wrapperProvider) + commontest.TestPutGet(t, wrapperProvider) + commontest.TestStoreGetTags(t, wrapperProvider) + commontest.TestStoreQuery(t, wrapperProvider) + commontest.TestStoreDelete(t, wrapperProvider) + commontest.TestStoreClose(t, wrapperProvider) + commontest.TestProviderClose(t, wrapperProvider) + commontest.TestStoreBatch(t, wrapperProvider) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) + t.Run("With prefix", func(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path, sqlite.WithDBPrefix("dbprefix_")) + require.NoError(t, err) + + wrapperProvider := storage.New(provider) + commontest.TestProviderOpenStoreSetGetConfig(t, wrapperProvider) + commontest.TestPutGet(t, wrapperProvider) + commontest.TestStoreGetTags(t, wrapperProvider) + commontest.TestStoreQuery(t, wrapperProvider) + commontest.TestStoreDelete(t, wrapperProvider) + commontest.TestStoreClose(t, wrapperProvider) + commontest.TestProviderClose(t, wrapperProvider) + commontest.TestStoreBatch(t, wrapperProvider) + + t.Cleanup(func() { + err := provider.Close() + require.NoError(t, err) + }) + }) +} + +func TestEnsureTagMapIsOnlyCreatedWhenNeeded(t *testing.T) { + path := setupSQLiteDB(t) + + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + // We defer creating the tag map entry until we actually have to. This saves on space if a client does not need + // to use tags + querying. The only thing that should cause the tag map entry to be created is if a Put is done + // with tags. + + testStore, err := provider.OpenStore("TestStore") + require.NoError(t, err) + + storeConfig := basestorage.StoreConfiguration{TagNames: []string{"TagName1"}} + storeConfigBytes, err := json.Marshal(storeConfig) + require.NoError(t, err) + + err = provider.SetStoreConfig("TestStore", storeConfigBytes) + require.NoError(t, err) + + value, err := testStore.Get("TagMap") + require.True(t, errors.Is(err, basestorage.ErrDataNotFound), "unexpected error or no error") + require.Nil(t, value) + + err = testStore.Put("Key", []byte("value"), []byte{}) + require.NoError(t, err) + + value, err = testStore.Get("TagMap") + require.True(t, errors.Is(err, basestorage.ErrDataNotFound)) + require.Nil(t, value) + + err = testStore.Delete("Key") + require.NoError(t, err) + + value, err = testStore.Get("TagMap") + require.True(t, errors.Is(err, basestorage.ErrDataNotFound), "unexpected error or no error") + require.Nil(t, value) + + tag := []basestorage.Tag{{Name: "TagName1"}} + tagBytes, err := json.Marshal(tag) + require.NoError(t, err) + + err = testStore.Put("Key", []byte("value"), tagBytes) + require.NoError(t, err) + + value, err = testStore.Get("TagMap") + require.NoError(t, err) + require.Equal(t, `{"TagName1":{"Key":{}}}`, string(value)) + + t.Cleanup(func() { + err = provider.Close() + require.NoError(t, err) + }) +} + +func TestStoreLargeData(t *testing.T) { + path := setupSQLiteDB(t) + provider, err := sqlite.NewProvider(path) + require.NoError(t, err) + + testStore, err := provider.OpenStore(randomStoreName()) + require.NoError(t, err) + + // Store 1 MiB worth of data. + err = testStore.Put("key", make([]byte, 1000000), []byte{}) + require.NoError(t, err) + + t.Cleanup(func() { + err = provider.Close() + require.NoError(t, err) + }) +} + +func randomStoreName() string { + return "store-" + uuid.New().String() +} diff --git a/cmd/aries-agent-mobile/pkg/component/storage/sqlite/testdata/nosqlite.txt b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/testdata/nosqlite.txt new file mode 100644 index 0000000000..25965255bf --- /dev/null +++ b/cmd/aries-agent-mobile/pkg/component/storage/sqlite/testdata/nosqlite.txt @@ -0,0 +1 @@ +file not valid \ No newline at end of file diff --git a/cmd/aries-agent-mobile/pkg/wrappers/storage/storage.go b/cmd/aries-agent-mobile/pkg/wrappers/storage/storage.go index 9690a58dbc..72c2c806d9 100644 --- a/cmd/aries-agent-mobile/pkg/wrappers/storage/storage.go +++ b/cmd/aries-agent-mobile/pkg/wrappers/storage/storage.go @@ -9,7 +9,6 @@ package storage import ( "encoding/json" - "errors" "fmt" "strings" "sync" @@ -365,7 +364,7 @@ func (i *iterator) Tags() ([]spi.Tag, error) { } func (i *iterator) TotalItems() (int, error) { - return -1, errors.New("not implemented") + return i.mobileBindingIterator.TotalItems() } func (i *iterator) Close() error {