diff --git a/build/docker-compose.yml b/build/docker-compose.yml index f06683a05..477ce1a14 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -1,4 +1,5 @@ version: "3.98" + services: web: build: @@ -10,8 +11,11 @@ services: - JAEGER_HTTP_URL=http://jaeger:14268/api/traces depends_on: - jaeger + - redis volumes: - ../config/compose.toml:/app/config/config.toml + networks: + - ssi_network swagger-ui: build: context: ../ @@ -34,4 +38,15 @@ services: ports: - "6831:6831/udp" - "16686:16686" - - "14268:14268" \ No newline at end of file + - "14268:14268" + redis: + image: redis:alpine + environment: + - ALLOW_EMPTY_PASSWORD=yes + # This allows for data to not be persisted on new runs + command: [sh, -c, "rm -f /data/dump.rdb && redis-server --save ''"] + networks: + - ssi_network + +networks: + ssi_network: \ No newline at end of file diff --git a/config/compose.toml b/config/compose.toml index 3899b3808..d1e5cc470 100644 --- a/config/compose.toml +++ b/config/compose.toml @@ -21,9 +21,10 @@ log_level = "info" enable_schema_caching = true +# Bolt Configuration [services] -storage = "bolt" service_endpoint = "http://localhost:8080" +storage = "bolt" # per-service configuration [services.keystore] diff --git a/config/config.toml b/config/config.toml index c1cc07130..4715f93d5 100644 --- a/config/config.toml +++ b/config/config.toml @@ -19,9 +19,10 @@ log_level = "debug" enable_schema_caching = true +# Bolt Configuration [services] -storage = "bolt" service_endpoint = "http://localhost:8080" +storage = "bolt" # per-service configuration [services.keystore] diff --git a/config/config.toml.example b/config/config.toml.example new file mode 100644 index 000000000..d9ddc5a24 --- /dev/null +++ b/config/config.toml.example @@ -0,0 +1,60 @@ +title = "SSI Service Config" + +svn = "0.0.1" +desc = "Default configuration to be used while running the service as a single go process." + +# http service configuration +[server] +api_host = "0.0.0.0:3000" +debug_host = "0.0.0.0:4000" + +# 5 seconds, time is in nanoseconds +read_timeout = 5000000000 +write_timeout = 5000000000 +shutdown_timeout = 5000000000 + +log_location = "logs" +# options: trace, debug, info, warning, error, fatal, panic +log_level = "debug" + +enable_schema_caching = true + +# Uncomment one of the following database configurations +# Bolt Configuration +# [services] +# service_endpoint = "http://localhost:8080" +# storage = "bolt" + +# Redis Configuration +# [services] +# service_endpoint = "http://localhost:8080" +# storage = "redis" + +# [services.storage_option] +# address = "redis:6379" +# password = "" + +# per-service configuration +[services.keystore] +name = "keystore" +password = "default-password" + +[services.did] +name = "did" +methods = ["key", "web"] +resolution_methods = ["key", "web", "pkh", "peer"] + +[services.schema] +name = "schema" + +[services.issuing] +name = "issuing" + +[services.credential] +name = "credential" + +[services.manifest] +name = "manifest" + +[services.presentation] +name = "presentation" diff --git a/go.mod b/go.mod index 99d75be3b..c0d650092 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,12 @@ go 1.19 require ( github.com/BurntSushi/toml v1.2.1 github.com/TBD54566975/ssi-sdk v0.0.2-alpha.0.20221214220628-e2badfb960fa + github.com/alicebob/miniredis/v2 v2.23.1 github.com/ardanlabs/conf v1.5.0 github.com/dimfeld/httptreemux/v5 v5.5.0 github.com/go-playground/locales v0.14.0 github.com/go-playground/universal-translator v0.18.0 + github.com/go-redis/redis/v8 v8.11.5 github.com/goccy/go-json v0.10.0 github.com/google/cel-go v0.13.0 github.com/google/go-cmp v0.5.9 @@ -33,10 +35,13 @@ require ( ) require ( + github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/bits-and-blooms/bitset v1.4.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/validator/v10 v10.11.1 // indirect @@ -49,12 +54,14 @@ require ( github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.1.0 // indirect github.com/multiformats/go-multicodec v0.7.0 // indirect + github.com/onsi/gomega v1.20.0 // indirect github.com/piprate/json-gold v0.5.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.1.1 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 // indirect golang.org/x/sys v0.3.0 // indirect golang.org/x/term v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect diff --git a/go.sum b/go.sum index 78f9f238c..a845be2e4 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,22 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/TBD54566975/ssi-sdk v0.0.2-alpha.0.20221209214116-fb01f46e7429 h1:YPGYl5SdnVF+b+7ccut1IuQ1xC4rEHuBqIXoy+379DE= -github.com/TBD54566975/ssi-sdk v0.0.2-alpha.0.20221209214116-fb01f46e7429/go.mod h1:h5Oa7RbqAzeGBlK3UR5qyCqU42+SMn0hb8hxjjAMkGs= github.com/TBD54566975/ssi-sdk v0.0.2-alpha.0.20221214220628-e2badfb960fa h1:d1O2F8c/AwCHmOOeQQFPySEz+Jou0EayyjM/66m6Vck= github.com/TBD54566975/ssi-sdk v0.0.2-alpha.0.20221214220628-e2badfb960fa/go.mod h1:GIfJZUiXKxnyFJs3H0SLoFL5tVezav6p5Zcs8TddKJE= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.23.1 h1:jR6wZggBxwWygeXcdNyguCOCIjPsZyNUNlAkTx2fu0U= +github.com/alicebob/miniredis/v2 v2.23.1/go.mod h1:84TWKZlxYkfgMucPBf5SOQBYJceZeQRFIaQgNMiCX6Q= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/ardanlabs/conf v1.5.0 h1:5TwP6Wu9Xi07eLFEpiCUF3oQXh9UzHMDVnD3u/I5d5c= github.com/ardanlabs/conf v1.5.0/go.mod h1:ILsMo9dMqYzCxDjDXTiwMI0IgxOJd0MOiucbQY2wlJw= github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8= github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -18,8 +25,11 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dimfeld/httptreemux/v5 v5.5.0 h1:p8jkiMrCuZ0CmhwYLcbNbl7DDo21fozhKHQ2PccwOFQ= github.com/dimfeld/httptreemux/v5 v5.5.0/go.mod h1:QeEylH57C0v3VO0tkKraVz9oD3Uu93CKPnTLbsidvSw= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -33,6 +43,8 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -46,6 +58,7 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -84,8 +97,12 @@ github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsI github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 h1:Yl0tPBa8QPjGmesFh1D0rDy+q1Twx6FyU7VWHi8wZbI= github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852/go.mod h1:eqOVx5Vwu4gd2mmMZvVZsgIqNSaW3xxRThUJ0k/TPk4= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/piprate/json-gold v0.5.0 h1:RmGh1PYboCFcchVFuh2pbSWAZy4XJaqTMU4KQYsApbM= github.com/piprate/json-gold v0.5.0/go.mod h1:WZ501QQMbZZ+3pXFPhQKzNwS1+jls0oqov3uQ2WasLs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -116,6 +133,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= +github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= go.einride.tech/aip v0.60.0 h1:h6bgabZ5BCfAptbGex8jbh3VvPBRLa6xq+pQ1CAjHYw= go.einride.tech/aip v0.60.0/go.mod h1:SdLbSbgSU60Xkb4TMkmsZEQPHeEWx0ikBoq5QnqZvdg= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= @@ -133,6 +152,8 @@ golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -165,6 +186,7 @@ gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8 gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/storage/bolt.go b/pkg/storage/bolt.go index 7014f9a92..a7549914b 100644 --- a/pkg/storage/bolt.go +++ b/pkg/storage/bolt.go @@ -14,7 +14,10 @@ import ( ) func init() { - _ = RegisterStorage(&BoltDB{}) + err := RegisterStorage(&BoltDB{}) + if err != nil { + panic(err) + } } const ( diff --git a/pkg/storage/bolt_test.go b/pkg/storage/bolt_test.go deleted file mode 100644 index 8c4e9d48a..000000000 --- a/pkg/storage/bolt_test.go +++ /dev/null @@ -1,339 +0,0 @@ -package storage - -import ( - "fmt" - "os" - "testing" - - "github.com/goccy/go-json" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBoltDB(t *testing.T) { - db := setupBoltDB(t) - - // create a name space and a message in it - namespace := "F1" - - team1 := "Red Bull" - players1 := []string{"Max Verstappen", "Sergio Pérez"} - p1Bytes, err := json.Marshal(players1) - assert.NoError(t, err) - - err = db.Write(namespace, team1, p1Bytes) - assert.NoError(t, err) - - // get it back - gotPlayers1, err := db.Read(namespace, team1) - assert.NoError(t, err) - - var players1Result []string - err = json.Unmarshal(gotPlayers1, &players1Result) - assert.NoError(t, err) - assert.EqualValues(t, players1, players1Result) - - // get a value from a namespace that doesn't exist - res, err := db.Read("bad", "worse") - assert.NoError(t, err) - assert.Empty(t, res) - - // get a value that doesn't exist in the namespace - noValue, err := db.Read(namespace, "Porsche") - assert.NoError(t, err) - assert.Empty(t, noValue) - - // create a second value in the namespace - team2 := "McLaren" - players2 := []string{"Lando Norris", "Daniel Ricciardo"} - p2Bytes, err := json.Marshal(players2) - assert.NoError(t, err) - - err = db.Write(namespace, team2, p2Bytes) - assert.NoError(t, err) - - // get all values from the namespace - gotAll, err := db.ReadAll(namespace) - assert.NoError(t, err) - assert.True(t, len(gotAll) == 2) - - _, gotRedBull := gotAll[team1] - assert.True(t, gotRedBull) - - _, gotMcLaren := gotAll[team2] - assert.True(t, gotMcLaren) - - // delete value in the namespace - err = db.Delete(namespace, team2) - assert.NoError(t, err) - - gotPlayers2, err := db.Read(namespace, team2) - assert.NoError(t, err) - assert.Empty(t, gotPlayers2) - - // delete value in a namespace that doesn't exist - err = db.Delete("bad", team2) - assert.Error(t, err) - assert.Contains(t, err.Error(), "namespace does not exist") - - // delete a namespace that doesn't exist - err = db.DeleteNamespace("bad") - assert.Contains(t, err.Error(), "could not delete namespace") - - // delete namespace - err = db.DeleteNamespace(namespace) - assert.NoError(t, err) - - res, err = db.Read(namespace, team1) - assert.NoError(t, err) - assert.Empty(t, res) -} - -func TestBoltDBPrefixAndKeys(t *testing.T) { - db := setupBoltDB(t) - - namespace := "blockchains" - - // set up prefix read test - - dummyData := []byte("dummy") - err := db.Write(namespace, "bitcoin-testnet", dummyData) - assert.NoError(t, err) - - err = db.Write(namespace, "bitcoin-mainnet", dummyData) - assert.NoError(t, err) - - err = db.Write(namespace, "tezos-testnet", dummyData) - assert.NoError(t, err) - - err = db.Write(namespace, "tezos-mainnet", dummyData) - assert.NoError(t, err) - - prefixValues, err := db.ReadPrefix(namespace, "bitcoin") - assert.NoError(t, err) - assert.Len(t, prefixValues, 2) - - keys := make([]string, 0, len(prefixValues)) - for k := range prefixValues { - keys = append(keys, k) - } - assert.Contains(t, keys, "bitcoin-testnet") - assert.Contains(t, keys, "bitcoin-mainnet") - - // read all keys - allKeys, err := db.ReadAllKeys(namespace) - - assert.NoError(t, err) - assert.NotEmpty(t, allKeys) - assert.Len(t, allKeys, 4) - assert.Contains(t, allKeys, "bitcoin-mainnet") - assert.Contains(t, allKeys, "tezos-mainnet") -} - -func setupBoltDB(t *testing.T) *BoltDB { - db, err := NewStorage(Bolt, "test.db") - assert.NoError(t, err) - assert.NotEmpty(t, db) - - t.Cleanup(func() { - _ = db.Close() - _ = os.Remove("test.db") - }) - return db.(*BoltDB) -} - -type testStruct struct { - Status int `json:"status"` - Reason string `json:"reason"` -} - -type operation struct { - Done bool `json:"done"` - Response []byte `json:"response"` -} - -func TestBoltDB_Update(t *testing.T) { - db := setupBoltDB(t) - namespace := "simple" - - data, err := json.Marshal(testStruct{ - Status: 0, - Reason: "", - }) - require.NoError(t, err) - require.NoError(t, db.Write(namespace, "123", data)) - - type args struct { - key string - values map[string]any - } - tests := []struct { - name string - args args - expectedData testStruct - expectedError assert.ErrorAssertionFunc - }{ - { - name: "simple update", - args: args{ - key: "123", - values: map[string]any{ - "status": 1, - "reason": "something here", - }, - }, - expectedData: testStruct{ - Status: 1, - Reason: "something here", - }, - expectedError: assert.NoError, - }, - { - name: "other key returns error", - args: args{ - key: "456", - values: map[string]any{ - "status": 1, - "reason": "something here", - }, - }, - expectedData: testStruct{}, - expectedError: assert.Error, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - data, err = db.Update(namespace, tt.args.key, tt.args.values) - if !tt.expectedError(t, err) { - return - } - var s testStruct - if tt.expectedData != s { - assert.NoError(t, json.Unmarshal(data, &s)) - assert.Equal(t, tt.expectedData, s) - } - }) - } -} - -type testOpUpdater struct { - UpdaterWithMap -} - -func (f testOpUpdater) SetUpdatedResponse(bytes []byte) { - f.UpdaterWithMap.Values["response"] = bytes -} - -func TestBoltDB_UpdatedSubmissionAndOperationTxFn(t *testing.T) { - db := setupBoltDB(t) - namespace := "simple" - opNamespace := "operation" - - data, err := json.Marshal(testStruct{ - Status: 0, - Reason: "", - }) - require.NoError(t, err) - require.NoError(t, db.Write(namespace, "123", data)) - - data, err = json.Marshal(operation{ - Done: false, - Response: nil, - }) - require.NoError(t, err) - require.NoError(t, db.Write(opNamespace, "op123", data)) - - type args struct { - namespace string - key string - updater Updater - opNamespace string - opKey string - } - tests := []struct { - name string - args args - wantFirst *testStruct - wantOpDone bool - wantOpResponse *testStruct - wantErr assert.ErrorAssertionFunc - }{ - { - name: "first and second get updated", - args: args{ - namespace: namespace, - key: "123", - updater: NewUpdater(map[string]any{ - "status": 1, - "reason": "hello", - }), - opNamespace: opNamespace, - opKey: "op123", - }, - wantFirst: &testStruct{ - Status: 1, - Reason: "hello", - }, - wantOpDone: true, - wantOpResponse: &testStruct{ - Status: 1, - Reason: "hello", - }, - wantErr: assert.NoError, - }, - { - name: "non-existent op key returns error", - args: args{ - namespace: namespace, - key: "123", - updater: NewUpdater(map[string]any{ - "status": 1, - "reason": "hello", - }), - opNamespace: opNamespace, - opKey: "crazy key", - }, - wantErr: assert.Error, - }, - { - name: "non-existent key returns error", - args: args{ - namespace: namespace, - key: "crazy key", - updater: NewUpdater(map[string]any{ - "status": 1, - "reason": "hello", - }), - opNamespace: opNamespace, - opKey: "op123", - }, - wantErr: assert.Error, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotFirstData, gotOpData, err := db.UpdateValueAndOperation(tt.args.namespace, tt.args.key, tt.args.updater, tt.args.opNamespace, tt.args.opKey, testOpUpdater{ - NewUpdater(map[string]any{ - "done": true, - }), - }) - if !tt.wantErr(t, err, fmt.Sprintf("UpdateValueAndOperation(%v, %v, %v, %v, %v)", tt.args.namespace, tt.args.key, tt.args.updater, tt.args.opNamespace, tt.args.opKey)) { - return - } - if tt.wantFirst == nil { - return - } - var gotFirst testStruct - assert.NoError(t, json.Unmarshal(gotFirstData, &gotFirst)) - assert.Equal(t, *tt.wantFirst, gotFirst) - - var gotOp operation - assert.NoError(t, json.Unmarshal(gotOpData, &gotOp)) - assert.Equal(t, tt.wantOpDone, gotOp.Done) - - var gotOpResponse testStruct - assert.NoError(t, json.Unmarshal(gotOp.Response, &gotOpResponse)) - assert.Equal(t, *tt.wantOpResponse, gotOpResponse) - }) - } -} diff --git a/pkg/storage/db_test.go b/pkg/storage/db_test.go new file mode 100644 index 000000000..855b54f47 --- /dev/null +++ b/pkg/storage/db_test.go @@ -0,0 +1,374 @@ +package storage + +import ( + "fmt" + "os" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/goccy/go-json" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getDBImplementations(t *testing.T) []ServiceStorage { + boltDB := setupBoltDB(t) + redisDB := setupRedisDB(t) + + dbImpls := make([]ServiceStorage, 0) + dbImpls = append(dbImpls, boltDB, redisDB) + return dbImpls +} + +func setupBoltDB(t *testing.T) *BoltDB { + db, err := NewStorage(Bolt, "test.db") + assert.NoError(t, err) + assert.NotEmpty(t, db) + + t.Cleanup(func() { + _ = db.Close() + _ = os.Remove("test.db") + }) + return db.(*BoltDB) +} + +func setupRedisDB(t *testing.T) *RedisDB { + server := miniredis.RunT(t) + options := make(map[string]interface{}) + options["address"] = server.Addr() + options["password"] = "" + + db, err := NewStorage(Redis, options) + assert.NoError(t, err) + assert.NotEmpty(t, db) + + t.Cleanup(func() { + _ = db.Close() + }) + + return db.(*RedisDB) +} + +func TestDB(t *testing.T) { + for _, dbImpl := range getDBImplementations(t) { + db := dbImpl + + // create a name space and a message in it + namespace := "F1" + + team1 := "Red Bull" + players1 := []string{"Max Verstappen", "Sergio Pérez"} + p1Bytes, err := json.Marshal(players1) + assert.NoError(t, err) + + err = db.Write(namespace, team1, p1Bytes) + assert.NoError(t, err) + + // get it back + gotPlayers1, err := db.Read(namespace, team1) + assert.NoError(t, err) + + var players1Result []string + err = json.Unmarshal(gotPlayers1, &players1Result) + assert.NoError(t, err) + assert.EqualValues(t, players1, players1Result) + + // get a value from a namespace that doesn't exist + res, err := db.Read("bad", "worse") + assert.NoError(t, err) + assert.Empty(t, res) + + // get a value that doesn't exist in the namespace + noValue, err := db.Read(namespace, "Porsche") + assert.NoError(t, err) + assert.Empty(t, noValue) + + // create a second value in the namespace + team2 := "McLaren" + players2 := []string{"Lando Norris", "Daniel Ricciardo"} + p2Bytes, err := json.Marshal(players2) + assert.NoError(t, err) + + err = db.Write(namespace, team2, p2Bytes) + assert.NoError(t, err) + + // get all values from the namespace + gotAll, err := db.ReadAll(namespace) + assert.NoError(t, err) + assert.True(t, len(gotAll) == 2) + + _, gotRedBull := gotAll[team1] + assert.True(t, gotRedBull) + + _, gotMcLaren := gotAll[team2] + assert.True(t, gotMcLaren) + + // delete value in the namespace + err = db.Delete(namespace, team2) + assert.NoError(t, err) + + gotPlayers2, err := db.Read(namespace, team2) + assert.NoError(t, err) + assert.Empty(t, gotPlayers2) + + // delete value in a namespace that doesn't exist + err = db.Delete("bad", team2) + assert.Error(t, err) + assert.Contains(t, err.Error(), "namespace does not exist") + + // delete a namespace that doesn't exist + err = db.DeleteNamespace("bad") + assert.Contains(t, err.Error(), "could not delete namespace") + + // delete namespace + err = db.DeleteNamespace(namespace) + assert.NoError(t, err) + + res, err = db.Read(namespace, team1) + assert.NoError(t, err) + assert.Empty(t, res) + } +} + +func TestDBPrefixAndKeys(t *testing.T) { + for _, dbImpl := range getDBImplementations(t) { + db := dbImpl + + namespace := "blockchains" + + // set up prefix read test + + dummyData := []byte("dummy") + err := db.Write(namespace, "bitcoin-testnet", dummyData) + assert.NoError(t, err) + + err = db.Write(namespace, "bitcoin-mainnet", dummyData) + assert.NoError(t, err) + + err = db.Write(namespace, "tezos-testnet", dummyData) + assert.NoError(t, err) + + err = db.Write(namespace, "tezos-mainnet", dummyData) + assert.NoError(t, err) + + prefixValues, err := db.ReadPrefix(namespace, "bitcoin") + assert.NoError(t, err) + assert.Len(t, prefixValues, 2) + + keys := make([]string, 0, len(prefixValues)) + for k := range prefixValues { + keys = append(keys, k) + } + assert.Contains(t, keys, "bitcoin-testnet") + assert.Contains(t, keys, "bitcoin-mainnet") + + // read all keys + allKeys, err := db.ReadAllKeys(namespace) + + assert.NoError(t, err) + assert.NotEmpty(t, allKeys) + assert.Len(t, allKeys, 4) + assert.Contains(t, allKeys, "bitcoin-mainnet") + assert.Contains(t, allKeys, "tezos-mainnet") + } +} + +type testStruct struct { + Status int `json:"status"` + Reason string `json:"reason"` +} + +type operation struct { + Done bool `json:"done"` + Response []byte `json:"response"` +} + +func TestDB_Update(t *testing.T) { + for _, dbImpl := range getDBImplementations(t) { + db := dbImpl + namespace := "simple" + + data, err := json.Marshal(testStruct{ + Status: 0, + Reason: "", + }) + require.NoError(t, err) + require.NoError(t, db.Write(namespace, "123", data)) + + type args struct { + key string + values map[string]any + } + tests := []struct { + name string + args args + expectedData testStruct + expectedError assert.ErrorAssertionFunc + }{ + { + name: "simple update", + args: args{ + key: "123", + values: map[string]any{ + "status": 1, + "reason": "something here", + }, + }, + expectedData: testStruct{ + Status: 1, + Reason: "something here", + }, + expectedError: assert.NoError, + }, + { + name: "other key returns error", + args: args{ + key: "456", + values: map[string]any{ + "status": 1, + "reason": "something here", + }, + }, + expectedData: testStruct{}, + expectedError: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err = db.Update(namespace, tt.args.key, tt.args.values) + if !tt.expectedError(t, err) { + return + } + var s testStruct + if tt.expectedData != s { + assert.NoError(t, json.Unmarshal(data, &s)) + assert.Equal(t, tt.expectedData, s) + } + }) + } + } +} + +type testOpUpdater struct { + UpdaterWithMap +} + +func (f testOpUpdater) SetUpdatedResponse(bytes []byte) { + f.UpdaterWithMap.Values["response"] = bytes +} + +func TestDB_UpdatedSubmissionAndOperationTxFn(t *testing.T) { + for _, dbImpl := range getDBImplementations(t) { + db := dbImpl + namespace := "simple" + opNamespace := "operation" + + data, err := json.Marshal(testStruct{ + Status: 0, + Reason: "", + }) + require.NoError(t, err) + require.NoError(t, db.Write(namespace, "123", data)) + + data, err = json.Marshal(operation{ + Done: false, + Response: nil, + }) + require.NoError(t, err) + require.NoError(t, db.Write(opNamespace, "op123", data)) + + type args struct { + namespace string + key string + updater Updater + opNamespace string + opKey string + } + tests := []struct { + name string + args args + wantFirst *testStruct + wantOpDone bool + wantOpResponse *testStruct + wantErr assert.ErrorAssertionFunc + }{ + { + name: "first and second get updated", + args: args{ + namespace: namespace, + key: "123", + updater: NewUpdater(map[string]any{ + "status": 1, + "reason": "hello", + }), + opNamespace: opNamespace, + opKey: "op123", + }, + wantFirst: &testStruct{ + Status: 1, + Reason: "hello", + }, + wantOpDone: true, + wantOpResponse: &testStruct{ + Status: 1, + Reason: "hello", + }, + wantErr: assert.NoError, + }, + { + name: "non-existent op key returns error", + args: args{ + namespace: namespace, + key: "123", + updater: NewUpdater(map[string]any{ + "status": 1, + "reason": "hello", + }), + opNamespace: opNamespace, + opKey: "crazy key", + }, + wantErr: assert.Error, + }, + { + name: "non-existent key returns error", + args: args{ + namespace: namespace, + key: "crazy key", + updater: NewUpdater(map[string]any{ + "status": 1, + "reason": "hello", + }), + opNamespace: opNamespace, + opKey: "op123", + }, + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFirstData, gotOpData, err := db.UpdateValueAndOperation(tt.args.namespace, tt.args.key, tt.args.updater, tt.args.opNamespace, tt.args.opKey, testOpUpdater{ + NewUpdater(map[string]any{ + "done": true, + }), + }) + if !tt.wantErr(t, err, fmt.Sprintf("UpdateValueAndOperation(%v, %v, %v, %v, %v)", tt.args.namespace, tt.args.key, tt.args.updater, tt.args.opNamespace, tt.args.opKey)) { + return + } + if tt.wantFirst == nil { + return + } + var gotFirst testStruct + assert.NoError(t, json.Unmarshal(gotFirstData, &gotFirst)) + assert.Equal(t, *tt.wantFirst, gotFirst) + + var gotOp operation + assert.NoError(t, json.Unmarshal(gotOpData, &gotOp)) + assert.Equal(t, tt.wantOpDone, gotOp.Done) + + var gotOpResponse testStruct + assert.NoError(t, json.Unmarshal(gotOp.Response, &gotOpResponse)) + assert.Equal(t, *tt.wantOpResponse, gotOpResponse) + }) + } + } +} diff --git a/pkg/storage/redis.go b/pkg/storage/redis.go new file mode 100644 index 000000000..177093ff3 --- /dev/null +++ b/pkg/storage/redis.go @@ -0,0 +1,288 @@ +package storage + +import ( + "context" + "fmt" + "strings" + + goredislib "github.com/go-redis/redis/v8" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + NamespaceKeySeparator = ":" + Pong = "PONG" + RedisScanBatchSize = 1000 +) + +func init() { + err := RegisterStorage(&RedisDB{}) + if err != nil { + panic(err) + } +} + +type RedisDB struct { + db *goredislib.Client + ctx context.Context +} + +func (b *RedisDB) Init(i interface{}) error { + options := i.(map[string]interface{}) + + client := goredislib.NewClient(&goredislib.Options{ + Addr: options["address"].(string), + Password: options["password"].(string), + }) + + b.db = client + b.ctx = context.Background() + + return nil +} + +func (b *RedisDB) URI() string { + return b.db.Options().Addr +} + +func (b *RedisDB) IsOpen() bool { + pong, err := b.db.Ping(b.ctx).Result() + if err != nil { + logrus.Error(err) + return false + } + + return pong == Pong +} + +func (b *RedisDB) Type() Type { + return Redis +} + +func (b *RedisDB) Close() error { + return b.db.Close() +} + +func (b *RedisDB) Write(namespace, key string, value []byte) error { + nameSpaceKey := getRedisKey(namespace, key) + // Zero expiration means the key has no expiration time. + return b.db.Set(b.ctx, nameSpaceKey, value, 0).Err() +} + +func (b *RedisDB) WriteMany(namespaces, keys []string, values [][]byte) error { + if len(namespaces) != len(keys) && len(namespaces) != len(values) { + return errors.New("namespaces, keys, and values, are not of equal length") + } + + nameSpaceKeys := make([]string, 0) + for i := range namespaces { + nameSpaceKeys = append(nameSpaceKeys, getRedisKey(namespaces[i], keys[i])) + } + + return b.db.MSet(b.ctx, nameSpaceKeys, values, 0).Err() +} + +func (b *RedisDB) Read(namespace, key string) ([]byte, error) { + nameSpaceKey := getRedisKey(namespace, key) + res, err := b.db.Get(b.ctx, nameSpaceKey).Bytes() + + // Nil reply returned by Redis when key does not exist. + if errors.Is(err, goredislib.Nil) { + return res, nil + } + + return res, err +} + +func (b *RedisDB) ReadPrefix(namespace, prefix string) (map[string][]byte, error) { + namespacePrefix := getRedisKey(namespace, prefix) + + keys, err := readAllKeys(namespacePrefix, b) + if err != nil { + return nil, errors.Wrap(err, "read all keys") + } + + return readAll(keys, b) +} + +func (b *RedisDB) ReadAll(namespace string) (map[string][]byte, error) { + keys, err := readAllKeys(namespace, b) + if err != nil { + return nil, errors.Wrap(err, "read all keys") + } + + return readAll(keys, b) +} + +// TODO: This potentially could dangerous as it might run out of memory as we populate result +func readAll(keys []string, b *RedisDB) (map[string][]byte, error) { + result := make(map[string][]byte, len(keys)) + + if len(keys) == 0 { + return nil, nil + } + + values, err := b.db.MGet(b.ctx, keys...).Result() + if err != nil { + return nil, errors.Wrap(err, "getting multiple keys") + } + + if len(keys) != len(values) { + return nil, errors.New("key length does not match value length") + } + + // result needs to take the namespace out of the key + namespaceDashIndex := strings.Index(keys[0], NamespaceKeySeparator) + for i, val := range values { + byteValue := []byte(fmt.Sprintf("%v", val)) + key := keys[i][namespaceDashIndex+1:] + result[key] = byteValue + } + + return result, nil +} + +func (b *RedisDB) ReadAllKeys(namespace string) ([]string, error) { + keys, err := readAllKeys(namespace, b) + if err != nil { + return nil, err + } + + if len(keys) == 0 { + return make([]string, 0), nil + } + + namespaceDashIndex := strings.Index(keys[0], NamespaceKeySeparator) + for i, key := range keys { + keyWithoutNamespace := key[namespaceDashIndex+1:] + keys[i] = keyWithoutNamespace + } + + return keys, nil +} + +// TODO: This potentially could dangerous as it might run out of memory as we populate allKeys +func readAllKeys(namespace string, b *RedisDB) ([]string, error) { + var cursor uint64 + + var allKeys []string + + for { + keys, nextCursor, err := b.db.Scan(b.ctx, cursor, namespace+"*", RedisScanBatchSize).Result() + if err != nil { + return nil, errors.Wrap(err, "scan error") + } + + allKeys = append(allKeys, keys...) + + if nextCursor == 0 { + break + } + + cursor = nextCursor + } + + return allKeys, nil +} + +func (b *RedisDB) Delete(namespace, key string) error { + nameSpaceKey := getRedisKey(namespace, key) + + if !namespaceExists(namespace, b) { + return errors.Errorf("namespace<%s> does not exist", namespace) + } + + res, err := b.db.GetDel(b.ctx, nameSpaceKey).Result() + if res == "" { + return errors.Wrapf(err, "key<%s> and namespace<%s> does not exist", key, namespace) + } + + return err + +} + +func (b *RedisDB) DeleteNamespace(namespace string) error { + keys, err := readAllKeys(namespace, b) + if err != nil { + return errors.Wrap(err, "read all keys") + } + + if len(keys) == 0 { + return errors.Errorf("could not delete namespace<%s>, namespace does not exist", namespace) + } + + return b.db.Del(b.ctx, keys...).Err() +} + +func (b *RedisDB) Update(namespace string, key string, values map[string]any) ([]byte, error) { + updatedData, err := txWithUpdater(namespace, key, NewUpdater(values), b) + return updatedData, err +} + +func (b *RedisDB) UpdateValueAndOperation(namespace, key string, updater Updater, opNamespace, opKey string, opUpdater ResponseSettingUpdater) (first, op []byte, err error) { + // The Pipeliner interface provided by the go-redis library guarantees that all the commands queued in the pipeline will either succeed or fail together. + _, err = b.db.TxPipelined(b.ctx, func(pipe goredislib.Pipeliner) error { + + firstTx, err := txWithUpdater(namespace, key, updater, b) + if err != nil { + return err + } + + opUpdater.SetUpdatedResponse(firstTx) + secondTx, err := txWithUpdater(opNamespace, opKey, opUpdater, b) + if err != nil { + return err + } + + first = firstTx + op = secondTx + + return nil + }) + + if err != nil { + return nil, nil, errors.Wrap(err, "failed to execute transaction") + } + + return first, op, err +} + +func txWithUpdater(namespace, key string, updater Updater, b *RedisDB) ([]byte, error) { + nameSpaceKey := getRedisKey(namespace, key) + v, err := b.db.Get(b.ctx, nameSpaceKey).Bytes() + if err != nil { + return nil, errors.Wrapf(err, "get error with namespace: %s key: %s", namespace, key) + } + if v == nil { + return nil, errors.Errorf("key not found %s", key) + } + if err := updater.Validate(v); err != nil { + return nil, errors.Wrapf(err, "validating update") + } + + data, err := updater.Update(v) + if err != nil { + return nil, err + } + + if err = b.db.Set(b.ctx, nameSpaceKey, data, 0).Err(); err != nil { + return nil, errors.Wrap(err, "writing to db") + } + + return data, nil +} + +func getRedisKey(namespace, key string) string { + return namespace + NamespaceKeySeparator + key +} + +func namespaceExists(namespace string, b *RedisDB) bool { + keys, _ := b.db.Scan(b.ctx, 0, namespace+"*", RedisScanBatchSize).Val() + + if len(keys) == 0 { + return false + } + + return true +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index b2417d114..27e453621 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -8,7 +8,8 @@ import ( type Type string const ( - Bolt Type = "bolt" + Bolt Type = "bolt" + Redis Type = "redis" ) var ( @@ -55,7 +56,7 @@ func RegisterStorage(storage ServiceStorage) error { // AvailableStorage returns the supported storage providers. func AvailableStorage() []Type { - return []Type{Bolt} + return []Type{Bolt, Redis} } // IsStorageAvailable determines whether a given storage provider is available for instantiation.