From d6b68dea7a633e9cb8416a5410e1778633a5780f Mon Sep 17 00:00:00 2001 From: Michael Kalantar Date: Tue, 14 Nov 2023 09:40:08 -0500 Subject: [PATCH] add redis as alternative metrics store (#1653) * add redis as alternative metrics store Signed-off-by: Michael Kalantar * fix lint problems Signed-off-by: Michael Kalantar * fix lint problems Signed-off-by: Michael Kalantar * test coverage Signed-off-by: Michael Kalantar * refactor Signed-off-by: Michael Kalantar * add package comment Signed-off-by: Michael Kalantar * refactor client config Signed-off-by: Michael Kalantar * update defaults Signed-off-by: Michael Kalantar * add comments Signed-off-by: Michael Kalantar * add comments Signed-off-by: Michael Kalantar * refactor names Signed-off-by: Michael Kalantar * comment fixes Signed-off-by: Michael Kalantar * rename files Signed-off-by: Michael Kalantar --------- Signed-off-by: Michael Kalantar --- abn/service.go | 15 +- abn/service_impl.go | 9 +- abn/service_impl_test.go | 5 +- abn/service_test.go | 36 ++- charts/controller/Chart.yaml | 2 +- .../templates/persistentvolumeclaim.yaml | 7 +- charts/controller/templates/statefulset.yaml | 8 +- charts/controller/values.yaml | 31 ++- go.mod | 6 + go.sum | 14 ++ metrics/server.go | 64 +++--- metrics/server_test.go | 34 +-- storage/badgerdb/{simple.go => badgerdb.go} | 101 +++------ .../{simple_test.go => badgerdb_test.go} | 61 +---- storage/client/client.go | 97 ++++++++ storage/client/client_test.go | 56 +++++ storage/interface.go | 10 +- storage/redis/redis.go | 212 ++++++++++++++++++ storage/redis/redis_test.go | 182 +++++++++++++++ storage/util.go | 74 ++++++ storage/util_test.go | 74 ++++++ 21 files changed, 877 insertions(+), 221 deletions(-) rename storage/badgerdb/{simple.go => badgerdb.go} (74%) rename storage/badgerdb/{simple_test.go => badgerdb_test.go} (74%) create mode 100644 storage/client/client.go create mode 100644 storage/client/client_test.go create mode 100644 storage/redis/redis.go create mode 100644 storage/redis/redis_test.go diff --git a/abn/service.go b/abn/service.go index 844e87f4d..2db7b282f 100644 --- a/abn/service.go +++ b/abn/service.go @@ -7,35 +7,24 @@ import ( "context" "fmt" "net" - "os" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/emptypb" - "github.com/dgraph-io/badger/v4" pb "github.com/iter8-tools/iter8/abn/grpc" util "github.com/iter8-tools/iter8/base" "github.com/iter8-tools/iter8/base/log" - "github.com/iter8-tools/iter8/storage" - "github.com/iter8-tools/iter8/storage/badgerdb" + storageclient "github.com/iter8-tools/iter8/storage/client" // auth package is necessary to enable authentication with various cloud providers _ "k8s.io/client-go/plugin/pkg/client/auth" ) const ( - // MetricsDirEnv is the environment variable identifying the directory with metrics storage - MetricsDirEnv = "METRICS_DIR" - configEnv = "ABN_CONFIG_FILE" defaultPortNumber = 50051 ) -var ( - // MetricsClient is the metrics client - MetricsClient storage.Interface -) - // newServer returns a new gRPC server func newServer() *abnServer { return &abnServer{} @@ -118,7 +107,7 @@ func LaunchGRPCServer(opts []grpc.ServerOption, stopCh <-chan struct{}) error { pb.RegisterABNServer(grpcServer, newServer()) // configure MetricsClient if needed - MetricsClient, err = badgerdb.GetClient(badger.DefaultOptions(os.Getenv(MetricsDirEnv)), badgerdb.AdditionalOptions{}) + storageclient.MetricsClient, err = storageclient.GetClient() if err != nil { log.Logger.Error("Unable to configure metrics storage client ", err) return err diff --git a/abn/service_impl.go b/abn/service_impl.go index e87de6c20..3e090619d 100644 --- a/abn/service_impl.go +++ b/abn/service_impl.go @@ -13,6 +13,7 @@ import ( util "github.com/iter8-tools/iter8/base" "github.com/iter8-tools/iter8/base/log" "github.com/iter8-tools/iter8/controllers" + storageclient "github.com/iter8-tools/iter8/storage/client" ) var allRoutemaps controllers.AllRouteMapsInterface = &controllers.DefaultRoutemaps{} @@ -47,10 +48,10 @@ func lookupInternal(application string, user string) (controllers.RoutemapInterf } // record user; ignore error if any; this is best effort - if MetricsClient == nil { + if storageclient.MetricsClient == nil { return nil, invalidVersion, fmt.Errorf("no metrics client") } - _ = MetricsClient.SetUser(application, versionNumber, *s.GetVersions()[versionNumber].GetSignature(), user) + _ = storageclient.MetricsClient.SetUser(application, versionNumber, *s.GetVersions()[versionNumber].GetSignature(), user) return s, versionNumber, nil } @@ -134,10 +135,10 @@ func writeMetricInternal(application, user, metric, valueStr string) error { v := s.GetVersions()[versionNumber] transaction := uuid.NewString() - if MetricsClient == nil { + if storageclient.MetricsClient == nil { return fmt.Errorf("no metrics client") } - err = MetricsClient.SetMetric( + err = storageclient.MetricsClient.SetMetric( application, versionNumber, *v.GetSignature(), metric, user, transaction, value) diff --git a/abn/service_impl_test.go b/abn/service_impl_test.go index 7824af83f..e60c613d8 100644 --- a/abn/service_impl_test.go +++ b/abn/service_impl_test.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" util "github.com/iter8-tools/iter8/base" "github.com/iter8-tools/iter8/storage/badgerdb" + storageclient "github.com/iter8-tools/iter8/storage/client" "github.com/stretchr/testify/assert" ) @@ -15,7 +16,7 @@ func TestLookupInternal(t *testing.T) { var err error // set up test metrics db for recording users tempDirPath := t.TempDir() - MetricsClient, err = badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) + storageclient.MetricsClient, err = badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) assert.NoError(t, err) // setup: add desired routemaps to allRoutemaps @@ -44,7 +45,7 @@ func TestWeights(t *testing.T) { // set up test metrics db for recording users tempDirPath := t.TempDir() - MetricsClient, err = badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) + storageclient.MetricsClient, err = badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) assert.NoError(t, err) // setup: add desired routemaps to allRoutemaps diff --git a/abn/service_test.go b/abn/service_test.go index 89905f704..c60ea725b 100644 --- a/abn/service_test.go +++ b/abn/service_test.go @@ -6,15 +6,16 @@ import ( "math/rand" "net" "os" - "path/filepath" "reflect" "testing" "time" + "github.com/alicebob/miniredis" "github.com/dgraph-io/badger/v4" pb "github.com/iter8-tools/iter8/abn/grpc" util "github.com/iter8-tools/iter8/base" "github.com/iter8-tools/iter8/storage/badgerdb" + storageclient "github.com/iter8-tools/iter8/storage/client" "github.com/stretchr/testify/assert" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -185,7 +186,7 @@ func setupGRPCService(t *testing.T) (*pb.ABNClient, func()) { grpcServer := grpc.NewServer(serverOptions...) pb.RegisterABNServer(grpcServer, newServer()) tempDirPath := t.TempDir() - MetricsClient, err = badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) + storageclient.MetricsClient, err = badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) assert.NoError(t, err) go func() { _ = grpcServer.Serve(lis) @@ -222,10 +223,10 @@ func getMetricsCount(t *testing.T, namespace string, name string, version int, m } // TODO: better error handling when there is no metrics client - if MetricsClient == nil { + if storageclient.MetricsClient == nil { return 0 } - versionmetrics, err := MetricsClient.GetMetrics(namespace+"/"+name, version, *signature) + versionmetrics, err := storageclient.MetricsClient.GetMetrics(namespace+"/"+name, version, *signature) if err != nil { return 0 } @@ -243,12 +244,31 @@ func TestLaunchGRPCServer(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - // define METRICS_DIR - err := os.Setenv(MetricsDirEnv, t.TempDir()) + server, _ := miniredis.Run() + assert.NotNil(t, server) + + abnConfig := `port: 50051` + + metricsConfig := `port: 8080 +implementation: redis +redis: + address: ` + server.Addr() + + af, err := os.CreateTemp("", "abn*.yaml") + assert.NoError(t, err) + abnConfigFile := af.Name() + _, err = af.WriteString(abnConfig) assert.NoError(t, err) - configFile := filepath.Clean(util.CompletePath("../testdata", "abninputs/config.yaml")) - err = os.Setenv("ABN_CONFIG_FILE", configFile) + mf, err := os.CreateTemp("", "metrics*.yaml") + assert.NoError(t, err) + metricsConfigFile := mf.Name() + _, err = mf.WriteString(metricsConfig) + assert.NoError(t, err) + + err = os.Setenv("ABN_CONFIG_FILE", abnConfigFile) + assert.NoError(t, err) + err = os.Setenv("METRICS_CONFIG_FILE", metricsConfigFile) assert.NoError(t, err) err = LaunchGRPCServer([]grpc.ServerOption{}, ctx.Done()) diff --git a/charts/controller/Chart.yaml b/charts/controller/Chart.yaml index 5419122fd..d89c04559 100644 --- a/charts/controller/Chart.yaml +++ b/charts/controller/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: controller -version: 0.18.3 +version: 0.18.4 description: Iter8 controller controller type: application keywords: diff --git a/charts/controller/templates/persistentvolumeclaim.yaml b/charts/controller/templates/persistentvolumeclaim.yaml index 34ef849b6..30a6a123d 100644 --- a/charts/controller/templates/persistentvolumeclaim.yaml +++ b/charts/controller/templates/persistentvolumeclaim.yaml @@ -1,3 +1,5 @@ + +{{- if or (not .Values.metrics) (not .Values.metrics.implementation) (eq "badgerdb" .Values.metrics.implementation) }} apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -8,5 +10,6 @@ spec: - ReadWriteOnce resources: requests: - storage: {{ .Values.storage }} - storageClassName: {{ .Values.storageClassName }} \ No newline at end of file + storage: {{ default "50Mi" .Values.metrics.badgerdb.storage }} + storageClassName: {{ default "standard" .Values.metrics.badgerdb.storageClassName }} +{{- end }} diff --git a/charts/controller/templates/statefulset.yaml b/charts/controller/templates/statefulset.yaml index e2d1be856..e054f418b 100644 --- a/charts/controller/templates/statefulset.yaml +++ b/charts/controller/templates/statefulset.yaml @@ -42,8 +42,10 @@ spec: - name: config mountPath: "/config" readOnly: true + {{- if or (not .Values.metrics) (not .Values.metrics.implementation) (eq "badgerdb" .Values.metrics.implementation) }} - name: metrics - mountPath: "/metrics" + mountPath: {{ default "/metrics" .Values.metrics.badgerdb.dir }} + {{- end }} resources: {{ toYaml .Values.resources | indent 10 | trim }} securityContext: @@ -58,6 +60,8 @@ spec: - name: config configMap: name: {{ .Release.Name }} + {{- if or (not .Values.metrics) (not .Values.metrics.implementation) (eq "badgerdb" .Values.metrics.implementation) }} - name: metrics persistentVolumeClaim: - claimName: {{ .Release.Name }} \ No newline at end of file + claimName: {{ .Release.Name }} + {{- end }} \ No newline at end of file diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml index 6add0639a..78427905b 100644 --- a/charts/controller/values.yaml +++ b/charts/controller/values.yaml @@ -57,13 +57,32 @@ resources: memory: "128Mi" cpu: "500m" -### PersistentVolumeClaim parameters -storage: 50Mi -storageClassName: standard - -### A/B/n service port +### A/B/n abn: + # port for Iter8 gRPC service port: 50051 -### Metrics service port + +### Metrics metrics: + # port on which HTTP service (for Grafana) should be exposed port: 8080 + # implementation technology for metrics storage + # Valid values are badgerdb (default) and redis + # The set of properties used to configure the metrics store depend on the + # implementation selected. + implementation: badgerdb + # default properties specific to BadgerDB + badgerdb: + # storage that should be created to support badger db + storage: 50Mi + storageClassName: standard + # location to mount storage + dir: /metrics + # default properties specific to Redis + redis: + address: redis:6379 + # password: (default - none) + # username: (default - none) + # db: (default 0) + + diff --git a/go.mod b/go.mod index 6e7f7d20b..9f98ec0b3 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( dario.cat/mergo v1.0.0 fortio.org/fortio v1.60.2 github.com/Masterminds/sprig v2.22.0+incompatible + github.com/alicebob/miniredis v2.5.0+incompatible github.com/antonmedv/expr v1.15.3 github.com/bojand/ghz v0.117.0 github.com/dgraph-io/badger/v4 v4.2.0 @@ -31,6 +32,7 @@ require ( github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/montanaflynn/stats v0.7.1 github.com/pkg/errors v0.9.1 + github.com/redis/go-redis/v9 v9.3.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 @@ -66,6 +68,7 @@ require ( github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect + github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bufbuild/protocompile v0.4.0 // indirect @@ -80,6 +83,7 @@ require ( github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/docker/cli v23.0.3+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker v23.0.3+incompatible // indirect @@ -111,6 +115,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.3 // indirect + github.com/gomodule/redigo v1.8.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -171,6 +176,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v1.2.0 // indirect + github.com/yuin/gopher-lua v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect diff --git a/go.sum b/go.sum index e33cbb843..12a37872a 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,10 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI= +github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/antonmedv/expr v1.15.3 h1:q3hOJZNvLvhqE8OHBs1cFRdbXFNKuA+bHmRaI+AmRmI= github.com/antonmedv/expr v1.15.3/go.mod h1:0E/6TxnOlRNp81GMzX9QfDPAmHo2Phg00y4JUv1ihsE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -73,6 +77,10 @@ github.com/bojand/ghz v0.117.0 h1:dTMxg+tUcLMw8BYi7vQPjXsrM2DJ20ns53hz1am1SbQ= github.com/bojand/ghz v0.117.0/go.mod h1:MXspmKdJie7NAS0IHzqG9X5h6zO3tIRGQ6Tkt8sAwa4= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= @@ -123,6 +131,8 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= github.com/docker/cli v23.0.3+incompatible h1:Zcse1DuDqBdgI7OQDV8Go7b83xLgfhW1eza4HfEdxpY= @@ -434,6 +444,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= +github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -484,6 +496,8 @@ github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= +github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= diff --git a/metrics/server.go b/metrics/server.go index 857e9a904..fd93586cc 100644 --- a/metrics/server.go +++ b/metrics/server.go @@ -12,11 +12,11 @@ import ( "time" "github.com/bojand/ghz/runner" - "github.com/iter8-tools/iter8/abn" util "github.com/iter8-tools/iter8/base" "github.com/iter8-tools/iter8/base/log" "github.com/iter8-tools/iter8/controllers" "github.com/iter8-tools/iter8/storage" + storageclient "github.com/iter8-tools/iter8/storage/client" "github.com/montanaflynn/stats" "gonum.org/v1/plot/plotter" @@ -25,17 +25,12 @@ import ( ) const ( - configEnv = "METRICS_CONFIG_FILE" - defaultPortNumber = 8080 - timeFormat = "02 Jan 06 15:04 MST" + // MetricsConfigFileEnv is name of environment variable containing the name metrics config file + MetricsConfigFileEnv = "METRICS_CONFIG_FILE" + defaultPortNumber = 8080 + timeFormat = "02 Jan 06 15:04 MST" ) -// metricsConfig defines the configuration of the controllers -type metricsConfig struct { - // Port is port number on which the metrics service should listen - Port *int `json:"port,omitempty"` -} - // versionSummarizedMetric adds version to summary data type versionSummarizedMetric struct { Version int @@ -132,11 +127,17 @@ type ghzDashboard struct { var allRoutemaps controllers.AllRouteMapsInterface = &controllers.DefaultRoutemaps{} +// metricsConfig is configuration of metrics service +type metricsServiceConfig struct { + // Port is port number on which the metrics service should listen + Port *int `json:"port,omitempty"` +} + // Start starts the HTTP server func Start(stopCh <-chan struct{}) error { // read configutation for metrics service - conf := &metricsConfig{} - err := util.ReadConfig(configEnv, conf, func() { + conf := &metricsServiceConfig{} + err := util.ReadConfig(MetricsConfigFileEnv, conf, func() { if nil == conf.Port { conf.Port = util.IntPointer(defaultPortNumber) } @@ -228,11 +229,11 @@ func getAbnDashboard(w http.ResponseWriter, r *http.Request) { continue } - if abn.MetricsClient == nil { + if storageclient.MetricsClient == nil { log.Logger.Error("no metrics client") continue } - versionmetrics, err := abn.MetricsClient.GetMetrics(namespaceApplication, v, *signature) + versionmetrics, err := storageclient.MetricsClient.GetMetrics(namespaceApplication, v, *signature) if err != nil { log.Logger.Debugf("no metrics found for application %s (version %d; signature %s)", namespaceApplication, v, *signature) continue @@ -256,12 +257,11 @@ func getAbnDashboard(w http.ResponseWriter, r *http.Request) { if err != nil { log.Logger.Debugf("unable to compute summaried metrics over transactions for application %s (version %d; signature %s)", namespaceApplication, v, *signature) continue - } else { - entry.SummaryOverTransactions = append(entry.SummaryOverTransactions, &versionSummarizedMetric{ - Version: v, - SummarizedMetric: smT, - }) } + entry.SummaryOverTransactions = append(entry.SummaryOverTransactions, &versionSummarizedMetric{ + Version: v, + SummarizedMetric: smT, + }) smU, err := calculateSummarizedMetric(metrics.MetricsOverUsers) if err != nil { @@ -298,11 +298,10 @@ func getAbnDashboard(w http.ResponseWriter, r *http.Request) { if err != nil { log.Logger.Debugf("unable to compute histogram over transactions for application %s (metric %s)", namespaceApplication, metric) continue - } else { - resultEntry := result[metric] - resultEntry.HistogramsOverTransactions = &hT - result[metric] = resultEntry } + resultEntry := result[metric] + resultEntry.HistogramsOverTransactions = &hT + result[metric] = resultEntry } for metric, byVersion := range byMetricOverUsers { @@ -310,11 +309,10 @@ func getAbnDashboard(w http.ResponseWriter, r *http.Request) { if err != nil { log.Logger.Debugf("unable to compute histogram over users for application %s (metric %s)", namespaceApplication, metric) continue - } else { - resultEntry := result[metric] - resultEntry.HistogramsOverUsers = &hT - result[metric] = resultEntry } + resultEntry := result[metric] + resultEntry.HistogramsOverUsers = &hT + result[metric] = resultEntry } // convert to JSON @@ -567,13 +565,13 @@ func getHTTPDashboard(w http.ResponseWriter, r *http.Request) { log.Logger.Tracef("getHTTPGrafana called for namespace %s and test %s", namespace, test) // get fortioResult from metrics client - if abn.MetricsClient == nil { + if storageclient.MetricsClient == nil { http.Error(w, "no metrics client", http.StatusInternalServerError) return } // get testResult from metrics client - testResult, err := abn.MetricsClient.GetExperimentResult(namespace, test) + testResult, err := storageclient.MetricsClient.GetExperimentResult(namespace, test) if err != nil { errorMessage := fmt.Sprintf("cannot get experiment result with namespace %s, test %s", namespace, test) log.Logger.Error(errorMessage) @@ -704,13 +702,13 @@ func getGRPCDashboard(w http.ResponseWriter, r *http.Request) { log.Logger.Tracef("getGRPCDashboard called for namespace %s and test %s", namespace, test) // get ghz result from metrics client - if abn.MetricsClient == nil { + if storageclient.MetricsClient == nil { http.Error(w, "no metrics client", http.StatusInternalServerError) return } // get testResult from metrics client - testResult, err := abn.MetricsClient.GetExperimentResult(namespace, test) + testResult, err := storageclient.MetricsClient.GetExperimentResult(namespace, test) if err != nil { errorMessage := fmt.Sprintf("cannot get experiment result with namespace %s, test %s", namespace, test) log.Logger.Error(errorMessage) @@ -784,12 +782,12 @@ func putExperimentResult(w http.ResponseWriter, r *http.Request) { return } - if abn.MetricsClient == nil { + if storageclient.MetricsClient == nil { http.Error(w, "no metrics client", http.StatusInternalServerError) return } - err = abn.MetricsClient.SetExperimentResult(namespace, experiment, &experimentResult) + err = storageclient.MetricsClient.SetExperimentResult(namespace, experiment, &experimentResult) if err != nil { errorMessage := fmt.Sprintf("cannot store result in storage client: %s: %e", string(body), err) log.Logger.Error(errorMessage) diff --git a/metrics/server_test.go b/metrics/server_test.go index 3d0e1e640..ed78a45bf 100644 --- a/metrics/server_test.go +++ b/metrics/server_test.go @@ -17,10 +17,10 @@ import ( "time" "github.com/dgraph-io/badger/v4" - "github.com/iter8-tools/iter8/abn" util "github.com/iter8-tools/iter8/base" "github.com/iter8-tools/iter8/controllers" "github.com/iter8-tools/iter8/storage/badgerdb" + storageclient "github.com/iter8-tools/iter8/storage/client" "github.com/stretchr/testify/assert" ) @@ -519,7 +519,7 @@ func TestStart(t *testing.T) { assert.NoError(t, err) }() - err = os.Setenv("METRICS_CONFIG_FILE", file.Name()) + err = os.Setenv(MetricsConfigFileEnv, file.Name()) assert.NoError(t, err) err = Start(ctx.Done()) @@ -534,10 +534,10 @@ func TestReadConfigDefaultPort(t *testing.T) { assert.NoError(t, err) }() - err = os.Setenv("METRICS_CONFIG_FILE", file.Name()) + err = os.Setenv(MetricsConfigFileEnv, file.Name()) assert.NoError(t, err) - conf := &metricsConfig{} - err = util.ReadConfig(configEnv, conf, func() { + conf := &metricsServiceConfig{} + err = util.ReadConfig(MetricsConfigFileEnv, conf, func() { if nil == conf.Port { conf.Port = util.IntPointer(defaultPortNumber) } @@ -560,10 +560,10 @@ func TestReadConfigSetPort(t *testing.T) { _, err = file.Write([]byte(fmt.Sprintf("port: %d", expectedPortNumber))) assert.NoError(t, err) - err = os.Setenv("METRICS_CONFIG_FILE", file.Name()) + err = os.Setenv(MetricsConfigFileEnv, file.Name()) assert.NoError(t, err) - conf := &metricsConfig{} - err = util.ReadConfig(configEnv, conf, func() { + conf := &metricsServiceConfig{} + err = util.ReadConfig(MetricsConfigFileEnv, conf, func() { if nil == conf.Port { conf.Port = util.IntPointer(defaultPortNumber) } @@ -619,7 +619,7 @@ func (cm *testRoutemaps) GetAllRoutemaps() controllers.RoutemapsInterface { func TestGetABNDashboard(t *testing.T) { testRM := testRoutemaps{ - allroutemaps: setupRoutemaps(t, *getTestRM("default", "test")), + allroutemaps: setupRoutemaps(*getTestRM("default", "test")), } allRoutemaps = &testRM @@ -650,7 +650,7 @@ func TestGetABNDashboard(t *testing.T) { err = client.SetMetric(app, version, signature, metric, user, transaction, value) assert.NoError(t, err) - abn.MetricsClient = client + storageclient.MetricsClient = client w := httptest.NewRecorder() rm := allRoutemaps.GetAllRoutemaps().GetRoutemapFromNamespaceName("default", "test") @@ -795,7 +795,7 @@ func TestCalculateHistogram(t *testing.T) { } } -func setupRoutemaps(t *testing.T, initialroutemaps ...testroutemap) testroutemaps { +func setupRoutemaps(initialroutemaps ...testroutemap) testroutemaps { routemaps := testroutemaps{ nsRoutemap: make(map[string]testroutemapsByName), } @@ -949,7 +949,7 @@ func TestPutExperimentResult(t *testing.T) { tempDirPath := t.TempDir() client, err := badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) assert.NoError(t, err) - abn.MetricsClient = client + storageclient.MetricsClient = client w := httptest.NewRecorder() @@ -988,7 +988,7 @@ func TestPutExperimentResult(t *testing.T) { }() // check to see if the result is stored in the metrics client - result, err := abn.MetricsClient.GetExperimentResult("default", "default") + result, err := storageclient.MetricsClient.GetExperimentResult("default", "default") assert.NoError(t, err) assert.Equal(t, &experimentResult, result) } @@ -1052,7 +1052,7 @@ func TestGetHTTPDashboard(t *testing.T) { tempDirPath := t.TempDir() client, err := badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) assert.NoError(t, err) - abn.MetricsClient = client + storageclient.MetricsClient = client // preload metric client with experiment result fortioResult := util.HTTPResult{} @@ -1070,7 +1070,7 @@ func TestGetHTTPDashboard(t *testing.T) { }, } - err = abn.MetricsClient.SetExperimentResult("default", "default", &experimentResult) + err = storageclient.MetricsClient.SetExperimentResult("default", "default", &experimentResult) assert.NoError(t, err) w := httptest.NewRecorder() @@ -1165,7 +1165,7 @@ func TestGetGRPCDashboard(t *testing.T) { tempDirPath := t.TempDir() client, err := badgerdb.GetClient(badger.DefaultOptions(tempDirPath), badgerdb.AdditionalOptions{}) assert.NoError(t, err) - abn.MetricsClient = client + storageclient.MetricsClient = client // preload metric client with experiment result ghzResult := util.GHZResult{} @@ -1183,7 +1183,7 @@ func TestGetGRPCDashboard(t *testing.T) { }, } - err = abn.MetricsClient.SetExperimentResult("default", "default", &experimentResult) + err = storageclient.MetricsClient.SetExperimentResult("default", "default", &experimentResult) assert.NoError(t, err) w := httptest.NewRecorder() diff --git a/storage/badgerdb/simple.go b/storage/badgerdb/badgerdb.go similarity index 74% rename from storage/badgerdb/simple.go rename to storage/badgerdb/badgerdb.go index 5d164d30e..f14e690f5 100644 --- a/storage/badgerdb/simple.go +++ b/storage/badgerdb/badgerdb.go @@ -1,4 +1,4 @@ -// Package badgerdb implements the storageclient interface with BadgerDB +// Package badgerdb implements the storage interface with BadgerDB package badgerdb import ( @@ -16,6 +16,13 @@ import ( "github.com/iter8-tools/iter8/storage" ) +// ClientConfig is configurable properties of a new BadgerDB client +type ClientConfig struct { + Storage *string `json:"storage,omitempty"` + StorageClassName *string `json:"storageClassName,omitempty"` + Dir *string `json:"dir,omitempty"` +} + // Client is a client for the BadgerDB type Client struct { db *badger.DB @@ -79,42 +86,10 @@ func getDefaultAdditionalOptions() AdditionalOptions { } } -func validateKeyToken(s string) error { - if strings.Contains(s, ":") { - return errors.New("key token contains \":\"") - } - - return nil -} - -func getMetricPrefix(applicationName string, version int, signature string) string { - return fmt.Sprintf("kt-metric::%s::%d::%s::", applicationName, version, signature) -} - -func getMetricKey(applicationName string, version int, signature, metric, user, transaction string) (string, error) { - if err := validateKeyToken(applicationName); err != nil { - return "", errors.New("application name cannot have \":\"") - } - if err := validateKeyToken(signature); err != nil { - return "", errors.New("signature cannot have \":\"") - } - if err := validateKeyToken(metric); err != nil { - return "", errors.New("metric name cannot have \":\"") - } - if err := validateKeyToken(user); err != nil { - return "", errors.New("user name cannot have \":\"") - } - if err := validateKeyToken(transaction); err != nil { - return "", errors.New("transaction ID cannot have \":\"") - } - - return fmt.Sprintf("%s%s::%s::%s", getMetricPrefix(applicationName, version, signature), metric, user, transaction), nil -} - // SetMetric sets a metric based on the app name, version, signature, metric type, user name, transaction ID, and metric value with BadgerDB // Example key: kt-metric::my-app::0::my-signature::my-metric::my-user::my-transaction-id -> my-metric-value func (cl Client) SetMetric(applicationName string, version int, signature, metric, user, transaction string, metricValue float64) error { - key, err := getMetricKey(applicationName, version, signature, metric, user, transaction) + key, err := storage.GetMetricKey(applicationName, version, signature, metric, user, transaction) if err != nil { return err } @@ -134,19 +109,10 @@ func (cl Client) SetMetric(applicationName string, version int, signature, metri return err } -func getUserPrefix(applicationName string, version int, signature string) string { - return fmt.Sprintf("kt-users::%s::%d::%s::", applicationName, version, signature) -} - -func getUserKey(applicationName string, version int, signature, user string) string { - // getUserKey() is just getUserPrefix() with the user appended at the end - return fmt.Sprintf("%s%s", getUserPrefix(applicationName, version, signature), user) -} - // SetUser sets a user based on the app name, version, signature, and user name with BadgerDB // Example key/value: kt-users::my-app::0::my-signature::my-user -> true func (cl Client) SetUser(applicationName string, version int, signature, user string) error { - key := getUserKey(applicationName, version, signature, user) + key := storage.GetUserKey(applicationName, version, signature, user) return cl.db.Update(func(txn *badger.Txn) error { e := badger.NewEntry([]byte(key), []byte("true")).WithTTL(cl.additionalOptions.TTL) @@ -165,7 +131,7 @@ func (cl Client) getUserCount(applicationName string, version int, signature str it := txn.NewIterator(opts) defer it.Close() - prefix := []byte(getUserPrefix(applicationName, version, signature)) + prefix := []byte(storage.GetUserKeyPrefix(applicationName, version, signature)) for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { count++ } @@ -219,7 +185,7 @@ func (cl Client) GetMetrics(applicationName string, version int, signature strin // iterate over all metrics of a particular application name, version, and signature it := txn.NewIterator(badger.DefaultIteratorOptions) defer it.Close() - prefix := []byte(getMetricPrefix(applicationName, version, signature)) + prefix := []byte(storage.GetMetricKeyPrefix(applicationName, version, signature)) for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { item := it.Item() key := string(item.Key()) @@ -315,11 +281,6 @@ func (cl Client) GetMetrics(applicationName string, version int, signature strin return &metrics, nil } -func getExperimentResultKey(namespace, experiment string) string { - // getExperimentResultKey() is just getUserPrefix() with the user appended at the end - return fmt.Sprintf("kt-result::%s::%s", namespace, experiment) -} - // SetExperimentResult sets the experiment result for a particular namespace and experiment name // the data is []byte in order to make this function reusable for different tasks func (cl Client) SetExperimentResult(namespace, experiment string, data *base.ExperimentResult) error { @@ -328,7 +289,7 @@ func (cl Client) SetExperimentResult(namespace, experiment string, data *base.Ex return fmt.Errorf("cannot JSON marshal ExperimentResult: %e", err) } - key := getExperimentResultKey(namespace, experiment) + key := storage.GetExperimentResultKey(namespace, experiment) return cl.db.Update(func(txn *badger.Txn) error { e := badger.NewEntry([]byte(key), dataBytes).WithTTL(cl.additionalOptions.TTL) err := txn.SetEntry(e) @@ -339,29 +300,23 @@ func (cl Client) SetExperimentResult(namespace, experiment string, data *base.Ex // GetExperimentResult sets the experiment result for a particular namespace and experiment name // the data is []byte in order to make this function reusable for different tasks func (cl Client) GetExperimentResult(namespace, experiment string) (*base.ExperimentResult, error) { - var valCopy []byte - err := cl.db.View(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(getExperimentResultKey(namespace, experiment))) - if err != nil { - return fmt.Errorf("cannot get ExperimentResult with name: \"%s\" and namespace: %s: %e", experiment, namespace, err) - } - valCopy, err = item.ValueCopy(nil) - if err != nil { - return fmt.Errorf("cannot copy value of ExperimentResult with name: \"%s\" and namespace: %s: %e", experiment, namespace, err) - } + return storage.GetExperimentResult(func() ([]byte, error) { + var valCopy []byte + err := cl.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(storage.GetExperimentResultKey(namespace, experiment))) + if err != nil { + return fmt.Errorf("cannot get ExperimentResult with name: \"%s\" and namespace: %s: %e", experiment, namespace, err) + } - return nil - }) - if err != nil { - return nil, err - } + valCopy, err = item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("cannot copy value of ExperimentResult with name: \"%s\" and namespace: %s: %e", experiment, namespace, err) + } - experimentResult := base.ExperimentResult{} - err = json.Unmarshal(valCopy, &experimentResult) - if err != nil { - return nil, fmt.Errorf("cannot JSON unmarshal ExperimentResult: \"%s\": %e", string(valCopy), err) - } + return nil + }) + return valCopy, err + }) - return &experimentResult, err } diff --git a/storage/badgerdb/simple_test.go b/storage/badgerdb/badgerdb_test.go similarity index 74% rename from storage/badgerdb/simple_test.go rename to storage/badgerdb/badgerdb_test.go index 8b737ecdf..5b5d604e1 100644 --- a/storage/badgerdb/simple_test.go +++ b/storage/badgerdb/badgerdb_test.go @@ -2,12 +2,12 @@ package badgerdb import ( "encoding/json" - "fmt" "strconv" "testing" "github.com/dgraph-io/badger/v4" "github.com/iter8-tools/iter8/base" + "github.com/iter8-tools/iter8/storage" "github.com/stretchr/testify/assert" ) @@ -59,7 +59,7 @@ func TestSetMetric(t *testing.T) { // get metric err = client.db.View(func(txn *badger.Txn) error { - key, err := getMetricKey(app, version, signature, metric, user, transaction) + key, err := storage.GetMetricKey(app, version, signature, metric, user, transaction) assert.NoError(t, err) item, err := txn.Get([]byte(key)) @@ -83,7 +83,7 @@ func TestSetMetric(t *testing.T) { // SetMetric() should also add a user err = client.db.View(func(txn *badger.Txn) error { - key := getUserKey(app, version, signature, user) + key := storage.GetUserKey(app, version, signature, user) item, err := txn.Get([]byte(key)) assert.NoError(t, err) assert.NotNil(t, item) @@ -126,7 +126,7 @@ func TestSetUser(t *testing.T) { // get user err = client.db.View(func(txn *badger.Txn) error { - key := getUserKey(app, version, signature, user) + key := storage.GetUserKey(app, version, signature, user) item, err := txn.Get([]byte(key)) assert.NoError(t, err) assert.NotNil(t, item) @@ -179,59 +179,6 @@ func TestGetMetricsWithExtraUsers(t *testing.T) { assert.Equal(t, "{\"my-metric\":{\"MetricsOverTransactions\":[25],\"MetricsOverUsers\":[25,10]},\"my-metric2\":{\"MetricsOverTransactions\":[50],\"MetricsOverUsers\":[50,0]}}", string(jsonMetrics)) } -type testmetrickey struct { - valid bool - application string - signature string - metric string - user string - transaction string -} - -func TestGetMetricKey(t *testing.T) { - for _, s := range []testmetrickey{ - {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, - {valid: false, application: "invalid:application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, - {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, - {valid: false, application: "application", signature: "invalid:signature", metric: "metric", user: "user", transaction: "transaction"}, - {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, - {valid: false, application: "application", signature: "signature", metric: "invalid:metric", user: "user", transaction: "transaction"}, - {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, - {valid: false, application: "application", signature: "signature", metric: "metric", user: "invalid:user", transaction: "transaction"}, - {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, - {valid: false, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "invalid:transaction"}, - } { - key, err := getMetricKey(s.application, 0, s.signature, s.metric, s.user, s.transaction) - if s.valid { - assert.NoError(t, err) - assert.Equal(t, fmt.Sprintf("%s%s::%s::%s", getMetricPrefix(s.application, 0, s.signature), s.metric, s.user, s.transaction), key) - } else { - assert.Error(t, err) - assert.Equal(t, "", key) - } - } -} - -func TestValidateKeyToken(t *testing.T) { - err := validateKeyToken("hello") - assert.NoError(t, err) - - err = validateKeyToken("::") - assert.Error(t, err) - - err = validateKeyToken("hello::world") - assert.Error(t, err) - - err = validateKeyToken("hello :: world") - assert.Error(t, err) - - err = validateKeyToken("hello:world") - assert.Error(t, err) - - err = validateKeyToken("hello : world") - assert.Error(t, err) -} - func TestGetMetrics(t *testing.T) { tempDirPath := t.TempDir() diff --git a/storage/client/client.go b/storage/client/client.go new file mode 100644 index 000000000..d86b7815b --- /dev/null +++ b/storage/client/client.go @@ -0,0 +1,97 @@ +// Package client implements an implementation independent storage client +package client + +import ( + "fmt" + "strings" + + "github.com/dgraph-io/badger/v4" + util "github.com/iter8-tools/iter8/base" + "github.com/iter8-tools/iter8/storage" + "github.com/iter8-tools/iter8/storage/badgerdb" + "github.com/iter8-tools/iter8/storage/redis" +) + +const ( + metricsConfigFileEnv = "METRICS_CONFIG_FILE" + defaultImplementation = "badgerdb" +) + +var ( + // MetricsClient is storage client + MetricsClient storage.Interface +) + +// metricsStorageConfig is configuration of metrics service +type metricsStorageConfig struct { + // Implementation method for metrics service + Implementation *string `json:"implementation,omitempty"` +} + +// GetClient creates a metric service client based on configuration +func GetClient() (storage.Interface, error) { + conf := &metricsStorageConfig{} + err := util.ReadConfig(metricsConfigFileEnv, conf, func() { + if conf.Implementation == nil { + conf.Implementation = util.StringPointer(defaultImplementation) + } + }) + if err != nil { + return nil, err + } + + switch strings.ToLower(*conf.Implementation) { + case "badgerdb": + // badgerConfig defines the configuration of a badgerDB based metrics service + type mConfig struct { + badgerdb.ClientConfig `json:"badgerdb,omitempty"` + } + + conf := &mConfig{} + err := util.ReadConfig(metricsConfigFileEnv, conf, func() { + if conf.ClientConfig.Storage == nil { + conf.ClientConfig.Storage = util.StringPointer("50Mi") + } + if conf.ClientConfig.StorageClassName == nil { + conf.ClientConfig.StorageClassName = util.StringPointer("standard") + } + if conf.ClientConfig.Dir == nil { + conf.ClientConfig.Dir = util.StringPointer("/metrics") + } + }) + if err != nil { + return nil, err + } + + cl, err := badgerdb.GetClient(badger.DefaultOptions(*conf.ClientConfig.Dir), badgerdb.AdditionalOptions{}) + if err != nil { + return nil, err + } + return cl, nil + + case "redis": + // redisConfig defines the configuration of a redis based metrics service + type mConfig struct { + redis.ClientConfig `json:"redis,omitempty"` + } + + conf := &mConfig{} + err := util.ReadConfig(metricsConfigFileEnv, conf, func() { + if conf.ClientConfig.Address == nil { + conf.ClientConfig.Address = util.StringPointer("redis:6379") + } + }) + if err != nil { + return nil, err + } + + cl, err := redis.GetClient(conf.ClientConfig) + if err != nil { + return nil, err + } + return cl, nil + + default: + return nil, fmt.Errorf("no metrics store implementation for %s", *conf.Implementation) + } +} diff --git a/storage/client/client_test.go b/storage/client/client_test.go new file mode 100644 index 000000000..0b6097613 --- /dev/null +++ b/storage/client/client_test.go @@ -0,0 +1,56 @@ +package client + +import ( + "os" + "testing" + + "github.com/alicebob/miniredis" + "github.com/stretchr/testify/assert" +) + +func TestGetClientRedis(t *testing.T) { + + server, _ := miniredis.Run() + assert.NotNil(t, server) + + metricsConfig := `port: 8080 +implementation: redis +redis: + address: ` + server.Addr() + + mf, err := os.CreateTemp("", "metrics*.yaml") + assert.NoError(t, err) + + err = os.Setenv(metricsConfigFileEnv, mf.Name()) + assert.NoError(t, err) + + _, err = mf.WriteString(metricsConfig) + assert.NoError(t, err) + + client, err := GetClient() + assert.NoError(t, err) + assert.NotNil(t, client) +} + +func TestGetClientBadger(t *testing.T) { + + tempDirPath := os.TempDir() + + metricsConfig := `port: 8080 +implementation: badgerdb +badgerdb: + dir: ` + tempDirPath + + mf, err := os.CreateTemp("", "metrics*.yaml") + assert.NoError(t, err) + + err = os.Setenv(metricsConfigFileEnv, mf.Name()) + assert.NoError(t, err) + + _, err = mf.WriteString(metricsConfig) + assert.NoError(t, err) + + client, err := GetClient() + assert.NoError(t, err) + assert.NotNil(t, client) +} diff --git a/storage/interface.go b/storage/interface.go index e75669196..243920fda 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -39,7 +39,8 @@ type VersionMetrics map[string]struct { // Interface enables interaction with a storage entity // Can be mocked in unit tests with fake implementation type Interface interface { - // returns a nested map of the metrics data for a particular application, version, and signature + // GetMerics returns all metrics for an app/version + // Returned result is a nested map of the metrics data // Example: // { // "my-metric": { @@ -59,16 +60,19 @@ type Interface interface { // } GetMetrics(applicationName string, version int, signature string) (*VersionMetrics, error) - // called by the A/B/n SDK gRPC API implementation (SDK for application clients) + // SetMetric records a metric value + // Called by the A/B/n SDK gRPC API implementation (SDK for application clients) // Example key: kt-metric::my-app::0::my-signature::my-metric::my-user::my-transaction-id -> my-metric-value (get the metric value with all the provided information) SetMetric(applicationName string, version int, signature, metric, user, transaction string, metricValue float64) error + // SetUser records the name of user // Example key: kt-users::my-app::0::my-signature::my-user -> true SetUser(applicationName string, version int, signature, user string) error - // get ExperimentResult for a particular namespace and experiment + // GetExperimentResult returns the experiment result for a particular namespace and experiment GetExperimentResult(namespace, experiment string) (*base.ExperimentResult, error) + // SetExperimentResult records an expeirment result // called by the A/B/n SDK gRPC API implementation (SDK for application clients) // Example key: kt-metric::my-app::0::my-signature::my-metric::my-user::my-transaction-id -> my-metric-value (get the metric value with all the provided information) SetExperimentResult(namespace, experiment string, data *base.ExperimentResult) error diff --git a/storage/redis/redis.go b/storage/redis/redis.go new file mode 100644 index 000000000..41dff008a --- /dev/null +++ b/storage/redis/redis.go @@ -0,0 +1,212 @@ +// Package redis implements the storage interface with Redis +package redis + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/iter8-tools/iter8/base" + "github.com/iter8-tools/iter8/storage" + "github.com/redis/go-redis/v9" +) + +// ClientConfig is configurable properties of a new BadgerDB client +type ClientConfig struct { + Address *string `json:"address,omitempty"` + Username *string `json:"username,omitempty"` + Password *string `json:"password,omitempty"` + DB *int `json:"db,omitempty"` +} + +// SetMetric records a metric value; see storage.Interface +func (cl Client) SetMetric(applicationName string, version int, signature, metric, user, transaction string, metricValue float64) error { + key, err := storage.GetMetricKey(applicationName, version, signature, metric, user, transaction) + if err != nil { + return err + } + + err = cl.rdb.Set(context.Background(), key, metricValue, 0).Err() + if err != nil { + return fmt.Errorf("cannot set metric with key \"%s\": %w", key, err) + } + + err = cl.SetUser(applicationName, version, signature, user) + return err +} + +// SetUser records the name of a user. See storage.Inferface +func (cl Client) SetUser(applicationName string, version int, signature, user string) error { + key := storage.GetUserKey(applicationName, version, signature, user) + + err := cl.rdb.Set(context.Background(), key, []byte("true"), 0).Err() + if err != nil { + return fmt.Errorf("cannot set metric with key \"%s\": %w", key, err) + } + return err +} + +// GetMetrics returns all metrics for an app/version. See storage.Inferface +func (cl Client) GetMetrics(applicationName string, version int, signature string) (*storage.VersionMetrics, error) { + metrics := storage.VersionMetrics{} + userCount, err := cl.getUserCount(applicationName, version, signature) + if err != nil { + return nil, err + } + + var currentMetric string + var currentUser string + + var cumulativeUserValue float64 + + var metricsOverTransactions []float64 + var metricsOverUsers []float64 + + prefix := storage.GetMetricKeyPrefix(applicationName, version, signature) + ctx := context.Background() + cursor := uint64(0) + it := cl.rdb.Scan(ctx, cursor, prefix+"*", int64(0)).Iterator() + for it.Next(ctx) { + key := it.Val() + tokens := strings.Split(key, "::") + if len(tokens) != 7 { + return nil, fmt.Errorf("incorrect number of tokens in metrics key") + } + metric := tokens[4] + user := tokens[5] + + value, err := cl.rdb.Get(ctx, key).Result() + if err != nil { + return nil, err + } + floatValue, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil, err + } + + if metric != currentMetric && currentMetric != "" { + metricsOverUsers = append(metricsOverUsers, cumulativeUserValue) + + // add 0s for all the users that did not produce metrics; for example, via Lookup() + diff := userCount - uint64(len(metricsOverUsers)) + for j := uint64(0); j < diff; j++ { + metricsOverUsers = append(metricsOverUsers, 0) + } + + metrics[currentMetric] = struct { + MetricsOverTransactions []float64 + MetricsOverUsers []float64 + }{ + MetricsOverTransactions: metricsOverTransactions, + MetricsOverUsers: metricsOverUsers, + } + + // currentMetric = "" + // currentUser = "" + cumulativeUserValue = 0 + metricsOverTransactions = []float64{} + metricsOverUsers = []float64{} + } + + metricsOverTransactions = append(metricsOverTransactions, floatValue) + + if user != currentUser && currentUser != "" { + metricsOverUsers = append(metricsOverUsers, cumulativeUserValue) + + cumulativeUserValue = 0 + } + cumulativeUserValue += floatValue + + currentMetric = metric + currentUser = user + } + + // flush last sequence of metric data + if currentMetric != "" || currentUser != "" { + metricsOverUsers = append(metricsOverUsers, cumulativeUserValue) + + // add 0s for all the users that did not produce metrics + // for example, via lookup() + if uint64(len(metricsOverUsers)) < userCount { + diff := userCount - uint64(len(metricsOverUsers)) + for j := uint64(0); j < diff; j++ { + metricsOverUsers = append(metricsOverUsers, 10) + } + } + + metrics[currentMetric] = struct { + MetricsOverTransactions []float64 + MetricsOverUsers []float64 + }{ + MetricsOverTransactions: metricsOverTransactions, + MetricsOverUsers: metricsOverUsers, + } + } + + return &metrics, nil +} + +// SetExperimentResult records an experiment result. See storage.Inferface +func (cl Client) SetExperimentResult(namespace, experiment string, data *base.ExperimentResult) error { + dataBytes, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("cannot JSON marshal ExperimentResult: %e", err) + } + + key := storage.GetExperimentResultKey(namespace, experiment) + err = cl.rdb.Set(context.Background(), key, dataBytes, 0).Err() + return err +} + +// GetExperimentResult returns an experiment result. See storage.Interface +func (cl Client) GetExperimentResult(namespace, experiment string) (*base.ExperimentResult, error) { + return storage.GetExperimentResult(func() ([]byte, error) { + return cl.rdb.Get(context.Background(), storage.GetExperimentResultKey(namespace, experiment)).Bytes() + }) +} + +// Client is a client for Redis +type Client struct { + rdb *redis.Client +} + +// GetClient returns a Redis client +func GetClient(config ClientConfig) (*Client, error) { + options := &redis.Options{} + options.Addr = *config.Address + options.Password = "" // default + options.DB = 0 // default + + if config.Username != nil { + options.Username = *config.Username + } + if config.Password != nil { + options.Password = *config.Password + } + if config.DB != nil { + options.DB = *config.DB + } + rdb := redis.NewClient(options) + + return &Client{ + rdb: rdb, + }, nil +} + +// getUserCount gets the number of users +func (cl Client) getUserCount(applicationName string, version int, signature string) (uint64, error) { + ctx := context.Background() + + count := uint64(0) + + prefix := storage.GetUserKeyPrefix(applicationName, version, signature) + cursor := uint64(0) + it := cl.rdb.Scan(ctx, cursor, prefix+"*", int64(0)).Iterator() + for it.Next(ctx) { + count++ + } + + return count, nil +} diff --git a/storage/redis/redis_test.go b/storage/redis/redis_test.go new file mode 100644 index 000000000..a6bb3baca --- /dev/null +++ b/storage/redis/redis_test.go @@ -0,0 +1,182 @@ +package redis + +import ( + "context" + "encoding/json" + "strconv" + "testing" + + "github.com/alicebob/miniredis" + "github.com/iter8-tools/iter8/base" + "github.com/iter8-tools/iter8/storage" + "github.com/stretchr/testify/assert" +) + +func TestSetMetric(t *testing.T) { + server, _ := miniredis.Run() + assert.NotNil(t, server) + + client, err := GetClient(ClientConfig{Address: base.StringPointer(server.Addr())}) + assert.NoError(t, err) + + app := "my-application" + version := 0 + signature := "my-signature" + metric := "my-metric" + user := "my-user" + transaction := "my-transaction" + value := 50.0 + + err = client.SetMetric(app, version, signature, metric, user, transaction, value) + assert.NoError(t, err) + + key, err := storage.GetMetricKey(app, version, signature, metric, user, transaction) + assert.NoError(t, err) + val, err := client.rdb.Get(context.Background(), key).Result() + assert.NoError(t, err) + fval, err := strconv.ParseFloat(string(val), 64) + assert.NoError(t, err) + + assert.Equal(t, value, fval) + + // SetMetric() should also add a user + userKey := storage.GetUserKey(app, version, signature, user) + u, err := client.rdb.Get(context.Background(), userKey).Result() + assert.NoError(t, err) + assert.Equal(t, "true", u) +} + +func TestSetMetricInvalid(t *testing.T) { + server, _ := miniredis.Run() + assert.NotNil(t, server) + + client, err := GetClient(ClientConfig{Address: base.StringPointer(server.Addr())}) + assert.NoError(t, err) + + err = client.SetMetric("invalid:application", 0, "signature", "metric", "user", "transaction", float64(0)) + assert.Error(t, err) +} + +func TestSetUser(t *testing.T) { + server, _ := miniredis.Run() + assert.NotNil(t, server) + + client, err := GetClient(ClientConfig{Address: base.StringPointer(server.Addr())}) + assert.NoError(t, err) + + app := "my-application" + version := 0 + signature := "my-signature" + user := "my-user" + + err = client.SetUser(app, version, signature, user) + assert.NoError(t, err) + + userKey := storage.GetUserKey(app, version, signature, user) + u, err := client.rdb.Get(context.Background(), userKey).Result() + assert.NoError(t, err) + assert.Equal(t, "true", u) +} + +// TestGetMetricsWithExtraUsers tests if GetMetrics adds 0 for all users that did not produce metrics +func TestGetMetricsWithExtraUsers(t *testing.T) { + server, _ := miniredis.Run() + assert.NotNil(t, server) + + client, err := GetClient(ClientConfig{Address: base.StringPointer(server.Addr())}) + assert.NoError(t, err) + + app := "my-application" + version := 0 + signature := "my-signature" + extraUser := "my-extra-user" + + err = client.SetUser(app, version, signature, extraUser) // extra user + assert.NoError(t, err) + + metric := "my-metric" + user := "my-user" + transaction := "my-transaction" + + err = client.SetMetric(app, version, signature, metric, user, transaction, 25) + assert.NoError(t, err) + + metric2 := "my-metric2" + + err = client.SetMetric(app, version, signature, metric2, user, transaction, 50) + assert.NoError(t, err) + + metrics, err := client.GetMetrics(app, version, signature) + assert.NoError(t, err) + + jsonMetrics, err := json.Marshal(metrics) + assert.NoError(t, err) + // 0s have been added to the MetricsOverUsers due to extraUser, [50,0] + assert.Equal(t, "{\"my-metric\":{\"MetricsOverTransactions\":[25],\"MetricsOverUsers\":[25,10]},\"my-metric2\":{\"MetricsOverTransactions\":[50],\"MetricsOverUsers\":[50,0]}}", string(jsonMetrics)) +} + +func TestGetMetrics(t *testing.T) { + server, _ := miniredis.Run() + assert.NotNil(t, server) + + client, err := GetClient(ClientConfig{Address: base.StringPointer(server.Addr())}) + assert.NoError(t, err) + + err = client.SetMetric("my-application", 0, "my-signature", "my-metric", "my-user", "my-transaction", 50.0) + assert.NoError(t, err) + err = client.SetMetric("my-application", 0, "my-signature", "my-metric", "my-user2", "my-transaction2", 10.0) + assert.NoError(t, err) + err = client.SetMetric("my-application", 1, "my-signature2", "my-metric2", "my-user", "my-transaction3", 20.0) + assert.NoError(t, err) + err = client.SetMetric("my-application", 2, "my-signature3", "my-metric3", "my-user2", "my-transaction4", 30.0) + assert.NoError(t, err) + err = client.SetMetric("my-application", 2, "my-signature3", "my-metric3", "my-user2", "my-transaction4", 40.0) // overwrites the previous set + assert.NoError(t, err) + + metrics, err := client.GetMetrics("my-application", 0, "my-signature") + assert.NoError(t, err) + jsonMetrics, err := json.Marshal(metrics) + assert.NoError(t, err) + assert.Equal(t, "{\"my-metric\":{\"MetricsOverTransactions\":[10,50],\"MetricsOverUsers\":[10,50]}}", string(jsonMetrics)) + + metrics, err = client.GetMetrics("my-application", 1, "my-signature2") + assert.NoError(t, err) + jsonMetrics, err = json.Marshal(metrics) + assert.NoError(t, err) + assert.Equal(t, "{\"my-metric2\":{\"MetricsOverTransactions\":[20],\"MetricsOverUsers\":[20]}}", string(jsonMetrics)) + + metrics, err = client.GetMetrics("my-application", 2, "my-signature3") + assert.NoError(t, err) + jsonMetrics, err = json.Marshal(metrics) + assert.NoError(t, err) + assert.Equal(t, "{\"my-metric3\":{\"MetricsOverTransactions\":[40],\"MetricsOverUsers\":[40]}}", string(jsonMetrics)) + + metrics, err = client.GetMetrics("my-application", 3, "my-signature") + assert.NoError(t, err) + jsonMetrics, err = json.Marshal(metrics) + assert.NoError(t, err) + assert.Equal(t, "{}", string(jsonMetrics)) +} + +func TestGetExperimentResult(t *testing.T) { + server, _ := miniredis.Run() + assert.NotNil(t, server) + + client, err := GetClient(ClientConfig{Address: base.StringPointer(server.Addr())}) + assert.NoError(t, err) + + namespace := "my-namespace" + experiment := "my-experiment" + + experimentResult := base.ExperimentResult{ + Name: experiment, + Namespace: namespace, + } + + err = client.SetExperimentResult(namespace, experiment, &experimentResult) + assert.NoError(t, err) + + result, err := client.GetExperimentResult(namespace, experiment) + assert.NoError(t, err) + assert.Equal(t, &experimentResult, result) +} diff --git a/storage/util.go b/storage/util.go index 492db8eaf..e612ec2db 100644 --- a/storage/util.go +++ b/storage/util.go @@ -1,6 +1,12 @@ package storage import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/iter8-tools/iter8/base" "golang.org/x/sys/unix" ) @@ -19,3 +25,71 @@ func GetVolumeUsage(path string) (uint64, uint64, error) { return availableBytes, totalBytes, nil } + +func validateKeyToken(s string) error { + if strings.Contains(s, ":") { + return errors.New("key token contains \":\"") + } + + return nil +} + +// GetMetricKeyPrefix returns the prefix of a metric key +func GetMetricKeyPrefix(applicationName string, version int, signature string) string { + return fmt.Sprintf("kt-metric::%s::%d::%s::", applicationName, version, signature) +} + +// GetMetricKey returns a metric key from the inputs +func GetMetricKey(applicationName string, version int, signature, metric, user, transaction string) (string, error) { + if err := validateKeyToken(applicationName); err != nil { + return "", errors.New("application name cannot have \":\"") + } + if err := validateKeyToken(signature); err != nil { + return "", errors.New("signature cannot have \":\"") + } + if err := validateKeyToken(metric); err != nil { + return "", errors.New("metric name cannot have \":\"") + } + if err := validateKeyToken(user); err != nil { + return "", errors.New("user name cannot have \":\"") + } + if err := validateKeyToken(transaction); err != nil { + return "", errors.New("transaction ID cannot have \":\"") + } + + return fmt.Sprintf("%s%s::%s::%s", GetMetricKeyPrefix(applicationName, version, signature), metric, user, transaction), nil +} + +// GetUserKeyPrefix returns the prefix of a user key +func GetUserKeyPrefix(applicationName string, version int, signature string) string { + prefix := fmt.Sprintf("kt-users::%s::%d::%s::", applicationName, version, signature) + return prefix +} + +// GetUserKey returns a user key from the inputs +func GetUserKey(applicationName string, version int, signature, user string) string { + key := fmt.Sprintf("%s%s", GetUserKeyPrefix(applicationName, version, signature), user) + return key +} + +// GetExperimentResultKey returns a performance experiment key from the inputs +func GetExperimentResultKey(namespace, experiment string) string { + // getExperimentResultKey() is just getUserPrefix() with the user appended at the end + return fmt.Sprintf("kt-result::%s::%s", namespace, experiment) +} + +// GetExperimentResult returns an experiment result retrieved from a key value store +func GetExperimentResult(fetch func() ([]byte, error)) (*base.ExperimentResult, error) { + value, err := fetch() + if err != nil { + return nil, err + } + + experimentResult := base.ExperimentResult{} + err = json.Unmarshal(value, &experimentResult) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal ExperimentResult: \"%s\": %e", string(value), err) + } + + return &experimentResult, err +} diff --git a/storage/util_test.go b/storage/util_test.go index 879972b8b..bc1bd212c 100644 --- a/storage/util_test.go +++ b/storage/util_test.go @@ -1,6 +1,7 @@ package storage import ( + "fmt" "os" "testing" @@ -25,3 +26,76 @@ func TestGetVolumeUsage(t *testing.T) { assert.Equal(t, uint64(0), totalBytes) assert.Equal(t, uint64(0), availableBytes) } + +type testmetrickey struct { + valid bool + application string + signature string + metric string + user string + transaction string +} + +func TestGetMetricKey(t *testing.T) { + for _, s := range []testmetrickey{ + {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, + {valid: false, application: "invalid:application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, + {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, + {valid: false, application: "application", signature: "invalid:signature", metric: "metric", user: "user", transaction: "transaction"}, + {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, + {valid: false, application: "application", signature: "signature", metric: "invalid:metric", user: "user", transaction: "transaction"}, + {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, + {valid: false, application: "application", signature: "signature", metric: "metric", user: "invalid:user", transaction: "transaction"}, + {valid: true, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "transaction"}, + {valid: false, application: "application", signature: "signature", metric: "metric", user: "user", transaction: "invalid:transaction"}, + } { + key, err := GetMetricKey(s.application, 0, s.signature, s.metric, s.user, s.transaction) + if s.valid { + assert.NoError(t, err) + assert.Equal(t, fmt.Sprintf("%s%s::%s::%s", GetMetricKeyPrefix(s.application, 0, s.signature), s.metric, s.user, s.transaction), key) + } else { + assert.Error(t, err) + assert.Equal(t, "", key) + } + } +} + +func TestValidateKeyToken(t *testing.T) { + err := validateKeyToken("hello") + assert.NoError(t, err) + + err = validateKeyToken("::") + assert.Error(t, err) + + err = validateKeyToken("hello::world") + assert.Error(t, err) + + err = validateKeyToken("hello :: world") + assert.Error(t, err) + + err = validateKeyToken("hello:world") + assert.Error(t, err) + + err = validateKeyToken("hello : world") + assert.Error(t, err) +} + +func TestGetUserPrefix(t *testing.T) { + assert.Equal(t, "kt-users::app::0::abc::", GetUserKeyPrefix("app", 0, "abc")) +} + +func TestGetUserKey(t *testing.T) { + assert.Equal(t, "kt-users::app::0::abc::user", GetUserKey("app", 0, "abc", "user")) +} + +func TestGetExperimentResultKey(t *testing.T) { + assert.Equal(t, "kt-result::ns::name", GetExperimentResultKey("ns", "name")) +} + +func TestGetExperimentResult(t *testing.T) { + _, err := GetExperimentResult(func() ([]byte, error) { return []byte{}, nil }) + assert.ErrorContains(t, err, "cannot unmarshal ExperimentResult") + + _, err = GetExperimentResult(func() ([]byte, error) { return []byte{}, fmt.Errorf("test error") }) + assert.ErrorContains(t, err, "test error") +}