Skip to content

Commit

Permalink
fix: remove missing 404 bug (#1006)
Browse files Browse the repository at this point in the history 
Signed-off-by: realanna <[email protected]>
  • Loading branch information
@RealAnna
RealAnna authored Mar 9, 2023
1 parent 4bf2919 commit e8c0f38
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 44 deletions.
30 changes: 19 additions & 11 deletions metrics-operator/pkg/metrics/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"sync"
Expand All @@ -16,8 +15,10 @@ import (
"github.com/gorilla/mux"
metricsapi "github.com/keptn/lifecycle-toolkit/metrics-operator/api/v1alpha2"
"github.com/open-feature/go-sdk/pkg/openfeature"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -27,8 +28,6 @@ type Metrics struct {
gauges map[string]prometheus.Gauge
}

var metrics Metrics

var instance *serverManager
var smOnce sync.Once

Expand All @@ -38,13 +37,13 @@ type serverManager struct {
ofClient *openfeature.Client
exposeMetrics bool
k8sClient client.Client
metrics Metrics
}

// StartServerManager starts a server manager to expose metrics and runs until
// the context is cancelled (i.e. an env variable gets changes and pod is restarted)
func StartServerManager(ctx context.Context, client client.Client, ofClient *openfeature.Client, exposeMetrics bool, interval time.Duration) {
smOnce.Do(func() {
metrics.gauges = make(map[string]prometheus.Gauge)
instance = &serverManager{
ticker: clock.New().Ticker(interval),
ofClient: ofClient,
Expand Down Expand Up @@ -108,6 +107,9 @@ func (m *serverManager) setup() error {
klog.Infof("Keptn Metrics server enabled: %v", serverEnabled)

if serverEnabled && m.server == nil {

m.metrics.gauges = make(map[string]prometheus.Gauge)

klog.Infof("serving Prometheus metrics at localhost:9999/metrics")
klog.Infof("serving KeptnMetrics at localhost:9999/api/v1/metrics/{namespace}/{metric}")

Expand Down Expand Up @@ -142,10 +144,15 @@ func (m *serverManager) returnMetric(w http.ResponseWriter, r *http.Request) {
namespace := vars["namespace"]
metric := vars["metric"]

metricObj := metricsapi.KeptnMetric{}
err := m.k8sClient.Get(context.Background(), types.NamespacedName{Name: metric, Namespace: namespace}, &metricObj)
metricObj := &metricsapi.KeptnMetric{}
err := m.k8sClient.Get(context.Background(), types.NamespacedName{Name: metric, Namespace: namespace}, metricObj)
if err != nil {
fmt.Println("failed to list keptn-metrics: " + err.Error())
//nolint:errorlint
if status, ok := err.(k8serrors.APIStatus); ok || errors.As(err, &status) {
w.WriteHeader(int(status.Status().Code))
}
return
}

data := map[string]string{
Expand All @@ -157,8 +164,9 @@ func (m *serverManager) returnMetric(w http.ResponseWriter, r *http.Request) {
err = json.NewEncoder(w).Encode(data)
if err != nil {
fmt.Println("failed to encode data")
os.Exit(1)
w.WriteHeader(http.StatusUnprocessableEntity)
}

}

func (m *serverManager) recordMetrics() {
Expand All @@ -174,15 +182,15 @@ func (m *serverManager) recordMetrics() {
}
for _, metric := range list.Items {
normName := normalizeMetricName(metric.Name)
if _, ok := metrics.gauges[normName]; !ok {
metrics.gauges[normName] = prometheus.NewGauge(prometheus.GaugeOpts{
if _, ok := m.metrics.gauges[normName]; !ok {
m.metrics.gauges[normName] = prometheus.NewGauge(prometheus.GaugeOpts{
Name: normName,
Help: metric.Name,
})
prometheus.MustRegister(metrics.gauges[normName])
prometheus.MustRegister(m.metrics.gauges[normName])
}
val, _ := strconv.ParseFloat(metric.Status.Value, 64)
metrics.gauges[normName].Set(val)
m.metrics.gauges[normName].Set(val)
}
<-time.After(10 * time.Second)
}
Expand Down
118 changes: 85 additions & 33 deletions metrics-operator/pkg/metrics/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import (
"bytes"
"context"
"net/http"
"os"
"testing"
"time"

"github.com/benbjohnson/clock"
metricsapi "github.com/keptn/lifecycle-toolkit/metrics-operator/api/v1alpha2"
"github.com/open-feature/go-sdk/pkg/openfeature"
"github.com/stretchr/testify/require"
Expand All @@ -15,8 +17,40 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

func TestMain(m *testing.M) {
err := metricsapi.AddToScheme(scheme.Scheme)
if err != nil {
panic("BAD SCHEME!")
}
code := m.Run()
os.Exit(code)
}

func TestMetricServer_disabledServer(t *testing.T) {

tInstance := &serverManager{
ticker: clock.New().Ticker(3 * time.Second),
ofClient: openfeature.NewClient("klt-test"),
exposeMetrics: false,
k8sClient: fake.NewClientBuilder().WithScheme(scheme.Scheme).Build(),
}
tInstance.start(context.Background())

var err error
require.Eventually(t, func() bool {
cli := &http.Client{}
req, _ := http.NewRequestWithContext(context.TODO(), http.MethodGet, "http://localhost:9999/metrics", nil)
_, err = cli.Do(req)
return err != nil
}, 30*time.Second, 3*time.Second)

require.Contains(t, err.Error(), "connection refused")

}

func TestMetricServer_happyPath(t *testing.T) {
metric := metricsapi.KeptnMetric{

var metric = metricsapi.KeptnMetric{
ObjectMeta: v1.ObjectMeta{
Name: "sample-metric",
Namespace: "keptn-lifecycle-toolkit-system",
Expand All @@ -28,25 +62,34 @@ func TestMetricServer_happyPath(t *testing.T) {
Query: "query",
FetchIntervalSeconds: 5,
},
Status: metricsapi.KeptnMetricStatus{
Value: "12",
RawValue: nil,
LastUpdated: v1.Time{
Time: time.Now(),
},
},
}

err := metricsapi.AddToScheme(scheme.Scheme)
require.Nil(t, err)
k8sClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(&metric).Build()

ctx, cancel := context.WithCancel(context.Background())

StartServerManager(ctx, k8sClient, openfeature.NewClient("klt-test"), true, 3*time.Second)
tInstance := &serverManager{
ticker: clock.New().Ticker(3 * time.Second),
ofClient: openfeature.NewClient("klt-test"),
exposeMetrics: true,
k8sClient: k8sClient,
}
tInstance.start(context.Background())

require.Eventually(t, func() bool {
return instance.server != nil
}, 10*time.Second, time.Second)
return tInstance.server != nil
}, 30*time.Second, time.Second)

var resp *http.Response
var err error

require.Eventually(t, func() bool {
cli := &http.Client{}
req, err2 := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:9999/metrics", nil)
req, err2 := http.NewRequestWithContext(context.TODO(), http.MethodGet, "http://localhost:9999/metrics", nil)
require.Nil(t, err2)
resp, err = cli.Do(req)
return err == nil
Expand All @@ -61,41 +104,50 @@ func TestMetricServer_happyPath(t *testing.T) {

require.Contains(t, newStr, "# TYPE sample_metric gauge")

cancel()

require.Eventually(t, func() bool {
return instance.server == nil
cli := &http.Client{}
req, _ := http.NewRequestWithContext(context.TODO(), http.MethodGet, "http://localhost:9999/api/v1/metrics/keptn-lifecycle-toolkit-system/sample-metric", nil)
resp, err = cli.Do(req)
return err == nil
}, 10*time.Second, time.Second)

defer resp.Body.Close()

buf = new(bytes.Buffer)
_, err = buf.ReadFrom(resp.Body)
require.Nil(t, err)
newStr = buf.String()

require.Contains(t, newStr, "\"metric\":\"sample-metric\",\"namespace\":\"keptn-lifecycle-toolkit-system\",\"value\":\"12\"")
}

func TestMetricServer_disabledServer(t *testing.T) {
err2 := metricsapi.AddToScheme(scheme.Scheme)
require.Nil(t, err2)
k8sClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()
func TestMetricServer_noMetric(t *testing.T) {

ctx, cancel := context.WithCancel(context.Background())
k8sClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()

StartServerManager(ctx, k8sClient, openfeature.NewClient("klt-test"), false, 3*time.Second)
tInstance := &serverManager{
ticker: clock.New().Ticker(3 * time.Second),
ofClient: openfeature.NewClient("klt-test"),
exposeMetrics: true,
k8sClient: k8sClient,
}
tInstance.start(context.Background())

require.Eventually(t, func() bool {
return instance.server == nil
}, 30*time.Second, 3*time.Second)
return tInstance.server != nil
}, 30*time.Second, time.Second)

var resp *http.Response
var err error

require.Eventually(t, func() bool {
cli := &http.Client{}
req, err2 := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:9999/metrics", nil)
require.Nil(t, err2)
_, err = cli.Do(req)
return err != nil
}, 30*time.Second, 3*time.Second)

require.Contains(t, err.Error(), "connection refused")
req, _ := http.NewRequestWithContext(context.TODO(), http.MethodGet, "http://localhost:9999/api/v1/metrics/default/sample", nil)
resp, err = cli.Do(req)
return err == nil
}, 10*time.Second, time.Second)

cancel()
defer resp.Body.Close()
stat := resp.StatusCode
require.Equal(t, 404, stat)

require.Eventually(t, func() bool {
return instance.server == nil
}, 30*time.Second, 3*time.Second)
}

0 comments on commit e8c0f38

Please sign in to comment.