diff --git a/e2e/go.mod b/e2e/go.mod index ffb76a011..5cb6ad649 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -10,6 +10,7 @@ require ( github.com/cisco-open/operator-tools v0.36.0 github.com/kube-logging/logging-operator v0.0.0-20240924153554-c26d75f96766 github.com/kube-logging/logging-operator/pkg/sdk v0.11.0 + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.1 github.com/spf13/cast v1.7.0 github.com/stretchr/testify v1.9.0 helm.sh/helm/v3 v3.16.1 @@ -18,6 +19,7 @@ require ( k8s.io/apimachinery v0.31.1 k8s.io/client-go v0.31.1 sigs.k8s.io/controller-runtime v0.19.0 + sigs.k8s.io/e2e-framework v0.5.0 ) require ( @@ -113,7 +115,6 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.1 // indirect github.com/prometheus/client_golang v1.20.2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect @@ -124,6 +125,7 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/vladimirvivien/gexe v0.3.0 // indirect github.com/wayneashleyberry/terminal-dimensions v1.1.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect diff --git a/e2e/go.sum b/e2e/go.sum index e7d2aac74..734ba5c33 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -382,6 +382,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vladimirvivien/gexe v0.3.0 h1:4xwiOwGrDob5OMR6E92B9olDXYDglXdHhzR1ggYtWJM= +github.com/vladimirvivien/gexe v0.3.0/go.mod h1:fp7cy60ON1xjhtEI/+bfSEIXX35qgmI+iRYlGOqbBFM= github.com/wayneashleyberry/terminal-dimensions v1.1.0 h1:EB7cIzBdsOzAgmhTUtTTQXBByuPheP/Zv1zL2BRPY6g= github.com/wayneashleyberry/terminal-dimensions v1.1.0/go.mod h1:2lc/0eWCObmhRczn2SdGSQtgBooLUzIotkkEGXqghyg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -572,6 +574,8 @@ oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/e2e-framework v0.5.0 h1:YLhk8R7EHuTFQAe6Fxy5eBzn5Vb+yamR5u8MH1Rq3cE= +sigs.k8s.io/e2e-framework v0.5.0/go.mod h1:jJSH8u2RNmruekUZgHAtmRjb5Wj67GErli9UjLSY7Zc= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g= diff --git a/e2e/metrics-servicemonitors/metrics_servicemonitors_test.go b/e2e/metrics-servicemonitors/metrics_servicemonitors_test.go new file mode 100644 index 000000000..1266c7bc7 --- /dev/null +++ b/e2e/metrics-servicemonitors/metrics_servicemonitors_test.go @@ -0,0 +1,253 @@ +// Copyright © 2021 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics_servicemonitors_test + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/cluster" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1" + + "github.com/kube-logging/logging-operator/e2e/common" + "github.com/kube-logging/logging-operator/e2e/common/setup" + "sigs.k8s.io/e2e-framework/third_party/helm" +) + +var serviceMonitorNames = map[string]bool{ + "metrics-test-syslog-ng-metrics": true, + "metrics-test-syslog-ng-buffer-metrics": true, + "metrics-test-fluentd-metrics": true, + "metrics-test-fluentbit-metrics": true, +} + +var TestTempDir string + +func init() { + var ok bool + TestTempDir, ok = os.LookupEnv("PROJECT_DIR") + if !ok { + TestTempDir = "../.." + } + TestTempDir = filepath.Join(TestTempDir, "build/_test") + err := os.MkdirAll(TestTempDir, os.FileMode(0755)) + if err != nil { + panic(err) + } +} + +func TestMetrics_ServiceMonitors(t *testing.T) { + common.Initialize(t) + ns := "test" + releaseNameOverride := "e2e" + common.WithCluster("syslog-ng-metrics", t, func(t *testing.T, c common.Cluster) { + setup.LoggingOperator(t, c, setup.LoggingOperatorOptionFunc(func(options *setup.LoggingOperatorOptions) { + options.Namespace = ns + options.NameOverride = releaseNameOverride + })) + + ctx := context.Background() + + err := installPrometheusOperator(c) + common.RequireNoError(t, err) + + logging := v1beta1.Logging{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + Namespace: ns, + }, + Spec: v1beta1.LoggingSpec{ + ControlNamespace: ns, + FluentbitSpec: &v1beta1.FluentbitSpec{ + Metrics: &v1beta1.Metrics{ + ServiceMonitor: true, + }, + }, + SyslogNGSpec: &v1beta1.SyslogNGSpec{ + Metrics: &v1beta1.Metrics{ + ServiceMonitor: true, + }, + BufferVolumeMetrics: &v1beta1.BufferMetrics{ + Metrics: v1beta1.Metrics{ + ServiceMonitor: true, + }, + }, + }, + }, + } + common.RequireNoError(t, c.GetClient().Create(ctx, &logging)) + + common.RequireNoError(t, checkServiceMonitorAvailability(t, ctx, c, ns)) + + defer startPortForwardingToService(t, "metrics-test-fluentbit-metrics", ns, "2020:2020") + rawOut, err := getRawOutputFromService(c, "2020", "/api/v1/metrics/prometheus") + common.RequireNoError(t, err) + + fmt.Print(rawOut) + + defer startPortForwardingToService(t, "metrics-test-syslog-ng-metrics", ns, "9577:9577") + rawOut, err = getRawOutputFromService(c, "9577", "/metrics") + common.RequireNoError(t, err) + + fmt.Print(rawOut) + + common.RequireNoError(t, c.GetClient().Delete(ctx, &logging)) + + loggingPatch := v1beta1.Logging{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + Namespace: ns, + }, + Spec: v1beta1.LoggingSpec{ + ControlNamespace: ns, + FluentbitSpec: &v1beta1.FluentbitSpec{ + Metrics: &v1beta1.Metrics{ + ServiceMonitor: true, + }, + }, + FluentdSpec: &v1beta1.FluentdSpec{ + Metrics: &v1beta1.Metrics{ + ServiceMonitor: true, + }, + }, + }, + } + common.RequireNoError(t, c.GetClient().Create(ctx, &loggingPatch)) + + defer startPortForwardingToService(t, "metrics-test-fluentd-metrics", ns, "24231:24231") + rawOut, err = getRawOutputFromService(c, "24231", "/metrics") + common.RequireNoError(t, err) + + fmt.Print(rawOut) + + }, func(t *testing.T, c common.Cluster) error { + path := filepath.Join(TestTempDir, fmt.Sprintf("cluster-%s.log", t.Name())) + t.Logf("Printing cluster logs to %s", path) + err := c.PrintLogs(common.PrintLogConfig{ + Namespaces: []string{ns, "default"}, + FilePath: path, + Limit: 100 * 1000, + }) + if err != nil { + return err + } + + loggingOperatorName := "logging-operator-" + releaseNameOverride + t.Logf("Collecting coverage files from logging-operator: %s/%s", ns, loggingOperatorName) + err = c.CollectTestCoverageFiles(ns, loggingOperatorName) + if err != nil { + t.Logf("Failed collecting coverage files: %s", err) + } + return err + + }, func(o *cluster.Options) { + if o.Scheme == nil { + o.Scheme = runtime.NewScheme() + } + common.RequireNoError(t, v1beta1.AddToScheme(o.Scheme)) + common.RequireNoError(t, apiextensionsv1.AddToScheme(o.Scheme)) + common.RequireNoError(t, appsv1.AddToScheme(o.Scheme)) + common.RequireNoError(t, batchv1.AddToScheme(o.Scheme)) + common.RequireNoError(t, corev1.AddToScheme(o.Scheme)) + common.RequireNoError(t, rbacv1.AddToScheme(o.Scheme)) + }) +} + +func installPrometheusOperator(c common.Cluster) error { + manager := helm.New(c.KubeConfigFilePath()) + + err := manager.RunRepo(helm.WithArgs("add", "prometheus-community", "https://prometheus-community.github.io/helm-charts")) + if err != nil { + return err + } + + err = manager.RunInstall( + helm.WithName("prometheus"), + helm.WithChart("prometheus-community/kube-prometheus-stack"), + helm.WithArgs("--create-namespace"), + helm.WithNamespace("monitoring"), + helm.WithArgs("--set", "prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false"), + helm.WithArgs("--set", "prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false"), + helm.WithWait(), + ) + if err != nil { + return err + } + + return nil +} + +func checkServiceMonitorAvailability(t *testing.T, ctx context.Context, c common.Cluster, ns string) error { + serviceMonitors := &v1.ServiceMonitorList{} + common.RequireNoError(t, c.GetClient().List(ctx, serviceMonitors)) + + for _, sm := range serviceMonitors.Items { + if sm.Namespace == ns { + delete(serviceMonitorNames, sm.Name) + } + } + + var errorMsg strings.Builder + for name := range serviceMonitorNames { + errorMsg.WriteString(fmt.Sprintf("ServiceMonitor %s not found\n", name)) + } + + if errorMsg.Len() > 0 { + return errors.New(errorMsg.String()) + } + + return nil +} + +func startPortForwardingToService(t *testing.T, svcName, ns, portMapping string) func() { + args := []string{"port-forward", fmt.Sprintf("svc/%s", svcName), portMapping, "-n", ns} + cmd := exec.Command("kubectl", args...) + cmd.Stderr = os.Stderr // Redirect stderr to test output + err := cmd.Start() + common.RequireNoError(t, err) + + // Wait for port forwarding to be established + time.Sleep(5 * time.Second) + + return func() { + _ = cmd.Process.Kill() + } +} + +func getRawOutputFromService(c common.Cluster, port, path string) ([]byte, error) { + cmd := common.CmdEnv(exec.Command("curl", fmt.Sprintf("localhost:%s%s", port, path)), c) + rawOut, err := cmd.Output() + if err != nil { + return nil, err + } + + return rawOut, nil +}