Skip to content

Commit

Permalink
Add metrics proxy
Browse files Browse the repository at this point in the history
  • Loading branch information
vthapar authored and tpantelis committed Sep 8, 2022
1 parent 831d21f commit 0104812
Show file tree
Hide file tree
Showing 9 changed files with 137 additions and 31 deletions.
29 changes: 10 additions & 19 deletions controllers/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,24 +34,11 @@ import (
controllerClient "sigs.k8s.io/controller-runtime/pkg/client"
)

func Setup(namespace string, owner metav1.Object, labels map[string]string, port int32,
client controllerClient.Client, config *rest.Config, scheme *runtime.Scheme,
reqLogger logr.Logger,
func Setup(serviceName, namespace, applicationKey, applicationName string, owner metav1.Object, port int32,
client controllerClient.Client, config *rest.Config, scheme *runtime.Scheme, reqLogger logr.Logger,
) error {
applicationKey := "app"
applicationName, ok := labels[applicationKey]

if !ok {
applicationKey = "name"
applicationName, ok = labels[applicationKey]

if !ok {
return fmt.Errorf("no app or name label in the provided labels, %v", labels)
}
}

metricsService, err := apply.Service(owner, newMetricsService(namespace, applicationKey, applicationName, port), reqLogger,
client, scheme)
metricsService, err := apply.Service(owner, newMetricsService(serviceName, namespace, applicationKey,
applicationName, port), reqLogger, client, scheme)
if err != nil {
return err // nolint:wrapcheck // No need to wrap here
}
Expand All @@ -76,11 +63,15 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port

// newMetricsService populates a Service providing access to metrics for the given application.
// The Service is named after the application name, suffixed with "-metrics".
func newMetricsService(namespace, appKey, appName string, port int32) *corev1.Service {
func newMetricsService(name, namespace, appKey, appName string, port int32) *corev1.Service {
labels := map[string]string{
appKey: appName,
}

if name == "" {
name = appName
}

servicePorts := []corev1.ServicePort{
{Port: port, Name: "metrics", Protocol: corev1.ProtocolTCP, TargetPort: intstr.IntOrString{
Type: intstr.Int,
Expand All @@ -92,7 +83,7 @@ func newMetricsService(namespace, appKey, appName string, port int32) *corev1.Se
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
Namespace: namespace,
Name: fmt.Sprintf("%s-metrics", appName),
Name: fmt.Sprintf("%s-metrics", name),
},
Spec: corev1.ServiceSpec{
Ports: servicePorts,
Expand Down
8 changes: 4 additions & 4 deletions controllers/servicediscovery/servicediscovery_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -692,8 +692,8 @@ func (r *Reconciler) ensureLightHouseAgent(instance *submarinerv1alpha1.ServiceD
return errors.Wrap(err, "error reconciling agent deployment")
}

err := metrics.Setup(instance.Namespace, instance, lightHouseAgent.GetLabels(), 8082, r.ScopedClient,
r.RestConfig, r.Scheme, reqLogger)
err := metrics.Setup(names.ServiceDiscoveryComponent, instance.Namespace, "app", names.ServiceDiscoveryComponent,
instance, 8082, r.ScopedClient, r.RestConfig, r.Scheme, reqLogger)
if err != nil {
return errors.Wrap(err, "error setting up metrics")
}
Expand All @@ -709,8 +709,8 @@ func (r *Reconciler) ensureLighthouseCoreDNSDeployment(instance *submarinerv1alp
return errors.Wrap(err, "error reconciling coredns deployment")
}

err := metrics.Setup(instance.Namespace, instance, lighthouseCoreDNSDeployment.GetLabels(), 9153, r.ScopedClient, r.RestConfig,
r.Scheme, reqLogger)
err := metrics.Setup(names.LighthouseCoreDNSComponent, instance.Namespace, "app", names.LighthouseCoreDNSComponent, instance,
9153, r.ScopedClient, r.RestConfig, r.Scheme, reqLogger)
if err != nil {
return errors.Wrap(err, "error setting up coredns metrics")
}
Expand Down
5 changes: 3 additions & 2 deletions controllers/submariner/gateway_resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner, name string, podSelectorLabe
{Name: "SUBMARINER_HEALTHCHECKENABLED", Value: strconv.FormatBool(healthCheckEnabled)},
{Name: "SUBMARINER_HEALTHCHECKINTERVAL", Value: strconv.FormatUint(healthCheckInterval, 10)},
{Name: "SUBMARINER_HEALTHCHECKMAXPACKETLOSSCOUNT", Value: strconv.FormatUint(healthCheckMaxPacketLossCount, 10)},
{Name: "SUBMARINER_METRICSPORT", Value: gatewayMetricsServerPort},
{Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
Expand Down Expand Up @@ -253,8 +254,8 @@ func (r *Reconciler) reconcileGatewayDaemonSet(
return nil, err
}

err = metrics.Setup(instance.Namespace, instance, daemonSet.GetLabels(), gatewayMetricsServerPort, r.config.ScopedClient,
r.config.RestConfig, r.config.Scheme, reqLogger)
err = metrics.Setup(names.GatewayComponent, instance.Namespace, "app", names.MetricsProxyComponent, instance, gatewayMetricsServicePort,
r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger)

return daemonSet, err
}
Expand Down
5 changes: 3 additions & 2 deletions controllers/submariner/globalnet_resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ func (r *Reconciler) reconcileGlobalnetDaemonSet(instance *v1alpha1.Submariner,
return nil, err
}

err = metrics.Setup(instance.Namespace, instance, daemonSet.GetLabels(), globalnetMetricsServerPort,
r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger)
err = metrics.Setup(names.GlobalnetComponent, instance.Namespace, "app", names.MetricsProxyComponent,
instance, globalnetMetricsServicePort, r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger)

return daemonSet, err
}
Expand Down Expand Up @@ -92,6 +92,7 @@ func newGlobalnetDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.DaemonS
{Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace},
{Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.ClusterID},
{Name: "SUBMARINER_EXCLUDENS", Value: "submariner-operator,kube-system,operators,openshift-monitoring,openshift-dns"},
{Name: "SUBMARINER_METRICSPORT", Value: globalnetMetricsServerPort},
{Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
Expand Down
97 changes: 97 additions & 0 deletions controllers/submariner/metrics_proxy_resources.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
SPDX-License-Identifier: Apache-2.0
Copyright Contributors to the Submariner project.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package submariner

import (
"fmt"

"github.com/go-logr/logr"
"github.com/submariner-io/submariner-operator/api/v1alpha1"
"github.com/submariner-io/submariner-operator/controllers/apply"
"github.com/submariner-io/submariner-operator/pkg/images"
"github.com/submariner-io/submariner-operator/pkg/names"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// nolint:wrapcheck // No need to wrap errors here.
func (r *Reconciler) reconcileMetricsProxyDaemonSet(instance *v1alpha1.Submariner, reqLogger logr.Logger) (*appsv1.DaemonSet,
error,
) {
return apply.DaemonSet(instance, newMetricsProxyDaemonSet(instance), reqLogger,
r.config.ScopedClient, r.config.Scheme)
}

func newMetricsProxyDaemonSet(cr *v1alpha1.Submariner) *appsv1.DaemonSet {
labels := map[string]string{
"app": names.MetricsProxyComponent,
"component": "metrics",
}

daemonSet := &appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: cr.Namespace,
Name: names.MetricsProxyComponent,
Labels: labels,
},
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
"app": names.MetricsProxyComponent,
}},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
*metricProxyContainer(cr, "gateway-metrics-proxy", fmt.Sprint(gatewayMetricsServicePort), gatewayMetricsServerPort),
},
NodeSelector: map[string]string{"submariner.io/gateway": "true"},
// The MetricsProxy Pod must be able to run on any flagged node, regardless of existing taints
Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}},
},
},
},
}

if cr.Spec.GlobalCIDR != "" {
daemonSet.Spec.Template.Spec.Containers = append(daemonSet.Spec.Template.Spec.Containers,
*metricProxyContainer(cr, "globalnet-metrics-proxy", fmt.Sprint(globalnetMetricsServicePort), globalnetMetricsServerPort))
}

return daemonSet
}

func metricProxyContainer(cr *v1alpha1.Submariner, name, hostPort, podPort string) *corev1.Container {
return &corev1.Container{
Name: name,
Image: getImagePath(cr, names.MetricsProxyImage, names.MetricsProxyComponent),
ImagePullPolicy: images.GetPullPolicy(cr.Spec.Version, cr.Spec.ImageOverrides[names.MetricsProxyComponent]),
Env: []corev1.EnvVar{
{Name: "NODE_IP", ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
}},
},
Command: []string{"/usr/bin/nc"},
Args: []string{"-v", "-lk", "-p", hostPort, "-e", "nc", "$(NODE_IP)", podPort},
}
}
12 changes: 10 additions & 2 deletions controllers/submariner/submariner_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,10 @@ import (
)

const (
gatewayMetricsServerPort = 8080
globalnetMetricsServerPort = 8081
gatewayMetricsServicePort = 8080
globalnetMetricsServicePort = 8081
gatewayMetricsServerPort = "32780"
globalnetMetricsServerPort = "32781"
)

var log = logf.Log.WithName("controller_submariner")
Expand Down Expand Up @@ -183,6 +185,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
}
}

if _, err = r.reconcileMetricsProxyDaemonSet(instance, reqLogger); err != nil {
return reconcile.Result{}, err
}

if err := r.reconcileNetworkPluginSyncerDeployment(instance, clusterNetwork, reqLogger); err != nil {
return reconcile.Result{}, err
}
Expand Down Expand Up @@ -227,6 +233,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
return reconcile.Result{}, err
}

// TODO: vthapar Add metrics-proxy status to Submariner CR so we can update it with daemonset status

if loadBalancer != nil {
instance.Status.LoadBalancerStatus.Status = &loadBalancer.Status.LoadBalancer
} else {
Expand Down
4 changes: 2 additions & 2 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func main() {
log.Info("Setting up metrics services and monitors")

// Setup the metrics services and service monitors
labels := map[string]string{"name": os.Getenv("OPERATOR_NAME")}
name := os.Getenv("OPERATOR_NAME")

// We need a new client using the manager's rest.Config because
// the manager's caches haven't started yet and it won't allow
Expand All @@ -163,7 +163,7 @@ func main() {
log.Error(err, "Error obtaining a Kubernetes client")
}

if err := metrics.Setup(namespace, nil, labels, metricsPort, metricsClient, cfg, scheme, log); err != nil {
if err := metrics.Setup(name, namespace, "name", name, nil, metricsPort, metricsClient, cfg, scheme, log); err != nil {
log.Error(err, "Error setting up metrics services and monitors")
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/names/names.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const (
OperatorComponent = "submariner-operator"
ServiceDiscoveryCrName = "service-discovery"
SubmarinerCrName = "submariner"
MetricsProxyComponent = "submariner-metrics-proxy"
)

/* These values are used by downstream distributions to override the component default image name. */
Expand All @@ -42,6 +43,7 @@ var (
ServiceDiscoveryImage = "lighthouse-agent"
LighthouseCoreDNSImage = "lighthouse-coredns"
OperatorImage = "submariner-operator"
MetricsProxyImage = "nettest"
)

var ValidImageNames = []string{
Expand Down
6 changes: 6 additions & 0 deletions release-notes/20220908-metrics-proxy.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<!-- markdownlint-disable MD041 -->
Users no longer need to open ports `8080` and `8081` on the host for querying metrics. A new `submariner-metrics-proxy`
DaemonSet runs pods on gateway nodes and forwards http requests for metrics services to gateway and globalnet pods running
on the nodes. Gateway and Globalnet pods now listen on ports `32780` and `32781` instead of well known ports `8080` and
`8081` to avoid conflict with any other services that might be using those ports. Users will continue to query existing
`submariner-gateway-metrics` and `submariner-globalnet-metrics` services to query the metrics.

0 comments on commit 0104812

Please sign in to comment.