Skip to content

Commit

Permalink
Add metrics proxy
Browse files Browse the repository at this point in the history
  • Loading branch information
vthapar committed Aug 18, 2022
1 parent d4eafa0 commit 3e51e77
Show file tree
Hide file tree
Showing 8 changed files with 150 additions and 10 deletions.
6 changes: 5 additions & 1 deletion controllers/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,8 @@ limitations under the License.

package constants

const CleanupFinalizer = "controllers.submariner.io/cleanup"
const (
CleanupFinalizer = "controllers.submariner.io/cleanup"
MetricsGWPort = "32780"
MetricsGNPort = "32781"
)
12 changes: 8 additions & 4 deletions controllers/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
controllerClient "sigs.k8s.io/controller-runtime/pkg/client"
)

func Setup(namespace string, owner metav1.Object, labels map[string]string, port int32,
func Setup(name, namespace string, owner metav1.Object, labels map[string]string, port int32,
client controllerClient.Client, config *rest.Config, scheme *runtime.Scheme,
reqLogger logr.Logger,
) error {
Expand All @@ -43,7 +43,7 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port
return fmt.Errorf("no app label in the provided labels, %v", labels)
}

metricsService, err := helpers.ReconcileService(owner, newMetricsService(namespace, app, port), reqLogger, client, scheme)
metricsService, err := helpers.ReconcileService(owner, newMetricsService(name, namespace, app, port), reqLogger, client, scheme)
if err != nil {
return err // nolint:wrapcheck // No need to wrap here
}
Expand All @@ -69,11 +69,15 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port
// newMetricsService populates a Service providing access to metrics for the given application.
// It is assumed that the application's resources are labeled with "app=" the given app name.
// The Service is named after the application name, suffixed with "-metrics".
func newMetricsService(namespace, app string, port int32) *corev1.Service {
func newMetricsService(name, namespace, app string, port int32) *corev1.Service {
labels := map[string]string{
"app": app,
}

if name == "" {
name = app
}

servicePorts := []corev1.ServicePort{
{Port: port, Name: "metrics", Protocol: corev1.ProtocolTCP, TargetPort: intstr.IntOrString{
Type: intstr.Int,
Expand All @@ -85,7 +89,7 @@ func newMetricsService(namespace, app string, port int32) *corev1.Service {
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
Namespace: namespace,
Name: fmt.Sprintf("%s-metrics", app),
Name: fmt.Sprintf("%s-metrics", name),
},
Spec: corev1.ServiceSpec{
Ports: servicePorts,
Expand Down
4 changes: 2 additions & 2 deletions controllers/servicediscovery/servicediscovery_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,7 @@ func (r *Reconciler) ensureLightHouseAgent(instance *submarinerv1alpha1.ServiceD
return errors.Wrap(err, "error reconciling agent deployment")
}

err := metrics.Setup(instance.Namespace, instance, lightHouseAgent.GetLabels(), 8082, r.config.Client,
err := metrics.Setup("", instance.Namespace, instance, lightHouseAgent.GetLabels(), 8082, r.config.Client,
r.config.RestConfig, r.config.Scheme, reqLogger)
if err != nil {
return errors.Wrap(err, "error setting up metrics")
Expand All @@ -741,7 +741,7 @@ func (r *Reconciler) ensureLighthouseCoreDNSDeployment(instance *submarinerv1alp
return errors.Wrap(err, "error reconciling coredns deployment")
}

err := metrics.Setup(instance.Namespace, instance, lighthouseCoreDNSDeployment.GetLabels(), 9153, r.config.Client, r.config.RestConfig,
err := metrics.Setup("", instance.Namespace, instance, lighthouseCoreDNSDeployment.GetLabels(), 9153, r.config.Client, r.config.RestConfig,
r.config.Scheme, reqLogger)
if err != nil {
return errors.Wrap(err, "error setting up coredns metrics")
Expand Down
10 changes: 8 additions & 2 deletions controllers/submariner/gateway_resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/pkg/errors"
"github.com/submariner-io/admiral/pkg/syncer/broker"
"github.com/submariner-io/submariner-operator/api/v1alpha1"
"github.com/submariner-io/submariner-operator/controllers/constants"
"github.com/submariner-io/submariner-operator/controllers/helpers"
"github.com/submariner-io/submariner-operator/controllers/metrics"
"github.com/submariner-io/submariner-operator/pkg/names"
Expand Down Expand Up @@ -199,6 +200,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner, name string, podSelectorLabe
{Name: "SUBMARINER_HEALTHCHECKENABLED", Value: strconv.FormatBool(healthCheckEnabled)},
{Name: "SUBMARINER_HEALTHCHECKINTERVAL", Value: strconv.FormatUint(healthCheckInterval, 10)},
{Name: "SUBMARINER_HEALTHCHECKMAXPACKETLOSSCOUNT", Value: strconv.FormatUint(healthCheckMaxPacketLossCount, 10)},
{Name: "SUBMARINER_GWMETRICSPORT", Value: constants.MetricsGWPort},
{Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
Expand Down Expand Up @@ -252,8 +254,12 @@ func (r *Reconciler) reconcileGatewayDaemonSet(
return nil, err
}

err = metrics.Setup(instance.Namespace, instance, daemonSet.GetLabels(), gatewayMetricsServerPort, r.config.ScopedClient,
r.config.RestConfig, r.config.Scheme, reqLogger)
metricsSvcLabels := map[string]string {
"app": names.MetricsProxyComponent,
}

err = metrics.Setup(names.GatewayComponent, instance.Namespace, instance, metricsSvcLabels, gatewayMetricsServerPort,
r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger)

return daemonSet, err
}
Expand Down
8 changes: 7 additions & 1 deletion controllers/submariner/globalnet_resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package submariner
import (
"github.com/go-logr/logr"
"github.com/submariner-io/submariner-operator/api/v1alpha1"
"github.com/submariner-io/submariner-operator/controllers/constants"
"github.com/submariner-io/submariner-operator/controllers/helpers"
"github.com/submariner-io/submariner-operator/controllers/metrics"
"github.com/submariner-io/submariner-operator/pkg/names"
Expand All @@ -40,7 +41,11 @@ func (r *Reconciler) reconcileGlobalnetDaemonSet(instance *v1alpha1.Submariner,
return nil, err
}

err = metrics.Setup(instance.Namespace, instance, daemonSet.GetLabels(), globalnetMetricsServerPort,
metricsSvcLabels := map[string]string {
"app": names.MetricsProxyComponent,
}

err = metrics.Setup(names.GlobalnetComponent, instance.Namespace, instance, metricsSvcLabels, globalnetMetricsServerPort,
r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger)

return daemonSet, err
Expand Down Expand Up @@ -91,6 +96,7 @@ func newGlobalnetDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.DaemonS
{Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace},
{Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.ClusterID},
{Name: "SUBMARINER_EXCLUDENS", Value: "submariner-operator,kube-system,operators,openshift-monitoring,openshift-dns"},
{Name: "SUBMARINER_GNMETRICSPORT", Value: constants.MetricsGNPort},
{Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
Expand Down
111 changes: 111 additions & 0 deletions controllers/submariner/metrics_proxy_resources.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
SPDX-License-Identifier: Apache-2.0
Copyright Contributors to the Submariner project.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package submariner

import (
"github.com/go-logr/logr"
"github.com/submariner-io/submariner-operator/api/v1alpha1"
"github.com/submariner-io/submariner-operator/controllers/constants"
"github.com/submariner-io/submariner-operator/controllers/helpers"
"github.com/submariner-io/submariner-operator/pkg/names"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// nolint:wrapcheck // No need to wrap errors here.
func (r *Reconciler) reconcileMetricsProxyDaemonSet(instance *v1alpha1.Submariner, reqLogger logr.Logger) (*appsv1.DaemonSet,
error,
) {
daemonSet, err := helpers.ReconcileDaemonSet(instance, newMetricsProxyDaemonSet(instance, names.MetricsProxyComponent), reqLogger,
r.config.ScopedClient, r.config.Scheme)
if err != nil {
return nil, err
}

return daemonSet, err
}

func newMetricsProxyDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.DaemonSet {
labels := map[string]string{
"app": name,
"component": "metrics",
}

daemonSet := &appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: cr.Namespace,
Name: name,
Labels: labels,
},
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
"app": name,
}},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "gateway-metrics-proxy",
Image: getImagePath(cr, names.MetricsProxyImage, names.MetricsProxyImage),
ImagePullPolicy: helpers.GetPullPolicy(cr.Spec.Version, cr.Spec.ImageOverrides[names.MetricsProxyImage]),
Env: []corev1.EnvVar{
{Name: "NODE_IP", ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
}},
},
Command: []string{"/usr/bin/nc"},
Args: []string{"-v", "-lk", "-p", "8080", "-e", "nc", "$(NODE_IP)", constants.MetricsGWPort},
},
},
//ServiceAccountName: names.MetricsProxyComponent,
NodeSelector: map[string]string{"submariner.io/gateway": "true"},
// The MetricsProxy Pod must be able to run on any flagged node, regardless of existing taints
Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}},
},
},
},
}

if cr.Spec.GlobalCIDR != "" {
gnContainer := corev1.Container{
Name: "globalnet-metrics-proxy",
Image: getImagePath(cr, names.MetricsProxyImage, names.MetricsProxyImage),
ImagePullPolicy: helpers.GetPullPolicy(cr.Spec.Version, cr.Spec.ImageOverrides[names.MetricsProxyImage]),
Env: []corev1.EnvVar{
{Name: "NODE_IP", ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
}},
},
Command: []string{"/usr/bin/nc"},
Args: []string{"-v", "-lk", "-p", "8081", "-e", "nc", "$(NODE_IP)", constants.MetricsGNPort},
}

daemonSet.Spec.Template.Spec.Containers = append(daemonSet.Spec.Template.Spec.Containers, gnContainer)
}

return daemonSet
}
7 changes: 7 additions & 0 deletions controllers/submariner/submariner_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
}
}

_, err = r.reconcileMetricsProxyDaemonSet(instance, reqLogger)
if err != nil {
return reconcile.Result{}, err
}

if err := r.reconcileNetworkPluginSyncerDeployment(instance, clusterNetwork, reqLogger); err != nil {
return reconcile.Result{}, err
}
Expand Down Expand Up @@ -230,6 +235,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
return reconcile.Result{}, err
}

// TODO: vthapar Do we need proxy status?

if loadBalancer != nil {
instance.Status.LoadBalancerStatus.Status = &loadBalancer.Status.LoadBalancer
} else {
Expand Down
2 changes: 2 additions & 0 deletions pkg/names/names.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const (
OperatorComponent = "submariner-operator"
ServiceDiscoveryCrName = "service-discovery"
SubmarinerCrName = "submariner"
MetricsProxyComponent = "submariner-metrics-proxy"
)

/* These values are used by downstream distributions to override the component default image name. */
Expand All @@ -42,6 +43,7 @@ var (
ServiceDiscoveryImage = "lighthouse-agent"
LighthouseCoreDNSImage = "lighthouse-coredns"
OperatorImage = "submariner-operator"
MetricsProxyImage = "nettest"
)

/* Deprecated: These values are used by downstream distributions to patch the image names by adding a prefix/suffix. */
Expand Down

0 comments on commit 3e51e77

Please sign in to comment.