diff --git a/controllers/metrics/metrics.go b/controllers/metrics/metrics.go index 61f8561d5..aaf144a03 100644 --- a/controllers/metrics/metrics.go +++ b/controllers/metrics/metrics.go @@ -34,24 +34,11 @@ import ( controllerClient "sigs.k8s.io/controller-runtime/pkg/client" ) -func Setup(namespace string, owner metav1.Object, labels map[string]string, port int32, - client controllerClient.Client, config *rest.Config, scheme *runtime.Scheme, - reqLogger logr.Logger, +func Setup(serviceName, namespace, applicationKey, applicationName string, owner metav1.Object, port int32, + client controllerClient.Client, config *rest.Config, scheme *runtime.Scheme, reqLogger logr.Logger, ) error { - applicationKey := "app" - applicationName, ok := labels[applicationKey] - - if !ok { - applicationKey = "name" - applicationName, ok = labels[applicationKey] - - if !ok { - return fmt.Errorf("no app or name label in the provided labels, %v", labels) - } - } - - metricsService, err := apply.Service(owner, newMetricsService(namespace, applicationKey, applicationName, port), reqLogger, - client, scheme) + metricsService, err := apply.Service(owner, newMetricsService(serviceName, namespace, applicationKey, + applicationName, port), reqLogger, client, scheme) if err != nil { return err // nolint:wrapcheck // No need to wrap here } @@ -76,11 +63,15 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port // newMetricsService populates a Service providing access to metrics for the given application. // The Service is named after the application name, suffixed with "-metrics". -func newMetricsService(namespace, appKey, appName string, port int32) *corev1.Service { +func newMetricsService(name, namespace, appKey, appName string, port int32) *corev1.Service { labels := map[string]string{ appKey: appName, } + if name == "" { + name = appName + } + servicePorts := []corev1.ServicePort{ {Port: port, Name: "metrics", Protocol: corev1.ProtocolTCP, TargetPort: intstr.IntOrString{ Type: intstr.Int, @@ -92,7 +83,7 @@ func newMetricsService(namespace, appKey, appName string, port int32) *corev1.Se ObjectMeta: metav1.ObjectMeta{ Labels: labels, Namespace: namespace, - Name: fmt.Sprintf("%s-metrics", appName), + Name: fmt.Sprintf("%s-metrics", name), }, Spec: corev1.ServiceSpec{ Ports: servicePorts, diff --git a/controllers/servicediscovery/servicediscovery_controller.go b/controllers/servicediscovery/servicediscovery_controller.go index ab68af9ce..625ac51d2 100644 --- a/controllers/servicediscovery/servicediscovery_controller.go +++ b/controllers/servicediscovery/servicediscovery_controller.go @@ -692,8 +692,8 @@ func (r *Reconciler) ensureLightHouseAgent(instance *submarinerv1alpha1.ServiceD return errors.Wrap(err, "error reconciling agent deployment") } - err := metrics.Setup(instance.Namespace, instance, lightHouseAgent.GetLabels(), 8082, r.ScopedClient, - r.RestConfig, r.Scheme, reqLogger) + err := metrics.Setup(names.ServiceDiscoveryComponent, instance.Namespace, "app", names.ServiceDiscoveryComponent, + instance, 8082, r.ScopedClient, r.RestConfig, r.Scheme, reqLogger) if err != nil { return errors.Wrap(err, "error setting up metrics") } @@ -709,8 +709,8 @@ func (r *Reconciler) ensureLighthouseCoreDNSDeployment(instance *submarinerv1alp return errors.Wrap(err, "error reconciling coredns deployment") } - err := metrics.Setup(instance.Namespace, instance, lighthouseCoreDNSDeployment.GetLabels(), 9153, r.ScopedClient, r.RestConfig, - r.Scheme, reqLogger) + err := metrics.Setup(names.LighthouseCoreDNSComponent, instance.Namespace, "app", names.LighthouseCoreDNSComponent, instance, + 9153, r.ScopedClient, r.RestConfig, r.Scheme, reqLogger) if err != nil { return errors.Wrap(err, "error setting up coredns metrics") } diff --git a/controllers/submariner/gateway_resources.go b/controllers/submariner/gateway_resources.go index 063424740..88b916c87 100644 --- a/controllers/submariner/gateway_resources.go +++ b/controllers/submariner/gateway_resources.go @@ -200,6 +200,7 @@ func newGatewayPodTemplate(cr *v1alpha1.Submariner, name string, podSelectorLabe {Name: "SUBMARINER_HEALTHCHECKENABLED", Value: strconv.FormatBool(healthCheckEnabled)}, {Name: "SUBMARINER_HEALTHCHECKINTERVAL", Value: strconv.FormatUint(healthCheckInterval, 10)}, {Name: "SUBMARINER_HEALTHCHECKMAXPACKETLOSSCOUNT", Value: strconv.FormatUint(healthCheckMaxPacketLossCount, 10)}, + {Name: "SUBMARINER_METRICSPORT", Value: gatewayMetricsServerPort}, {Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ FieldPath: "spec.nodeName", @@ -253,8 +254,8 @@ func (r *Reconciler) reconcileGatewayDaemonSet( return nil, err } - err = metrics.Setup(instance.Namespace, instance, daemonSet.GetLabels(), gatewayMetricsServerPort, r.config.ScopedClient, - r.config.RestConfig, r.config.Scheme, reqLogger) + err = metrics.Setup(names.GatewayComponent, instance.Namespace, "app", names.MetricsProxyComponent, instance, gatewayMetricsServicePort, + r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger) return daemonSet, err } diff --git a/controllers/submariner/globalnet_resources.go b/controllers/submariner/globalnet_resources.go index 50bec91a3..c75d5f8d0 100644 --- a/controllers/submariner/globalnet_resources.go +++ b/controllers/submariner/globalnet_resources.go @@ -41,8 +41,8 @@ func (r *Reconciler) reconcileGlobalnetDaemonSet(instance *v1alpha1.Submariner, return nil, err } - err = metrics.Setup(instance.Namespace, instance, daemonSet.GetLabels(), globalnetMetricsServerPort, - r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger) + err = metrics.Setup(names.GlobalnetComponent, instance.Namespace, "app", names.MetricsProxyComponent, + instance, globalnetMetricsServicePort, r.config.ScopedClient, r.config.RestConfig, r.config.Scheme, reqLogger) return daemonSet, err } @@ -92,6 +92,7 @@ func newGlobalnetDaemonSet(cr *v1alpha1.Submariner, name string) *appsv1.DaemonS {Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace}, {Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.ClusterID}, {Name: "SUBMARINER_EXCLUDENS", Value: "submariner-operator,kube-system,operators,openshift-monitoring,openshift-dns"}, + {Name: "SUBMARINER_METRICSPORT", Value: globalnetMetricsServerPort}, {Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ FieldPath: "spec.nodeName", diff --git a/controllers/submariner/metrics_proxy_resources.go b/controllers/submariner/metrics_proxy_resources.go new file mode 100644 index 000000000..c65795768 --- /dev/null +++ b/controllers/submariner/metrics_proxy_resources.go @@ -0,0 +1,97 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package submariner + +import ( + "fmt" + + "github.com/go-logr/logr" + "github.com/submariner-io/submariner-operator/api/v1alpha1" + "github.com/submariner-io/submariner-operator/controllers/apply" + "github.com/submariner-io/submariner-operator/pkg/images" + "github.com/submariner-io/submariner-operator/pkg/names" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// nolint:wrapcheck // No need to wrap errors here. +func (r *Reconciler) reconcileMetricsProxyDaemonSet(instance *v1alpha1.Submariner, reqLogger logr.Logger) (*appsv1.DaemonSet, + error, +) { + return apply.DaemonSet(instance, newMetricsProxyDaemonSet(instance), reqLogger, + r.config.ScopedClient, r.config.Scheme) +} + +func newMetricsProxyDaemonSet(cr *v1alpha1.Submariner) *appsv1.DaemonSet { + labels := map[string]string{ + "app": names.MetricsProxyComponent, + "component": "metrics", + } + + daemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cr.Namespace, + Name: names.MetricsProxyComponent, + Labels: labels, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "app": names.MetricsProxyComponent, + }}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + *metricProxyContainer(cr, "gateway-metrics-proxy", fmt.Sprint(gatewayMetricsServicePort), gatewayMetricsServerPort), + }, + NodeSelector: map[string]string{"submariner.io/gateway": "true"}, + // The MetricsProxy Pod must be able to run on any flagged node, regardless of existing taints + Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, + }, + }, + }, + } + + if cr.Spec.GlobalCIDR != "" { + daemonSet.Spec.Template.Spec.Containers = append(daemonSet.Spec.Template.Spec.Containers, + *metricProxyContainer(cr, "globalnet-metrics-proxy", fmt.Sprint(globalnetMetricsServicePort), globalnetMetricsServerPort)) + } + + return daemonSet +} + +func metricProxyContainer(cr *v1alpha1.Submariner, name, hostPort, podPort string) *corev1.Container { + return &corev1.Container{ + Name: name, + Image: getImagePath(cr, names.MetricsProxyImage, names.MetricsProxyComponent), + ImagePullPolicy: images.GetPullPolicy(cr.Spec.Version, cr.Spec.ImageOverrides[names.MetricsProxyComponent]), + Env: []corev1.EnvVar{ + {Name: "NODE_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.hostIP", + }, + }}, + }, + Command: []string{"/usr/bin/nc"}, + Args: []string{"-v", "-lk", "-p", hostPort, "-e", "nc", "$(NODE_IP)", podPort}, + } +} diff --git a/controllers/submariner/submariner_controller.go b/controllers/submariner/submariner_controller.go index 8328d3c5b..4fe8030e7 100644 --- a/controllers/submariner/submariner_controller.go +++ b/controllers/submariner/submariner_controller.go @@ -55,8 +55,10 @@ import ( ) const ( - gatewayMetricsServerPort = 8080 - globalnetMetricsServerPort = 8081 + gatewayMetricsServicePort = 8080 + globalnetMetricsServicePort = 8081 + gatewayMetricsServerPort = "32780" + globalnetMetricsServerPort = "32781" ) var log = logf.Log.WithName("controller_submariner") @@ -183,6 +185,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( } } + if _, err = r.reconcileMetricsProxyDaemonSet(instance, reqLogger); err != nil { + return reconcile.Result{}, err + } + if err := r.reconcileNetworkPluginSyncerDeployment(instance, clusterNetwork, reqLogger); err != nil { return reconcile.Result{}, err } @@ -227,6 +233,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{}, err } + // TODO: vthapar Add metrics-proxy status to Submariner CR so we can update it with daemonset status + if loadBalancer != nil { instance.Status.LoadBalancerStatus.Status = &loadBalancer.Status.LoadBalancer } else { diff --git a/main.go b/main.go index 7747b1d2a..8c42e3c05 100644 --- a/main.go +++ b/main.go @@ -153,7 +153,7 @@ func main() { log.Info("Setting up metrics services and monitors") // Setup the metrics services and service monitors - labels := map[string]string{"name": os.Getenv("OPERATOR_NAME")} + name := os.Getenv("OPERATOR_NAME") // We need a new client using the manager's rest.Config because // the manager's caches haven't started yet and it won't allow @@ -163,7 +163,7 @@ func main() { log.Error(err, "Error obtaining a Kubernetes client") } - if err := metrics.Setup(namespace, nil, labels, metricsPort, metricsClient, cfg, scheme, log); err != nil { + if err := metrics.Setup(name, namespace, "name", name, nil, metricsPort, metricsClient, cfg, scheme, log); err != nil { log.Error(err, "Error setting up metrics services and monitors") } diff --git a/pkg/names/names.go b/pkg/names/names.go index 448f20e6a..91950f592 100644 --- a/pkg/names/names.go +++ b/pkg/names/names.go @@ -31,6 +31,7 @@ const ( OperatorComponent = "submariner-operator" ServiceDiscoveryCrName = "service-discovery" SubmarinerCrName = "submariner" + MetricsProxyComponent = "submariner-metrics-proxy" ) /* These values are used by downstream distributions to override the component default image name. */ @@ -42,6 +43,7 @@ var ( ServiceDiscoveryImage = "lighthouse-agent" LighthouseCoreDNSImage = "lighthouse-coredns" OperatorImage = "submariner-operator" + MetricsProxyImage = "nettest" ) var ValidImageNames = []string{ diff --git a/release-notes/20220908-metrics-proxy.md b/release-notes/20220908-metrics-proxy.md new file mode 100644 index 000000000..3f8a8b696 --- /dev/null +++ b/release-notes/20220908-metrics-proxy.md @@ -0,0 +1,6 @@ + +Users no longer need to open ports `8080` and `8081` on the host for querying metrics. A new `submariner-metrics-proxy` +DaemonSet runs pods on gateway nodes and forwards http requests for metrics services to gateway and globalnet pods running +on the nodes. Gateway and Globalnet pods now listen on ports `32780` and `32781` instead of well known ports `8080` and +`8081` to avoid conflict with any other services that might be using those ports. Users will continue to query existing +`submariner-gateway-metrics` and `submariner-globalnet-metrics` services to query the metrics.