MGDCTRS-1727: Propagate metadata information as metrics tag to fleets…

…hard components
lburgazzoli · Dec 14, 2022 · efd52dd · efd52dd
1 parent f449a96
commit efd52dd
Show file tree

Hide file tree

Showing 42 changed files with 1,009 additions and 109 deletions.
diff --git a/...fleetshard-api/model/src/main/java/org/bf2/cos/fleetshard/api/ManagedConnectorStatus.java b/...fleetshard-api/model/src/main/java/org/bf2/cos/fleetshard/api/ManagedConnectorStatus.java
@@ -2,6 +2,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -96,6 +97,16 @@ public enum PhaseType {
         Stopped,
         Transferring,
         Transferred,
-        Error,
+        Error;
+
+        private final String id;
+
+        PhaseType() {
+            this.id = name().toLowerCase(Locale.US);
+        }
+
+        public String getId() {
+            return id;
+        }
     }
 }
diff --git a/...etshard-it/cucumber/src/main/java/org/bf2/cos/fleetshard/it/cucumber/MicrometerSteps.java b/...etshard-it/cucumber/src/main/java/org/bf2/cos/fleetshard/it/cucumber/MicrometerSteps.java
@@ -1,6 +1,7 @@
 package org.bf2.cos.fleetshard.it.cucumber;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
@@ -12,8 +13,10 @@
 import io.cucumber.java.en.And;
 import io.cucumber.java.en.Then;
 import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Meter;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Tag;
+import io.micrometer.core.instrument.search.Search;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -146,6 +149,62 @@ public void counter_wait_isGreaterThanOrEqualTo(String name, int expected) {
         });
     }
 
+    @And("the meters has entry with name {string} and tags:")
+    public void meter_exists_with_tags(String name, Map<String, String> entry) {
+        Search s = registry.find(name);
+
+        assertThat(s).isNotNull();
+        assertThat(s.meters()).isNotEmpty();
+
+        assertThat(s.meters()).anySatisfy(m -> {
+            for (var expectedTag : entry.entrySet()) {
+                final String tagName = ctx.resolvePlaceholders(expectedTag.getKey());
+                final String expected = ctx.resolvePlaceholders(expectedTag.getValue());
+                final String current = m.getId().getTag(tagName);
+
+                assertThat(current)
+                    .withFailMessage(() -> String.format("Meter with id '%s' does not have a tag '%s' with value '%s'", name,
+                        tagName, expected))
+                    .isEqualTo(expected);
+            }
+        });
+    }
+
+    @And("the meters has entries with name matching {string} and tags:")
+    public void meters_exists_with_tags(String regex, Map<String, String> entry) {
+        List<Meter> meters = registry.getMeters();
+        assertThat(meters).isNotEmpty();
+
+        int matches = 0;
+
+        for (Meter meter : meters) {
+            if (!meter.getId().getName().matches(regex)) {
+                continue;
+            }
+
+            matches++;
+
+            assertThat(meter).satisfies(m -> {
+                for (var expectedTag : entry.entrySet()) {
+                    final String tagName = ctx.resolvePlaceholders(expectedTag.getKey());
+                    final String expected = ctx.resolvePlaceholders(expectedTag.getValue());
+                    final String current = m.getId().getTag(tagName);
+
+                    assertThat(current)
+                        .withFailMessage(() -> String.format("Meter with name '%s' does not have a tag '%s' with value '%s'",
+                            meter.getId().getName(),
+                            tagName,
+                            expected))
+                        .isEqualTo(expected);
+                }
+            });
+        }
+
+        assertThat(matches)
+            .withFailMessage(() -> String.format("No meters matching '%s'", regex))
+            .isGreaterThan(0);
+    }
+
     // ***********************************
     //
     // timers

diff --git a/...el-it/src/test/java/org/bf2/cos/fleetshard/operator/it/camel/CamelConnectorReifyTest.java b/...el-it/src/test/java/org/bf2/cos/fleetshard/operator/it/camel/CamelConnectorReifyTest.java
@@ -43,6 +43,13 @@ public Map<String, String> getConfigOverrides() {
             configs.put("cos.operator.camel.exchange-pooling.exchange-factory", "pooled");
             configs.put("cos.operator.camel.exchange-pooling.exchange-factory-capacity", "31");
             configs.put("cos.operator.camel.exchange-pooling.exchange-factory-statistics-enabled", "true");
+
+            configs.put("cos.metrics.connector-operand.enabled", "true");
+            configs.put("cos.metrics.recorder.tags.common.foo", "bar");
+            configs.put("cos.metrics.recorder.tags.annotations[0]", "my.cos.bf2.org/connector-group");
+            configs.put("cos.metrics.recorder.tags.labels[0]", "cos.bf2.org/organisation-id");
+            configs.put("cos.metrics.recorder.tags.labels[1]", "cos.bf2.org/pricing-tier");
+
             return configs;
         }
     }

diff --git a/...-it/src/test/java/org/bf2/cos/fleetshard/operator/it/camel/glues/KameletBindingSteps.java b/...-it/src/test/java/org/bf2/cos/fleetshard/operator/it/camel/glues/KameletBindingSteps.java
@@ -263,7 +263,32 @@ public void klb_target_labels_contains(String value) {
         assertThat(res.getMetadata().getAnnotations())
             .containsKey(CamelConstants.TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_LABELS);
 
-        String labels = res.getMetadata().getAnnotations().get(CamelConstants.TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_LABELS);
+        String labels = res.getMetadata()
+            .getAnnotations()
+            .get(CamelConstants.TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_LABELS);
+
+        assertThatJson(labels)
+            .isArray()
+            .contains(value);
+    }
+
+    @And("the klb has target-annotations containing {string}")
+    public void klb_target_annotations_contains(String value) {
+        KameletBinding res = klb();
+
+        assertThat(res)
+            .isNotNull();
+        assertThat(res.getMetadata().getAnnotations())
+            .isNotEmpty();
+
+        assertThat(res.getMetadata().getAnnotations())
+            .isNotEmpty();
+        assertThat(res.getMetadata().getAnnotations())
+            .containsKey(CamelConstants.TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_ANNOTATIONS);
+
+        String labels = res.getMetadata()
+            .getAnnotations()
+            .get(CamelConstants.TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_ANNOTATIONS);
 
         assertThatJson(labels)
             .isArray()

diff --git a/cos-fleetshard-operator-camel-it/src/test/resources/CamelConnectorReify.feature b/cos-fleetshard-operator-camel-it/src/test/resources/CamelConnectorReify.feature
@@ -16,14 +16,24 @@ Feature: Camel Connector Reify
       | operator.id                 | cos-fleetshard-operator-camel   |
       | operator.type               | camel-connector-operator        |
       | operator.version            | [1.0.0,2.0.0)                   |
-    And with sample camel connector
+
+     And with sample camel connector
+
+     And set connector label "cos.bf2.org/organisation-id" to "20000000"
+     And set connector label "cos.bf2.org/pricing-tier" to "essential"
+     And set connector annotation "my.cos.bf2.org/connector-group" to "baz"
 
     When deploy
     Then the connector exists
      And the connector secret exists
      And the connector is in phase "Monitor"
 
     Then the klb exists
+
+     And the klb has labels containing:
+          | cos.bf2.org/organisation-id | 20000000 |
+          | cos.bf2.org/pricing-tier    | essential |
+
      And the klb has annotations containing:
           | camel.apache.org/operator.id                              | ${cos.operator.id}                         |
           | trait.camel.apache.org/container.image                    | quay.io/lburgazzoli/mci:0.1.2-log-sink-0.1 |
@@ -42,11 +52,16 @@ Feature: Camel Connector Reify
           | trait.camel.apache.org/prometheus.pod-monitor             | false                                      |
           | trait.camel.apache.org/deployment.enabled                 | true                                       |
           | trait.camel.apache.org/deployment.strategy                | Recreate                                   |
+          | my.cos.bf2.org/connector-group                            | baz                                        |
 
      And the klb has target-labels containing "cos.bf2.org/deployment.id"
      And the klb has target-labels containing "cos.bf2.org/connector.id"
      And the klb has target-labels containing "cos.bf2.org/connector.type.id"
      And the klb has target-labels containing "cos.bf2.org/operator.type"
+     And the klb has target-labels containing "cos.bf2.org/organisation-id"
+     And the klb has target-labels containing "cos.bf2.org/pricing-tier"
+
+    And the klb has target-annotations containing "my.cos.bf2.org/connector-group"
 
      And the klb has labels containing:
           | cos.bf2.org/cluster.id                |                               |
@@ -99,6 +114,7 @@ Feature: Camel Connector Reify
           | camel.main.exchange-factory                         | pooled                     |
           | camel.main.exchange-factory-capacity                | 31                         |
           | camel.main.exchange-factory-statistics-enabled      | true                       |
+
      And the klb secret has labels containing:
           | cos.bf2.org/cluster.id                | ${cos.cluster.id}             |
           | cos.bf2.org/connector.id              | ${cos.connector.id}           |
@@ -113,3 +129,15 @@ Feature: Camel Connector Reify
     And the klb secret has an entry at path "$.metadata.ownerReferences[0].apiVersion" with value "cos.bf2.org/v1alpha1"
     And the klb secret has an entry at path "$.metadata.ownerReferences[0].kind" with value "ManagedConnector"
 
+    And the meters has entries with name matching "cos.fleetshard.controller.connectors.reconcile\..*" and tags:
+      | foo                       | bar       |
+      | connector_group           | baz       |
+      | organisation_id           | 20000000  |
+      | pricing_tier              | essential |
+
+    And the meters has entries with name matching "cos.fleetshard.controller.event.operators.operand\..*" and tags:
+      | foo                       | bar       |
+      | connector_group           | baz       |
+      | organisation_id           | 20000000  |
+      | pricing_tier              | essential |
+
diff --git a/...rd-operator-camel/src/main/java/org/bf2/cos/fleetshard/operator/camel/CamelConstants.java b/...rd-operator-camel/src/main/java/org/bf2/cos/fleetshard/operator/camel/CamelConstants.java
@@ -1,6 +1,7 @@
 package org.bf2.cos.fleetshard.operator.camel;
 
-import org.bf2.cos.fleetshard.support.json.JacksonUtil;
+import java.util.Set;
+
 import org.bf2.cos.fleetshard.support.resources.Resources;
 
 import static org.bf2.cos.fleetshard.support.resources.Resources.LABEL_CONNECTOR_ID;
@@ -57,7 +58,7 @@ public final class CamelConstants {
     public static final String ERROR_HANDLER_TYPE_STOP = "stop";
     public static final String ERROR_HANDLER_TYPE_DLQ = "dead_letter_queue";
 
-    public static final String LABELS_TO_TRANSFER = JacksonUtil.asArrayString(
+    public static final Set<String> LABELS_TO_TRANSFER = Set.of(
         Resources.LABEL_OPERATOR_TYPE,
         LABEL_DEPLOYMENT_ID,
         LABEL_CONNECTOR_ID,

diff --git a/...tor-camel/src/main/java/org/bf2/cos/fleetshard/operator/camel/CamelOperandController.java b/...tor-camel/src/main/java/org/bf2/cos/fleetshard/operator/camel/CamelOperandController.java
@@ -2,19 +2,23 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.TreeSet;
 
 import javax.inject.Singleton;
 
 import org.bf2.cos.fleetshard.api.ManagedConnector;
 import org.bf2.cos.fleetshard.api.Operator;
 import org.bf2.cos.fleetshard.api.ServiceAccountSpec;
+import org.bf2.cos.fleetshard.operator.FleetShardOperatorConfig;
 import org.bf2.cos.fleetshard.operator.camel.model.CamelShardMetadata;
 import org.bf2.cos.fleetshard.operator.camel.model.KameletBinding;
 import org.bf2.cos.fleetshard.operator.camel.model.KameletBindingSpec;
 import org.bf2.cos.fleetshard.operator.camel.model.KameletEndpoint;
 import org.bf2.cos.fleetshard.operator.connector.ConnectorConfiguration;
 import org.bf2.cos.fleetshard.operator.operand.AbstractOperandController;
+import org.bf2.cos.fleetshard.support.json.JacksonUtil;
 import org.bf2.cos.fleetshard.support.resources.Resources;
 import org.bf2.cos.fleetshard.support.resources.Secrets;
 import org.slf4j.Logger;
@@ -29,7 +33,34 @@
 import io.fabric8.kubernetes.client.KubernetesClient;
 import io.fabric8.kubernetes.client.dsl.base.ResourceDefinitionContext;
 
-import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.*;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.APPLICATION_PROPERTIES;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.CONNECTOR_TYPE_SINK;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.CONNECTOR_TYPE_SOURCE;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.KAMEL_OPERATOR_ID;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.LABELS_TO_TRANSFER;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.SA_CLIENT_ID_PLACEHOLDER;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.SA_CLIENT_SECRET_PLACEHOLDER;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_CONTAINER_IMAGE;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_DEPLOYMENT_ENABLED;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_DEPLOYMENT_STRATEGY;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_ENABLED;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_LIVENESS_FAILURE_THRESHOLD;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_LIVENESS_PERIOD;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_LIVENESS_PROBE_ENABLED;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_LIVENESS_SUCCESS_THRESHOLD;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_LIVENESS_TIMEOUT;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_READINESS_FAILURE_THRESHOLD;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_READINESS_PERIOD;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_READINESS_PROBE_ENABLED;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_READINESS_SUCCESS_THRESHOLD;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_HEALTH_READINESS_TIMEOUT;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_JVM_ENABLED;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_KAMELETS_ENABLED;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_LOGGING_JSON;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_ANNOTATIONS;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_LABELS;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_PROMETHEUS_ENABLED;
+import static org.bf2.cos.fleetshard.operator.camel.CamelConstants.TRAIT_CAMEL_APACHE_ORG_PROMETHEUS_POD_MONITOR;
 import static org.bf2.cos.fleetshard.operator.camel.CamelOperandSupport.computeStatus;
 import static org.bf2.cos.fleetshard.operator.camel.CamelOperandSupport.configureKameletProperties;
 import static org.bf2.cos.fleetshard.operator.camel.CamelOperandSupport.createErrorHandler;
@@ -46,8 +77,12 @@ public class CamelOperandController extends AbstractOperandController<CamelShard
 
     private final CamelOperandConfiguration configuration;
 
-    public CamelOperandController(KubernetesClient kubernetesClient, CamelOperandConfiguration configuration) {
-        super(kubernetesClient, CamelShardMetadata.class, ObjectNode.class, ObjectNode.class);
+    public CamelOperandController(
+        FleetShardOperatorConfig config,
+        KubernetesClient kubernetesClient,
+        CamelOperandConfiguration configuration) {
+
+        super(config, kubernetesClient, CamelShardMetadata.class, ObjectNode.class, ObjectNode.class);
 
         this.configuration = configuration;
     }
@@ -66,6 +101,7 @@ protected List<HasMetadata> doReify(
         ServiceAccountSpec serviceAccountSpec) {
 
         final Map<String, String> properties = createSecretsData(
+            getFleetShardOperatorConfig(),
             connector,
             connectorConfiguration,
             serviceAccountSpec,
@@ -179,6 +215,7 @@ protected List<HasMetadata> doReify(
         binding.setMetadata(new ObjectMeta());
         binding.getMetadata().setName(connector.getMetadata().getName());
         binding.getMetadata().setAnnotations(new TreeMap<>());
+        binding.getMetadata().setLabels(new TreeMap<>());
         binding.setSpec(new KameletBindingSpec());
         binding.getSpec().setSource(source);
         binding.getSpec().setSink(sink);
@@ -201,7 +238,6 @@ protected List<HasMetadata> doReify(
         annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_KAMELETS_ENABLED, "false");
         annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_JVM_ENABLED, "false");
         annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_LOGGING_JSON, "false");
-        annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_LABELS, LABELS_TO_TRANSFER);
         annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_PROMETHEUS_ENABLED, "true");
         annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_PROMETHEUS_POD_MONITOR, "false");
 
@@ -210,6 +246,31 @@ protected List<HasMetadata> doReify(
         annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_HEALTH_READINESS_PROBE_ENABLED, "true");
         annotations.putIfAbsent(TRAIT_CAMEL_APACHE_ORG_HEALTH_LIVENESS_PROBE_ENABLED, "true");
 
+        Set<String> labelsToTransfer = new TreeSet<>(LABELS_TO_TRANSFER);
+        Set<String> annotationsToTransfer = new TreeSet<>();
+
+        getFleetShardOperatorConfig().metrics().recorder().tags().labels().stream().flatMap(List::stream).forEach(k -> {
+            String v = Resources.getLabel(connector, k);
+            if (v != null) {
+                binding.getMetadata().getLabels().put(k, v);
+                labelsToTransfer.add(k);
+            }
+        });
+        getFleetShardOperatorConfig().metrics().recorder().tags().annotations().stream().flatMap(List::stream).forEach(k -> {
+            String v = Resources.getAnnotation(connector, k);
+            if (v != null) {
+                binding.getMetadata().getAnnotations().put(k, v);
+                annotationsToTransfer.add(k);
+            }
+        });
+
+        annotations.putIfAbsent(
+            TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_LABELS,
+            JacksonUtil.asArrayString(labelsToTransfer));
+        annotations.putIfAbsent(
+            TRAIT_CAMEL_APACHE_ORG_OWNER_TARGET_ANNOTATIONS,
+            JacksonUtil.asArrayString(annotationsToTransfer));
+
         CamelOperandConfiguration.Health health = configuration.health();
         if (health != null) {
             annotations.putIfAbsent(