Update otel configuration files

jobs/metric-store-nozzle/templates/bpm.yml.erb

@@ -31,7 +31,7 @@ processes:
     LOGS_PROVIDER_KEY_PATH:  "<%= "#{cert_dir}/logs_provider.key" %>"
 
     # Otel Provider
-    OTEL_ADDR:               "<%= "0.0.0.0:#{p('otel_port')}" %>"
+    OTEL_ADDR:               "<%= "#{p('otel_port')}" %>"
     OTEL_PROVIDER_CA_PATH:   "<%= "#{cert_dir}/otel_provider_ca.crt" %>"
     OTEL_PROVIDER_CERT_PATH: "<%= "#{cert_dir}/otel_provider.crt" %>"
     OTEL_PROVIDER_KEY_PATH:  "<%= "#{cert_dir}/otel_provider.key" %>"

scripts/configure-otel-tas10

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(cd $(dirname $0) && pwd)
+
+if [ -z "${OM_TARGET}" ] || [ -z "${OM_USERNAME}" ] || [ -z "${OM_PASSWORD}" ]; then
+  echo "OM_TARGET, OM_USERNAME, and OM_PASSWORD must be set"
+  return
+fi
+
+echo "Configuring $OM_TARGET"
+
+CERTS_DIR=$SCRIPT_DIR/certs
+mkdir "$CERTS_DIR"
+
+export AVAILABLE_PRODUCTS=$(om products --deployed  -f json)
+
+function get-certs() {
+  echo "Getting $PRODUCT cert of $CERT_NAME"
+
+  export HAS_PRODUCT=$( echo "$AVAILABLE_PRODUCTS" | grep "\"$PRODUCT\"")
+  if [ -n "$HAS_PRODUCT" ]; then
+    export KEY_FILE=$CERTS_DIR/key.pem
+    om -k credentials -p "${PRODUCT}" -c "${CERT_NAME}" -f private_key_pem > "${KEY_FILE}"
+    export private_key_pem="$(awk '{printf "%s\\n", $0}' "${KEY_FILE}" )"
+    echo "received $private_key_pem"
+
+    export CERT_FILE=$CERTS_DIR/cert.pem
+    om -k credentials -p "${PRODUCT}" -c "${CERT_NAME}" -f cert_pem > "${CERT_FILE}"
+    export cert_pem="$(awk '{printf "%s\\n", $0}' "${CERT_FILE}" )"
+    echo "received $cert_pem"
+  fi
+}
+
+export SYS_DOMAIN=$(cat "${ENVIRONMENT_LOCK_METADATA}" | jq -r .sys_domain)
+export ca_cert="$(om certificate-authorities -f json | jq '.[0].cert_pem')"
+export CA_FILE=$CERTS_DIR/ca_cert.pem
+cat "${ca_cert}" > "$CA_FILE"
+
+
+cat <<-HEREDOC > otel-config.yaml
+---
+product-name: cf
+product-properties:
+  .properties.otel_collector_config:
+    value:
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+              endpoint: 127.0.0.1:9100
+              tls:
+                client_ca_file: "/var/vcap/jobs/otel-collector/config/certs/otel-collector-ca.crt"
+                cert_file: "/var/vcap/jobs/otel-collector/config/certs/otel-collector.crt"
+                key_file: "/var/vcap/jobs/otel-collector/config/certs/otel-collector.key"
+                min_version: '1.3'
+      exporters:
+HEREDOC
+
+
+PRODUCT="metric-store" CERT_NAME=".properties.otel_provider_mtls" get-certs
+if [ -n "$HAS_PRODUCT" ]; then
+metricStoreMetricExporter="- otlp/metric-store-metrics"
+metricStoreTraceExporter="- otlp/metric-store-traces"
+cat <<-HEREDOC > metric-store-config.yaml
+        otlp/metric-store-metrics:
+          endpoint: metric-store.service.internal:6062
+          retry_on_failure:
+            enabled: true
+            max_interval: 90s
+            max_elapsed_time: 600s
+          balancer_name: round_robin
+          tls:
+            insecure_skip_verify: true
+            ca_pem: ${ca_cert}
+            cert_pem: "${cert_pem}"
+            key_pem: "${private_key_pem}"
+        otlp/metric-store-traces:
+          endpoint: metric-store.service.internal:6062
+          retry_on_failure:
+            enabled: true
+            max_interval: 90s
+            max_elapsed_time: 600s
+          balancer_name: round_robin
+          tls:
+            insecure_skip_verify: true
+            ca_pem: ${ca_cert}
+            cert_pem: "${cert_pem}"
+            key_pem: "${private_key_pem}"
+HEREDOC
+  cat metric-store-config.yaml >> otel-config.yaml
+fi
+
+PRODUCT="p-healthwatch2" CERT_NAME=".properties.healthwatch_exporter_client_mtls" get-certs
+if [ -n "$HAS_PRODUCT" ]; then
+healthwatchMetricExporter="- prometheus/healthwatch"
+cat <<-HEREDOC > prometheus-config.yaml
+        prometheus/healthwatch:
+          endpoint: ":65331"
+          add_metric_suffixes: false
+          tls:
+            ca_pem: ${ca_cert}
+            cert_pem: "${cert_pem}"
+            key_pem: "${private_key_pem}"
+HEREDOC
+  cat prometheus-config.yaml >> otel-config.yaml
+fi
+
+cat <<-HEREDOC > services-config.yaml
+      service:
+        telemetry:
+          metrics:
+            level: basic
+            address: 127.0.0.1:14830
+        pipelines:
+          metrics:
+            receivers:
+            - otlp
+            exporters:
+            ${metricStoreMetricExporter}
+            ${healthwatchMetricExporter}
+          traces:
+            receivers:
+            - otlp
+            exporters:
+            ${metricStoreTraceExporter}
+
+HEREDOC
+
+cat services-config.yaml >> otel-config.yaml
+
+# curl --cert "${CERT_FILE}" --key "${KEY_FILE}" --cacert "${CA_FILE}"
+
+echo "CF Config"
+echo "==================================="
+cat otel-config.yaml
+echo "==================================="
+om configure-product -p cf -c otel-config.yaml #&& om apply-changes --product-name cf

scripts/configure-otel-tas6

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(cd $(dirname $0) && pwd)
+
+if [ -z "${OM_TARGET}" ] || [ -z "${OM_USERNAME}" ] || [ -z "${OM_PASSWORD}" ]; then
+  echo "OM_TARGET, OM_USERNAME, and OM_PASSWORD must be set"
+  return
+fi
+
+echo "Configuring $OM_TARGET"
+
+CERTS_DIR=$SCRIPT_DIR/certs
+mkdir "$CERTS_DIR"
+
+export AVAILABLE_PRODUCTS=$(om products --deployed  -f json)
+
+function get-certs() {
+  echo "Getting $PRODUCT cert of $CERT_NAME"
+
+  export HAS_PRODUCT=$( echo "$AVAILABLE_PRODUCTS" | grep "\"$PRODUCT\"")
+  if [ -n "$HAS_PRODUCT" ]; then
+    export KEY_FILE=$CERTS_DIR/key.pem
+    om -k credentials -p "${PRODUCT}" -c "${CERT_NAME}" -f private_key_pem > "${KEY_FILE}"
+    export private_key_pem="$(awk '{printf "%s\\n", $0}' "${KEY_FILE}" )"
+    echo "received $private_key_pem"
+
+    export CERT_FILE=$CERTS_DIR/cert.pem
+    om -k credentials -p "${PRODUCT}" -c "${CERT_NAME}" -f cert_pem > "${CERT_FILE}"
+    export cert_pem="$(awk '{printf "%s\\n", $0}' "${CERT_FILE}" )"
+    echo "received $cert_pem"
+  fi
+}
+
+export SYS_DOMAIN=$(cat "${ENVIRONMENT_LOCK_METADATA}" | jq -r .sys_domain)
+export ca_cert="$(om certificate-authorities -f json | jq '.[0].cert_pem')"
+export CA_FILE=$CERTS_DIR/ca_cert.pem
+cat "${ca_cert}" > "$CA_FILE"
+
+cat <<-HEREDOC > otel-config.yaml
+---
+product-name: cf
+product-properties:
+  .properties.otel_collector_metric_exporters_config:
+    value: |-
+HEREDOC
+
+PRODUCT="p-healthwatch2" CERT_NAME=".properties.healthwatch_exporter_client_mtls" get-certs
+if [ -n "$HAS_PRODUCT" ]; then
+cat <<-HEREDOC > prometheus-config.yaml
+        prometheus/healthwatch:
+          endpoint: ":65331"
+          add_metric_suffixes: false
+          tls:
+            ca_pem: ${ca_cert}
+            cert_pem: "${cert_pem}"
+            key_pem: "${private_key_pem}"
+HEREDOC
+  cat prometheus-config.yaml >> otel-config.yaml
+fi
+
+PRODUCT="metric-store" CERT_NAME=".properties.otel_provider_mtls" get-certs
+if [ -n "$HAS_PRODUCT" ]; then
+cat <<-HEREDOC > metric-store-config.yaml
+        otlp/metric-store-metrics:
+          endpoint: dns:metric-store.service.internal:6061
+          retry_on_failure:
+            enabled: true
+            max_interval: 90s
+            max_elapsed_time: 600s
+          balancer_name: round_robin
+          tls:
+            insecure_skip_verify: true
+            ca_pem: ${ca_cert}
+            cert_pem: "${cert_pem}"
+            key_pem: "${private_key_pem}"
+HEREDOC
+
+cat metric-store-config.yaml >> otel-config.yaml
+
+cat <<-HEREDOC > trace-config.yaml
+  .properties.otel_collector_trace_exporter_config:
+    value: |-
+        otlp/metric-store-traces:
+          endpoint: dns:metric-store.service.internal:6061
+          retry_on_failure:
+            enabled: true
+            max_interval: 90s
+            max_elapsed_time: 600s
+          balancer_name: round_robin
+          tls:
+            insecure_skip_verify: true
+            ca_pem: ${ca_cert}
+            cert_pem: "${cert_pem}"
+            key_pem: "${private_key_pem}"
+HEREDOC
+
+cat trace-config.yaml >> otel-config.yaml
+fi
+
+# curl --cert "${CERT_FILE}" --key "${KEY_FILE}" --cacert "${CA_FILE}"
+
+echo "CF Config"
+echo "==================================="
+cat otel-config.yaml
+echo "==================================="
+om configure-product -p cf -c otel-config.yaml #&& om apply-changes --product-name cf

sha

@@ -1 +1 @@
-da19787a73e08e848fd3e2b5b3cdbd0d0ced2413
+0df81d43bb5fa458d480984903d84efd8c09c712

src/internal/nozzle/nozzle.go

@@ -52,7 +52,7 @@ type StreamConnector interface {
 
 const (
 	BATCH_FLUSH_INTERVAL = 500 * time.Millisecond
-	BATCH_CHANNEL_SIZE   = 512
+	BATCH_CHANNEL_SIZE   = 1024
 )
 
 func NewNozzle(c StreamConnector, ingressAddr string, tlsConfig *tls.Config, shardId string, nodeIndex int, filterMetrics bool, allowListTags []string, opts ...Option) *Nozzle {
@@ -263,7 +263,7 @@ func (n *Nozzle) timerEmitter() {
 			points = append(points, pointsBatch.Points...)
 			size += pointsBatch.Size
 
-			if size >= ingressclient.MAX_BATCH_SIZE_IN_BYTES {
+			if size >= ingressclient.MAX_BATCH_SIZE_IN_BYTES || len(points) >= BATCH_CHANNEL_SIZE {
 				points = n.writeToChannelOrDiscard(points)
 				size = 0
 			}
@@ -273,7 +273,7 @@ func (n *Nozzle) timerEmitter() {
 			points = append(points, pointsBatch.Points...)
 			size += pointsBatch.Size
 
-			if size >= ingressclient.MAX_BATCH_SIZE_IN_BYTES {
+			if size >= ingressclient.MAX_BATCH_SIZE_IN_BYTES || len(points) >= BATCH_CHANNEL_SIZE {
 				points = n.writeToChannelOrDiscard(points)
 				size = 0
 			}