diff --git a/infra/charts/feast-feature-server/opentelemetry.md b/infra/charts/feast-feature-server/opentelemetry.md new file mode 100644 index 00000000000..28dce79a02d --- /dev/null +++ b/infra/charts/feast-feature-server/opentelemetry.md @@ -0,0 +1,107 @@ +## Adding Monitoring +To add monitoring to the Feast Feature Server, follow these steps: + +### Workflow + +Feast instrumentation Using OpenTelemetry and Prometheus - +![Workflow](samples/workflow.png) + +### Deploy Prometheus Operator +Navigate to OperatorHub and install the stable version of the Prometheus Operator + +### Deploy OpenTelemetry Operator +Before installing OTEL Operator, install `cert-manager` and validate the `pods` should spin up -- +``` +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml +``` + +Then navigate to OperatorHub and install the stable version of the community OpenTelemetry Operator + + +### Configure OpenTelemetry Collector +Add the OpenTelemetry Collector configuration under the metrics section in your values.yaml file. + +Example values.yaml: + +``` +metrics: + enabled: true + otelCollector: + endpoint: "otel-collector.default.svc.cluster.local:4317" #sample + headers: + api-key: "your-api-key" +``` + +### Add instrumentation annotation and environment variables in the deployment.yaml + +``` +template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + instrumentation.opentelemetry.io/inject-python: "true" +``` + +``` +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://{{ .Values.service.name }}-collector.{{ .Release.namespace }}.svc.cluster.local:{{ .Values.metrics.endpoint.port}} +- name: OTEL_EXPORTER_OTLP_INSECURE + value: "true" +``` + +### Add checks +Add metric checks to all manifests and deployment file - + +``` +{{ if .Values.metrics.enabled }} +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: feast-instrumentation +spec: + exporter: + endpoint: http://{{ .Values.service.name }}-collector.{{ .Release.Namespace }}.svc.cluster.local:4318 # This is the default port for the OpenTelemetry Collector + env: + propagators: + - tracecontext + - baggage + python: + env: + - name: OTEL_METRICS_EXPORTER + value: console,otlp_proto_http + - name: OTEL_LOGS_EXPORTER + value: otlp_proto_http + - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED + value: "true" +{{end}} +``` + +### Add manifests to the chart +Add Instrumentation, OpenTelemetryCollector, ServiceMonitors, Prometheus Instance and RBAC rules as provided in the samples/ directory. + +For latest updates please refer the official repository - https://github.com/open-telemetry/opentelemetry-operator + +### Deploy Feast +Deploy Feast and set `metrics` value to `true`. + +Example - +``` +helm install feast-release infra/charts/feast-feature-server --set metric=true --set feature_store_yaml_base64="" +``` + +## See logs +Once the opentelemetry is deployed, you can search the logs to see the required metrics - + +``` +oc logs otelcol-collector-0 | grep "Name: feast_feature_server_memory_usage\|Value: 0.*" +oc logs otelcol-collector-0 | grep "Name: feast_feature_server_cpu_usage\|Value: 0.*" +``` +``` + -> Name: feast_feature_server_memory_usage +Value: 0.579426 +``` +``` +-> Name: feast_feature_server_cpu_usage +Value: 0.000000 +``` diff --git a/infra/charts/feast-feature-server/samples/instrumentation.yml b/infra/charts/feast-feature-server/samples/instrumentation.yml index 7c2ca328662..43333501f06 100644 --- a/infra/charts/feast-feature-server/samples/instrumentation.yml +++ b/infra/charts/feast-feature-server/samples/instrumentation.yml @@ -1,4 +1,3 @@ -{{- if .Values.metrics }} apiVersion: opentelemetry.io/v1alpha1 kind: Instrumentation metadata: diff --git a/infra/charts/feast-feature-server/samples/otel-collector.yml b/infra/charts/feast-feature-server/samples/otel-collector.yml index a37875f7f18..7ef22765d46 100644 --- a/infra/charts/feast-feature-server/samples/otel-collector.yml +++ b/infra/charts/feast-feature-server/samples/otel-collector.yml @@ -1,6 +1,5 @@ # API reference https://github.com/open-telemetry/opentelemetry-operator/blob/main/docs/api.md # Refs for v1beta1 config: https://github.com/open-telemetry/opentelemetry-operator/issues/3011#issuecomment-2154118998 -{{- if .Values.metrics }} apiVersion: opentelemetry.io/v1beta1 kind: OpenTelemetryCollector metadata: diff --git a/infra/charts/feast-feature-server/samples/otel-sm.yml b/infra/charts/feast-feature-server/samples/otel-sm.yml index 8543aecfadf..24d0904212e 100644 --- a/infra/charts/feast-feature-server/samples/otel-sm.yml +++ b/infra/charts/feast-feature-server/samples/otel-sm.yml @@ -1,4 +1,3 @@ -{{- if .Values.metrics }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: diff --git a/infra/charts/feast-feature-server/samples/prometheus.yaml b/infra/charts/feast-feature-server/samples/prometheus.yaml index e684fd540c1..24141362b52 100644 --- a/infra/charts/feast-feature-server/samples/prometheus.yaml +++ b/infra/charts/feast-feature-server/samples/prometheus.yaml @@ -1,4 +1,3 @@ -{{- if .Values.metrics }} apiVersion: monitoring.coreos.com/v1 kind: Prometheus metadata: diff --git a/infra/charts/feast-feature-server/samples/rbac.yml b/infra/charts/feast-feature-server/samples/rbac.yml index 15f38f4cf53..8258331c04d 100644 --- a/infra/charts/feast-feature-server/samples/rbac.yml +++ b/infra/charts/feast-feature-server/samples/rbac.yml @@ -1,4 +1,3 @@ -{{- if .Values.metrics }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/infra/charts/feast-feature-server/samples/service-monitor.yml b/infra/charts/feast-feature-server/samples/service-monitor.yml index c1cbf7125fc..eb5c58e6b8c 100644 --- a/infra/charts/feast-feature-server/samples/service-monitor.yml +++ b/infra/charts/feast-feature-server/samples/service-monitor.yml @@ -1,4 +1,3 @@ -{{- if .Values.metrics }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: diff --git a/infra/charts/feast-feature-server/samples/workflow.png b/infra/charts/feast-feature-server/samples/workflow.png new file mode 100644 index 00000000000..f2d8d3d52e6 Binary files /dev/null and b/infra/charts/feast-feature-server/samples/workflow.png differ diff --git a/ui/yarn.lock b/ui/yarn.lock index 89107de0b89..26c833fa11f 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -11640,14 +11640,14 @@ write-file-atomic@^3.0.0: typedarray-to-buffer "^3.1.5" ws@^7.4.6: - version "7.5.6" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.6.tgz#e59fc509fb15ddfb65487ee9765c5a51dec5fe7b" - integrity sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA== + version "7.5.10" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" + integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== ws@^8.1.0: - version "8.4.2" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.4.2.tgz#18e749868d8439f2268368829042894b6907aa0b" - integrity sha512-Kbk4Nxyq7/ZWqr/tarI9yIt/+iNNFOjBXEWgTb4ydaNHBNGgvf2QHbS9fdfsndfjFlFwEd4Al+mw83YkaD10ZA== + version "8.17.1" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b" + integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ== xml-name-validator@^3.0.0: version "3.0.0"