feat: add OTLP/Traces output support

Dockerfile

@@ -3,7 +3,7 @@ ARG BASE_IMAGE=alpine:3.17
 FROM ${BASE_IMAGE} AS final-stage
 LABEL MAINTAINER "Thomas Labarussias <[email protected]>"
 
-RUN apk add --update --no-cache ca-certificates
+RUN apk add --update --no-cache ca-certificates gcompat
 
 # Create user falcosidekick
 RUN addgroup -S falcosidekick && adduser -u 1234 -S falcosidekick -G falcosidekick

Makefile

@@ -45,6 +45,9 @@ GOLANGCI_LINT_VER := v1.52.2
 GOLANGCI_LINT_BIN := golangci-lint
 GOLANGCI_LINT := $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER)
 
+# Docker
+IMAGE_TAG := falcosecurity/falcosidekick:latest
+
 ## --------------------------------------
 ## Build
 ## --------------------------------------
@@ -61,7 +64,11 @@ falcosidekick-linux-amd64:
 
 .PHONY: build-image
 build-image: falcosidekick-linux-amd64
-	$(DOCKER) build -t falcosecurity/falcosidekick:latest .
+	$(DOCKER) build -t $(IMAGE_TAG) .
+
+.PHONY: push-image
+push-image:
+	$(DOCKER) push $(IMAGE_TAG)
 
 ## --------------------------------------
 ## Test

README.md

@@ -37,6 +37,7 @@ It works as a single endpoint for as many as you want `Falco` instances :
     - [Web](#web)
     - [SIEM](#siem)
     - [Workflow](#workflow)
+    - [Traces](#traces)
     - [Other](#other)
   - [Installation](#installation)
     - [Localhost](#localhost)
@@ -168,6 +169,11 @@ Follow the links to get the configuration of each output.
 
 - [**n8n**](https://github.com/falcosecurity/falcosidekick/blob/master/docs/outputs/n8n.md)
 
+### Traces
+
+- [**OTEL Traces**](https://github.com/falcosecurity/falcosidekick/blob/master/docs/outputs/otlp_traces.md)
+
+
 ### Other
 - [**Policy Report**](https://github.com/falcosecurity/falcosidekick/blob/master/docs/outputs/policy-reporter.md)
 

config.go

@@ -50,6 +50,7 @@ func getConfig() *types.Configuration {
 		Alertmanager:    types.AlertmanagerOutputConfig{ExtraLabels: make(map[string]string), ExtraAnnotations: make(map[string]string), CustomSeverityMap: make(map[types.PriorityType]string)},
 		CloudEvents:     types.CloudEventsOutputConfig{Extensions: make(map[string]string)},
 		GCP:             types.GcpOutputConfig{PubSub: types.GcpPubSub{CustomAttributes: make(map[string]string)}},
+		OTLP:            types.OTLPOutputConfig{Traces: types.OTLPTraces{ExtraEnvVars: make(map[string]string)}},
 	}
 
 	configFile := kingpin.Flag("config-file", "config file").Short('c').ExistingFile()
@@ -511,6 +512,21 @@ func getConfig() *types.Configuration {
 	v.SetDefault("Dynatrace.CheckCert", true)
 	v.SetDefault("Dynatrace.MinimumPriority", "")
 
+	v.SetDefault("OTLP.Traces.Endpoint", "")
+	v.SetDefault("OTLP.Traces.Protocol", "http/json")
+	// NOTE: we don't need to parse the OTLP.Traces.Headers field, as use it to
+	// set OTEL_EXPORTER_OTLP_TRACES_HEADERS (at otlp_init.go), which is then
+	// parsed by the OTLP SDK libs, see
+	// https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/#otel_exporter_otlp_traces_headers
+	v.SetDefault("OTLP.Traces.Headers", "")
+	v.SetDefault("OTLP.Traces.Timeout", 10000)
+	v.SetDefault("OTLP.Traces.Synced", false)
+	v.SetDefault("OTLP.Traces.MinimumPriority", "")
+	v.SetDefault("OTLP.Traces.CheckCert", true)
+	// NB: Unfortunately falco events don't provide endtime, artificially set
+	// it to 1000ms by default, override-able via OTLP_DURATION environment variable.
+	v.SetDefault("OTLP.Traces.Duration", 1000)
+
 	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	v.AutomaticEnv()
 	if *configFile != "" {
@@ -536,6 +552,7 @@ func getConfig() *types.Configuration {
 	v.GetStringMapString("AlertManager.ExtraAnnotations")
 	v.GetStringMapString("AlertManager.CustomSeverityMap")
 	v.GetStringMapString("GCP.PubSub.CustomAttributes")
+	v.GetStringMapString("OTLP.Traces.ExtraEnvVars")
 	if err := v.Unmarshal(c); err != nil {
 		log.Printf("[ERROR] : Error unmarshalling config : %s", err)
 	}
@@ -658,6 +675,21 @@ func getConfig() *types.Configuration {
 		}
 	}
 
+	if value, present := os.LookupEnv("OTLP_TRACES_EXTRAENVVARS"); present {
+		extraEnvVars := strings.Split(value, ",")
+		for _, extraEnvVarData := range extraEnvVars {
+			envName, envValue, found := strings.Cut(extraEnvVarData, ":")
+			envName, envValue = strings.TrimSpace(envName), strings.TrimSpace(envValue)
+			if !promKVNameRegex.MatchString(envName) {
+				log.Printf("[ERROR] : OTLPTraces - Extra Env Var name '%v' is not valid", envName)
+			} else if found {
+				c.OTLP.Traces.ExtraEnvVars[envName] = envValue
+			} else {
+				c.OTLP.Traces.ExtraEnvVars[envName] = ""
+			}
+		}
+	}
+
 	if c.AWS.SecurityLake.Interval < 5 {
 		c.AWS.SecurityLake.Interval = 5
 	}
@@ -777,6 +809,7 @@ func getConfig() *types.Configuration {
 	c.Mattermost.MessageFormatTemplate = getMessageFormatTemplate("Mattermost", c.Mattermost.MessageFormat)
 	c.Googlechat.MessageFormatTemplate = getMessageFormatTemplate("Googlechat", c.Googlechat.MessageFormat)
 	c.Cliq.MessageFormatTemplate = getMessageFormatTemplate("Cliq", c.Cliq.MessageFormat)
+
 	return c
 }
 

config_example.yaml

@@ -504,3 +504,17 @@ sumologic:
   # name: "" # Override the default Sumologic Source Name
   # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
   # checkcert: true # check if ssl certificate of the output is valid (default: true)
+
+otlp:
+  traces:
+    # endpoint: "" # OTLP endpoint in the form of http://{domain or ip}:4318/v1/traces
+    # protocol: "" # OTLP protocol http/json, http/protobuf, grpc (default: "" which uses SDK default: http/json)
+    # timeout: "" # OTLP timeout: timeout value in milliseconds (default: "" which uses SDK default: 10000)
+    # headers: "" # OTLP headers: list of headers to apply to all outgoing traces in the form of "some-key=some-value,other-key=other-value" (default: "")
+    # synced: false # Set to true if you want traces to be sent synchronously (default: false)
+    # duration: 1000 # Artificial span duration in milliseconds (default: 1000)
+    # extraenvvars: # Extra env vars (override the other settings)
+      # OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: 10000
+      # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
+    # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+    # checkcert: true # Set if you want to skip TLS certificate validation (default: true)

docs/outputs/otlp_traces.md

@@ -0,0 +1,242 @@
+# OTEL Traces
+
+- **Category**: Traces
+- **Website**: <https://opentelemetry.io/docs/concepts/signals/traces/>
+
+## Table of content
+
+- [OTEL Traces](#otel-traces)
+  - [Table of content](#table-of-content)
+  - [Configuration](#configuration)
+  - [Example of config.yaml](#example-of-configyaml)
+  - [Additional info](#additional-info)
+  - [Running a whole stack with docker-compose](#running-a-whole-stack-with-docker-compose)
+
+## Configuration
+
+|            Setting            |            Env var            |   Default value    |                                                             Description                                                             |
+| ----------------------------- | ----------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- |
+| `otlp.traces.endpoint`        | `OTLP_TRACES_ENDPOINT`        |                    | OTLP endpoint in the form of http://{domain or ip}:4318/v1/traces                                                                   |
+| `otlp.traces.protocol`        | `OTLP_TRACES_PROTOCOL`        | `http` (from SDK)  | OTLP Protocol                                                                                                                       |
+| `otlp.traces.timeout`         | `OTLP_TRACES_TIMEOUT`         | `10000` (from SDK) | Timeout value in milliseconds                                                                                                       |
+| `otlp.traces.headers`         | `OTLP_TRACES_HEADERS`         |                    | List of headers to apply to all outgoing traces in the form of "some-key=some-value,other-key=other-value"                          |
+| `otlp.traces.synced`          | `OTLP_TRACES_SYNCED`          | `false`            | Set to `true` if you want traces to be sent synchronously                                                                           |
+| `otlp.traces.minimumpriority` | `OTLP_TRACES_MINIMUMPRIORITY` | `""` (=`debug`)    | minimum priority of event for using this output, order is `emergency,alert,critical,error,warning,notice,informational,debug or ""` |
+| `otlp.traces.checkcert`       | `OTLP_TRACES_CHECKCERT`       | `false`            | Set if you want to skip TLS certificate validation                                                                                  |
+| `otlp.traces.duration`        | `OTLP_TRACES_DURATION`        | `1000`             | Artificial span duration in milliseconds (as Falco doesn't provide an ending timestamp)                                             |
+| `otlp.traces.extraenvvars`    | `OTLP_TRACES_EXTRAENVVARS`    |                    | Extra env vars (override the other settings)                                                                                        |
+
+> **Note**
+For the extra Env Vars values see [standard `OTEL_*` environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/):
+
+## Example of config.yaml
+
+```yaml
+otlp:
+  traces:
+    # endpoint: "" # OTLP endpoint in the form of http://{domain or ip}:4318/v1/traces
+    # protocol: "" # OTLP protocol http/json, http/protobuf, grpc (default: "" which uses SDK default: http/json)
+    # timeout: "" # OTLP timeout: timeout value in milliseconds (default: "" which uses SDK default: 10000)
+    # headers: "" # OTLP headers: list of headers to apply to all outgoing traces in the form of "some-key=some-value,other-key=other-value" (default: "")
+    # synced: false # Set to true if you want traces to be sent synchronously (default: false)
+    # duration: 1000 # Artificial span duration in milliseconds (default: 1000)
+    # extraenvvars: # Extra env vars (override the other settings)
+      # OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: 10000
+      # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
+    # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+    # checkcert: true # Set if you want to skip TLS certificate validation (default: true)
+```
+
+## Additional info
+
+> **Note**
+The OTLP Traces are only available for the source: `syscalls`.
+
+## Running a whole stack with docker-compose
+
+Below `docker-compose` file runs a stack of:
+
+- `falco`
+- `falcosidekick`
+- `events-generator` to generate arbitrary falco events
+- [Tempo](https://grafana.com/oss/tempo/) as OTLP traces backend
+- [Grafana](https://grafana.com/oss/grafana/) for visualization
+
+### Requirements
+
+A local Linux kernel capable of running `falco`--modern-bpf`, see
+<https://falco.org/blog/falco-modern-bpf/>.
+
+### Configuration files
+
+You need to create these files:
+
+- `./docker-compose.yaml`: minimal docker-compose configuration
+
+```yaml
+---
+version: "3.9"
+services:
+  falco:
+    image: falcosecurity/falco-no-driver:latest
+    privileged: true
+    command: "falco --modern-bpf -r /etc/falco/rules"
+    volumes:
+      - /var/run/docker.sock:/host/var/run/docker.sock
+      - /dev:/host/dev
+      - /proc:/host/proc:ro
+      - /boot:/host/boot:ro
+      - /lib/modules:/host/lib/modules:ro
+      - ./etc/falco:/etc/falco:ro
+
+  falcosidekick:
+    # Build from locally cloned repository
+    build: ../../../
+    volumes:
+      - ./etc/falco:/etc/falco:ro
+    command: -c /etc/falco/falcosidekick.yaml
+    ports:
+      - 2801:2801
+    environment:
+      - OTLP_TRACES_ENDPOINT=http://traces-backend:4318/v1/traces
+      - OTLP_HEADERS=X-Scope-OrgID=1
+      - OTLP_TRACES_SYNCED=true
+  traces-backend:
+    image: grafana/tempo:latest
+    ports:
+      - 4317
+      - 4318
+      - 3200
+    volumes:
+      - ./etc/tempo:/etc/tempo:ro
+    command: "-config.file /etc/tempo/config.yaml"
+    restart: always
+
+  grafana:
+    image: grafana/grafana:10.0.3
+    volumes:
+      - ./etc/grafana/provisioning:/etc/grafana/provisioning:ro
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor
+    ports:
+      - "3000:3000"
+  event-generator:
+    image: falcosecurity/event-generator
+    command: run
+    restart: always
+  trigger:
+    image: alpine
+    command: ["sh", "-c", "while true; do cat /etc/shadow > /dev/null; sleep 5; done"]
+```
+
+- `./etc/falco/falco.yaml`: minimal falco configuration
+
+```yaml
+---
+debug: true
+outputs:
+  rate: 1
+  max_burst: 1000
+json_output: true
+http_output:
+  enabled: true
+  url: http://falcosidekick:2801
+  #url: http://172.17.0.1:2801
+  user_agent: "falcosecurity/falco"
+  # Tell Falco to not verify the remote server.
+  insecure: true
+
+plugins:
+  - name: json
+    library_path: libjson.so
+
+stdout_output:
+  enabled: true
+log_stderr: true
+
+syscall_buf_size_preset: 4
+```
+
+- `./etc/falco/rules/` folder: from upstream
+  <https://github.com/falcosecurity/rules.git>
+
+```shell
+mkdir -p ./etc/falco/upstream-rules
+git clone --depth 1 https://github.com/falcosecurity/rules/ ./etc/falco/upstream-rules
+ln -s upstream-rules/rules ./etc/falco/rules
+```
+
+- `./etc/grafana/provisioning/datasources/datasources.yaml`: provisioning Tempo
+  backend as Grafana datasource
+
+```yaml
+apiVersion: 1
+
+datasources:
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://traces-backend:3200
+  basicAuth: false
+  isDefault: true
+  version: 1
+  editable: false
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus
+```
+
+- `./etc/tempo/config.yaml`: minimal tempo configuration
+
+```yaml
+---
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+        grpc:
+  log_received_spans:
+    enabled: true
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo/blocks
+```
+
+### Run it
+
+To bring up the stack, and peek at how Grafana shows it:
+
+1. Bring up the stack
+
+  ```shell
+  docker-compose up
+  ```
+
+1. Navigate to <http://localhost:3000/> to start browsing the local Grafana UI
+
+1. Navigate to [/explore](http://localhost:3000/explore/), choose `Tempo` datasource, and query `{}`, or just click [here](http://localhost:3000/explore?orgId=1&left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22tempo%22,%22uid%22:%22tempo%22%7D,%22queryType%22:%22traceql%22,%22limit%22:20,%22query%22:%22%7B%7D%22%7D%5D) for such already crafted query.
+   ![Grafana explore](images/otlp_traces-grafana_explore.png)
+
+1. Click on any of the shown traces on the left panel, you should see something
+   similar to the below attached screenshot.
+   ![Falco traces view](images/otlp_traces-traces_view.png)
+
+1. Bring down the stack
+
+  ```shell
+  docker-compose down
+  ```