feat: distributed trace by otel and cloud run sidecar (#64)

* feat: set otel tracer

* fix: lint

* feat: update CD

* chore: comment

* feat: otel collector

* feat: trace

* feat: export trace

.clouddeploy/run-prod.yaml

@@ -26,3 +26,5 @@ spec:
               value: dummy # from-param: ${gcp_project_id}
             - name: PUBSUB_TOPIC_ANALYZE_IMAGE
               value: fitness-supporter-prd-analyze-image-event
+        - image: asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/sidecar/otel:latest
+          name: sidecar-otel

.github/workflows/deploy.yml

@@ -45,6 +45,11 @@ jobs:
           docker build . -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/fitness-supporter/prd:latest"
           docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/fitness-supporter/prd:latest"
 
+      - name: "Build and push sidecar"
+        run: |-
+          docker build .otelcollector/ -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/fitness-supporter/sidecar/otel:latest"
+          docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/fitness-supporter/sidecar/otel:latest"
+
       - name: "Create release name"
         run: |-
           echo "RELEASE_NAME=${{ env.APP }}-${GITHUB_SHA::7}-${GITHUB_RUN_NUMBER}" >> ${GITHUB_ENV}

.otelcollector/Dockerfile

@@ -0,0 +1,4 @@
+FROM otel/opentelemetry-collector-contrib:0.94.0
+COPY config.yaml /etc/otelcol-contrib/config.yaml
+EXPOSE 4317
+CMD ["--config", "/etc/otelcol-contrib/config.yaml"]

.otelcollector/config.yaml

@@ -0,0 +1,61 @@
+# [START cloudrun_mc_custom_metrics_otel_config]
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+processors:
+  batch:
+    # batch metrics before sending to reduce API usage
+    send_batch_max_size: 200
+    send_batch_size: 200
+    timeout: 5s
+
+  memory_limiter:
+    # drop metrics if memory usage gets too high
+    check_interval: 1s
+    limit_percentage: 65
+    spike_limit_percentage: 20
+
+  # automatically detect Cloud Run resource metadata
+  resourcedetection:
+    detectors: [env, gcp]
+    timeout: 2s
+    override: false
+
+  resource:
+    attributes:
+      # add instance_id as a resource attribute
+      - key: service.instance.id
+        from_attribute: faas.id
+        action: upsert
+        # parse service name from K_SERVICE Cloud Run variable
+      - key: service.name
+        value: ${env:K_SERVICE}
+        action: insert
+
+exporters:
+  googlemanagedprometheus: # Note: this is intentionally left blank
+  googlecloud:
+  # otlphttp/openobserve:
+  #   endpoint: https://api.openobserve.ai/api/kiyong_organization_2077_A0vz8lYUhIFWpjS/
+  #   headers:
+  #     Authorization: TODO:
+  #     stream-name: default
+
+extensions:
+  health_check:
+
+service:
+  extensions: [health_check]
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [batch, memory_limiter, resourcedetection, resource]
+      exporters: [googlemanagedprometheus, googlecloud]
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [googlecloud]
+# [END cloudrun_mc_custom_metrics_otel_config]

Makefile

@@ -43,9 +43,15 @@ deploy.apply:
 
 .PHONY: deploy.build
 deploy.build:
+	@echo "build fitness-supporter"
 	@-docker image rm asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/prd
 	@docker buildx build . --platform linux/amd64 --no-cache --tag asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/prd:latest
 	@docker push asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/prd:latest
+	@echo "build sidecar"
+	@-docker image rm asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/sidecar/otel:latest
+	@docker buildx build .otelcollector/ --platform linux/amd64 --no-cache --tag asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/sidecar/otel:latest
+	@docker push asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/sidecar/otel:latest
+
 
 .PHONY: deploy.release
 deploy.release:

cmd/server/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log/slog"
 	"net/http"
@@ -10,6 +11,7 @@ import (
 	"github.com/kyong0612/fitness-supporter/handler"
 	"github.com/kyong0612/fitness-supporter/infra/config"
 	"github.com/kyong0612/fitness-supporter/infra/logger"
+	"github.com/kyong0612/fitness-supporter/infra/trace"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 )
@@ -21,6 +23,19 @@ func main() {
 		os.Exit(1) // Exit with error.
 	}
 
+	// Init tracer
+	tp, err := trace.InitTracer(config.Get().ENV)
+	if err != nil {
+		slog.Error("failed to init tracer", slog.Any("err", err))
+		os.Exit(1)
+	}
+
+	defer func() {
+		if err := tp.Shutdown(context.Background()); err != nil {
+			slog.Error("failed to tracer shutdown", slog.Any("err", err))
+		}
+	}()
+
 	// Init logger
 	logger.Init()
 
@@ -39,6 +54,5 @@ func main() {
 
 	if err := srv.ListenAndServe(); err != nil {
 		slog.Error(err.Error())
-		os.Exit(1) // Exit with error.
 	}
 }

docs/7-otel.md

@@ -0,0 +1,23 @@
+# OpenTelemetry
+
+## References
+
+- <https://zenn.dev/google_cloud_jp/articles/20230516-cloud-run-otel>
+- <https://github.com/GoogleCloudPlatform/devrel-demos/tree/main/devops/otel-col-cloud-run-multicontainer>
+- <https://cloud.google.com/run/docs/tutorials/custom-metrics-opentelemetry-sidecar?hl=ja>
+- <https://opentelemetry.io/docs/collector/configuration/>
+
+## build sidecar container
+
+- ref:<https://cloud.google.com/run/docs/deploying?hl=ja#multicontainer-yaml>
+
+```bash
+docker buildx build .otelcollector/ --platform linux/amd64 --no-cache --tag asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/sidecar/otel:latest
+docker push asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/sidecar/otel:latest
+# docker image rm asia-northeast1-docker.pkg.dev/kyong0612-lab/fitness-supporter/sidecar/otel:latest
+```
+
+## replace config
+
+- cloud runのyaml fileを編集する
+<https://share.cleanshot.com/ncrCNj1T>

go.mod

@@ -12,13 +12,18 @@ require (
 	github.com/cosmtrek/air v1.49.0
 	github.com/fullstorydev/grpcurl v1.8.9
 	github.com/go-chi/chi v1.5.5
+	github.com/go-chi/chi/v5 v5.0.10
 	github.com/golangci/golangci-lint v1.55.1
 	github.com/google/generative-ai-go v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/jarcoal/httpmock v1.3.1
 	github.com/line/line-bot-sdk-go/v8 v8.4.0
+	github.com/riandyrn/otelchi v0.5.1
 	github.com/testcontainers/testcontainers-go v0.27.0
 	github.com/testcontainers/testcontainers-go/modules/gcloud v0.27.0
+	go.opentelemetry.io/otel v1.22.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0
+	go.opentelemetry.io/otel/sdk v1.21.0
 	golang.org/x/net v0.21.0
 	golang.org/x/vuln v1.0.2
 	google.golang.org/api v0.163.0
@@ -113,7 +118,6 @@ require (
 	github.com/fzipp/gocyclo v0.6.0 // indirect
 	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/ghostiam/protogetter v0.2.3 // indirect
-	github.com/go-chi/chi/v5 v5.0.10 // indirect
 	github.com/go-critic/go-critic v0.9.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -154,6 +158,7 @@ require (
 	github.com/gostaticanalysis/comment v1.4.2 // indirect
 	github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect
 	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
@@ -276,12 +281,13 @@ require (
 	gitlab.com/bosi/decorder v0.4.1 // indirect
 	go-simpler.org/sloglint v0.1.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib v1.0.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
-	go.opentelemetry.io/otel v1.22.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect
 	go.opentelemetry.io/otel/metric v1.22.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.tmz.dev/musttag v0.7.2 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect