From 049aa267cb0d8e7bbc77f3cee52578e6a295f584 Mon Sep 17 00:00:00 2001 From: Yoshi Yamaguchi Date: Wed, 24 Jan 2024 17:00:37 +0900 Subject: [PATCH 1/3] feat: add OpenTelemetry Collector for logs demo * add working Kubernetes sample --- devops/otel-logs/create-gke-cluster.sh | 29 +++++ devops/otel-logs/manifests/collector.yaml | 149 ++++++++++++++++++++++ devops/otel-logs/manifests/namespace.yaml | 21 +++ devops/otel-logs/skaffold.yaml | 30 +++++ devops/otel-logs/src/app/Dockerfile | 24 ++++ devops/otel-logs/src/app/go.mod | 3 + devops/otel-logs/src/app/main.go | 54 ++++++++ 7 files changed, 310 insertions(+) create mode 100755 devops/otel-logs/create-gke-cluster.sh create mode 100644 devops/otel-logs/manifests/collector.yaml create mode 100644 devops/otel-logs/manifests/namespace.yaml create mode 100644 devops/otel-logs/skaffold.yaml create mode 100644 devops/otel-logs/src/app/Dockerfile create mode 100644 devops/otel-logs/src/app/go.mod create mode 100644 devops/otel-logs/src/app/main.go diff --git a/devops/otel-logs/create-gke-cluster.sh b/devops/otel-logs/create-gke-cluster.sh new file mode 100755 index 00000000..94a0c633 --- /dev/null +++ b/devops/otel-logs/create-gke-cluster.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2024 Yoshi Yamaguchi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +gcloud container clusters create otel-collector-log-demo \ +--region asia-east1 \ +--release-channel rapid \ +--logging SYSTEM,WORKLOAD \ +--monitoring SYSTEM \ +--preemptible \ +--enable-autoprovisioning \ +--max-cpu 16 \ +--max-memory 32 \ +--enable-autoscaling \ +--max-nodes 1 \ +--no-enable-ip-alias \ +--scopes cloud-platform diff --git a/devops/otel-logs/manifests/collector.yaml b/devops/otel-logs/manifests/collector.yaml new file mode 100644 index 00000000..84a150fe --- /dev/null +++ b/devops/otel-logs/manifests/collector.yaml @@ -0,0 +1,149 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config + labels: + app: otel-collector + component: otel-collector-config + namespace: otel-collector-log-demo + +data: + otel-collector-config: | + receivers: + filelog: + include: + - /var/log/otel-json.log + start_at: beginning + id: json-log + include_file_path: true + operators: + - type: json_parser + id: parser-app-json + timestamp: + parse_from: attributes.time + layout_type: gotime + layout: '2006-01-02T15:04:05.999999999Z07:00' + + processors: + batch: + + exporters: + googlecloud: + log: + default_log_name: opentelemetry.io/collector-exported-log + + service: + pipelines: + logs: + receivers: [filelog] + processors: [batch] + exporters: [googlecloud] + +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + labels: + app: otel-collector + component: otel-collector + namespace: otel-collector-log-demo +spec: + selector: + app: otel-collector + ports: + - port: 4317 + name: otlp-grpc + protocol: TCP + targetPort: 4317 + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: otel-collector + labels: + app: otel-collector + component: otel-collector + namespace: otel-collector-log-demo +spec: + selector: + matchLabels: + app: otel-collector + component: otel-collector + template: + metadata: + labels: + app: otel-collector + component: otel-collector + spec: + containers: + - name: log-app + image: app + ports: + - containerPort: 8080 + volumeMounts: + - name: json-log-vol + mountPath: /var/log + resources: + limits: + memory: "128Mi" + cpu: "100m" + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.92.0-amd64 + command: + - "/otelcol-contrib" + - "--config=/conf/otel-collector-config.yaml" + resources: + limits: + memory: "128Mi" + cpu: "250m" + volumeMounts: + - name: otel-collector-config-vol + mountPath: /conf + readOnly: true + - name: json-log-vol + mountPath: /var/log + readOnly: true + volumes: + - name: otel-collector-config-vol + configMap: + name: otel-collector-config + items: + - key: otel-collector-config + path: otel-collector-config.yaml + - name: json-log-vol + emptyDir: {} + +--- +apiVersion: v1 +kind: Service +metadata: + name: log-app-external + namespace: otel-collector-log-demo + labels: + app: otel-collector +spec: + type: LoadBalancer + selector: + app: otel-collector + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP diff --git a/devops/otel-logs/manifests/namespace.yaml b/devops/otel-logs/manifests/namespace.yaml new file mode 100644 index 00000000..c76cedf5 --- /dev/null +++ b/devops/otel-logs/manifests/namespace.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Yoshi Yamaguchi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: otel-collector-log-demo + labels: + name: otel-collector-log-demo diff --git a/devops/otel-logs/skaffold.yaml b/devops/otel-logs/skaffold.yaml new file mode 100644 index 00000000..54f002b6 --- /dev/null +++ b/devops/otel-logs/skaffold.yaml @@ -0,0 +1,30 @@ +# Copyright 2024 Yoshi Yamaguchi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: skaffold/v3 +kind: Config +metadata: + name: otel-collector-log-demo +build: + googleCloudBuild: + concurrency: 0 + artifacts: + - image: app + context: src/app + tagPolicy: + gitCommit: {} + +manifests: + rawYaml: + - manifests/collector.yaml diff --git a/devops/otel-logs/src/app/Dockerfile b/devops/otel-logs/src/app/Dockerfile new file mode 100644 index 00000000..e2f89483 --- /dev/null +++ b/devops/otel-logs/src/app/Dockerfile @@ -0,0 +1,24 @@ +# Copyright 2024 Yoshi Yamaguchi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.21.6-bookworm as builder +WORKDIR /dist +COPY go.mod main.go ./ +RUN go mod download +RUN go build -o app + +FROM gcr.io/distroless/base-debian12 +COPY --from=builder /dist/app /app +EXPOSE 8080 +CMD ["/app"] diff --git a/devops/otel-logs/src/app/go.mod b/devops/otel-logs/src/app/go.mod new file mode 100644 index 00000000..5ad6b4a3 --- /dev/null +++ b/devops/otel-logs/src/app/go.mod @@ -0,0 +1,3 @@ +module app + +go 1.21.6 diff --git a/devops/otel-logs/src/app/main.go b/devops/otel-logs/src/app/main.go new file mode 100644 index 00000000..6923ee9a --- /dev/null +++ b/devops/otel-logs/src/app/main.go @@ -0,0 +1,54 @@ +// Copyright 2024 Yoshi Yamaguchi +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log/slog" + "net/http" + "os" +) + +const logPath = "/var/log/otel-json.log" + +var counter int +var logger *slog.Logger + +func main() { + f, err := os.Create(logPath) + if err != nil { + panic(err) + } + defer f.Close() + logger = slog.New(slog.NewJSONHandler(f, &slog.HandlerOptions{})) + + http.HandleFunc("/", rootHandler) + http.HandleFunc("/reset", resetHandler) + if err := http.ListenAndServe(":8080", nil); err != nil { + panic(err) + } +} + +func rootHandler(w http.ResponseWriter, r *http.Request) { + counter++ + logger.Info("successfully incremented counter", "count", counter) + fmt.Fprintf(w, "counter: %d\n", counter) +} + +func resetHandler(w http.ResponseWriter, r *http.Request) { + counter = 0 + logger.Info("reset counter", "count", counter) + fmt.Fprintln(w, "counter reset") +} From 8db73debe978ef844751bb0200934efc80f2bb49 Mon Sep 17 00:00:00 2001 From: Yoshi Yamaguchi Date: Thu, 25 Jan 2024 17:15:41 +0900 Subject: [PATCH 2/3] docs: add README --- devops/otel-logs/README.md | 101 +++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 devops/otel-logs/README.md diff --git a/devops/otel-logs/README.md b/devops/otel-logs/README.md new file mode 100644 index 00000000..dcfc1ded --- /dev/null +++ b/devops/otel-logs/README.md @@ -0,0 +1,101 @@ +# Log forwarding with OpenTelemetry Collector + +This demo shows the minimum example of log forwarding using OpenTelemetry Collector from a GKE pod. + +## Pre-requisites + +* `gcloud` command +* `skaffold` command +* `kubectl` command +* Google Cloud project account enabled with: + * Google Kubernetes Engine + * Artifact Registry + * Cloud Build + * Cloud Logging + +You can set up `skaffold` and `kubectl` via `gcloud` command. + +``` +gcloud components install skaffold kubectl +``` + +Also you can enable the APIs via `gcloud` command: + +``` +gcloud services enable +``` + +The required services for this demo are: + +* `container.googleapis.com` +* `artifactregistry.googleapis.com` +* `containerregistry.googleapis.com` +* `cloudbuild.googleapis.com` +* `logging.googleapis.com` + +## How to run + +``` +./create-cluster.sh +skaffold config set default-repo +``` + +For example, if you are using Container Registry, you can set up `skaffold` like: + +``` +skaffold config set default-repo asia-east1-docker.pkg.dev/$(gcloud config get-value project)/ +``` + +To confirm the registry URL, you should find it on Cloud Console or by the following command: + +``` +gcloud artifacts repositories describe +``` + +After setting up default-repo of `skaffold`, run the project: + +``` +skaffold run +``` + +This command sends project files to your Cloud Build environment, and remotely build and deploy the project to GKE. + +## How to confirm the result + +First, you need to generate logs. Find the external IP of the cluster and access the IP in HTTP GET: + +``` +$ kubectl get services --namespace otel-collector-log-demo +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +log-app-external LoadBalancer 10.3.251.149 35.201.245.242 80:32229/TCP 4m58s +otel-collector ClusterIP 10.3.245.40 4317/TCP 4m58s + +$ curl 35.201.245.242 +counter: 46 +``` + +Check the Logs Explorer in Cloud Logging menu. The log name of logs via OpenTelemetry is labeled as `projects/yoshifumi-demo/logs/opentelemetry.io%2Fcollector-exported-log`, so you can filter the logs with the following Log query (please replace `PROJECT_NAME` with your project ID): + +``` +logName="projects/PROJECT_NAME/logs/opentelemetry.io%2Fcollector-exported-log" +``` + +## Clean up + +Once you try and understand how it works, run the following commands to delete the resources you used so that you won't be charged for unnecessary workloads. + +``` +skaffold delete +``` + +Also, you can turn down the Kubernetes cluster: + +``` +gcloud container cluster delete +``` + +If you prepared a new project dedicated for this demo, you can delete it. + +``` +gcloud projects delete +``` From cb5465c97280e64f83e51cd4d79502acd383c310 Mon Sep 17 00:00:00 2001 From: Yoshi Yamaguchi Date: Fri, 26 Jan 2024 08:23:21 +0900 Subject: [PATCH 3/3] nit: change directory name --- devops/{otel-logs => otel-col-logs}/README.md | 0 devops/{otel-logs => otel-col-logs}/create-gke-cluster.sh | 0 devops/{otel-logs => otel-col-logs}/manifests/collector.yaml | 0 devops/{otel-logs => otel-col-logs}/manifests/namespace.yaml | 0 devops/{otel-logs => otel-col-logs}/skaffold.yaml | 0 devops/{otel-logs => otel-col-logs}/src/app/Dockerfile | 0 devops/{otel-logs => otel-col-logs}/src/app/go.mod | 0 devops/{otel-logs => otel-col-logs}/src/app/main.go | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename devops/{otel-logs => otel-col-logs}/README.md (100%) rename devops/{otel-logs => otel-col-logs}/create-gke-cluster.sh (100%) rename devops/{otel-logs => otel-col-logs}/manifests/collector.yaml (100%) rename devops/{otel-logs => otel-col-logs}/manifests/namespace.yaml (100%) rename devops/{otel-logs => otel-col-logs}/skaffold.yaml (100%) rename devops/{otel-logs => otel-col-logs}/src/app/Dockerfile (100%) rename devops/{otel-logs => otel-col-logs}/src/app/go.mod (100%) rename devops/{otel-logs => otel-col-logs}/src/app/main.go (100%) diff --git a/devops/otel-logs/README.md b/devops/otel-col-logs/README.md similarity index 100% rename from devops/otel-logs/README.md rename to devops/otel-col-logs/README.md diff --git a/devops/otel-logs/create-gke-cluster.sh b/devops/otel-col-logs/create-gke-cluster.sh similarity index 100% rename from devops/otel-logs/create-gke-cluster.sh rename to devops/otel-col-logs/create-gke-cluster.sh diff --git a/devops/otel-logs/manifests/collector.yaml b/devops/otel-col-logs/manifests/collector.yaml similarity index 100% rename from devops/otel-logs/manifests/collector.yaml rename to devops/otel-col-logs/manifests/collector.yaml diff --git a/devops/otel-logs/manifests/namespace.yaml b/devops/otel-col-logs/manifests/namespace.yaml similarity index 100% rename from devops/otel-logs/manifests/namespace.yaml rename to devops/otel-col-logs/manifests/namespace.yaml diff --git a/devops/otel-logs/skaffold.yaml b/devops/otel-col-logs/skaffold.yaml similarity index 100% rename from devops/otel-logs/skaffold.yaml rename to devops/otel-col-logs/skaffold.yaml diff --git a/devops/otel-logs/src/app/Dockerfile b/devops/otel-col-logs/src/app/Dockerfile similarity index 100% rename from devops/otel-logs/src/app/Dockerfile rename to devops/otel-col-logs/src/app/Dockerfile diff --git a/devops/otel-logs/src/app/go.mod b/devops/otel-col-logs/src/app/go.mod similarity index 100% rename from devops/otel-logs/src/app/go.mod rename to devops/otel-col-logs/src/app/go.mod diff --git a/devops/otel-logs/src/app/main.go b/devops/otel-col-logs/src/app/main.go similarity index 100% rename from devops/otel-logs/src/app/main.go rename to devops/otel-col-logs/src/app/main.go