Merge pull request #133 from ymotongpoo/otel-logs

feat: add sample of OpenTelemetry Collector for log collection

devops/otel-col-logs/README.md

@@ -0,0 +1,101 @@
+# Log forwarding with OpenTelemetry Collector
+
+This demo shows the minimum example of log forwarding using OpenTelemetry Collector from a GKE pod.
+
+## Pre-requisites
+
+* `gcloud` command
+* `skaffold` command
+* `kubectl` command
+* Google Cloud project account enabled with:
+  * Google Kubernetes Engine
+  * Artifact Registry
+  * Cloud Build
+  * Cloud Logging
+
+You can set up `skaffold` and `kubectl` via `gcloud` command.
+
+```
+gcloud components install skaffold kubectl
+```
+
+Also you can enable the APIs via `gcloud` command:
+
+```
+gcloud services enable <service name>
+```
+
+The required services for this demo are:
+
+* `container.googleapis.com`
+* `artifactregistry.googleapis.com`
+* `containerregistry.googleapis.com`
+* `cloudbuild.googleapis.com`
+* `logging.googleapis.com`
+
+## How to run
+
+```
+./create-cluster.sh
+skaffold config set default-repo <path to container registry>
+```
+
+For example, if you are using Container Registry, you can set up `skaffold` like:
+
+```
+skaffold config set default-repo asia-east1-docker.pkg.dev/$(gcloud config get-value project)/<repo name>
+```
+
+To confirm the registry URL, you should find it on Cloud Console or by the following command:
+
+```
+gcloud artifacts repositories describe <repo name>
+```
+
+After setting up default-repo of `skaffold`, run the project:
+
+```
+skaffold run
+```
+
+This command sends project files to your Cloud Build environment, and remotely build and deploy the project to GKE.
+
+## How to confirm the result
+
+First, you need to generate logs. Find the external IP of the cluster and access the IP in HTTP GET:
+
+```
+$ kubectl get services --namespace otel-collector-log-demo
+NAME               TYPE           CLUSTER-IP     EXTERNAL-IP      PORT(S)        AGE
+log-app-external   LoadBalancer   10.3.251.149   35.201.245.242   80:32229/TCP   4m58s
+otel-collector     ClusterIP      10.3.245.40    <none>           4317/TCP       4m58s
+
+$ curl 35.201.245.242
+counter: 46
+```
+
+Check the Logs Explorer in Cloud Logging menu. The log name of logs via OpenTelemetry is labeled as `projects/yoshifumi-demo/logs/opentelemetry.io%2Fcollector-exported-log`, so you can filter the logs with the following Log query (please replace `PROJECT_NAME` with your project ID):
+
+```
+logName="projects/PROJECT_NAME/logs/opentelemetry.io%2Fcollector-exported-log"
+```
+
+## Clean up
+
+Once you try and understand how it works, run the following commands to delete the resources you used so that you won't be charged for unnecessary workloads.
+
+```
+skaffold delete
+```
+
+Also, you can turn down the Kubernetes cluster:
+
+```
+gcloud container cluster delete <cluster name>
+```
+
+If you prepared a new project dedicated for this demo, you can delete it.
+
+```
+gcloud projects delete <project ID or number>
+```

devops/otel-col-logs/create-gke-cluster.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2024 Yoshi Yamaguchi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+gcloud container clusters create otel-collector-log-demo \
+--region asia-east1 \
+--release-channel rapid \
+--logging SYSTEM,WORKLOAD \
+--monitoring SYSTEM \
+--preemptible \
+--enable-autoprovisioning \
+--max-cpu 16 \
+--max-memory 32 \
+--enable-autoscaling \
+--max-nodes 1 \
+--no-enable-ip-alias \
+--scopes cloud-platform

devops/otel-col-logs/manifests/collector.yaml

@@ -0,0 +1,149 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-config
+  labels:
+    app: otel-collector
+    component: otel-collector-config
+  namespace: otel-collector-log-demo
+
+data:
+  otel-collector-config: |
+    receivers:
+      filelog:
+        include:
+          - /var/log/otel-json.log
+        start_at: beginning
+        id: json-log
+        include_file_path: true
+        operators:
+          - type: json_parser
+            id: parser-app-json
+            timestamp:
+              parse_from: attributes.time
+              layout_type: gotime
+              layout: '2006-01-02T15:04:05.999999999Z07:00'
+
+    processors:
+      batch:
+
+    exporters:
+      googlecloud:
+        log:
+          default_log_name: opentelemetry.io/collector-exported-log
+
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [batch]
+          exporters: [googlecloud]
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  labels:
+    app: otel-collector
+    component: otel-collector
+  namespace: otel-collector-log-demo
+spec:
+  selector:
+    app: otel-collector
+  ports:
+    - port: 4317
+      name: otlp-grpc
+      protocol: TCP
+      targetPort: 4317
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: otel-collector
+  labels:
+    app: otel-collector
+    component: otel-collector
+  namespace: otel-collector-log-demo
+spec:
+  selector:
+    matchLabels:
+      app: otel-collector
+      component: otel-collector
+  template:
+    metadata:
+      labels:
+        app: otel-collector
+        component: otel-collector
+    spec:
+      containers:
+        - name: log-app
+          image: app
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: json-log-vol
+              mountPath: /var/log
+          resources:
+            limits:
+              memory: "128Mi"
+              cpu: "100m"
+        - name: otel-collector
+          image: otel/opentelemetry-collector-contrib:0.92.0-amd64
+          command:
+            - "/otelcol-contrib"
+            - "--config=/conf/otel-collector-config.yaml"
+          resources:
+            limits:
+              memory: "128Mi"
+              cpu: "250m"
+          volumeMounts:
+            - name: otel-collector-config-vol
+              mountPath: /conf
+              readOnly: true
+            - name: json-log-vol
+              mountPath: /var/log
+              readOnly: true
+      volumes:
+        - name: otel-collector-config-vol
+          configMap:
+            name: otel-collector-config
+            items:
+              - key: otel-collector-config
+                path: otel-collector-config.yaml
+        - name: json-log-vol
+          emptyDir: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: log-app-external
+  namespace: otel-collector-log-demo
+  labels:
+    app: otel-collector
+spec:
+  type: LoadBalancer
+  selector:
+    app: otel-collector
+  ports:
+    - name: http
+      port: 80
+      targetPort: 8080
+      protocol: TCP

devops/otel-col-logs/manifests/namespace.yaml

@@ -0,0 +1,21 @@
+# Copyright 2024 Yoshi Yamaguchi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: otel-collector-log-demo
+  labels:
+    name: otel-collector-log-demo

devops/otel-col-logs/skaffold.yaml

@@ -0,0 +1,30 @@
+# Copyright 2024 Yoshi Yamaguchi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skaffold/v3
+kind: Config
+metadata:
+  name: otel-collector-log-demo
+build:
+  googleCloudBuild:
+    concurrency: 0
+  artifacts:
+    - image: app
+      context: src/app
+  tagPolicy:
+    gitCommit: {}
+
+manifests:
+  rawYaml:
+    - manifests/collector.yaml

devops/otel-col-logs/src/app/Dockerfile

@@ -0,0 +1,24 @@
+# Copyright 2024 Yoshi Yamaguchi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.21.6-bookworm as builder
+WORKDIR /dist
+COPY go.mod main.go ./
+RUN go mod download
+RUN go build -o app
+
+FROM gcr.io/distroless/base-debian12
+COPY --from=builder /dist/app /app
+EXPOSE 8080
+CMD ["/app"]

devops/otel-col-logs/src/app/go.mod

@@ -0,0 +1,3 @@
+module app
+
+go 1.21.6