[RTC-367] Add Loki to Jellyroom deployment (#39)

* Initial version of loki support * Adjust workflows for promtail and loki * Reorder volumes * Seperate loki and promtail * Update dashboard * Add retention for Loki * Review fixes * Review chagnes
fishjam-dev · Nov 10, 2023 · 425940f · 425940f
1 parent 2965300
commit 425940f
Show file tree

Hide file tree

Showing 11 changed files with 231 additions and 18 deletions.
diff --git a/.github/workflows/production_build_and_deploy.yml b/.github/workflows/production_build_and_deploy.yml
@@ -44,6 +44,7 @@ jobs:
           BE_JF_ADDRESSES=${{env.JF1_HOST}} ${{env.JF2_HOST}}
           PROM_JELLYFISH_TARGETS=jellyfish1:9568,${{secrets.SANDBOX2_IP}}:9568
           PROM_NODE_TARGETS=172.17.0.1:9100,${{secrets.SANDBOX2_IP}}:9100
+          LOKI_ADDRESS=loki
           GF_SECURITY_ADMIN_PASSWORD=${{secrets.GF_SECURITY_ADMIN_PASSWORD}}
           GF_SECURITY_ADMIN_USER=${{secrets.GF_SECURITY_ADMIN_USER}}" > .env
 
@@ -63,7 +64,7 @@ jobs:
           ssh_public_key: ${{ secrets.SSH_PUB_KEY }}
           ssh_port: ${{ secrets.SSH_PROD_PORT }}
           stack_file_name: ${{ env.COMPOSE_FILE_NAME }}
-          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish1 node-exporter1 frontend backend prometheus grafana
+          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish1 node-exporter1 frontend backend prometheus grafana promtail loki
 
   deploy2:
     runs-on: ubuntu-latest
@@ -83,6 +84,7 @@ jobs:
           JF_PORT=5002
           JF1_IP=${{secrets.PRODUCTION_IP}}
           JF2_IP=${{secrets.PRODUCTION2_IP}}
+          LOKI_ADDRESS=${{secrets.PRODUCTION_IP}}
           JF2_HOST=${{env.JF2_HOST}}" > .env
       - name: Remove old containers on second host
         uses: JimCronqvist/action-ssh@master
@@ -100,4 +102,4 @@ jobs:
           ssh_public_key: ${{ secrets.SSH_PUB_KEY }}
           ssh_port: ${{ secrets.SSH_PROD_PORT }}
           stack_file_name: ${{ env.COMPOSE_FILE_NAME }}
-          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish2 node-exporter2
+          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish2 node-exporter2 promtail
diff --git a/.github/workflows/sandbox_build_and_deploy.yml b/.github/workflows/sandbox_build_and_deploy.yml
@@ -46,6 +46,7 @@ jobs:
           BE_JF_ADDRESSES=${{env.JF1_HOST}} ${{env.JF2_HOST}}
           PROM_JELLYFISH_TARGETS=jellyfish1:9568,${{secrets.SANDBOX2_IP}}:9568
           PROM_NODE_TARGETS=172.17.0.1:9100,${{secrets.SANDBOX2_IP}}:9100
+          LOKI_ADDRESS=loki
           GF_SECURITY_ADMIN_PASSWORD=${{secrets.GF_SECURITY_ADMIN_PASSWORD}}
           GF_SECURITY_ADMIN_USER=${{secrets.GF_SECURITY_ADMIN_USER}}" > .env
 
@@ -64,7 +65,7 @@ jobs:
           ssh_private_key: ${{ secrets.SSH_PRIV_KEY }}
           ssh_public_key: ${{ secrets.SSH_PUB_KEY }}
           stack_file_name: ${{ env.COMPOSE_FILE_NAME }}
-          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish1 node-exporter1 frontend backend prometheus grafana
+          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish1 node-exporter1 frontend backend prometheus grafana promtail loki
 
   deploy2:
     runs-on: ubuntu-latest
@@ -84,6 +85,7 @@ jobs:
           JF_PORT=5002
           JF1_IP=${{secrets.SANDBOX_IP}}
           JF2_IP=${{secrets.SANDBOX2_IP}}
+          LOKI_ADDRESS=${{secrets.SANDBOX_IP}}
           JF2_HOST=${{env.JF2_HOST}}" > .env
       - name: Remove old containers on second host
         uses: JimCronqvist/action-ssh@master
@@ -100,4 +102,4 @@ jobs:
           ssh_private_key: ${{ secrets.SSH_PRIV_KEY }}
           ssh_public_key: ${{ secrets.SSH_PUB_KEY }}
           stack_file_name: ${{ env.COMPOSE_FILE_NAME }}
-          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish2 node-exporter2
+          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish2 node-exporter2 promtail
diff --git a/.github/workflows/staging_build_and_deploy.yml b/.github/workflows/staging_build_and_deploy.yml
@@ -44,6 +44,7 @@ jobs:
           BE_JF_ADDRESSES=${{env.JF1_HOST}} ${{env.JF2_HOST}}
           PROM_JELLYFISH_TARGETS=jellyfish1:9568,${{secrets.SANDBOX2_IP}}:9568
           PROM_NODE_TARGETS=172.17.0.1:9100,${{secrets.SANDBOX2_IP}}:9100
+          LOKI_ADDRESS=loki
           GF_SECURITY_ADMIN_PASSWORD=${{secrets.GF_SECURITY_ADMIN_PASSWORD}}
           GF_SECURITY_ADMIN_USER=${{secrets.GF_SECURITY_ADMIN_USER}}" > .env
 
@@ -62,7 +63,7 @@ jobs:
           ssh_private_key: ${{ secrets.SSH_PRIV_KEY }}
           ssh_public_key: ${{ secrets.SSH_PUB_KEY }}
           stack_file_name: ${{ env.COMPOSE_FILE_NAME }}
-          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish1 node-exporter1 frontend backend prometheus grafana
+          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish1 node-exporter1 frontend backend prometheus grafana promtail loki
 
   deploy2:
     runs-on: ubuntu-latest
@@ -82,6 +83,7 @@ jobs:
           JF_PORT=5002
           JF1_IP=${{secrets.STAGING_IP}}
           JF2_IP=${{secrets.STAGING2_IP}}
+          LOKI_ADDRESS=${{secrets.STAGING_IP}}
           JF2_HOST=${{env.JF2_HOST}}" > .env
       - name: Remove old containers on second host
         uses: JimCronqvist/action-ssh@master
@@ -98,4 +100,4 @@ jobs:
           ssh_private_key: ${{ secrets.SSH_PRIV_KEY }}
           ssh_public_key: ${{ secrets.SSH_PUB_KEY }}
           stack_file_name: ${{ env.COMPOSE_FILE_NAME }}
-          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish2 node-exporter2
+          args: -p ${{ env.PROJECT }} --env-file .env up -d --remove-orphans --build jellyfish2 node-exporter2 promtail
diff --git a/docker-compose-deploy.yaml b/docker-compose-deploy.yaml
@@ -127,6 +127,28 @@ services:
     <<: *node-exporter-template
     container_name: node_exporter2
 
+  loki:
+    image: grafana/loki:2.9.0
+    build:
+      context: ./loki
+      dockerfile: Dockerfile
+    volumes:
+      - loki_data:/loki
+    ports:
+      - "3100:3100"
+
+  promtail:
+    build:
+      context: ./promtail
+      dockerfile: Dockerfile
+      args:
+        LOKI_ADDRESS: $LOKI_ADDRESS
+    volumes:
+      # to read container labels and logs
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/containers:/var/lib/docker/containers
+
 volumes:
   grafana_data: {}
   prometheus_data: {}
+  loki_data: {}
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -130,7 +130,27 @@ services:
       - --path.rootfs=/host
     volumes:
       - /:/host:ro,rslave
+  loki:
+    build:
+      context: ./loki
+      dockerfile: Dockerfile
+    volumes:
+      - loki_data:/loki
+    ports:
+      - "3100:3100"
+
+  promtail:
+    build:
+      context: ./promtail
+      dockerfile: Dockerfile
+      args:
+        LOKI_ADDRESS: loki
+    volumes:
+      # to read container labels and logs
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/containers:/var/lib/docker/containers
 
 volumes:
   grafana_data: {}
   prometheus_data: {}
+  loki_data: {}
diff --git a/grafana/provisioning/dashboards/metrics.json b/grafana/provisioning/dashboards/metrics.json
@@ -59,7 +59,7 @@
                         "h": 8,
                         "w": 10,
                         "x": 0,
-                        "y": 1
+                        "y": 9
                     },
                     "id": 6,
                     "options": {
@@ -156,7 +156,7 @@
                         "h": 8,
                         "w": 14,
                         "x": 10,
-                        "y": 1
+                        "y": 9
                     },
                     "id": 1,
                     "options": {
@@ -265,7 +265,7 @@
                         "h": 8,
                         "w": 12,
                         "x": 0,
-                        "y": 2
+                        "y": 10
                     },
                     "id": 3,
                     "options": {
@@ -361,7 +361,7 @@
                         "h": 8,
                         "w": 12,
                         "x": 12,
-                        "y": 2
+                        "y": 10
                     },
                     "id": 2,
                     "options": {
@@ -473,7 +473,7 @@
                         "h": 10,
                         "w": 24,
                         "x": 0,
-                        "y": 10
+                        "y": 18
                     },
                     "id": 8,
                     "options": {
@@ -601,7 +601,7 @@
                         "h": 8,
                         "w": 24,
                         "x": 0,
-                        "y": 11
+                        "y": 19
                     },
                     "id": 10,
                     "options": {
@@ -693,7 +693,7 @@
                         "h": 8,
                         "w": 24,
                         "x": 0,
-                        "y": 19
+                        "y": 27
                     },
                     "id": 14,
                     "options": {
@@ -785,7 +785,7 @@
                         "h": 8,
                         "w": 24,
                         "x": 0,
-                        "y": 27
+                        "y": 35
                     },
                     "id": 15,
                     "options": {
@@ -877,7 +877,7 @@
                         "h": 8,
                         "w": 24,
                         "x": 0,
-                        "y": 35
+                        "y": 43
                     },
                     "id": 16,
                     "options": {
@@ -1347,8 +1347,7 @@
                                 "mode": "absolute",
                                 "steps": [
                                     {
-                                        "color": "green",
-                                        "value": null
+                                        "color": "green"
                                     }
                                 ]
                             },
@@ -1395,6 +1394,58 @@
             ],
             "title": "Node stats",
             "type": "row"
+        },
+        {
+            "collapsed": true,
+            "gridPos": {
+                "h": 1,
+                "w": 24,
+                "x": 0,
+                "y": 4
+            },
+            "id": 25,
+            "panels": [
+                {
+                    "datasource": {
+                        "type": "loki",
+                        "uid": "${logs_ds}"
+                    },
+                    "description": "Logs from selected containers",
+                    "gridPos": {
+                        "h": 10,
+                        "w": 24,
+                        "x": 0,
+                        "y": 5
+                    },
+                    "id": 26,
+                    "options": {
+                        "dedupStrategy": "none",
+                        "enableLogDetails": false,
+                        "prettifyLogMessage": false,
+                        "showCommonLabels": false,
+                        "showLabels": true,
+                        "showTime": false,
+                        "sortOrder": "Descending",
+                        "wrapLogMessage": true
+                    },
+                    "targets": [
+                        {
+                            "datasource": {
+                                "type": "loki",
+                                "uid": "${logs_ds}"
+                            },
+                            "editorMode": "code",
+                            "expr": "{container=~\"$container\"}",
+                            "queryType": "range",
+                            "refId": "A"
+                        }
+                    ],
+                    "title": "Logs",
+                    "type": "logs"
+                }
+            ],
+            "title": "Logs",
+            "type": "row"
         }
     ],
     "refresh": "",
@@ -1484,6 +1535,62 @@
                 "skipUrlSync": false,
                 "sort": 0,
                 "type": "query"
+            },
+            {
+                "current": {
+                    "selected": false,
+                    "text": "Loki",
+                    "value": "f1146f90-cd6e-40a3-97bc-834ec46d3167"
+                },
+                "description": "",
+                "hide": 0,
+                "includeAll": false,
+                "label": "Logs Data Source",
+                "multi": false,
+                "name": "logs_ds",
+                "options": [],
+                "query": "loki",
+                "refresh": 1,
+                "regex": "",
+                "skipUrlSync": false,
+                "type": "datasource"
+            },
+            {
+                "current": {
+                    "selected": true,
+                    "text": [
+                        "backend",
+                        "jellyfish1",
+                        "jellyfish2"
+                    ],
+                    "value": [
+                        "backend",
+                        "jellyfish1",
+                        "jellyfish2"
+                    ]
+                },
+                "datasource": {
+                    "type": "loki",
+                    "uid": "f1146f90-cd6e-40a3-97bc-834ec46d3167"
+                },
+                "definition": "",
+                "hide": 0,
+                "includeAll": true,
+                "label": "Container",
+                "multi": true,
+                "name": "container",
+                "options": [],
+                "query": {
+                    "label": "container",
+                    "refId": "LokiVariableQueryEditor-VariableQuery",
+                    "stream": "",
+                    "type": 1
+                },
+                "refresh": 1,
+                "regex": "",
+                "skipUrlSync": false,
+                "sort": 0,
+                "type": "query"
             }
         ]
     },
@@ -1495,6 +1602,6 @@
     "timezone": "",
     "title": "Jellyfishes dashboard",
     "uid": "ef7bdb94-c912-49ac-a273-e7a31598001a",
-    "version": 8,
+    "version": 37,
     "weekStart": ""
 }
diff --git a/grafana/provisioning/datasources/datasource.yml b/grafana/provisioning/datasources/datasource.yml
@@ -14,3 +14,8 @@ datasources:
     editable: true
     basicAuth: false
     isDefault: true
+
+  - name: Loki
+    type: loki
+    url: http://loki:3100
+    isDefault: false
diff --git a/loki/Dockerfile b/loki/Dockerfile
@@ -0,0 +1,2 @@
+FROM grafana/loki:2.9.0
+COPY ./loki-config.yaml /etc/loki/config.yaml
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		FROM grafana/loki:2.9.0
		COPY ./loki-config.yaml /etc/loki/config.yaml