promtail

scripts/deploy_cluster/.gitignore

@@ -1,4 +1,5 @@
+promtail.yaml
 gen_ansible.ini
 gen_docker_stack.yml
 compose_*
-/test_dir
+/test_dir

scripts/deploy_cluster/_gen_each_node_compose.py

@@ -13,17 +13,15 @@ def read_yaml(file_path):
 
 
 def generate_docker_compose(ip, nodes):
-    template = """
-    loki:
-        image: grafana/loki:latest
-        ports:
-        - "3100:3100"
-        command: -config.file=/etc/loki/local-config.yaml
-        volumes:
-        - ./loki.yaml:/etc/loki/local-config.yaml
-        restart: always
-    """
-    services = yaml.safe_load(template)
+    services =yaml.safe_load('''
+        promtail:
+            image: "grafana/promtail"
+            volumes:
+                - /var/lib/docker/containers:/var/lib/docker/containers:ro
+                - /var/run/docker.sock:/var/run/docker.sock
+                - /root/wasm_serverless_deploy/promtail.yaml:/etc/promtail/promtail.yaml
+            command: -config.file=/etc/promtail/promtail.yaml
+        ''')
 
     for key, node in nodes.items():
         service_name = f"node{key}"
@@ -42,20 +40,57 @@ def generate_docker_compose(ip, nodes):
                     }
                 }
             },
-
-            'volumes': ['/root/wasm_serverless_deploy:/etc/wasm_serverless'],
+            'volumes': [
+                '/root/wasm_serverless_deploy:/etc/wasm_serverless',
+            ],
             'environment': {
                 'WASM_SERVERLESS_NODEID': key
             },
-            'privileged': True # for tc control
+            'privileged': True, # for tc control,
+            'logging': {
+                # 设置日志驱动程序和选项
+                'driver': "json-file",
+                'options':{
+                    'max-size': "10m",
+                    'max-file': "5"
+                }
+            },
+            'labels':["log_promtail"]
         }
+
 
     compose_data = {'version': '3', 'services': services}
     compose_file_name = os.path.join(DEPLOY_CLUSTER_DIR, f"compose_{ip}.yml")
 
     with open(compose_file_name, 'w') as file:
         yaml.dump(compose_data, file, default_flow_style=False)
 
+def promtail_config(lokiaddr):
+    PROMTAIL_CONFIG = f'''
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://{lokiaddr}/loki/api/v1/push
+
+scrape_configs:
+  - job_name: flog_scrape
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        filters:
+          - name: label
+            values: ["logging=promtail"]
+    '''
+    path=os.path.join(DEPLOY_CLUSTER_DIR, 'promtail.yaml')
+    with open(path, 'w') as f:
+        f.write(PROMTAIL_CONFIG)
+
+
 
 def main():
     data = read_yaml(NODE_CONFIG)
@@ -69,6 +104,8 @@ def main():
 
     for ip, nodes in grouped_nodes.items():
         generate_docker_compose(ip, nodes)
+
+    promtail_config(data['loki']['addr'])
 
 
 if __name__ == "__main__":

scripts/deploy_cluster/ans_redeploy_cluster.yml

@@ -7,9 +7,11 @@
     - name: Build sys and demo apps
       include_tasks: ../build/_ans_build_sys_with_apps.yml
 
+
     - name: Generate Docker Compose File
       shell: python3 _gen_each_node_compose.py
 
+    # Preparing for next zip
     - name: Remove Zip
       file:
         path: deploy_pack.zip
@@ -38,6 +40,17 @@
       loop:
         - /root/wasm_serverless_deploy
 
+    - name: Stop & Remove container
+      block:
+      - name: Stop containers
+        shell: docker ps -a | grep "wasm_serverless_deploy" | awk '{print $1}' | xargs docker stop
+        become: yes
+        ignore_errors: yes
+      - name: Remove containers
+        shell: docker ps -a | grep "wasm_serverless_deploy" | awk '{print $1}' | xargs docker rm
+        become: yes
+        ignore_errors: yes
+
     - name: Clear the target directory
       shell: rm -rf /root/wasm_serverless_deploy/*
       become: yes
@@ -80,6 +93,10 @@
       shell: mv /root/wasm_serverless_deploy/scripts/deploy_cluster/test_dir /root/wasm_serverless_deploy/
       become: yes
 
+    - name: Move promtail.yaml to root
+      shell: mv /root/wasm_serverless_deploy/scripts/deploy_cluster/promtail.yaml /root/wasm_serverless_deploy/
+      become: yes
+
     - name: Move bin to test_dir
       shell: mv /root/wasm_serverless_deploy/target/release/wasm_serverless /root/wasm_serverless_deploy/
       become: yes
@@ -109,11 +126,22 @@
           chdir: /root/wasm_serverless_deploy
         become: yes
 
+      - name: Kill processes occupying specific ports
+        shell: sudo kill $(sudo lsof -t -i:{{ item }})
+        become: yes
+        ignore_errors: yes
+        loop:
+          - 2500
+          - 2501
+          - 2505
+          - 2506
+
       - name: docker-compose
         command: docker-compose -v
         args:
           chdir: /root/wasm_serverless_deploy
         become: yes
+
 
       - name: Docker compose up
         command: docker-compose --compatibility up -d

scripts/deploy_cluster/node_config.yaml

@@ -17,3 +17,6 @@ nodes:
   6: 
     addr: 192.168.31.96:2505
     spec: [meta,worker]
+
+loki:
+  addr: 192.168.31.56:3100

scripts/deploy_monitor/docker-compose.yaml

@@ -1,5 +1,14 @@
 version: '3'
 services:
+  loki:
+    image: grafana/loki:latest
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    volumes:
+      - ./loki.yaml:/etc/loki/local-config.yaml:ro
+    restart: always
+
   prometheus:
     image: prom/prometheus
     ports:

scripts/deploy_monitor/loki.yaml

@@ -0,0 +1,66 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
+  wal:
+    dir: /loki/.cache/loki/wal/
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/boltdb-shipper-active
+    cache_location: /loki/boltdb-shipper-cache
+    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
+  filesystem:
+    directory: /loki/chunks
+
+compactor:
+  working_directory: /loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true