diff --git a/production/README.md b/production/README.md index bf2d660b0eb0..410236ef25ec 100644 --- a/production/README.md +++ b/production/README.md @@ -41,6 +41,8 @@ Until this is fixed we recommend [building and running from source](#build-and-r For instructions on how to query Loki, see [our usage docs](../docs/logql.md). +To deploy a cluster of loki locally, please refer to this [doc](./docker/) + ## Using Helm to deploy on Kubernetes There is a [Helm chart](helm) to deploy Loki and Promtail to Kubernetes. diff --git a/production/docker/README.md b/production/docker/README.md new file mode 100644 index 000000000000..7d5fecf38274 --- /dev/null +++ b/production/docker/README.md @@ -0,0 +1,37 @@ +# Loki cluster using docker-compose + +To deploy a cluster of loki nodes on a local machine (as shown below), you could use the `docker-compose-ha-member.yaml` file. + + + +Some features of the deployment: + +- Backend: 3 Loki servers enabled with distributor, ingester, querier module +- Together they form a cluster ring based on memberlist mechanism (if using consul/etcd, modules can be separate for further separate read/write workloads) +- Index data are stored and replicated through botldb-shipper +- Replication_factor=2: the receiving distributor sends log data to 2 ingesters based on consistent hashing +- Chunk storage is a shared directory mounted from the same host directory (to simulate S3 or gcs) +- Query are performed through the two query frontend servers +- An nginx gateway to route the write and read workloads from clients (Grafana, promtail) + +1. Ensure you have the most up-to-date Docker container images: + + ```bash + docker-compose pull + ``` + +1. Run the stack on your local Docker: + + ```bash + docker-compose -f ./docker-compose-ha-memberlist.yaml up + ``` + +1. When adding data source in the grafana dashboar, using `http://loki-gateway:3100` for the URL field. + +1. To clean up + + ```bash + docker-compose -f ./docker-compose-ha-memberlist.yaml down + ``` + + Remove the chunk data under `./chunks/`. diff --git a/production/docker/chunks/.gitignore b/production/docker/chunks/.gitignore new file mode 100755 index 000000000000..f59ec20aabf5 --- /dev/null +++ b/production/docker/chunks/.gitignore @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/production/docker/config/loki-docker-memberlist-boltdb-shipper.yaml b/production/docker/config/loki-docker-memberlist-boltdb-shipper.yaml new file mode 100644 index 000000000000..a68cb6f53e03 --- /dev/null +++ b/production/docker/config/loki-docker-memberlist-boltdb-shipper.yaml @@ -0,0 +1,108 @@ +auth_enabled: false + +http_prefix: + +server: + http_listen_address: 0.0.0.0 + grpc_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_port: 9095 + log_level: debug + +memberlist: + join_members: ["loki-1", "loki_2", "loki_3"] + dead_node_reclaim_time: 30s + gossip_to_dead_nodes_time: 15s + left_ingesters_timeout: 30s + bind_addr: ['0.0.0.0'] + bind_port: 7946 + +ingester: + lifecycler: + join_after: 60s + observe_period: 5s + ring: + replication_factor: 2 + kvstore: + store: memberlist + final_sleep: 0s + chunk_idle_period: 1h + max_chunk_age: 1h + chunk_retain_period: 30s + chunk_encoding: snappy + chunk_target_size: 0 + chunk_block_size: 262144 + # chunk_target_size: 1.572864e+06 + +# Only needed for global rate strategy +# distributor: +# ring: +# kvstore: +# store: memberlist + +schema_config: + configs: + - from: 2020-08-01 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + # shared_store: s3 + shared_store: filesystem + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/boltdb-cache + + #aws: + # s3: s3://us-east-1/mybucket + # sse_encryption: true + # insecure: false + # s3forcepathstyle: true + filesystem: + directory: /loki/chunks + + +limits_config: + max_cache_freshness_per_query: '10m' + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 30m + ingestion_rate_mb: 10 + ingestion_burst_size_mb: 20 + +chunk_store_config: + max_look_back_period: 336h + +table_manager: + retention_deletes_enabled: true + retention_period: 336h + +query_range: + # make queries more cache-able by aligning them with their step intervals + align_queries_with_step: true + max_retries: 5 + # parallelize queries in 15min intervals + split_queries_by_interval: 15m + parallelise_shardable_queries: true + cache_results: true + + results_cache: + cache: + # We're going to use the in-process "FIFO" cache + enable_fifocache: true + fifocache: + size: 1024 + validity: 24h + +frontend: + log_queries_longer_than: 5s + # downstream_url: http://loki-1:3100 + downstream_url: http://loki-gateway:3100 + compress_responses: true + +querier: + query_ingesters_within: 2h diff --git a/production/docker/config/nginx-loki-gateway.conf b/production/docker/config/nginx-loki-gateway.conf new file mode 100644 index 000000000000..5df6822a20cc --- /dev/null +++ b/production/docker/config/nginx-loki-gateway.conf @@ -0,0 +1,67 @@ +error_log /dev/stderr; +pid /tmp/nginx.pid; +worker_rlimit_nofile 8192; + +events { + worker_connections 4096; ## Default: 1024 +} + +http { + + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /dev/stderr main; + sendfile on; + tcp_nopush on; + + upstream distributor { + server loki-1:3100; + server loki-2:3100; + server loki-3:3100; + } + + upstream querier { + server loki-1:3100; + server loki-2:3100; + server loki-3:3100; + } + + upstream query-frontend { + server loki-frontend:3100; + } + + server { + listen 80; + proxy_set_header X-Scope-OrgID docker-ha; + + location = /loki/api/v1/push { + proxy_pass http://distributor$request_uri; + } + + location = /ring { + proxy_pass http://distributor$request_uri; + } + + location = /loki/api/v1/tail { + proxy_pass http://querier$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + location ~ /loki/api/.* { + proxy_pass http://query-frontend$request_uri; + } + } + + server { + listen 3100; + proxy_set_header X-Scope-OrgID docker-ha; + + location ~ /loki/api/.* { + proxy_pass http://querier$request_uri; + } + + } +} diff --git a/production/docker/config/promtail-gateway.yaml b/production/docker/config/promtail-gateway.yaml new file mode 100644 index 000000000000..9ce3d2652fc5 --- /dev/null +++ b/production/docker/config/promtail-gateway.yaml @@ -0,0 +1,19 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + log_level: "debug" + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki-gateway:80/loki/api/v1/push + +scrape_configs: + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log diff --git a/production/docker/docker-compose-ha-diagram.png b/production/docker/docker-compose-ha-diagram.png new file mode 100644 index 000000000000..5033674be92f Binary files /dev/null and b/production/docker/docker-compose-ha-diagram.png differ diff --git a/production/docker/docker-compose-ha-memberlist.yaml b/production/docker/docker-compose-ha-memberlist.yaml new file mode 100644 index 000000000000..72573ba376aa --- /dev/null +++ b/production/docker/docker-compose-ha-memberlist.yaml @@ -0,0 +1,88 @@ +version: "3.8" + +networks: + loki: + +services: + + grafana: + image: grafana/grafana:7.2.0 + ports: + - "3000:3000" + networks: + - loki + + promtail: + image: grafana/promtail:2.0.0 + volumes: + - /var/log:/var/log + - ./config:/etc/promtail/ + ports: + - "9080:9080" + command: -config.file=/etc/promtail/promtail-gateway.yaml + networks: + - loki + + loki-gateway: + image: nginx:1.19 + volumes: + - ./config/nginx-loki-gateway.conf:/etc/nginx/nginx.conf + ports: + - "80" + - "3100" + networks: + - loki + + loki-frontend: + image: grafana/loki:2.0.0 + volumes: + - ./config:/etc/loki/ + ports: + - "3100" + command: "-config.file=/etc/loki/loki-docker-memberlist-boltdb-shipper.yaml -target=query-frontend" + networks: + - loki + deploy: + mode: replicated + replicas: 2 + + loki-1: + image: grafana/loki:1.6.1 + volumes: + - ./config:/etc/loki/ + - ./chunks:/loki/chunks/ + ports: + - "3100" + - "7946" + command: "-config.file=/etc/loki/loki-docker-memberlist-boltdb-shipper.yaml -target=all" + networks: + - loki + restart: on-failure + + loki-2: + image: grafana/loki:1.6.1 + volumes: + - ./config:/etc/loki/ + - ./chunks:/loki/chunks/ + ports: + - "3100" + - "7946" + command: "-config.file=/etc/loki/loki-docker-memberlist-boltdb-shipper.yaml -target=all" + # command: "-config.file=/etc/loki/loki-config.yaml" + networks: + - loki + restart: on-failure + + loki-3: + image: grafana/loki:1.6.1 + volumes: + - ./config:/etc/loki/ + - ./chunks:/loki/chunks/ + ports: + - "3100" + - "7946" + command: "-config.file=/etc/loki/loki-docker-memberlist-boltdb-shipper.yaml -target=all" + # command: "-config.file=/etc/loki/loki-config.yaml" + networks: + - loki + restart: on-failure