From fa7dac68cb7556b6480b2f4b126da9dace22b56c Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez Date: Tue, 11 Jul 2023 18:56:48 +0200 Subject: [PATCH 1/2] Configure default head_chunks_write_queue_size to 1 million to avoid high latency when shipping Signed-off-by: Friedrich Gonzalez --- CHANGELOG.md | 1 + cortex/ingester.libsonnet | 20 +------------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aefd628..4648f8c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## master / unreleased * [CHANGE] Use cortex v1.15.3 * [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility +* [ENHANCEMENT] Configure head_chunks_write_queue_size to 1 million to avoid high latency when shipping ## 1.14.1 / 2023-07-11 diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet index 818716e1..7b86dc7e 100644 --- a/cortex/ingester.libsonnet +++ b/cortex/ingester.libsonnet @@ -36,6 +36,7 @@ 'blocks-storage.tsdb.dir': '/data/tsdb', 'blocks-storage.tsdb.block-ranges-period': '2h', + 'blocks-storage.tsdb.head-chunks-write-queue-size': 1e6, 'blocks-storage.tsdb.retention-period': '96h', // 4 days protection against blocks not being uploaded from ingesters. 'blocks-storage.tsdb.ship-interval': '1m', @@ -44,18 +45,6 @@ 'ingester.tokens-file-path': '/data/tokens', }, - ingester_statefulset_args:: - $._config.grpcConfig - { - 'ingester.wal-enabled': true, - 'ingester.checkpoint-enabled': true, - 'ingester.recover-from-wal': true, - 'ingester.wal-dir': $._config.ingester.wal_dir, - 'ingester.checkpoint-duration': '15m', - '-log.level': 'info', - 'ingester.tokens-file-path': $._config.ingester.wal_dir + '/tokens', - }, - ingester_ports:: $.util.defaultPorts, local name = 'ingester', @@ -72,13 +61,6 @@ local volumeMount = $.core.v1.volumeMount, - ingester_statefulset_container:: - $.ingester_container + - container.withArgsMixin($.util.mapToFlags($.ingester_statefulset_args)) + - container.withVolumeMountsMixin([ - volumeMount.new('ingester-pvc', $._config.ingester.wal_dir), - ]), - ingester_deployment_labels:: {}, local ingester_pvc = From 58b973fcbeb416eb24c6b5f46e481665716b58ae Mon Sep 17 00:00:00 2001 From: Charlie Le <3375195+CharlieTLe@users.noreply.github.com> Date: Sun, 21 Jul 2024 10:56:36 -0700 Subject: [PATCH 2/2] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81ca7179..8728dcf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * [CHANGE] Remove chunks support for dashboards * [ENHANCEMENT] Configure `-ingester.client.grpc-compression` to be `snappy-block` * [ENHANCEMENT] Support Grafana 11 in Cortex Service Scaling Dashboard +* [ENHANCEMENT] Configure head_chunks_write_queue_size to 1 million to avoid high latency when shipping ## 1.16.1 * [CHANGE] Upgrade memcached to 1.6.23-alpine and memcached-exporter to v0.14.2 @@ -23,7 +24,6 @@ * [CHANGE] Add default tenant shard sizes * [CHANGE] Use cortex v1.15.3 * [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility -* [ENHANCEMENT] Configure head_chunks_write_queue_size to 1 million to avoid high latency when shipping ## 1.14.1 / 2023-07-11