Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add pattern ingester support in SSD mode #12685

Merged
merged 3 commits into from
Apr 19, 2024
Merged

feat: Add pattern ingester support in SSD mode #12685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
12ec6ed
feat: Add pattern ingester support in SSD mode
cyriltovena Apr 18, 2024
339caac
make documentation-helm-reference-check
cyriltovena Apr 18, 2024
3b36a16
bump chart and changelog
cyriltovena Apr 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion docs/sources/setup/install/helm/reference.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5211,7 +5211,7 @@ null
"replication_factor": 3
},
"compactor": {},
"config": "{{- if .Values.enterprise.enabled}}\n{{- tpl .Values.enterprise.config . }}\n{{- else }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\n{{- end }}\n\n{{- with .Values.loki.server }}\nserver:\n {{- toYaml . | nindent 2}}\n{{- end}}\n\nmemberlist:\n{{- if .Values.loki.memberlistConfig }}\n {{- toYaml .Values.loki.memberlistConfig | nindent 2 }}\n{{- else }}\n{{- if .Values.loki.extraMemberlistConfig}}\n{{- toYaml .Values.loki.extraMemberlistConfig | nindent 2}}\n{{- end }}\n join_members:\n - {{ include \"loki.memberlist\" . }}\n {{- with .Values.migrate.fromDistributed }}\n {{- if .enabled }}\n - {{ .memberlistService }}\n {{- end }}\n {{- end }}\n{{- end }}\n\n{{- with .Values.loki.ingester }}\ningester:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- if .Values.loki.commonConfig}}\ncommon:\n{{- toYaml .Values.loki.commonConfig | nindent 2}}\n storage:\n {{- include \"loki.commonStorageConfig\" . | nindent 4}}\n{{- end}}\n\n{{- with .Values.loki.limits_config }}\nlimits_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\nruntime_config:\n file: /etc/loki/runtime-config/runtime-config.yaml\n\n{{- with .Values.chunksCache }}\n{{- if .enabled }}\nchunk_store_config:\n chunk_cache_config:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached:\n batch_size: {{ .batchSize }}\n parallelism: {{ .parallelism }}\n memcached_client:\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-chunks-cache.{{ $.Release.Namespace }}.svc\n consistent_hash: true\n timeout: {{ .timeout }}\n max_idle_conns: 72\n{{- end }}\n{{- end }}\n\n{{- if .Values.loki.schemaConfig }}\nschema_config:\n{{- toYaml .Values.loki.schemaConfig | nindent 2}}\n{{- end }}\n\n{{- if .Values.loki.useTestSchema }}\nschema_config:\n{{- toYaml .Values.loki.testSchemaConfig | nindent 2}}\n{{- end }}\n\n{{ include \"loki.rulerConfig\" . }}\n\n{{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }}\ntable_manager:\n retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }}\n retention_period: {{ .Values.tableManager.retention_period }}\n{{- end }}\n\nquery_range:\n align_queries_with_step: true\n {{- with .Values.loki.query_range }}\n {{- tpl (. | toYaml) $ | nindent 4 }}\n {{- end }}\n {{- if .Values.resultsCache.enabled }}\n {{- with .Values.resultsCache }}\n cache_results: true\n results_cache:\n cache:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached_client:\n consistent_hash: true\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-results-cache.{{ $.Release.Namespace }}.svc\n timeout: {{ .timeout }}\n update_interval: 1m\n {{- end }}\n {{- end }}\n\n{{- with .Values.loki.storage_config }}\nstorage_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.query_scheduler }}\nquery_scheduler:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.compactor }}\ncompactor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.analytics }}\nanalytics:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.querier }}\nquerier:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.index_gateway }}\nindex_gateway:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend }}\nfrontend:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend_worker }}\nfrontend_worker:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.distributor }}\ndistributor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\ntracing:\n enabled: {{ .Values.loki.tracing.enabled }}\n",
"config": "{{- if .Values.enterprise.enabled}}\n{{- tpl .Values.enterprise.config . }}\n{{- else }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\n{{- end }}\n\n{{- with .Values.loki.server }}\nserver:\n {{- toYaml . | nindent 2}}\n{{- end}}\n\npattern_ingester:\n enabled: {{ .Values.loki.pattern_ingester.enabled }}\n\nmemberlist:\n{{- if .Values.loki.memberlistConfig }}\n {{- toYaml .Values.loki.memberlistConfig | nindent 2 }}\n{{- else }}\n{{- if .Values.loki.extraMemberlistConfig}}\n{{- toYaml .Values.loki.extraMemberlistConfig | nindent 2}}\n{{- end }}\n join_members:\n - {{ include \"loki.memberlist\" . }}\n {{- with .Values.migrate.fromDistributed }}\n {{- if .enabled }}\n - {{ .memberlistService }}\n {{- end }}\n {{- end }}\n{{- end }}\n\n{{- with .Values.loki.ingester }}\ningester:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- if .Values.loki.commonConfig}}\ncommon:\n{{- toYaml .Values.loki.commonConfig | nindent 2}}\n storage:\n {{- include \"loki.commonStorageConfig\" . | nindent 4}}\n{{- end}}\n\n{{- with .Values.loki.limits_config }}\nlimits_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\nruntime_config:\n file: /etc/loki/runtime-config/runtime-config.yaml\n\n{{- with .Values.chunksCache }}\n{{- if .enabled }}\nchunk_store_config:\n chunk_cache_config:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached:\n batch_size: {{ .batchSize }}\n parallelism: {{ .parallelism }}\n memcached_client:\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-chunks-cache.{{ $.Release.Namespace }}.svc\n consistent_hash: true\n timeout: {{ .timeout }}\n max_idle_conns: 72\n{{- end }}\n{{- end }}\n\n{{- if .Values.loki.schemaConfig }}\nschema_config:\n{{- toYaml .Values.loki.schemaConfig | nindent 2}}\n{{- end }}\n\n{{- if .Values.loki.useTestSchema }}\nschema_config:\n{{- toYaml .Values.loki.testSchemaConfig | nindent 2}}\n{{- end }}\n\n{{ include \"loki.rulerConfig\" . }}\n\n{{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }}\ntable_manager:\n retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }}\n retention_period: {{ .Values.tableManager.retention_period }}\n{{- end }}\n\nquery_range:\n align_queries_with_step: true\n {{- with .Values.loki.query_range }}\n {{- tpl (. | toYaml) $ | nindent 4 }}\n {{- end }}\n {{- if .Values.resultsCache.enabled }}\n {{- with .Values.resultsCache }}\n cache_results: true\n results_cache:\n cache:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached_client:\n consistent_hash: true\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-results-cache.{{ $.Release.Namespace }}.svc\n timeout: {{ .timeout }}\n update_interval: 1m\n {{- end }}\n {{- end }}\n\n{{- with .Values.loki.storage_config }}\nstorage_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.query_scheduler }}\nquery_scheduler:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.compactor }}\ncompactor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.analytics }}\nanalytics:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.querier }}\nquerier:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.index_gateway }}\nindex_gateway:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend }}\nfrontend:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend_worker }}\nfrontend_worker:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.distributor }}\ndistributor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\ntracing:\n enabled: {{ .Values.loki.tracing.enabled }}\n",
cyriltovena marked this conversation as resolved.
Show resolved Hide resolved
"configObjectName": "{{ include \"loki.name\" . }}",
"configStorageType": "ConfigMap",
"containerSecurityContext": {
Expand Down Expand Up @@ -5269,6 +5269,9 @@ null
"timeout": "500ms"
}
},
"pattern_ingester": {
"enabled": false
},
"podAnnotations": {},
"podLabels": {},
"podSecurityContext": {
Expand Down Expand Up @@ -5627,6 +5630,17 @@ null
}
}
</pre>
</td>
</tr>
<tr>
<td>loki.pattern_ingester</td>
<td>object</td>
<td>Optional pattern ingester configuration</td>
<td><pre lang="json">
{
"enabled": false
}
</pre>
</td>
</tr>
<tr>
Expand Down
2 changes: 1 addition & 1 deletion pkg/loki/loki.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,7 +695,7 @@ func (t *Loki) setupModuleManager() error {
MemberlistKV: {Server},

Read: {QueryFrontend, Querier},
Write: {Ingester, Distributor},
Write: {Ingester, Distributor, PatternIngester},
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how does the distributor find the pattern ingester? is it using a ring?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes correct they have their own ring, and the distributor uses that via the tee component.

Backend: {QueryScheduler, Ruler, Compactor, IndexGateway, BloomGateway, BloomCompactor},

All: {QueryScheduler, QueryFrontend, Querier, Ingester, PatternIngester, Distributor, Ruler, Compactor},
Expand Down
10 changes: 4 additions & 6 deletions production/helm/loki/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ Entries should include a reference to the pull request that introduced the chang

[//]: # (<AUTOMATED_UPDATES_LOCATOR> : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.)

## 6.2.2

- [FEATURE] Add support for enabling pattern ingester config via `loki.pattern_ingester.enabled`.

## 6.2.1

- [BUGFIX] Removed duplicate bucketNames from documentation and fixed key name `deploymentMode`
Expand Down Expand Up @@ -44,7 +48,6 @@ Entries should include a reference to the pull request that introduced the chang

- [CHANGE] Changed version of Loki to 2.9.6


## 5.46.0

- [CHANGE] Changed version of Loki to 2.9.5
Expand Down Expand Up @@ -202,7 +205,6 @@ Entries should include a reference to the pull request that introduced the chang

- [CHANGE] Changed version of Grafana Enterprise Logs to v1.8.4


## 5.32.0

- [CHANGE] Grafana dashboards are no longer created solely in scalable mode and with external cloud storage enabled.
Expand Down Expand Up @@ -454,12 +456,10 @@ Entries should include a reference to the pull request that introduced the chang

- [CHANGE] Changed version of Grafana Enterprise Logs to v1.7.2


## 5.4.0

- [CHANGE] Changed version of Loki to 2.8.2


- [CHANGE] Change default GEL and Loki versions to 1.7.1 and 2.8.1 respectively
- [BUGFIX] Fix dns port in network-policy

Expand All @@ -469,12 +469,10 @@ Entries should include a reference to the pull request that introduced the chang

- [BUGFIX] Add projected volume type to psp


## 4.9.0

- [CHANGE] Changed version of Loki to 2.7.5


- [BUGFIX] Fix role/PSP mapping

- [BUGFIX] Fix service/ingress mapping
Expand Down
2 changes: 1 addition & 1 deletion production/helm/loki/Chart.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: loki
description: Helm chart for Grafana Loki in simple, scalable mode
type: application
appVersion: 3.0.0
version: 6.2.1
version: 6.2.2
home: https://grafana.github.io/helm-charts
sources:
- https://github.com/grafana/loki
Expand Down
2 changes: 1 addition & 1 deletion production/helm/loki/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# loki

![Version: 6.2.1](https://img.shields.io/badge/Version-6.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square)
![Version: 6.2.2](https://img.shields.io/badge/Version-6.2.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square)

Helm chart for Grafana Loki in simple, scalable mode

Expand Down
10 changes: 6 additions & 4 deletions production/helm/loki/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,6 @@ loki:
# The value will be passed through tpl.
generatedConfigObjectName: '{{ include "loki.name" . }}'


# -- Config file contents for Loki
# @default -- See values.yaml
config: |
Expand All @@ -137,6 +136,9 @@ loki:
{{- toYaml . | nindent 2}}
{{- end}}

pattern_ingester:
enabled: {{ .Values.loki.pattern_ingester.enabled }}

memberlist:
{{- if .Values.loki.memberlistConfig }}
{{- toYaml .Values.loki.memberlistConfig | nindent 2 }}
Expand Down Expand Up @@ -415,6 +417,9 @@ loki:
up_to: 3
# -- Optional compactor configuration
compactor: {}
# -- Optional pattern ingester configuration
pattern_ingester:
enabled: false
# -- Optional analytics configuration
analytics: {}
# -- Optional querier configuration
Expand Down Expand Up @@ -851,7 +856,6 @@ adminApi:
# -- Grace period to allow the admin-api to shutdown before it is killed
terminationGracePeriodSeconds: 60


######################################################################################################################
#
# Gateway and Ingress
Expand Down Expand Up @@ -1175,7 +1179,6 @@ ingress:
# - loki.example.com
# secretName: loki-distributed-tls


######################################################################################################################
#
# Migration
Expand All @@ -1192,7 +1195,6 @@ migrate:
# memberlist service DNS so the new deployment can join its ring.
memberlistService: ""


######################################################################################################################
#
# Single Binary Deployment
Expand Down
Loading