From 421df27faf0f3e2318b2ebdb3aa602d6b0d41461 Mon Sep 17 00:00:00 2001 From: Krzysztof Kwapisiewicz Date: Wed, 31 Aug 2022 11:03:08 +0200 Subject: [PATCH] Remove batchMetricsFast processor --- .../agent/files/aperture-agent-config.yaml | 3 --- manifests/charts/agent/values.yaml | 7 ------- manifests/charts/aperture-operator/README.md | 2 -- .../aperture-operator/templates/agent.yaml | 5 ----- manifests/charts/aperture-operator/values.yaml | 6 ------ operator/api/v1alpha1/agent_types.go | 5 ----- operator/api/v1alpha1/zz_generated.deepcopy.go | 1 - operator/controllers/agent_config.tpl | 3 --- .../agent_config_with_fluxninja_plugin_test.tpl | 3 --- operator/controllers/configmaps.go | 2 -- operator/controllers/configmaps_test.go | 4 ---- pkg/otel/config.go | 16 ++++------------ 12 files changed, 4 insertions(+), 53 deletions(-) diff --git a/manifests/charts/agent/files/aperture-agent-config.yaml b/manifests/charts/agent/files/aperture-agent-config.yaml index dcd0988259..5c91d646b7 100644 --- a/manifests/charts/agent/files/aperture-agent-config.yaml +++ b/manifests/charts/agent/files/aperture-agent-config.yaml @@ -14,9 +14,6 @@ otel: batch_postrollup: timeout: {{ .Values.agent.batchPostrollup.timeout }} send_batch_size: {{ .Values.agent.batchPostrollup.sendBatchSize }} - batch_metrics_fast: - timeout: {{ .Values.agent.batchMetricsFast.timeout }} - send_batch_size: {{ .Values.agent.batchMetricsFast.sendBatchSize }} {{- end }} log: diff --git a/manifests/charts/agent/values.yaml b/manifests/charts/agent/values.yaml index 316f067b8a..aeb54ce0ef 100644 --- a/manifests/charts/agent/values.yaml +++ b/manifests/charts/agent/values.yaml @@ -288,13 +288,6 @@ agent: timeout: 1s sendBatchSize: 10000 - ## @param agent.batchMetricsFast.timeout Timeout for batch metrics/fast processor. - ## @param agent.batchMetricsFast.sendBatchSize Size of a batch in metrics/fast processor which after hit, will trigger it to be sent. - ## - batchMetricsFast: - timeout: 1s - sendBatchSize: 1000 - ## @section Controller Parameters ## Agent Controller container diff --git a/manifests/charts/aperture-operator/README.md b/manifests/charts/aperture-operator/README.md index 3a8d8ef0aa..a483aaecb4 100644 --- a/manifests/charts/aperture-operator/README.md +++ b/manifests/charts/aperture-operator/README.md @@ -136,8 +136,6 @@ Aperture Operator | `agent.batchPrerollup.sendBatchSize` | Size of a batch in prerollup processor which after hit, will trigger it to be sent. | `nil` | | `agent.batchPostrollup.timeout` | Timeout for batch postrollup processor. | `nil` | | `agent.batchPostrollup.sendBatchSize` | Size of a batch in postrollup processor which after hit, will trigger it to be sent. | `nil` | -| `agent.batchMetricsFast.timeout` | Timeout for batch metrics/fast processor. | `nil` | -| `agent.batchMetricsFast.sendBatchSize` | Size of a batch in metrics/fast processor which after hit, will trigger it to be sent. | `nil` | | `agent.log.prettyConsole` | Additional log writer: pretty console (stdout) logging (not recommended for prod environments). Defaults to false. | `nil` | | `agent.log.nonBlocking` | Use non-blocking log writer (can lose logs at high throughput). Defaults to True. | `nil` | | `agent.log.level` | Log level. Keywords allowed - ["debug", "info", "warn", "fatal", "panic", "trace"]. Defaults to 'info'. | `nil` | diff --git a/manifests/charts/aperture-operator/templates/agent.yaml b/manifests/charts/aperture-operator/templates/agent.yaml index 558391b488..83757d875a 100644 --- a/manifests/charts/aperture-operator/templates/agent.yaml +++ b/manifests/charts/aperture-operator/templates/agent.yaml @@ -36,11 +36,6 @@ spec: timeout: {{ .Values.agent.batchPostrollup.timeout }} sendBatchSize: {{ .Values.agent.batchPostrollup.sendBatchSize }} {{- end }} - {{- if or .Values.agent.batchMetricsFast.timeout .Values.agent.batchMetricsFast.sendBatchSize }} - batchMetricsFast: - timeout: {{ .Values.agent.batchMetricsFast.timeout }} - sendBatchSize: {{ .Values.agent.batchMetricsFast.sendBatchSize }} - {{- end }} {{- if or .Values.agent.log.prettyConsole .Values.agent.log.nonBlocking .Values.agent.log.level .Values.agent.log.file }} log: {{ .Values.agent.log | toYaml | nindent 4 }} {{- end }} diff --git a/manifests/charts/aperture-operator/values.yaml b/manifests/charts/aperture-operator/values.yaml index 1fbe7f5a41..81a829bf53 100644 --- a/manifests/charts/aperture-operator/values.yaml +++ b/manifests/charts/aperture-operator/values.yaml @@ -392,12 +392,6 @@ agent: batchPostrollup: timeout: ~ sendBatchSize: ~ - ## @param agent.batchMetricsFast.timeout Timeout for batch metrics/fast processor. - ## @param agent.batchMetricsFast.sendBatchSize Size of a batch in metrics/fast processor which after hit, will trigger it to be sent. - ## - batchMetricsFast: - timeout: ~ - sendBatchSize: ~ ## @param agent.log.prettyConsole Additional log writer: pretty console (stdout) logging (not recommended for prod environments). Defaults to false. ## @param agent.log.nonBlocking Use non-blocking log writer (can lose logs at high throughput). Defaults to True. ## @param agent.log.level Log level. Keywords allowed - ["debug", "info", "warn", "fatal", "panic", "trace"]. Defaults to 'info'. diff --git a/operator/api/v1alpha1/agent_types.go b/operator/api/v1alpha1/agent_types.go index da6d12997d..6fcaa6a040 100644 --- a/operator/api/v1alpha1/agent_types.go +++ b/operator/api/v1alpha1/agent_types.go @@ -65,11 +65,6 @@ type AgentSpec struct { //+kubebuilder:validation:Optional //+kubebuilder:default:={timeout:"1s",sendBatchSize:10000} BatchPostrollup Batch `json:"batchPostrollup"` - - // Batch metrics/fast processor configuration. - //+kubebuilder:validation:Optional - //+kubebuilder:default:={timeout:"1s",sendBatchSize:1000} - BatchMetricsFast Batch `json:"batchMetricsFast"` } // AgentStatus defines the observed state of Agent. diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 0e6e62fe6c..8549ab13a4 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -126,7 +126,6 @@ func (in *AgentSpec) DeepCopyInto(out *AgentSpec) { in.Sidecar.DeepCopyInto(&out.Sidecar) out.BatchPrerollup = in.BatchPrerollup out.BatchPostrollup = in.BatchPostrollup - out.BatchMetricsFast = in.BatchMetricsFast } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentSpec. diff --git a/operator/controllers/agent_config.tpl b/operator/controllers/agent_config.tpl index 28d5684eb0..b64f073f21 100644 --- a/operator/controllers/agent_config.tpl +++ b/operator/controllers/agent_config.tpl @@ -14,9 +14,6 @@ otel: batch_postrollup: timeout: {{ .BatchPostrollup.Timeout }} send_batch_size: {{ .BatchPostrollup.SendBatchSize }} - batch_metrics_fast: - timeout: {{ .BatchMetricsFast.Timeout }} - send_batch_size: {{ .BatchMetricsFast.SendBatchSize }} {{- end }} log: diff --git a/operator/controllers/agent_config_with_fluxninja_plugin_test.tpl b/operator/controllers/agent_config_with_fluxninja_plugin_test.tpl index 78d53aacc7..ff8997dcd2 100644 --- a/operator/controllers/agent_config_with_fluxninja_plugin_test.tpl +++ b/operator/controllers/agent_config_with_fluxninja_plugin_test.tpl @@ -13,9 +13,6 @@ otel: batch_postrollup: timeout: 1s send_batch_size: 10000 - batch_metrics_fast: - timeout: 1s - send_batch_size: 10000 log: pretty_console: false diff --git a/operator/controllers/configmaps.go b/operator/controllers/configmaps.go index 688b86ba1a..e869355eeb 100644 --- a/operator/controllers/configmaps.go +++ b/operator/controllers/configmaps.go @@ -55,7 +55,6 @@ func filledAgentConfig(instance *v1alpha1.Agent) (string, error) { Ingestion v1alpha1.Ingestion `json:"ingestion"` BatchPrerollup v1alpha1.Batch BatchPostrollup v1alpha1.Batch - BatchMetricsFast v1alpha1.Batch }{ ServerPort: instance.Spec.ServerPort, DistributedCachePort: instance.Spec.DistributedCachePort, @@ -66,7 +65,6 @@ func filledAgentConfig(instance *v1alpha1.Agent) (string, error) { PrometheusAddress: checkPrometheusAddress(instance.Spec.Prometheus.Address, instance.GetName(), instance.GetNamespace()), BatchPrerollup: instance.Spec.BatchPrerollup, BatchPostrollup: instance.Spec.BatchPostrollup, - BatchMetricsFast: instance.Spec.BatchMetricsFast, } var config bytes.Buffer diff --git a/operator/controllers/configmaps_test.go b/operator/controllers/configmaps_test.go index 51af076756..136d9d86a4 100644 --- a/operator/controllers/configmaps_test.go +++ b/operator/controllers/configmaps_test.go @@ -90,10 +90,6 @@ var _ = Describe("ConfigMap for Agent", func() { Timeout: "1s", SendBatchSize: 10000, }, - BatchMetricsFast: v1alpha1.Batch{ - Timeout: "1s", - SendBatchSize: 10000, - }, DistributedCachePort: 3320, MemberListPort: 3322, }, diff --git a/pkg/otel/config.go b/pkg/otel/config.go index 95b5af716d..59d0bcfd86 100644 --- a/pkg/otel/config.go +++ b/pkg/otel/config.go @@ -2,7 +2,6 @@ package otel import ( "fmt" - "time" promapi "github.com/prometheus/client_golang/api" "go.opentelemetry.io/collector/config/configgrpc" @@ -34,9 +33,6 @@ const ( // ProcessorBatchPostrollup batches data after rolling up, as roll up process // shrinks number of data points significantly. ProcessorBatchPostrollup = "batch/postrollup" - // ProcessorBatchMetricsFast batches metrics in small and fast packages. Used - // in flow control policy. - ProcessorBatchMetricsFast = "batch/metrics-fast" // ProcessorRollup rolls up data to decrease cardinality. ProcessorRollup = "rollup" @@ -52,10 +48,9 @@ type otelConfig struct { // Addr is an address on which this app is serving metrics. // TODO this should be inherited from the listener.Listener config, but it's // not initialized at the provide state of app. - Addr string `json:"addr" validate:"hostname_port" default:":8080"` - BatchPrerollup Batch `json:"batch_prerollup"` - BatchPostrollup Batch `json:"batch_postrollup"` - BatchMetricsFast Batch `json:"batch_metrics_fast"` + Addr string `json:"addr" validate:"hostname_port" default:":8080"` + BatchPrerollup Batch `json:"batch_prerollup"` + BatchPostrollup Batch `json:"batch_postrollup"` } // Batch defines configuration for OTEL batch processor. @@ -147,13 +142,11 @@ func addLogsAndTracesPipelines(config *otelcollector.OTELConfig, cfg otelConfig) func addMetricsPipeline(config *otelcollector.OTELConfig, promClient promapi.Client, cfg otelConfig) { addPrometheusReceiver(config, cfg) config.AddProcessor(ProcessorEnrichment, nil) - config.AddBatchProcessor(ProcessorBatchMetricsFast, cfg.BatchMetricsFast.Timeout.Duration.AsDuration(), cfg.BatchMetricsFast.SendBatchSize) addPrometheusRemoteWriteExporter(config, promClient) config.Service.AddPipeline("metrics/fast", otelcollector.Pipeline{ Receivers: []string{ReceiverPrometheus}, Processors: []string{ ProcessorEnrichment, - ProcessorBatchMetricsFast, }, Exporters: []string{ExporterPrometheusRemoteWrite}, }) @@ -161,11 +154,10 @@ func addMetricsPipeline(config *otelcollector.OTELConfig, promClient promapi.Cli func addControllerMetricsPipeline(config *otelcollector.OTELConfig, promClient promapi.Client, cfg otelConfig) { addControllerPrometheusReceiver(config, cfg) - config.AddBatchProcessor(ProcessorBatchMetricsFast, 1*time.Second, 1000) addPrometheusRemoteWriteExporter(config, promClient) config.Service.AddPipeline("metrics/controller-fast", otelcollector.Pipeline{ Receivers: []string{ReceiverPrometheus}, - Processors: []string{ProcessorBatchMetricsFast}, + Processors: []string{}, Exporters: []string{ExporterPrometheusRemoteWrite}, }) }