High memory utilization

[P0lskay]

A note for the community

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

Problem

We are currently testing Vector in our k8s clusters to replace the Fluent Bit log collector. However, we are faced with high memory consumption relative to the Fluent Bit. At the same load, the Fluent Bit consumes 25 MiB of RAM, while Vector consumes up to 150 MiB. Why is there such a big difference?

Configuration

api:
  address: 0.0.0.0:8686
  enabled: true
data_dir: /vector-data-dir
sinks:
  main-rules-prom_exporter:
    address: 0.0.0.0:9598
    inputs:
      - main-rules-internal_metrics
    type: prometheus_exporter
  main-rules-vector_output:
    address: http://${VECTOR_SERVICE_HOST}:6000
    batch:
      max_events: 3000
      timeout_secs: 3
    buffer:
      max_events: 50000
      type: memory
      when_full: block
    healthcheck:
      enabled: false
    inputs:
      - main-rules-container_tag
      - main-rules-api_tag
      - main-rules-event_tag
      - main-rules-node_tag
    request:
      adaptive_concurrency:
        initial_concurrency: 50
        max_concurrency_limit: 200
    type: vector
  throttle-rule-container_output:
    address: http://${VECTOR_SERVICE_HOST}:6000
    batch:
      max_events: 3000
      timeout_secs: 3
    buffer:
      max_events: 50000
      type: memory
      when_full: block
    healthcheck:
      enabled: false
    inputs:
      - throttle-rule-container_tag
    request:
      adaptive_concurrency:
        initial_concurrency: 50
        max_concurrency_limit: 200
    type: vector
sources:
  main-rules-api_input:
    file_key: ''
    host_key: ''
    include:
      - /var/log/kubernetes/audit/kube-apiserver-audit.log
    max_line_bytes: 500000
    type: file
  main-rules-container_input:
    delay_deletion_ms: 60000
    exclude_paths_glob_patterns:
      - /var/log/pods/k8s-event-logger-*.log
      - /var/log/containers/k8s-event-logger-*.log
      - '**/*.gz'
      - '**/*.tmp'
    extra_namespace_label_selector: vector.dev/throttle-logs!=true
    glob_minimum_cooldown_ms: 5000
    namespace_annotation_fields:
      namespace_labels: ''
    node_annotation_fields:
      node_labels: ''
    oldest_first: true
    pod_annotation_fields:
      container_image_id: ''
      pod_annotations: ''
      pod_ip: ''
      pod_ips: ''
      pod_labels: .k8s.labels
      pod_namespace: .kubernetes.namespace_name
      pod_node_name: .hostname
      pod_owner: ''
      pod_uid: ''
    type: kubernetes_logs
  main-rules-event_input:
    file_key: ''
    host_key: ''
    include:
      - /var/log/containers/k8s-event-logger-*.log
    max_line_bytes: 500000
    type: file
  main-rules-host_input:
    file_key: ''
    host_key: ''
    include:
      - /var/log/messages
    max_line_bytes: 500000
    type: file
  main-rules-internal_metrics:
    scrape_interval_secs: 30
    type: internal_metrics
  throttle-rule-container_input:
    delay_deletion_ms: 60000
    exclude_paths_glob_patterns:
      - /var/log/pods/k8s-event-logger-*.log
      - '**/*.gz'
      - '**/*.tmp'
    extra_namespace_label_selector: vector.dev/throttle-logs=true
    glob_minimum_cooldown_ms: 5000
    namespace_annotation_fields:
      namespace_labels: ''
    node_annotation_fields:
      node_labels: ''
    oldest_first: true
    pod_annotation_fields:
      container_image_id: ''
      pod_annotations: ''
      pod_ip: ''
      pod_ips: ''
      pod_labels: .k8s.labels
      pod_namespace: .kubernetes.namespace_name
      pod_node_name: .hostname
      pod_owner: ''
      pod_uid: ''
    type: kubernetes_logs
transforms:
  main-rules-api_tag:
    inputs:
      - main-rules-api_input
    source: |-
      . = parse_json!(.message)
      del(.requestObject)
      del(.responseObject)
      .hostname = get_env_var!("VECTOR_SELF_NODE_NAME")
      .log_type = "api"
    type: remap
  main-rules-container_tag:
    inputs:
      - main-rules-container_input
    source: .log_type = "container"
    type: remap
  main-rules-event_tag:
    inputs:
      - main-rules-event_input
    source: >-
      . = parse_regex!(.message, r'^(?P<ts>.*?) (?P<stream>.*?) (?P<logtag>.*?)
      (?P<message>.*?)$')

      .event_parsed = parse_json!(.message)

      del(.event_parsed.metadata.managedFields)

      del(.message)

      .hostname = get_env_var!("VECTOR_SELF_NODE_NAME")

      .log_type = "event"
    type: remap
  main-rules-node_tag:
    inputs:
      - main-rules-host_input
    source: |-
      .log = del(.message)
      .hostname = get_env_var!("VECTOR_SELF_NODE_NAME")
      .log_type = "node"
    type: remap
  throttle-rule-container_tag:
    inputs:
      - throttle-rule-throttle-*
    source: .log_type = "container"
    type: remap
  throttle-rule-routing:
    inputs:
      - throttle-rule-container_input
    reroute_unmatched: false
    route:
      personal-aekralin: .kubernetes.namespace_name == "personal-aekralin"
      personal-akkocheshkov: .kubernetes.namespace_name == "personal-akkocheshkov"
      personal-anmakarov: .kubernetes.namespace_name == "personal-anmakarov"
      personal-annikulin: .kubernetes.namespace_name == "personal-annikulin"
      personal-aszubkov: .kubernetes.namespace_name == "personal-aszubkov"
      personal-avshevchuk: .kubernetes.namespace_name == "personal-avshevchuk"
      personal-avvorobev2: .kubernetes.namespace_name == "personal-avvorobev2"
      personal-birkganms: .kubernetes.namespace_name == "personal-birkganms"
      personal-mskondratev: .kubernetes.namespace_name == "personal-mskondratev"
      system-ingress: .kubernetes.namespace_name == "system-ingress"
    type: route
  throttle-rule-throttle-personal-aekralin:
    inputs:
      - throttle-rule-routing.personal-aekralin
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-akkocheshkov:
    inputs:
      - throttle-rule-routing.personal-akkocheshkov
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-anmakarov:
    inputs:
      - throttle-rule-routing.personal-anmakarov
    threshold: 5000000
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-annikulin:
    inputs:
      - throttle-rule-routing.personal-annikulin
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-aszubkov:
    inputs:
      - throttle-rule-routing.personal-aszubkov
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-avshevchuk:
    inputs:
      - throttle-rule-routing.personal-avshevchuk
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-avvorobev2:
    inputs:
      - throttle-rule-routing.personal-avvorobev2
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-birkganms:
    inputs:
      - throttle-rule-routing.personal-birkganms
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-personal-mskondratev:
    inputs:
      - throttle-rule-routing.personal-mskondratev
    threshold: 100
    type: throttle
    window_secs: 60
  throttle-rule-throttle-system-ingress:
    inputs:
      - throttle-rule-routing.system-ingress
    threshold: 10000
    type: throttle
    window_secs: 60

Version

0.42.0

Debug Output

No response

Example Data

No response

Additional Context

No response

References

No response