Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

APM Server permissions for Kibana appears to be broken #8037

Open
up2neck opened this issue Sep 2, 2024 · 2 comments
Open

APM Server permissions for Kibana appears to be broken #8037

up2neck opened this issue Sep 2, 2024 · 2 comments
Labels

Comments

@up2neck
Copy link

up2neck commented Sep 2, 2024

When APM Server is deployed with ECK it has no sufficient permissions for Kibana API:

Image

Transaction JSON
{
  "_index": ".ds-traces-apm-epm_paas-2024.08.24-000002",
  "_id": "x61jsZEB26iDE9PXHEpU",
  "_version": 1,
  "_score": 0,
  "_source": {
    "parent": {
      "id": "154a123cf3b4f8ce"
    },
    "agent": {
      "name": "go",
      "version": "2.6.0"
    },
    "process": {
      "args": [
        "apm-server",
        "run",
        "-e",
        "-c",
        "config/config-secret/apm-server.yml"
      ],
      "pid": 1,
      "title": "apm-server"
    },
    "destination": {
      "address": "kibana-v1-kb-http.apm-sandbox.svc",
      "port": 5601
    },
    "processor": {
      "event": "span"
    },
    "url": {
      "original": "https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm"
    },
    "labels": {
      "project": "epm-paas"
    },
    "cloud": {
      "availability_zone": "europe-west3-a",
      "instance": {
        "name": "gke-epm-iass-elastic-europe-w-generic-c4e5a328-nebn",
        "id": "3933813931648332798"
      },
      "provider": "gcp",
      "project": {
        "id": "or2-ms-epm-iass-elastic-t1iylu"
      },
      "region": "europe-west3"
    },
    "observer": {
      "hostname": "apm-server-v2-apm-server-56c7746446-m7dzp",
      "type": "apm-server",
      "version": "8.14.3"
    },
    "trace": {
      "id": "154a123cf3b4f8ce0fb856d2d80a0416"
    },
    "@timestamp": "2024-09-02T06:19:00.318Z",
    "data_stream": {
      "namespace": "epm_paas",
      "type": "traces",
      "dataset": "apm"
    },
    "service": {
      "node": {
        "name": "apm-server-v2-apm-server-56c7746446-m7dzp"
      },
      "environment": "sandbox-latest",
      "name": "apm-server",
      "runtime": {
        "name": "gc",
        "version": "go1.22.5"
      },
      "language": {
        "name": "go",
        "version": "go1.22.5"
      },
      "version": "8.14.3",
      "target": {
        "name": "kibana-v1-kb-http.apm-sandbox.svc:5601",
        "type": "http"
      }
    },
    "host": {
      "hostname": "apm-server-v2-apm-server-56c7746446-m7dzp",
      "os": {
        "platform": "linux"
      },
      "name": "apm-server-v2-apm-server-56c7746446-m7dzp",
      "architecture": "amd64"
    },
    "http": {
      "response": {
        "status_code": 403
      }
    },
    "event": {
      "agent_id_status": "missing",
      "ingested": "2024-09-02T06:19:08Z",
      "success_count": 0,
      "outcome": "failure"
    },
    "transaction": {
      "id": "154a123cf3b4f8ce"
    },
    "span": {
      "duration": {
        "us": 102475
      },
      "representative_count": 1,
      "stacktrace": [
        {
          "exclude_from_grouping": false,
          "library_frame": true,
          "filename": "span.go",
          "line": {
            "number": 442
          },
          "function": "(*Span).End",
          "module": "go.elastic.co/apm/v2"
        },
        {
          "exclude_from_grouping": false,
          "library_frame": true,
          "filename": "client.go",
          "line": {
            "number": 198
          },
          "function": "(*responseBody).endSpan",
          "module": "go.elastic.co/apm/module/apmhttp/v2"
        },
        {
          "exclude_from_grouping": false,
          "library_frame": true,
          "filename": "client.go",
          "line": {
            "number": 187
          },
          "function": "(*responseBody).Read",
          "module": "go.elastic.co/apm/module/apmhttp/v2"
        },
        {
          "exclude_from_grouping": false,
          "library_frame": true,
          "filename": "client.go",
          "line": {
            "number": 963
          },
          "function": "(*cancelTimerBody).Read",
          "module": "net/http"
        },
        {
          "exclude_from_grouping": false,
          "library_frame": true,
          "filename": "io.go",
          "line": {
            "number": 712
          },
          "function": "ReadAll",
          "module": "io"
        },
        {
          "exclude_from_grouping": false,
          "filename": "checkintegration.go",
          "line": {
            "number": 94
          },
          "function": "checkIntegrationInstalledKibana",
          "module": "github.com/elastic/apm-server/internal/beater"
        },
        {
          "exclude_from_grouping": false,
          "filename": "checkintegration.go",
          "line": {
            "number": 57
          },
          "function": "checkIntegrationInstalled",
          "module": "github.com/elastic/apm-server/internal/beater"
        },
        {
          "exclude_from_grouping": false,
          "filename": "beater.go",
          "line": {
            "number": 629
          },
          "function": "(*Runner).waitReady.func3",
          "module": "github.com/elastic/apm-server/internal/beater"
        },
        {
          "exclude_from_grouping": false,
          "filename": "beater.go",
          "line": {
            "number": 638
          },
          "function": "(*Runner).waitReady.func4",
          "module": "github.com/elastic/apm-server/internal/beater"
        },
        {
          "exclude_from_grouping": false,
          "filename": "waitready.go",
          "line": {
            "number": 59
          },
          "function": "waitReady",
          "module": "github.com/elastic/apm-server/internal/beater"
        },
        {
          "exclude_from_grouping": false,
          "filename": "beater.go",
          "line": {
            "number": 644
          },
          "function": "(*Runner).waitReady",
          "module": "github.com/elastic/apm-server/internal/beater"
        },
        {
          "exclude_from_grouping": false,
          "filename": "beater.go",
          "line": {
            "number": 331
          },
          "function": "(*Runner).Run.func4",
          "module": "github.com/elastic/apm-server/internal/beater"
        },
        {
          "exclude_from_grouping": false,
          "filename": "errgroup.go",
          "line": {
            "number": 78
          },
          "function": "(*Group).Go.func1",
          "module": "golang.org/x/sync/errgroup"
        },
        {
          "exclude_from_grouping": false,
          "library_frame": true,
          "filename": "asm_amd64.s",
          "line": {
            "number": 1695
          },
          "function": "goexit",
          "module": "runtime"
        }
      ],
      "subtype": "http",
      "destination": {
        "service": {
          "resource": "kibana-v1-kb-http.apm-sandbox.svc:5601",
          "name": "https://kibana-v1-kb-http.apm-sandbox.svc:5601",
          "type": "external"
        }
      },
      "name": "GET kibana-v1-kb-http.apm-sandbox.svc:5601",
      "id": "b9dd9b517374b4cf",
      "type": "external"
    },
    "timestamp": {
      "us": 1725257940318438
    }
  },
  "fields": {
    "host.hostname": [
      "apm-server-v2-apm-server-56c7746446-m7dzp"
    ],
    "url.original.text": [
      "https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm"
    ],
    "process.pid": [
      1
    ],
    "service.language.name": [
      "go"
    ],
    "cloud.availability_zone": [
      "europe-west3-a"
    ],
    "process.title.text": [
      "apm-server"
    ],
    "transaction.id": [
      "154a123cf3b4f8ce"
    ],
    "processor.event": [
      "span"
    ],
    "labels.project": [
      "epm-paas"
    ],
    "agent.name": [
      "go"
    ],
    "destination.address": [
      "kibana-v1-kb-http.apm-sandbox.svc"
    ],
    "host.name": [
      "apm-server-v2-apm-server-56c7746446-m7dzp"
    ],
    "event.agent_id_status": [
      "missing"
    ],
    "http.response.status_code": [
      403
    ],
    "event.outcome": [
      "failure"
    ],
    "cloud.region": [
      "europe-west3"
    ],
    "service.runtime.version": [
      "go1.22.5"
    ],
    "span.id": [
      "b9dd9b517374b4cf"
    ],
    "data_stream.type": [
      "traces"
    ],
    "span.type": [
      "external"
    ],
    "host.architecture": [
      "amd64"
    ],
    "cloud.provider": [
      "gcp"
    ],
    "timestamp.us": [
      1725257940318438
    ],
    "observer.type": [
      "apm-server"
    ],
    "observer.version": [
      "8.14.3"
    ],
    "agent.version": [
      "2.6.0"
    ],
    "parent.id": [
      "154a123cf3b4f8ce"
    ],
    "span.destination.service.name": [
      "https://kibana-v1-kb-http.apm-sandbox.svc:5601"
    ],
    "process.title": [
      "apm-server"
    ],
    "span.representative_count": [
      1
    ],
    "span.destination.service.type": [
      "external"
    ],
    "span.name": [
      "GET kibana-v1-kb-http.apm-sandbox.svc:5601"
    ],
    "destination.port": [
      5601
    ],
    "service.node.name": [
      "apm-server-v2-apm-server-56c7746446-m7dzp"
    ],
    "cloud.instance.id": [
      "3933813931648332798"
    ],
    "trace.id": [
      "154a123cf3b4f8ce0fb856d2d80a0416"
    ],
    "span.duration.us": [
      102475
    ],
    "span.stacktrace": [
      {
        "exclude_from_grouping": false,
        "library_frame": true,
        "filename": "span.go",
        "line": {
          "number": 442
        },
        "function": "(*Span).End",
        "module": "go.elastic.co/apm/v2"
      },
      {
        "exclude_from_grouping": false,
        "library_frame": true,
        "filename": "client.go",
        "line": {
          "number": 198
        },
        "function": "(*responseBody).endSpan",
        "module": "go.elastic.co/apm/module/apmhttp/v2"
      },
      {
        "exclude_from_grouping": false,
        "library_frame": true,
        "filename": "client.go",
        "line": {
          "number": 187
        },
        "function": "(*responseBody).Read",
        "module": "go.elastic.co/apm/module/apmhttp/v2"
      },
      {
        "exclude_from_grouping": false,
        "library_frame": true,
        "filename": "client.go",
        "line": {
          "number": 963
        },
        "function": "(*cancelTimerBody).Read",
        "module": "net/http"
      },
      {
        "exclude_from_grouping": false,
        "library_frame": true,
        "filename": "io.go",
        "line": {
          "number": 712
        },
        "function": "ReadAll",
        "module": "io"
      },
      {
        "exclude_from_grouping": false,
        "filename": "checkintegration.go",
        "line": {
          "number": 94
        },
        "function": "checkIntegrationInstalledKibana",
        "module": "github.com/elastic/apm-server/internal/beater"
      },
      {
        "exclude_from_grouping": false,
        "filename": "checkintegration.go",
        "line": {
          "number": 57
        },
        "function": "checkIntegrationInstalled",
        "module": "github.com/elastic/apm-server/internal/beater"
      },
      {
        "exclude_from_grouping": false,
        "filename": "beater.go",
        "line": {
          "number": 629
        },
        "function": "(*Runner).waitReady.func3",
        "module": "github.com/elastic/apm-server/internal/beater"
      },
      {
        "exclude_from_grouping": false,
        "filename": "beater.go",
        "line": {
          "number": 638
        },
        "function": "(*Runner).waitReady.func4",
        "module": "github.com/elastic/apm-server/internal/beater"
      },
      {
        "exclude_from_grouping": false,
        "filename": "waitready.go",
        "line": {
          "number": 59
        },
        "function": "waitReady",
        "module": "github.com/elastic/apm-server/internal/beater"
      },
      {
        "exclude_from_grouping": false,
        "filename": "beater.go",
        "line": {
          "number": 644
        },
        "function": "(*Runner).waitReady",
        "module": "github.com/elastic/apm-server/internal/beater"
      },
      {
        "exclude_from_grouping": false,
        "filename": "beater.go",
        "line": {
          "number": 331
        },
        "function": "(*Runner).Run.func4",
        "module": "github.com/elastic/apm-server/internal/beater"
      },
      {
        "exclude_from_grouping": false,
        "filename": "errgroup.go",
        "line": {
          "number": 78
        },
        "function": "(*Group).Go.func1",
        "module": "golang.org/x/sync/errgroup"
      },
      {
        "exclude_from_grouping": false,
        "library_frame": true,
        "filename": "asm_amd64.s",
        "line": {
          "number": 1695
        },
        "function": "goexit",
        "module": "runtime"
      }
    ],
    "event.success_count": [
      0
    ],
    "service.target.type": [
      "http"
    ],
    "service.environment": [
      "sandbox-latest"
    ],
    "service.name": [
      "apm-server"
    ],
    "data_stream.namespace": [
      "epm_paas"
    ],
    "service.runtime.name": [
      "gc"
    ],
    "process.args": [
      "apm-server",
      "run",
      "-e",
      "-c",
      "config/config-secret/apm-server.yml"
    ],
    "span.subtype": [
      "http"
    ],
    "service.target.name": [
      "kibana-v1-kb-http.apm-sandbox.svc:5601"
    ],
    "observer.hostname": [
      "apm-server-v2-apm-server-56c7746446-m7dzp"
    ],
    "event.ingested": [
      "2024-09-02T06:19:08.000Z"
    ],
    "url.original": [
      "https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm"
    ],
    "@timestamp": [
      "2024-09-02T06:19:00.318Z"
    ],
    "service.version": [
      "8.14.3"
    ],
    "host.os.platform": [
      "linux"
    ],
    "data_stream.dataset": [
      "apm"
    ],
    "service.language.version": [
      "go1.22.5"
    ],
    "span.destination.service.resource": [
      "kibana-v1-kb-http.apm-sandbox.svc:5601"
    ],
    "cloud.instance.name": [
      "gke-epm-iass-elastic-europe-w-generic-c4e5a328-nebn"
    ],
    "cloud.project.id": [
      "or2-ms-epm-iass-elastic-t1iylu"
    ]
  }
}
@botelastic botelastic bot added the triage label Sep 2, 2024
@barkbay
Copy link
Contributor

barkbay commented Sep 2, 2024

Could you please provide the manifests you are using, this would help me reproduce. Thanks!

@up2neck
Copy link
Author

up2neck commented Sep 3, 2024

Could you please provide the manifests you are using, this would help me reproduce. Thanks!

It's slightly sanitized from sensitive data: affinity, specific labels, but cross-resource refers are persisted "as-is"

apiVersion: apm.k8s.elastic.co/v1
kind: ApmServer
metadata:
  labels:
    module/name: apm-server
    package/name: intake
    package/overlay: base
    package/version: "2"
  name: apm-server-v2
  namespace: apm-sandbox
spec:
  config:
    apm-server:
      auth:
        anonymous:
          allow_agent:
          - rum-js
          - rum-js-dpeo
          - js-base
          - java
          - dotnet
          - php
          - opentelemetry/cpp
          - python
          - otlp
          - go
          - opentelemetry
          - opentelemetry/webjs
          - opentelemetry/js
          - opentelemetry/go
          - opentelemetry/java
          - opentelemetry/nodejs
          - opentelemetry/dotnet
          - nodejs
          - '@microlabs/otel-workers-sdk/js'
          enabled: true
          rate_limit:
            event_limit: 8000
            ip_limit: 1000
        api_key:
          enabled: false
          limit: 100
      capture_personal_data: true
      default_service_environment: undefined
      expvar.enabled: false
      host: 0.0.0.0:8200
      idle_timeout: 45s
      max_connections: 0
      max_event_size: 307200
      max_header_size: 1048576
      pprof.enabled: false
      read_timeout: 30s
      rum:
        allow_headers:
        - x-requested-with
        - access-control-request-private-network
        - access-control-allow-origin
        - xmlhttprequest
        - request-origin
        allow_origins:
        - '*'
        enabled: true
        exclude_from_grouping: ^/webpack
        library_pattern: node_modules|bower_components|~
      shutdown_timeout: 30s
      ssl:
        supported_protocols:
        - TLSv1.2
        - TLSv1.3
      write_timeout: 30s
    logging.level: warning
    monitoring.elasticsearch: {}
  count: 2
  elasticsearchRef:
    name: elasticsearch-v1
  http:
    service:
      metadata:
        labels:
          module/name: apm-server
          package/name: intake
          package/version: "2"
      spec:
        ports:
        - appProtocol: HTTPS
          name: https
          port: 8200
          protocol: TCP
          targetPort: 8200
    tls:
      certificate: {}
      selfSignedCertificate:
        subjectAltNames:
        - dns: apm-server
  kibanaRef:
    name: kibana-v1
  podTemplate:
    metadata:
      creationTimestamp: null
      labels:
        module/name: apm-server
        package/name: intake
        package/version: "2"
    spec:
      containers:
      - env:
        - name: ELASTIC_APM_GLOBAL_LABELS
          value: project=dummy
        - name: ELASTIC_APM_CAPTURE_BODY
          value: all
        - name: ELASTICSEARCH_HOST
          value: https://elasticsearch:9200
        name: apm-server
        resources:
          limits:
            cpu: 1
            memory: 1Gi
          requests:
            cpu: 1
            memory: 1Gi
      topologySpreadConstraints:
      - labelSelector:
          matchLabels:
            module/name: apm-server
            package/name: intake
            package/version: "2"
        maxSkew: 1
        nodeAffinityPolicy: Honor
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: DoNotSchedule
  version: 8.14.3
---
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
  labels:
    package/name: elastic-stack-eck
    package/type: component
    package/version: "1"
  name: kibana-v1
  namespace: apm-sandbox
spec:
  config:
    elasticsearch.requestHeadersWhitelist:
    - authorization
    elasticsearch.requestTimeout: 60000
    elasticsearch.shardTimeout: 60000
    server:
      customResponseHeaders:
        X-Content-Type-Options: nosniff
        X-Frame-Options: SAMEORIGIN
        X-XSS-Protection: 1; mode=block
    telemetry.optIn: false
    xpack.fleet.agentPolicies:
    - id: eck-fleet-server
      monitoring_enabled:
      - logs
      - metrics
      name: Fleet Server on ECK policy
      namespace: default
      package_policies:
      - id: fleet_server-1
        name: fleet_server-1
        package:
          name: fleet_server
      unenroll_timeout: 900
    xpack.fleet.agents.fleet_server.hosts:
    - https://fleet:8220
    xpack.fleet.outputs:
    - config:
        ssl.verification_mode: none
      hosts:
      - https://elasticsearch:9200
      id: fleet-default-output
      is_default: "true"
      is_default_monitoring: "true"
      name: default
      type: elasticsearch
    xpack.fleet.packages:
    - name: system
      version: latest
    - name: elastic_agent
      version: latest
    - name: fleet_server
      version: latest
    - name: apm
      version: latest
    - name: kubernetes
      version: latest
    - name: cloudflare
      version: latest
    - name: synthetics
      version: latest
    - name: cloudflare_logpush
      version: latest
    - name: gcp_pubsub
      version: latest
    xpack.reporting.roles.enabled: false
    xpack.spaces.maxSpaces: 1000
    xpack.task_manager.max_workers: 100
    xpack.task_manager.monitored_stats_health_verbose_log.enabled: true
  count: 3
  elasticsearchRef:
    name: elasticsearch-v1
  enterpriseSearchRef: {}
  http:
    service:
      metadata:
        labels:
          package/name: elastic-stack-eck
          package/type: component
          package/version: "1"
      spec:
        - name: https
          port: 5601
          protocol: TCP
          targetPort: 5601
    tls:
      certificate: {}
      selfSignedCertificate:
        subjectAltNames:
        - dns: kibana
  monitoring:
    logs: {}
    metrics: {}
  podTemplate:
    metadata:
      creationTimestamp: null
      labels:
        package/name: elastic-stack-eck
        package/type: component
        package/version: "1"
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                common.k8s.elastic.co/type: kibana
                package/name: elastic-stack-eck
                package/type: component
                package/version: "1"
            topologyKey: kubernetes.io/hostname
      containers:
      - name: kibana
        resources:
          limits:
            cpu: 1
            memory: 1Gi
          requests:
            cpu: 500m
            memory: 1Gi
  version: 8.14.3
---
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  labels:
    package/name: elastic-stack-eck
    package/type: component
    package/version: "1"
  name: elasticsearch-v1
spec:
  auth: {}
  http:
    service:
      metadata: {}
      spec: {}
    tls:
      certificate: {}
      selfSignedCertificate:
        subjectAltNames:
        - dns: elasticsearch
  monitoring:
    logs: {}
    metrics: {}
  nodeSets:
  - config:
      node.roles:
      - master
      - remote_cluster_client
      xpack.security.authc:
        anonymous:
          roles: monitoring_user
          username: anon
    count: 3
    name: master
    podTemplate:
      metadata:
        labels:
          package/name: elastic-stack-eck
          package/type: component
          package/version: "1"
      spec:
        containers:
        - name: elasticsearch
          readinessProbe:
            httpGet:
              port: 9200
              scheme: HTTPS
          resources:
            limits:
              cpu: 2
              memory: 10Gi
            requests:
              cpu: 1
              memory: 10Gi
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 10Gi
        storageClassName: standard-rwo
  - config:
      node.roles:
      - remote_cluster_client
      - data_content
      - data_hot
      - ingest
      - transform
      xpack.security.authc:
        anonymous:
          roles: monitoring_user
          username: anon
    count: 4
    name: data
    podTemplate:
      metadata:
        labels:
          package/name: elastic-stack-eck
          package/type: component
          package/version: "1"
      spec:
        containers:
        - name: elasticsearch
          readinessProbe:
            httpGet:
              port: 9200
              scheme: HTTPS
          resources:
            limits:
              cpu: 7
              memory: 54Gi
            requests:
              cpu: 6
              memory: 54Gi
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 2Ti
        storageClassName: premium-rwo
  - config:
      node.roles:
      - remote_cluster_client
      - data_cold
      - data_warm
      xpack.security.authc:
        anonymous:
          roles: monitoring_user
          username: anon
    count: 3
    name: data-cold
    podTemplate:
      metadata:
        labels:
          package/name: elastic-stack-eck
          package/type: component
          package/version: "1"
      spec:
        containers:
        - name: elasticsearch
          readinessProbe:
            httpGet:
              port: 9200
              scheme: HTTPS
          resources:
            limits:
              cpu: 4
              memory: 16Gi
            requests:
              cpu: 2
              memory: 16Gi
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 3096Gi
        storageClassName: standard-rwo
  podDisruptionBudget:
    metadata: {}
    spec:
      maxUnavailable: 1
      selector:
        matchLabels:
          common.k8s.elastic.co/type: elasticsearch
          package/name: elastic-stack-eck
          package/type: component
          package/version: "1"
  transport:
    service:
      metadata: {}
      spec: {}
    tls:
      certificate: {}
      certificateAuthorities: {}
  updateStrategy:
    changeBudget:
      maxUnavailable: 1
  version: 8.14.3

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
Projects
None yet
Development

No branches or pull requests

2 participants