[fleet integration] Support apm-server config options and adapt defau…

…lts. (#5444)

* Add missing settings that should be supported for APM Server Integration.
* Adapt default values to be aligned with cloud values where possible.

closes #4528

(cherry picked from commit 17166cd)

# Conflicts:
#	changelogs/head.asciidoc

apmpackage/apm/agent/input/template.yml.hbs

@@ -1,30 +1,56 @@
 apm-server:
-    host: {{host}}
-    auth:
-        secret_token: {{secret_token}}
-        api_key:
-            enabled: {{api_key_enabled}}
-            limit: {{api_key_limit}}
-    max_event_size: {{max_event_bytes}}
+    api_key:
+        enabled: {{api_key_enabled}}
+        limit: {{api_key_limit}}
     capture_personal_data: {{capture_personal_data}}
+    idle_timeout: {{idle_timeout}}
     default_service_environment: {{default_service_environment}}
+    expvar.enabled: {{expvar_enabled}}
+    host: {{host}}
+    max_connections: {{max_connections}}
+    max_event_size: {{max_event_bytes}}
+    max_header_size: {{max_header_bytes}}
+    read_timeout: {{read_timeout}}
+    response_headers: {{response_headers}}
     rum:
-        enabled: {{enable_rum}}
-        source_mapping.elasticsearch.api_key: {{sourcemap_api_key}}
-        allow_service_names:
-        {{#each rum_allow_service_names}}
+        allow_headers:
+        {{#each rum_allow_headers}}
           - {{this}}
         {{/each}}
         allow_origins:
         {{#each rum_allow_origins}}
           - {{this}}
         {{/each}}
-        allow_headers:
-        {{#each rum_allow_headers}}
+        allow_service_names:
+        {{#each rum_allow_service_names}}
           - {{this}}
         {{/each}}
-        library_pattern: {{rum_library_pattern}}
-        exclude_from_grouping: {{rum_exclude_from_grouping}}
-        response_headers: {{rum_response_headers}}
+        enabled: {{enable_rum}}
         event_rate.limit: {{rum_event_rate_limit}}
         event_rate.lru_size: {{rum_event_rate_lru_size}}
+        exclude_from_grouping: {{rum_exclude_from_grouping}}
+        library_pattern: {{rum_library_pattern}}
+        response_headers: {{rum_response_headers}}
+        source_mapping.elasticsearch.api_key: {{sourcemap_api_key}}
+    secret_token: {{secret_token}}
+    shutdown_timeout: {{shutdown_timeout}}
+    {{#if tls_enabled}}
+    ssl:
+        enabled: {{tls_enabled}}
+        certificate: {{tls_certificate}}
+        key: {{tls_key}}
+        key_passphrase: {{tls_key_passphrase}}
+        supported_protocols: {{tls_supported_protocols}}
+        {{#each tls_supported_protocols}}
+          - {{this}}
+        {{/each}}
+        cipher_suites: {{tls_cipher_suites}}
+        {{#each tls_cipher_suites}}
+          - {{this}}
+        {{/each}}
+        curve_types: {{tls_curve_types}}
+        {{#each tls_curve_types}}
+          - {{this}}
+        {{/each}}
+    {{/if}}
+    write_timeout: {{write_timeout}}

apmpackage/apm/manifest.yml

@@ -71,7 +71,7 @@ policy_templates:
             description: Enable Real User Monitoring (RUM).
             required: true
             show_user: true
-            default: false
+            default: true
           - name: default_service_environment
             type: text
             title: Default Service Environment
@@ -112,14 +112,14 @@ policy_templates:
             description: Maximum number of events allowed per IP per second.
             required: false
             show_user: false
-            default: 300
+            default: 10
           - name: rum_event_rate_lru_size
             type: integer
             title: RUM - Rate limit cache size
             description: Number of unique IPs to be cached for the rate limiter.
             required: false
             show_user: false
-            default: 1000
+            default: 10000
           - name: sourcemap_api_key
             type: text
             title: RUM - API Key for Sourcemaps
@@ -146,6 +146,93 @@ policy_templates:
             required: false
             show_user: false
             default: true
+          - name: max_header_bytes
+            type: integer
+            title: Maximum size of a request's header (bytes)
+            required: false
+            show_user: false
+            default: 1048576
+          - name: idle_timeout
+            type: text
+            title: Idle time before underlying connection is closed
+            required: false
+            show_user: false
+            default: "45s"
+          - name: read_timeout
+            type: text
+            title: Maximum duration for reading an entire request
+            required: false
+            show_user: false
+            default: "3600s"
+          - name: shutdown_timeout
+            type: text
+            title: Maximum duration before releasing resources when shutting down
+            required: false
+            show_user: false
+            default: "30s"
+          - name: write_timeout
+            type: text
+            title: Maximum duration for writing a response
+            required: false
+            show_user: false
+            default: "30s"
+          - name: max_connections
+            type: integer
+            title: Simultaneously accepted connections
+            description: 0 for unlimited
+            required: false
+            show_user: false
+            default: 0
+          - name: response_headers
+            type: yaml
+            title: Custom HTTP headers added to HTTP responses
+            description: Might be used for security policy compliance.
+            required: false
+            show_user: false
+          - name: expvar_enabled
+            type: bool
+            title: Enable APM Server Golang expvar support
+            description: exposed under /debug/vars
+            required: false
+            show_user: false
+            default: false
+          - name: tls_enabled
+            type: bool
+            title: Enable TLS
+            required: false
+            show_user: false
+            default: false
+          - name: tls_certificate
+            type: text
+            title: File path to server certificate
+            description: Required when TLS is enabled.
+            required: false
+            show_user: false
+          - name: tls_key
+            type: text
+            title: File path to server certificate key
+            description: Required when TLS is enabled.
+            required: false
+            show_user: false
+          - name: tls_supported_protocols
+            type: text
+            multi: true
+            title: Supported protocol versions
+            required: false
+            show_user: false
+          - name: tls_cipher_suites
+            type: text
+            multi: true
+            title: Cipher suites for TLS connections.
+            description: Not configurable for TLS 1.3.
+            required: false
+            show_user: false
+          - name: tls_curve_types
+            type: text
+            multi: true
+            title: Curve types for ECDHE based cipher suites
+            required: false
+            show_user: false
         template_path: template.yml.hbs
 owner:
   github: elastic/apm-server

changelogs/head.asciidoc

@@ -0,0 +1,40 @@
+[[release-notes-head]]
+== APM Server version HEAD
+
+https://github.com/elastic/apm-server/compare/7.13\...master[View commits]
+
+[float]
+==== Breaking Changes
+* Removed monitoring counters `apm-server.processor.stream.errors.{queue,server,closed}` {pull}5453[5453]
+
+[float]
+==== Bug fixes
+* Fix panic due to misaligned 64-bit access on 32-bit architectures {pull}5277[5277]
+* Fixed tail-based sampling pubsub to use _seq_no correctly {pull}5126[5126]
+* Removed service name from dataset {pull}5451[5451]
+
+[float]
+==== Intake API Changes
+
+[float]
+==== Added
+* Support setting agent configuration from apm-server.yml {pull}5177[5177]
+* Add metric_type and unit to field metadata of system metrics {pull}5230[5230]
+* Display apm-server url in fleet ui's apm-server integration {pull}4895[4895]
+* Translate otel messaging.* semantic conventions to ECS {pull}5334[5334]
+* Add support for dynamic histogram metrics {pull}5239[5239]
+* Tail-sampling processor now resumes subscription from previous position after restart {pull}5350[5350]
+* Add support for histograms to metrics intake {pull}5360[5360]
+* Upgrade Go to 1.16.5 {pull}5454[5454]
+* Add units to metric fields {pull}5395[5395]
+* Add support for adjusting OTel event timestamps using `telemetry.sdk.elastic_export_timestamp` {pull}5433[5433]
+* Add support for OpenTelemetry labels describing mobile connectivity {pull}5436[5436]
+* Introduce `apm-server.auth.*` config {pull}5457[5457]
+* Add debug logging of OpenTelemetry payloads {pull}5474[5474]
+* Add support for more input variables in fleet integration {pull}5444[5444]
+
+[float]
+==== Deprecated
+* Make destination.service.name and destination.service.type fields optional and deprecated {pull}5468[5468]
+* `apm-server.secret_token` is now `apm-server.auth.secret_token` {pull}5457[5457]
+* `apm-server.api_key` is now `apm-server.auth.api_key` {pull}5457[5457]