Add new hostname information on Intake API and in ES. (elastic#2540)

Deprecate existing `system.hostname` and add `system.detected_hostname` and `system.configured_hostname` to the Intake API.

follow up on elastic#2502

beater/api/config_agent_integration_test.go

@@ -32,7 +32,7 @@ import (
 	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/apm-server/beater/headers"
 	"github.com/elastic/apm-server/beater/request"
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 )
 
 func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) {
@@ -42,7 +42,7 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) {
 		rec := requestToConfigAgentHandler(t, cfg)
 
 		assert.Equal(t, http.StatusUnauthorized, rec.Code)
-		tests.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
 	})
 
 	t.Run("Authorized", func(t *testing.T) {
@@ -55,7 +55,7 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) {
 		h(c)
 
 		assert.Equal(t, http.StatusServiceUnavailable, rec.Code)
-		tests.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
 	})
 }
 
@@ -64,15 +64,15 @@ func TestConfigAgentHandler_KillSwitchMiddleware(t *testing.T) {
 		rec := requestToConfigAgentHandler(t, config.DefaultConfig(beatertest.MockBeatVersion()))
 
 		assert.Equal(t, http.StatusForbidden, rec.Code)
-		tests.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
 
 	})
 
 	t.Run("On", func(t *testing.T) {
 		rec := requestToConfigAgentHandler(t, configEnabledConfigAgent())
 
 		assert.Equal(t, http.StatusServiceUnavailable, rec.Code)
-		tests.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
 	})
 }
 
@@ -84,7 +84,7 @@ func TestConfigAgentHandler_PanicMiddleware(t *testing.T) {
 	c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil))
 	h(c)
 	assert.Equal(t, http.StatusInternalServerError, rec.StatusCode)
-	tests.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
+	approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes())
 }
 
 func TestConfigAgentHandler_MonitoringMiddleware(t *testing.T) {

beater/api/intake/handler_test.go

@@ -38,7 +38,7 @@ import (
 	"github.com/elastic/apm-server/model"
 	"github.com/elastic/apm-server/processor/stream"
 	"github.com/elastic/apm-server/publish"
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 	"github.com/elastic/apm-server/tests/loader"
 	"github.com/elastic/apm-server/transform"
 )
@@ -154,7 +154,7 @@ func TestIntakeHandler(t *testing.T) {
 				assert.NotNil(t, tc.w.Body.Len())
 			}
 			body := tc.w.Body.Bytes()
-			tests.AssertApproveResult(t, "test_approved/"+name, body)
+			approvals.AssertApproveResult(t, "test_approved/"+name, body)
 		})
 	}
 }

beater/api/intake_backend_integration_test.go

@@ -30,7 +30,7 @@ import (
 	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/apm-server/beater/headers"
 	"github.com/elastic/apm-server/beater/request"
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 )
 
 func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) {
@@ -40,7 +40,7 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) {
 		rec := requestToIntakeBackendHandler(t, cfg)
 
 		assert.Equal(t, http.StatusUnauthorized, rec.Code)
-		tests.AssertApproveResult(t, approvalPathIntakeBackend(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathIntakeBackend(t.Name()), rec.Body.Bytes())
 	})
 
 	t.Run("Authorized", func(t *testing.T) {
@@ -53,7 +53,7 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) {
 		h(c)
 
 		assert.Equal(t, http.StatusBadRequest, rec.Code)
-		tests.AssertApproveResult(t, approvalPathIntakeBackend(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathIntakeBackend(t.Name()), rec.Body.Bytes())
 	})
 }
 
@@ -65,7 +65,7 @@ func TestIntakeBackendHandler_PanicMiddleware(t *testing.T) {
 	c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil))
 	h(c)
 	assert.Equal(t, http.StatusInternalServerError, rec.StatusCode)
-	tests.AssertApproveResult(t, approvalPathIntakeBackend(t.Name()), rec.Body.Bytes())
+	approvals.AssertApproveResult(t, approvalPathIntakeBackend(t.Name()), rec.Body.Bytes())
 }
 
 func TestIntakeBackendHandler_MonitoringMiddleware(t *testing.T) {

beater/api/intake_rum_integration_test.go

@@ -31,7 +31,7 @@ import (
 	"github.com/elastic/apm-server/beater/headers"
 	"github.com/elastic/apm-server/beater/middleware"
 	"github.com/elastic/apm-server/beater/request"
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 )
 
 func TestOPTIONS(t *testing.T) {
@@ -73,14 +73,14 @@ func TestRUMHandler_KillSwitchMiddleware(t *testing.T) {
 		rec := requestToIntakeRUMHandler(t, config.DefaultConfig(beatertest.MockBeatVersion()))
 
 		assert.Equal(t, http.StatusForbidden, rec.Code)
-		tests.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes())
 	})
 
 	t.Run("On", func(t *testing.T) {
 		rec := requestToIntakeRUMHandler(t, cfgEnabledRUM())
 
 		assert.Equal(t, http.StatusBadRequest, rec.Code)
-		tests.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes())
 	})
 }
 
@@ -104,7 +104,7 @@ func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) {
 	c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil))
 	h(c)
 	assert.Equal(t, http.StatusInternalServerError, rec.StatusCode)
-	tests.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes())
+	approvals.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes())
 }
 
 func TestRumHandler_MonitoringMiddleware(t *testing.T) {

beater/api/root_integration_test.go

@@ -30,7 +30,7 @@ import (
 	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/apm-server/beater/headers"
 	"github.com/elastic/apm-server/beater/request"
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 )
 
 func TestRootHandler_AuthorizationMiddleware(t *testing.T) {
@@ -53,7 +53,7 @@ func TestRootHandler_AuthorizationMiddleware(t *testing.T) {
 		h(c)
 
 		assert.Equal(t, http.StatusOK, rec.Code)
-		tests.AssertApproveResult(t, approvalPathRoot(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathRoot(t.Name()), rec.Body.Bytes())
 	})
 }
 
@@ -66,7 +66,7 @@ func TestRootHandler_PanicMiddleware(t *testing.T) {
 	h(c)
 
 	assert.Equal(t, http.StatusInternalServerError, rec.StatusCode)
-	tests.AssertApproveResult(t, approvalPathRoot(t.Name()), rec.Body.Bytes())
+	approvals.AssertApproveResult(t, approvalPathRoot(t.Name()), rec.Body.Bytes())
 }
 
 func TestRootHandler_MonitoringMiddleware(t *testing.T) {

beater/api/sourcemap_handler_integration_test.go

@@ -30,7 +30,7 @@ import (
 	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/apm-server/beater/headers"
 	"github.com/elastic/apm-server/beater/request"
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 )
 
 func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) {
@@ -40,7 +40,7 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) {
 		rec := requestToSourcemapHandler(t, cfg)
 
 		assert.Equal(t, http.StatusUnauthorized, rec.Code)
-		tests.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
 	})
 
 	t.Run("Authorized", func(t *testing.T) {
@@ -53,7 +53,7 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) {
 		h(c)
 
 		assert.Equal(t, http.StatusBadRequest, rec.Code)
-		tests.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
 	})
 }
 
@@ -62,7 +62,7 @@ func TestSourcemapHandler_KillSwitchMiddleware(t *testing.T) {
 		rec := requestToSourcemapHandler(t, config.DefaultConfig(beatertest.MockBeatVersion()))
 
 		assert.Equal(t, http.StatusForbidden, rec.Code)
-		tests.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
 	})
 
 	t.Run("OffSourcemap", func(t *testing.T) {
@@ -73,14 +73,14 @@ func TestSourcemapHandler_KillSwitchMiddleware(t *testing.T) {
 		rec := requestToSourcemapHandler(t, cfg)
 
 		assert.Equal(t, http.StatusForbidden, rec.Code)
-		tests.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
 	})
 
 	t.Run("On", func(t *testing.T) {
 		rec := requestToSourcemapHandler(t, cfgEnabledRUM())
 
 		assert.Equal(t, http.StatusBadRequest, rec.Code)
-		tests.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
+		approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
 	})
 }
 
@@ -92,7 +92,7 @@ func TestSourcemapHandler_PanicMiddleware(t *testing.T) {
 	c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil))
 	h(c)
 	assert.Equal(t, http.StatusInternalServerError, rec.StatusCode)
-	tests.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
+	approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes())
 }
 
 func TestSourcemapHandler_MonitoringMiddleware(t *testing.T) {

beater/integration_test.go

@@ -30,7 +30,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/apm-server/beater/api"
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 	"github.com/elastic/apm-server/tests/loader"
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
@@ -124,7 +124,7 @@ func TestPublishIntegration(t *testing.T) {
 			b, err := loader.LoadDataAsBytes(filepath.Join("../testdata/intake-v2/", tc.payload))
 			require.NoError(t, err)
 			docs := testPublish(t, apm, events, api.IntakePath, bytes.NewReader(b))
-			tests.AssertApproveResult(t, "test_approved_es_documents/TestPublishIntegration"+tc.name, docs)
+			approvals.AssertApproveResult(t, "test_approved_es_documents/TestPublishIntegration"+tc.name, docs)
 		})
 	}
 }

docs/spec/system.json

@@ -9,12 +9,17 @@
             "maxLength": 1024
         },
         "hostname": {
-            "description": "Hostname of the system the agent is running on. Will be ignored if kubernetes information is set.",
+            "description": "Deprecated. Hostname of the system the agent is running on. Will be ignored if kubernetes information is set.",
             "type": ["string", "null"],
             "maxLength": 1024
         },
-        "name": {
-            "description": "Name of the system the agent is running on. Will be set to hostname or derived from kubernetes information if not provided.",
+        "detected_hostname": {
+            "description": "Hostname of the host the monitored service is running on. It normally contains what the hostname command returns on the host machine. Will be ignored if kubernetes information is set, otherwise should always be set.",
+            "type": ["string", "null"],
+            "maxLength": 1024
+        },
+        "configured_hostname": {
+            "description": "Name of the host the monitored service is running on. It should only be set when configured by the user. If empty, will be set to detected_hostname or derived from kubernetes information if provided.",
             "type": ["string", "null"],
             "maxLength": 1024
         },

model/context_test.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/common"
 
-	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/apm-server/tests/approvals"
 	"github.com/elastic/apm-server/utility"
 )
 
@@ -168,7 +168,7 @@ func TestDecodeContext(t *testing.T) {
 				resultName := fmt.Sprintf("test_approved_model/context_%s", name)
 				resultJSON, err := json.Marshal(out)
 				require.NoError(t, err)
-				tests.AssertApproveResult(t, resultName, resultJSON)
+				approvals.AssertApproveResult(t, resultName, resultJSON)
 			}
 		})
 	}

model/metadata/generated/schema/metadata.go

@@ -160,12 +160,17 @@ const ModelSchema = `{
             "maxLength": 1024
         },
         "hostname": {
-            "description": "Hostname of the system the agent is running on. Will be ignored if kubernetes information is set.",
+            "description": "Deprecated. Hostname of the system the agent is running on. Will be ignored if kubernetes information is set.",
             "type": ["string", "null"],
             "maxLength": 1024
         },
-        "name": {
-            "description": "Name of the system the agent is running on. Will be set to hostname or derived from kubernetes information if not provided.",
+        "detected_hostname": {
+            "description": "Hostname of the host the monitored service is running on. It normally contains what the hostname command returns on the host machine. Will be ignored if kubernetes information is set, otherwise should always be set.",
+            "type": ["string", "null"],
+            "maxLength": 1024
+        },
+        "configured_hostname": {
+            "description": "Name of the host the monitored service is running on. It should only be set when configured by the user. If empty, will be set to detected_hostname or derived from kubernetes information if provided.",
             "type": ["string", "null"],
             "maxLength": 1024
         },

model/metadata/metadata_test.go

@@ -96,7 +96,7 @@ func TestDecodeMetadata(t *testing.T) {
 						Version: &agentVersion,
 					},
 				},
-				&System{Hostname: &host},
+				&System{DetectedHostname: &host},
 				&Process{Pid: pid},
 				&User{Id: &uid, Email: &mail},
 				common.MapStr{"k": "v", "n": 1, "f": 1.5, "b": false},
@@ -132,7 +132,7 @@ func TestMetadata_Set(t *testing.T) {
 						Version: &agentVersion,
 					},
 				},
-				&System{Hostname: &host},
+				&System{DetectedHostname: &host},
 				&Process{Pid: pid},
 				&User{Id: &uid, Email: &mail},
 				nil,
@@ -181,7 +181,7 @@ func TestMetadata_SetMinimal(t *testing.T) {
 						Version: &agentVersion,
 					},
 				},
-				&System{Hostname: &host},
+				&System{DetectedHostname: &host},
 				&Process{Pid: pid},
 				&User{Id: &uid, Email: &mail},
 				nil,
@@ -203,7 +203,7 @@ func TestMetadata_SetMinimal(t *testing.T) {
 						Version: &agentVersion,
 					},
 				},
-				&System{Hostname: &host},
+				&System{DetectedHostname: &host},
 				&Process{Pid: pid},
 				&User{Id: &uid},
 				common.MapStr{},