[Elastic-Agent] Use data subfolder as default for process logs (elast…

…ic#17960)

[Elastic-Agent] Use data subfolder as default for process logs (elastic#17960)

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -37,3 +37,5 @@
 - Expose stream.* variables in events {pull}17468[17468]
 - Monitoring configuration reloadable {pull}17855[17855]
 - Enable Filebeat input: S3, Azureeventhub, cloudfoundry, httpjson, netflow, o365audit. {pull}17909[17909]
+- Use data subfolder as default for process logs {pull}17960[17960]
+- Enable debug log level for Metricbeat and Filebeat when run under the Elastic Agent. {pull}17935[17935]

x-pack/elastic-agent/pkg/agent/application/global_config.go

@@ -5,23 +5,12 @@
 package application
 
 import (
-	"os"
-	"path/filepath"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 )
 
-var (
-	homePath string
-	dataPath string
-)
-
-func init() {
-	homePath = retrieveExecutablePath()
-	dataPath = retrieveDataPath()
-}
-
 // InjectAgentConfig injects config to a provided configuration.
 func InjectAgentConfig(c *config.Config) error {
 	globalConfig := AgentGlobalConfig()
@@ -37,24 +26,8 @@ func InjectAgentConfig(c *config.Config) error {
 func AgentGlobalConfig() map[string]interface{} {
 	return map[string]interface{}{
 		"path": map[string]interface{}{
-			"data": dataPath,
-			"home": homePath,
+			"data": paths.Data(),
+			"home": paths.Home(),
 		},
 	}
 }
-
-// retrieveExecutablePath returns a directory where binary lives
-// Executable is not supported on nacl.
-func retrieveExecutablePath() string {
-	execPath, err := os.Executable()
-	if err != nil {
-		panic(err)
-	}
-
-	return filepath.Dir(execPath)
-}
-
-// retrieveHomePath returns a home directory of current user
-func retrieveDataPath() string {
-	return filepath.Join(retrieveExecutablePath(), "data")
-}

x-pack/elastic-agent/pkg/agent/application/paths/paths.go

@@ -0,0 +1,45 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package paths
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+)
+
+var (
+	homePath string
+	dataPath string
+)
+
+func init() {
+	exePath := retrieveExecutablePath()
+
+	fs := flag.CommandLine
+	fs.StringVar(&homePath, "path.home", exePath, "Agent root path")
+	fs.StringVar(&dataPath, "path.data", filepath.Join(exePath, "data"), "Data path contains Agent managed binaries")
+}
+
+// Home returns a directory where binary lives
+// Executable is not supported on nacl.
+func Home() string {
+	return homePath
+}
+
+// Data returns a home directory of current user
+func Data() string {
+	return dataPath
+}
+
+func retrieveExecutablePath() string {
+
+	execPath, err := os.Executable()
+	if err != nil {
+		panic(err)
+	}
+
+	return filepath.Dir(execPath)
+}

x-pack/elastic-agent/pkg/agent/cmd/common.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/basecmd"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
 )
@@ -27,8 +28,8 @@ type globalFlags struct {
 }
 
 func (f *globalFlags) Config() string {
-	if len(f.PathConfigFile) == 0 {
-		return filepath.Join(f.PathHome, defaultConfig)
+	if len(f.PathConfigFile) == 0 || f.PathConfigFile == defaultConfig {
+		return filepath.Join(paths.Home(), defaultConfig)
 	}
 	return f.PathConfigFile
 }

x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/drop_test.go

@@ -15,22 +15,22 @@ type testCase struct {
 
 func TestMonitoringDrops(t *testing.T) {
 	cases := []testCase{
-		testCase{`/var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`npipe://drop`, ""},
-		testCase{`http+npipe://drop`, ""},
-		testCase{`\\.\pipe\drop`, ""},
-		testCase{`unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`http+unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`file:///var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`http://localhost/stats`, ""},
-		testCase{`localhost/stats`, ""},
-		testCase{`http://localhost:8080/stats`, ""},
-		testCase{`localhost:8080/stats`, ""},
-		testCase{`http://1.2.3.4/stats`, ""},
-		testCase{`http://1.2.3.4:5678/stats`, ""},
-		testCase{`1.2.3.4:5678/stats`, ""},
-		testCase{`http://hithere.com:5678/stats`, ""},
-		testCase{`hithere.com:5678/stats`, ""},
+		{`/var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`npipe://drop`, ""},
+		{`http+npipe://drop`, ""},
+		{`\\.\pipe\drop`, ""},
+		{`unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`http+unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`file:///var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`http://localhost/stats`, ""},
+		{`localhost/stats`, ""},
+		{`http://localhost:8080/stats`, ""},
+		{`localhost:8080/stats`, ""},
+		{`http://1.2.3.4/stats`, ""},
+		{`http://1.2.3.4:5678/stats`, ""},
+		{`1.2.3.4:5678/stats`, ""},
+		{`http://hithere.com:5678/stats`, ""},
+		{`hithere.com:5678/stats`, ""},
 	}
 
 	for _, c := range cases {

x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/monitoring.go

@@ -6,16 +6,18 @@ package beats
 
 import (
 	"fmt"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 )
 
 const (
-	// args: pipeline name, application name
-	logFileFormat = "/var/log/elastic-agent/%s/%s"
-	// args: install path, pipeline name, application name
-	logFileFormatWin = "%s\\logs\\elastic-agent\\%s\\%s"
+	// args: data path, pipeline name, application name
+	logFileFormat = "%s/logs/%s/%s"
+	// args: data path, install path, pipeline name, application name
+	logFileFormatWin = "%s\\logs\\%s\\%s"
 
 	// args: pipeline name, application name
-	mbEndpointFileFormat = "unix:///var/run/elastic-agent/%s/%s/%s.sock"
+	mbEndpointFileFormat = "unix://%s/run/%s/%s/%s.sock"
 	// args: pipeline name, application name
 	mbEndpointFileFormatWin = `npipe:///%s-%s`
 )
@@ -25,13 +27,13 @@ func getMonitoringEndpoint(program, operatingSystem, pipelineID string) string {
 		return fmt.Sprintf(mbEndpointFileFormatWin, pipelineID, program)
 	}
 
-	return fmt.Sprintf(mbEndpointFileFormat, pipelineID, program, program)
+	return fmt.Sprintf(mbEndpointFileFormat, paths.Data(), pipelineID, program, program)
 }
 
 func getLoggingFile(program, operatingSystem, installPath, pipelineID string) string {
 	if operatingSystem == "windows" {
-		return fmt.Sprintf(logFileFormatWin, installPath, pipelineID, program)
+		return fmt.Sprintf(logFileFormatWin, paths.Data(), pipelineID, program)
 	}
 
-	return fmt.Sprintf(logFileFormat, pipelineID, program)
+	return fmt.Sprintf(logFileFormat, paths.Data(), pipelineID, program)
 }

x-pack/elastic-agent/pkg/core/plugin/app/start.go

@@ -15,6 +15,7 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/plugin/authority"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/plugin/process"
@@ -210,12 +211,7 @@ func (a *Application) checkGrpcHTTP(ctx context.Context, address string, ca *aut
 }
 
 func injectDataPath(args []string, pipelineID, id string) []string {
-	wd := ""
-	if w, err := os.Getwd(); err == nil {
-		wd = w
-	}
-
-	dataPath := filepath.Join(wd, "data", pipelineID, id)
+	dataPath := filepath.Join(paths.Data(), pipelineID, id)
 	return append(args, "-E", "path.data="+dataPath)
 }