cli: enable loading default profiles turnkey (#943)

* cli: enable loading default profiles turnkey Fixes #933 * update NOTICE * Fix additional lint issues in ingest_manager_test * Cleanup comment and trace log in GetComposeFile * Provide better trace feedback if missing docker-compose * Update cli/config/config.go * chore: add back traces when extracting the files from the box (#946) * fix: use a more comprehensive initialisation method for configs As go init() method is not deterministic, I found that the logger init was not called at the right time. With change we ensure that the Init is: 1) called first 2) existing it the config was already populated * chore: add back traces when extracting the files from the box Signed-off-by: Adam Stokes <[email protected]> Co-authored-by: Manuel de la Peña <[email protected]>
elastic · Mar 29, 2021 · ae5b9fe · ae5b9fe
1 parent 91a9785
commit ae5b9fe
Show file tree

Hide file tree

Showing 11 changed files with 1,363 additions and 169 deletions.
diff --git a/cli/NOTICE.txt b/cli/NOTICE.txt
diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go
@@ -17,7 +17,7 @@ import (
 var deployToProfile string
 
 func init() {
-	config.InitConfig()
+	config.Init()
 
 	for k := range config.AvailableServices() {
 		// deploy command

diff --git a/cli/cmd/run.go b/cli/cmd/run.go
@@ -19,7 +19,7 @@ var servicesToRun string
 var versionToRun string
 
 func init() {
-	config.InitConfig()
+	config.Init()
 
 	rootCmd.AddCommand(runCmd)
 

diff --git a/cli/cmd/stop.go b/cli/cmd/stop.go
@@ -17,7 +17,7 @@ import (
 var versionToStop string
 
 func init() {
-	config.InitConfig()
+	config.Init()
 
 	rootCmd.AddCommand(stopCmd)
 

diff --git a/cli/cmd/sync.go b/cli/cmd/sync.go
@@ -27,7 +27,7 @@ var excludedBlocks = []string{"build"}
 var remote = "elastic:master"
 
 func init() {
-	config.InitConfig()
+	config.Init()
 
 	syncIntegrationsCmd.Flags().BoolVarP(&deleteRepository, "delete", "d", false, "Will delete the existing Beats repository before cloning it again (default false)")
 	syncIntegrationsCmd.Flags().StringVarP(&remote, "remote", "r", "elastic:master", "Sets the remote for Beats, using 'user:branch' as format (i.e. elastic:master)")

diff --git a/...es/fleet/configurations/kibana.config.yml → ...es/fleet/configurations/kibana.config.yml b/...es/fleet/configurations/kibana.config.yml → ...es/fleet/configurations/kibana.config.yml
diff --git a/cli/config/compose/profiles/fleet/docker-compose.yml b/cli/config/compose/profiles/fleet/docker-compose.yml
@@ -33,7 +33,7 @@ services:
     ports:
       - "5601:5601"
     volumes:
-      - ${kibanaConfigPath}:/usr/share/kibana/config/kibana.yml
+      - ./configurations/kibana.config.yml:/usr/share/kibana/config/kibana.yml
   package-registry:
     image: docker.elastic.co/package-registry/distribution:staging
     healthcheck:

diff --git a/cli/config/config.go b/cli/config/config.go
@@ -6,6 +6,7 @@ package config
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -71,7 +72,7 @@ func FileExists(configFile string) (bool, error) {
 }
 
 // GetComposeFile returns the path of the compose file, looking up the
-// tool's workdir or in the static resources already packaged in the binary
+// tool's workdir
 func GetComposeFile(isProfile bool, composeName string) (string, error) {
 	composeFileName := "docker-compose.yml"
 	serviceType := "services"
@@ -94,9 +95,8 @@ func GetComposeFile(isProfile bool, composeName string) (string, error) {
 		"composeFilePath": composeFilePath,
 		"error":           err,
 		"type":            serviceType,
-	}).Trace("Compose file not found at workdir. Extracting from binary resources")
+	}).Trace("Compose file not found. Please make sure the file exists at the location")
 
-	composeBytes, err := opComposeBox.Find(path.Join(serviceType, composeName, composeFileName))
 	if err != nil {
 		log.WithFields(log.Fields{
 			"composeFileName": composeFileName,
@@ -108,23 +108,6 @@ func GetComposeFile(isProfile bool, composeName string) (string, error) {
 		return "", err
 	}
 
-	// create parent directory for the compose file
-	err = io.MkdirAll(filepath.Dir(composeFilePath))
-	if err != nil {
-		return composeFilePath, err
-	}
-
-	err = io.WriteFile(composeBytes, composeFilePath)
-	if err != nil {
-		return composeFilePath, err
-	}
-
-	log.WithFields(log.Fields{
-		"composeFilePath": composeFilePath,
-		"isProfile":       isProfile,
-		"type":            serviceType,
-	}).Trace("Compose file generated at workdir.")
-
 	return composeFilePath, nil
 }
 
@@ -135,6 +118,10 @@ func GetServiceConfig(service string) (Service, bool) {
 
 // Init creates this tool workspace under user's home, in a hidden directory named ".op"
 func Init() {
+	if Op != nil {
+		return
+	}
+
 	configureLogger()
 
 	binaries := []string{
@@ -296,53 +283,72 @@ func newConfig(workspace string) {
 	}
 	Op = &opConfig
 
-	box := packComposeFiles(Op)
+	box := packFiles(Op)
 	if box == nil {
 		log.WithFields(log.Fields{
 			"workspace": workspace,
 		}).Error("Could not get packaged compose files")
 		return
 	}
 
+	// initialize included profiles/services
+	extractProfileServiceConfig(Op, box)
+
 	// add file system services and profiles
 	readFilesFromFileSystem("services")
 	readFilesFromFileSystem("profiles")
 
 	opComposeBox = box
 }
 
-func packComposeFiles(op *OpConfig) *packr.Box {
-	box := packr.New("Compose Files", "./compose")
-
-	err := box.Walk(func(boxedPath string, f packr.File) error {
-		// there must be three tokens: i.e. 'services/aerospike/docker-compose.yml'
-		tokens := strings.Split(boxedPath, string(os.PathSeparator))
-		composeType := tokens[0]
-		composeName := tokens[1]
-
-		log.WithFields(log.Fields{
-			"service": composeName,
-			"path":    boxedPath,
-		}).Trace("Boxed file")
-
-		if composeType == "profiles" {
-			op.Profiles[composeName] = Profile{
-				Name: composeName,
-				Path: boxedPath,
-			}
-		} else if composeType == "services" {
-			op.Services[composeName] = Service{
-				Name: composeName,
-				Path: boxedPath,
+// Extract packaged profiles and services for use with cli runner
+// all default configs/profiles/services will be overwritten. In order to customize
+// the default deployments, create a new directory and copy the existing files over.
+func extractProfileServiceConfig(op *OpConfig, box *packr.Box) error {
+	var walkFn = func(s string, file packr.File) error {
+		p := filepath.Join(op.Workspace, "compose", s)
+		dir := filepath.Dir(p)
+		if _, err := os.Stat(dir); os.IsNotExist(err) {
+			err := os.MkdirAll(dir, 0755)
+			if err != nil {
+				return err
 			}
 		}
 
-		return nil
-	})
-	if err != nil {
-		return nil
+		log.WithFields(log.Fields{
+			"file": p,
+			"dir":  dir,
+		}).Trace("Extracting boxed file")
+
+		return ioutil.WriteFile(p, []byte(file.String()), 0644)
 	}
 
+	if err := box.Walk(walkFn); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Packs all files required for starting profiles/services
+// if a configurations directory exists within the parent directory for each compose file
+// it will be included in the package binary but skipped in determining the correct paths
+// for each compose file
+//
+// Directory format for profiles/services are as follows:
+// compose/
+//  profiles/
+//    fleet/
+//      configurations/
+//      docker-compose.yml
+//  services/
+//    apache/
+//      configurations/
+//      docker-compose.yml
+//
+// configurations/ directory is optional and only needed if docker-compose.yml needs to reference
+// any filelike object within its parent directory
+func packFiles(op *OpConfig) *packr.Box {
+	box := packr.New("Compose Files", "./compose")
 	return box
 }
 

diff --git a/cli/go.mod b/cli/go.mod
@@ -11,22 +11,23 @@ require (
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/gobuffalo/packr/v2 v2.7.1
 	github.com/gogo/protobuf v1.3.1 // indirect
-	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.5.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.6.1
 	github.com/testcontainers/testcontainers-go v0.7.0
-	go.elastic.co/go-licence-detector v0.5.0 // indirect
-	golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf // indirect
-	golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 // indirect
+	go.elastic.co/apm v1.11.0
+	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 // indirect
 	google.golang.org/grpc v1.24.0 // indirect