feat: sync Docker Compose files with Beats Integrations

.ci/scripts/functional-test.sh

@@ -23,6 +23,9 @@ make -C metricbeat-tests fetch-binary
 # Build runtime dependencies
 STACK_VERSION=${STACK_VERSION} make -C metricbeat-tests run-elastic-stack
 
+# Sync integrations
+make -C metricbeat-tests sync-integrations
+
 rm -rf outputs || true
 mkdir -p outputs
 REPORT=outputs/junit-functional-tests

.vscode/launch.json

@@ -13,6 +13,18 @@
             },
             "args": ["run", "-h"]
         },
+        {
+            "name": "Debug cloning a repository",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cli",
+            "env": {
+                "GO111MODULE": "on",
+                "OP_LOG_LEVEL": "DEBUG",
+            },
+            "args": ["sync", "integrations", "--remote", "elastic:master"]
+        },
         {
             "name": "Debug Running Services",
             "type": "go",
@@ -25,6 +37,18 @@
             },
             "args": ["run", "service", "mysql", "--version", "5.6"]
         },
+        {
+            "name": "Debug Stopping Services",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cli",
+            "env": {
+                "GO111MODULE": "on",
+                "OP_LOG_LEVEL": "DEBUG",
+            },
+            "args": ["stop", "service", "mysql", "--version", "5.6"]
+        },
         {
             "name": "Debug Deploying a Service",
             "type": "go",
@@ -35,7 +59,7 @@
                 "GO111MODULE": "on",
                 "OP_LOG_LEVEL": "DEBUG",
             },
-            "args": ["deploy", "mysql", "--version", "5.6asda", "--stack", "metricbeat"]
+            "args": ["deploy", "redis", "--version", "4.0.11", "--stack", "metricbeat"]
         },
         {
             "name": "Debug Undeploying a Service",
@@ -47,7 +71,7 @@
                 "GO111MODULE": "on",
                 "OP_LOG_LEVEL": "DEBUG",
             },
-            "args": ["undeploy", "mysql", "--stack", "metricbeat"]
+            "args": ["undeploy", "redis", "--stack", "metricbeat"]
         },
         {
             "name": "Debug Running Stacks",
@@ -59,7 +83,19 @@
                 "GO111MODULE": "on",
                 "OP_LOG_LEVEL": "DEBUG",
             },
-            "args": ["run", "stack", "metricbeat", "--withServices", "apache:2.2,redis:latest"]
+            "args": ["run", "stack", "metricbeat", "-v", "7.5.0", "--withServices", "apache:2.2,redis:3.2.12"]
+        },
+        {
+            "name": "Debug Stopping Stacks",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cli",
+            "env": {
+                "GO111MODULE": "on",
+                "OP_LOG_LEVEL": "DEBUG",
+            },
+            "args": ["stop", "stack", "metricbeat"]
         },
         {
             "name": "Godog Tests",

cli/README.md

@@ -34,21 +34,27 @@ $ ./op stop stack observability
 >By the way, `op` comes from `Observability Provisioner`.
 
 ## Configuring the CLI
-The CLI uses a set of YAML files where the configuration is stored, defining a precedence in those files so that it's possible to override or append new configurations to the CLI.
+The **default configuration file** is located under `$HOME/.op`, which is the workspace for the tool. If this directory does not exist when the CLI is run, it will create it under the hood, populating the default services already bundled in the tool.
 
-The **default configuration file** is located under `$HOME/.op`, which is the workspace for the tool. If this directory does not exist when the CLI is run, it will create it under the hood, populating the configuration file (`config.yml`) with the default values.
+>Those default services are defined at [config.go](./config/config.go).
 
->Those default values are defined at [config.go](./config/config.go).
+### Updating services from Beats
+The CLI includes a command to fetch Beats integrations from its GitHub repository, making possible to add them to the list of available services. Tu run this command:
 
-A **second layer for the configuration file** is defined by CLI's execution path, so if a `config.yml` exists at the location where the tool is executed, then the CLI will merge this file into the default one, with higher precedence over defaults.
-
-**Last layer for the configuration file** is defined by the `OP_CONFIG_PATH` environment variable, so if a `config.yml` exists at the location defined by that environment variable, then the CLI will merge this file into the previous ones, with higher precedence over them.
+```
+$ ./op sync integrations -h
+Sync services from Beats, checking out current version of the services from GitHub
 
-This way, the configuration precedence is defined by:
+Usage:
+  op sync integrations [flags]
 
-`$HOME/.op/config.yml < $(pwd)/config.yml < $OP_CONFIG_PATH/config.yml`
+Flags:
+  -h, --help            help for integrations
+  -r, --remote string   Sets the remote for Beats, using 'user:branch' as format (i.e. elastic:master) (default "elastic:master")
+  -R, --remove          Will remove the existing Beats repository before cloning it again
+```
 
-A clear benefit of this layered configuration is to be able to define custom services/stacks at the higher layers of the configuration: the user could define a service or a stack just for that execution, which could be shared accross teams simply copying the configuration file in the proper path. If that configuration is valuable enough, it could be contributed to the tool.
+It's possible to update the services from a different remote, using the `--remote` flag, as described above.
 
 ## Logging
 The CLI uses [`Logrus`](https://github.com/sirupsen/logrus) as default Logger, so it's possible to configure the logger using [Logging levels](https://github.com/sirupsen/logrus#level-logging) to enrich the output of the tool.

cli/cmd/deploy.go

@@ -59,9 +59,8 @@ func buildDeployServiceCommand(srv string) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			serviceManager := services.NewServiceManager()
 
-			env := map[string]string{
-				srv + "Tag": versionToRun,
-			}
+			env := map[string]string{}
+			env = config.PutServiceEnvironment(env, srv, versionToRun)
 
 			err := serviceManager.AddServicesToCompose(deployToStack, []string{srv}, env)
 			if err != nil {
@@ -82,7 +81,10 @@ func buildUndeployServiceCommand(srv string) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			serviceManager := services.NewServiceManager()
 
-			err := serviceManager.RemoveServicesFromCompose(deployToStack, []string{srv})
+			env := map[string]string{}
+			env = config.PutServiceEnvironment(env, srv, versionToRun)
+
+			err := serviceManager.RemoveServicesFromCompose(deployToStack, []string{srv}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"stack":    deployToStack,

cli/cmd/run.go

@@ -57,9 +57,7 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			serviceManager := services.NewServiceManager()
 
-			env := map[string]string{
-				srv + "Tag": versionToRun,
-			}
+			env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun)
 
 			err := serviceManager.RunCompose(false, []string{srv}, env)
 			if err != nil {
@@ -91,7 +89,6 @@ func buildRunStackCommand(key string, stack config.Stack) *cobra.Command {
 			}
 
 			composeNames := []string{}
-			env = map[string]string{}
 			if servicesToRun != "" {
 				services := strings.Split(servicesToRun, ",")
 
@@ -100,7 +97,7 @@ func buildRunStackCommand(key string, stack config.Stack) *cobra.Command {
 					image := arr[0]
 					tag := arr[1]
 
-					env[image+"Tag"] = tag
+					env = config.PutServiceEnvironment(env, image, tag)
 					composeNames = append(composeNames, image)
 				}
 

cli/cmd/sync.go

@@ -0,0 +1,117 @@
+package cmd
+
+import (
+	"errors"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/elastic/metricbeat-tests-poc/cli/config"
+	git "github.com/elastic/metricbeat-tests-poc/cli/internal"
+	io "github.com/elastic/metricbeat-tests-poc/cli/internal"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var remote = "elastic:master"
+var remove = false
+
+func init() {
+	config.InitConfig()
+
+	syncIntegrationsCmd.Flags().StringVarP(&remote, "remote", "r", "elastic:master", "Sets the remote for Beats, using 'user:branch' as format (i.e. elastic:master)")
+	syncIntegrationsCmd.Flags().BoolVarP(&remove, "remove", "R", false, "Will remove the existing Beats repository before cloning it again")
+
+	syncCmd.AddCommand(syncIntegrationsCmd)
+	rootCmd.AddCommand(syncCmd)
+}
+
+var syncCmd = &cobra.Command{
+	Use:   "sync",
+	Short: "Sync services from Beats",
+	Long:  "Subcommands will allow synchronising services",
+	Run: func(cmd *cobra.Command, args []string) {
+		// NOOP
+	},
+}
+
+var syncIntegrationsCmd = &cobra.Command{
+	Use:   "integrations",
+	Short: "Sync services from Beats",
+	Long:  "Sync services from Beats, checking out current version of the services from GitHub",
+	Args: func(cmd *cobra.Command, args []string) error {
+		arr := strings.Split(remote, ":")
+		if len(arr) == 2 {
+			return nil
+		}
+		return errors.New("invalid 'user:branch' format: " + remote + ". Example: 'elastic:master'")
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		workspace := config.Op.Workspace
+
+		// BeatsRepo default object representing Beats project
+		var BeatsRepo = git.ProjectBuilder.
+			WithBaseWorkspace(path.Join(workspace, "git")).
+			WithGitProtocol().
+			WithDomain("github.com").
+			WithName("beats").
+			WithRemote(remote).
+			Build()
+
+		if remove {
+			repoDir := path.Join(workspace, "git", BeatsRepo.Name)
+
+			log.WithFields(log.Fields{
+				"path": repoDir,
+			}).Debug("Removing repository")
+			os.RemoveAll(repoDir)
+			log.WithFields(log.Fields{
+				"path": repoDir,
+			}).Debug("Repository removed")
+		}
+
+		git.Clone(BeatsRepo)
+
+		copyIntegrationsComposeFiles(BeatsRepo, workspace)
+	},
+}
+
+// CopyComposeFiles copies only those services that has a supported-versions.yml
+// file from Beats integrations, and we will need to copy them into a directory
+// named as the original service (i.e. aerospike) under this tool's workspace,
+// alongside the services. Besides that, the method will copy the _meta directory
+// for each service
+func copyIntegrationsComposeFiles(beats git.Project, target string) {
+	pattern := path.Join(
+		beats.GetWorkspace(), "metricbeat", "module", "*", "_meta", "supported-versions.yml")
+
+	files := io.FindFiles(pattern)
+
+	for _, file := range files {
+		metaDir := filepath.Dir(file)
+		serviceDir := filepath.Dir(metaDir)
+		service := filepath.Base(serviceDir)
+
+		composeFile := filepath.Join(serviceDir, "docker-compose.yml")
+		targetFile := filepath.Join(
+			target, "compose", "services", service, "docker-compose.yml")
+
+		err := io.CopyFile(composeFile, targetFile, 10000)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+				"file":  file,
+			}).Warn("File was not copied")
+		}
+
+		targetMetaDir := filepath.Join(target, "compose", "services", service, "_meta")
+		err = io.CopyDir(metaDir, targetMetaDir)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+				"_meta": metaDir,
+			}).Warn("Meta dir was not copied")
+		}
+	}
+}

cli/config/compose/services/apm-server.yml → cli/config/compose/services/apm-server/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   apm-server:
     environment:

cli/config/compose/services/elasticsearch.yml → cli/config/compose/services/elasticsearch/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   elasticsearch:
     environment:

cli/config/compose/services/kibana.yml → cli/config/compose/services/kibana/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   kibana:
     environment:

cli/config/compose/services/metricbeat.yml → cli/config/compose/services/metricbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   metricbeat:
     command: [

cli/config/compose/services/opbeans-go.yml → cli/config/compose/services/opbeans-go/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   opbeans-go:
     environment:

cli/config/compose/services/opbeans-java.yml → cli/config/compose/services/opbeans-java/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   opbeans-java:
     environment:

cli/config/compose/services/vsphere.yml → cli/config/compose/services/vsphere/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   vsphere:
     image: "nimmis/vcsim:${vsphereTag}"

cli/config/compose/stacks/metricbeat.yml → cli/config/compose/stacks/metricbeat/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.3'
 services:
   elasticsearch:
     environment: