diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index b77472b97b..061a6da8e0 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -23,6 +23,9 @@ make -C metricbeat-tests fetch-binary # Build runtime dependencies STACK_VERSION=${STACK_VERSION} make -C metricbeat-tests run-elastic-stack +# Sync integrations +make -C metricbeat-tests sync-integrations + rm -rf outputs || true mkdir -p outputs REPORT=outputs/junit-functional-tests diff --git a/.vscode/launch.json b/.vscode/launch.json index 3ca11a783e..3ef27a18c1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,6 +13,18 @@ }, "args": ["run", "-h"] }, + { + "name": "Debug cloning a repository", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cli", + "env": { + "GO111MODULE": "on", + "OP_LOG_LEVEL": "DEBUG", + }, + "args": ["sync", "integrations", "--remote", "elastic:master"] + }, { "name": "Debug Running Services", "type": "go", @@ -25,6 +37,18 @@ }, "args": ["run", "service", "mysql", "--version", "5.6"] }, + { + "name": "Debug Stopping Services", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cli", + "env": { + "GO111MODULE": "on", + "OP_LOG_LEVEL": "DEBUG", + }, + "args": ["stop", "service", "mysql", "--version", "5.6"] + }, { "name": "Debug Deploying a Service", "type": "go", @@ -35,7 +59,7 @@ "GO111MODULE": "on", "OP_LOG_LEVEL": "DEBUG", }, - "args": ["deploy", "mysql", "--version", "5.6asda", "--stack", "metricbeat"] + "args": ["deploy", "redis", "--version", "4.0.11", "--stack", "metricbeat"] }, { "name": "Debug Undeploying a Service", @@ -47,7 +71,7 @@ "GO111MODULE": "on", "OP_LOG_LEVEL": "DEBUG", }, - "args": ["undeploy", "mysql", "--stack", "metricbeat"] + "args": ["undeploy", "redis", "--stack", "metricbeat"] }, { "name": "Debug Running Stacks", @@ -59,7 +83,19 @@ "GO111MODULE": "on", "OP_LOG_LEVEL": "DEBUG", }, - "args": ["run", "stack", "metricbeat", "--withServices", "apache:2.2,redis:latest"] + "args": ["run", "stack", "metricbeat", "-v", "7.5.0", "--withServices", "apache:2.2,redis:3.2.12"] + }, + { + "name": "Debug Stopping Stacks", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cli", + "env": { + "GO111MODULE": "on", + "OP_LOG_LEVEL": "DEBUG", + }, + "args": ["stop", "stack", "metricbeat"] }, { "name": "Godog Tests", diff --git a/cli/README.md b/cli/README.md index fbd4f8560a..cee2c7df96 100644 --- a/cli/README.md +++ b/cli/README.md @@ -34,21 +34,27 @@ $ ./op stop stack observability >By the way, `op` comes from `Observability Provisioner`. ## Configuring the CLI -The CLI uses a set of YAML files where the configuration is stored, defining a precedence in those files so that it's possible to override or append new configurations to the CLI. +The **default configuration file** is located under `$HOME/.op`, which is the workspace for the tool. If this directory does not exist when the CLI is run, it will create it under the hood, populating the default services already bundled in the tool. -The **default configuration file** is located under `$HOME/.op`, which is the workspace for the tool. If this directory does not exist when the CLI is run, it will create it under the hood, populating the configuration file (`config.yml`) with the default values. +>Those default services are defined at [config.go](./config/config.go). ->Those default values are defined at [config.go](./config/config.go). +### Updating services from Beats +The CLI includes a command to fetch Beats integrations from its GitHub repository, making it possible to add them to the list of available services. To run this command: -A **second layer for the configuration file** is defined by CLI's execution path, so if a `config.yml` exists at the location where the tool is executed, then the CLI will merge this file into the default one, with higher precedence over defaults. - -**Last layer for the configuration file** is defined by the `OP_CONFIG_PATH` environment variable, so if a `config.yml` exists at the location defined by that environment variable, then the CLI will merge this file into the previous ones, with higher precedence over them. +``` +$ ./op sync integrations -h +Sync services from Beats, checking out current version of the services from GitHub -This way, the configuration precedence is defined by: +Usage: + op sync integrations [flags] -`$HOME/.op/config.yml < $(pwd)/config.yml < $OP_CONFIG_PATH/config.yml` +Flags: + -h, --help help for integrations + -d, --delete Will delete the existing Beats repository before cloning it again (default false) + -r, --remote string Sets the remote for Beats, using 'user:branch' as format (i.e. elastic:master) (default "elastic:master") +``` -A clear benefit of this layered configuration is to be able to define custom services/stacks at the higher layers of the configuration: the user could define a service or a stack just for that execution, which could be shared accross teams simply copying the configuration file in the proper path. If that configuration is valuable enough, it could be contributed to the tool. +It's possible to update the services from a different remote, using the `--remote` flag, as described above. ## Logging The CLI uses [`Logrus`](https://github.com/sirupsen/logrus) as default Logger, so it's possible to configure the logger using [Logging levels](https://github.com/sirupsen/logrus#level-logging) to enrich the output of the tool. diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index 61315e5782..226903af19 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -59,9 +59,8 @@ func buildDeployServiceCommand(srv string) *cobra.Command { Run: func(cmd *cobra.Command, args []string) { serviceManager := services.NewServiceManager() - env := map[string]string{ - srv + "Tag": versionToRun, - } + env := map[string]string{} + env = config.PutServiceEnvironment(env, srv, versionToRun) err := serviceManager.AddServicesToCompose(deployToStack, []string{srv}, env) if err != nil { @@ -82,7 +81,10 @@ func buildUndeployServiceCommand(srv string) *cobra.Command { Run: func(cmd *cobra.Command, args []string) { serviceManager := services.NewServiceManager() - err := serviceManager.RemoveServicesFromCompose(deployToStack, []string{srv}) + env := map[string]string{} + env = config.PutServiceEnvironment(env, srv, versionToRun) + + err := serviceManager.RemoveServicesFromCompose(deployToStack, []string{srv}, env) if err != nil { log.WithFields(log.Fields{ "stack": deployToStack, diff --git a/cli/cmd/run.go b/cli/cmd/run.go index b92ba7fc33..d445303c25 100644 --- a/cli/cmd/run.go +++ b/cli/cmd/run.go @@ -57,9 +57,7 @@ func buildRunServiceCommand(srv string) *cobra.Command { Run: func(cmd *cobra.Command, args []string) { serviceManager := services.NewServiceManager() - env := map[string]string{ - srv + "Tag": versionToRun, - } + env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun) err := serviceManager.RunCompose(false, []string{srv}, env) if err != nil { @@ -91,7 +89,6 @@ func buildRunStackCommand(key string, stack config.Stack) *cobra.Command { } composeNames := []string{} - env = map[string]string{} if servicesToRun != "" { services := strings.Split(servicesToRun, ",") @@ -100,7 +97,7 @@ func buildRunStackCommand(key string, stack config.Stack) *cobra.Command { image := arr[0] tag := arr[1] - env[image+"Tag"] = tag + env = config.PutServiceEnvironment(env, image, tag) composeNames = append(composeNames, image) } diff --git a/cli/cmd/sync.go b/cli/cmd/sync.go new file mode 100644 index 0000000000..0649a82723 --- /dev/null +++ b/cli/cmd/sync.go @@ -0,0 +1,117 @@ +package cmd + +import ( + "errors" + "os" + "path" + "path/filepath" + "strings" + + "github.com/elastic/metricbeat-tests-poc/cli/config" + git "github.com/elastic/metricbeat-tests-poc/cli/internal" + io "github.com/elastic/metricbeat-tests-poc/cli/internal" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var deleteRepository = false +var remote = "elastic:master" + +func init() { + config.InitConfig() + + syncIntegrationsCmd.Flags().BoolVarP(&deleteRepository, "delete", "d", false, "Will delete the existing Beats repository before cloning it again (default false)") + syncIntegrationsCmd.Flags().StringVarP(&remote, "remote", "r", "elastic:master", "Sets the remote for Beats, using 'user:branch' as format (i.e. elastic:master)") + + syncCmd.AddCommand(syncIntegrationsCmd) + rootCmd.AddCommand(syncCmd) +} + +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "Sync services from Beats", + Long: "Subcommands will allow synchronising services", + Run: func(cmd *cobra.Command, args []string) { + // NOOP + }, +} + +var syncIntegrationsCmd = &cobra.Command{ + Use: "integrations", + Short: "Sync services from Beats", + Long: "Sync services from Beats, checking out current version of the services from GitHub", + Args: func(cmd *cobra.Command, args []string) error { + arr := strings.Split(remote, ":") + if len(arr) == 2 { + return nil + } + return errors.New("invalid 'user:branch' format: " + remote + ". Example: 'elastic:master'") + }, + Run: func(cmd *cobra.Command, args []string) { + workspace := config.Op.Workspace + + // BeatsRepo default object representing Beats project + var BeatsRepo = git.ProjectBuilder. + WithBaseWorkspace(path.Join(workspace, "git")). + WithGitProtocol(). + WithDomain("github.com"). + WithName("beats"). + WithRemote(remote). + Build() + + if deleteRepository { + repoDir := path.Join(workspace, "git", BeatsRepo.Name) + + log.WithFields(log.Fields{ + "path": repoDir, + }).Debug("Removing repository") + os.RemoveAll(repoDir) + log.WithFields(log.Fields{ + "path": repoDir, + }).Debug("Repository removed") + } + + git.Clone(BeatsRepo) + + copyIntegrationsComposeFiles(BeatsRepo, workspace) + }, +} + +// CopyComposeFiles copies only those services that has a supported-versions.yml +// file from Beats integrations, and we will need to copy them into a directory +// named as the original service (i.e. aerospike) under this tool's workspace, +// alongside the services. Besides that, the method will copy the _meta directory +// for each service +func copyIntegrationsComposeFiles(beats git.Project, target string) { + pattern := path.Join( + beats.GetWorkspace(), "metricbeat", "module", "*", "_meta", "supported-versions.yml") + + files := io.FindFiles(pattern) + + for _, file := range files { + metaDir := filepath.Dir(file) + serviceDir := filepath.Dir(metaDir) + service := filepath.Base(serviceDir) + + composeFile := filepath.Join(serviceDir, "docker-compose.yml") + targetFile := filepath.Join( + target, "compose", "services", service, "docker-compose.yml") + + err := io.CopyFile(composeFile, targetFile, 10000) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": file, + }).Warn("File was not copied") + } + + targetMetaDir := filepath.Join(target, "compose", "services", service, "_meta") + err = io.CopyDir(metaDir, targetMetaDir) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "_meta": metaDir, + }).Warn("Meta dir was not copied") + } + } +} diff --git a/cli/config/compose/services/apache.yml b/cli/config/compose/services/apache.yml deleted file mode 100644 index cf1abd7b60..0000000000 --- a/cli/config/compose/services/apache.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3' -services: - apache: - image: "docker.elastic.co/integrations-ci/beats-apache:${apacheTag}-1" - ports: - - "80:80" \ No newline at end of file diff --git a/cli/config/compose/services/apm-server.yml b/cli/config/compose/services/apm-server/docker-compose.yml similarity index 97% rename from cli/config/compose/services/apm-server.yml rename to cli/config/compose/services/apm-server/docker-compose.yml index db70807e83..17a0898369 100644 --- a/cli/config/compose/services/apm-server.yml +++ b/cli/config/compose/services/apm-server/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: apm-server: environment: diff --git a/cli/config/compose/services/elasticsearch.yml b/cli/config/compose/services/elasticsearch/docker-compose.yml similarity index 96% rename from cli/config/compose/services/elasticsearch.yml rename to cli/config/compose/services/elasticsearch/docker-compose.yml index 5871c1e1a4..63130ee350 100644 --- a/cli/config/compose/services/elasticsearch.yml +++ b/cli/config/compose/services/elasticsearch/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: elasticsearch: environment: diff --git a/cli/config/compose/services/kafka.yml b/cli/config/compose/services/kafka.yml deleted file mode 100644 index aebf911e65..0000000000 --- a/cli/config/compose/services/kafka.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3' -services: - kafka: - image: "docker.elastic.co/integrations-ci/beats-kafka:${kafkaTag}-2" - ports: - - "9092:9092" \ No newline at end of file diff --git a/cli/config/compose/services/kibana.yml b/cli/config/compose/services/kibana/docker-compose.yml similarity index 94% rename from cli/config/compose/services/kibana.yml rename to cli/config/compose/services/kibana/docker-compose.yml index 9a0a2fa4cf..19330b571c 100644 --- a/cli/config/compose/services/kibana.yml +++ b/cli/config/compose/services/kibana/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: kibana: environment: diff --git a/cli/config/compose/services/metricbeat.yml b/cli/config/compose/services/metricbeat/docker-compose.yml similarity index 97% rename from cli/config/compose/services/metricbeat.yml rename to cli/config/compose/services/metricbeat/docker-compose.yml index 58fb863658..1f9ccde3a4 100644 --- a/cli/config/compose/services/metricbeat.yml +++ b/cli/config/compose/services/metricbeat/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: metricbeat: command: [ diff --git a/cli/config/compose/services/mongodb.yml b/cli/config/compose/services/mongodb.yml deleted file mode 100644 index d7ca856e70..0000000000 --- a/cli/config/compose/services/mongodb.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3' -services: - mongodb: - image: "mongo:${mongodbTag}" - ports: - - "27017:27017" \ No newline at end of file diff --git a/cli/config/compose/services/mysql.yml b/cli/config/compose/services/mysql.yml deleted file mode 100644 index a61a2b03cb..0000000000 --- a/cli/config/compose/services/mysql.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '3' -services: - mysql: - environment: - - MYSQL_ROOT_PASSWORD=secret - image: "docker.elastic.co/integrations-ci/beats-mysql:${mysqlTag}-1" - ports: - - "3306:3306" \ No newline at end of file diff --git a/cli/config/compose/services/opbeans-go.yml b/cli/config/compose/services/opbeans-go/docker-compose.yml similarity index 96% rename from cli/config/compose/services/opbeans-go.yml rename to cli/config/compose/services/opbeans-go/docker-compose.yml index 7be11124a9..6c9e4f6602 100644 --- a/cli/config/compose/services/opbeans-go.yml +++ b/cli/config/compose/services/opbeans-go/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: opbeans-go: environment: diff --git a/cli/config/compose/services/opbeans-java.yml b/cli/config/compose/services/opbeans-java/docker-compose.yml similarity index 96% rename from cli/config/compose/services/opbeans-java.yml rename to cli/config/compose/services/opbeans-java/docker-compose.yml index b7b3f8c837..78af7b63ea 100644 --- a/cli/config/compose/services/opbeans-java.yml +++ b/cli/config/compose/services/opbeans-java/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: opbeans-java: environment: diff --git a/cli/config/compose/services/redis.yml b/cli/config/compose/services/redis.yml deleted file mode 100644 index 5e4a7eac40..0000000000 --- a/cli/config/compose/services/redis.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3' -services: - redis: - image: "docker.elastic.co/integrations-ci/beats-redis:${redisTag}-1" - ports: - - "6379:6379" \ No newline at end of file diff --git a/cli/config/compose/services/vsphere.yml b/cli/config/compose/services/vsphere/docker-compose.yml similarity index 94% rename from cli/config/compose/services/vsphere.yml rename to cli/config/compose/services/vsphere/docker-compose.yml index ca54aa9723..7bbedc5380 100644 --- a/cli/config/compose/services/vsphere.yml +++ b/cli/config/compose/services/vsphere/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: vsphere: image: "nimmis/vcsim:${vsphereTag}" diff --git a/cli/config/compose/stacks/metricbeat.yml b/cli/config/compose/stacks/metricbeat/docker-compose.yml similarity index 97% rename from cli/config/compose/stacks/metricbeat.yml rename to cli/config/compose/stacks/metricbeat/docker-compose.yml index 2d73360c3c..86f5e73e7b 100644 --- a/cli/config/compose/stacks/metricbeat.yml +++ b/cli/config/compose/stacks/metricbeat/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2.3' services: elasticsearch: environment: diff --git a/cli/config/config.go b/cli/config/config.go index 994c1cb92d..798e0320b5 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -1,7 +1,7 @@ package config import ( - "io/ioutil" + "fmt" "os" "os/exec" "os/user" @@ -9,8 +9,12 @@ import ( "path/filepath" "strings" + io "github.com/elastic/metricbeat-tests-poc/cli/internal" + packr "github.com/gobuffalo/packr/v2" log "github.com/sirupsen/logrus" + + "gopkg.in/yaml.v2" ) // opComposeBox the tool's static files where we will embed default Docker compose @@ -27,6 +31,13 @@ type OpConfig struct { Workspace string `mapstructure:"workspace"` } +// GetServiceConfig configuration of a service +func (c *OpConfig) GetServiceConfig(service string) (Service, bool) { + srv, exists := c.Services[service] + + return srv, exists +} + // Service represents the configuration for a service type Service struct { Name string `mapstructure:"Name"` @@ -50,17 +61,17 @@ func AvailableStacks() map[string]Stack { return Op.Stacks } -// GetPackedCompose returns the path of the compose file, looking up the +// GetComposeFile returns the path of the compose file, looking up the // tool's workdir or in the static resources already packaged in the binary -func GetPackedCompose(isStack bool, composeName string) (string, error) { - composeFileName := composeName + ".yml" +func GetComposeFile(isStack bool, composeName string) (string, error) { + composeFileName := "docker-compose.yml" serviceType := "services" if isStack { serviceType = "stacks" } - composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeFileName) - found, err := exists(composeFilePath) + composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName) + found, err := io.Exists(composeFilePath) if found && err == nil { log.WithFields(log.Fields{ "composeFilePath": composeFilePath, @@ -75,7 +86,7 @@ func GetPackedCompose(isStack bool, composeName string) (string, error) { "type": serviceType, }).Debug("Compose file not found at workdir. Extracting from binary resources") - composeBytes, err := opComposeBox.Find(path.Join(serviceType, composeFileName)) + composeBytes, err := opComposeBox.Find(path.Join(serviceType, composeName, composeFileName)) if err != nil { log.WithFields(log.Fields{ "composeFileName": composeFileName, @@ -87,15 +98,14 @@ func GetPackedCompose(isStack bool, composeName string) (string, error) { return "", err } - err = ioutil.WriteFile(composeFilePath, composeBytes, 0755) + // create parent directory for the compose file + err = io.MkdirAll(filepath.Dir(composeFilePath)) if err != nil { - log.WithFields(log.Fields{ - "composeFilePath": composeFilePath, - "error": err, - "isStack": isStack, - "type": serviceType, - }).Error("Cannot write file at workdir.") + return composeFilePath, err + } + err = io.WriteFile(composeBytes, composeFilePath) + if err != nil { return composeFilePath, err } @@ -113,13 +123,6 @@ func GetServiceConfig(service string) (Service, bool) { return Op.GetServiceConfig(service) } -// GetServiceConfig configuration of a service -func (c *OpConfig) GetServiceConfig(service string) (Service, bool) { - srv, exists := c.Services[service] - - return srv, exists -} - // Init creates this tool workspace under user's home, in a hidden directory named ".op" func Init() { configureLogger() @@ -142,18 +145,82 @@ func InitConfig() { newConfig(w) } -func checkConfigDirectory(dir string) { - found, err := exists(dir) - if found && err == nil { - return +// PutServiceEnvironment puts the environment variables for the service, replacing "SERVICE_" +// with service name in uppercase. The variables are: +// - SERVICE_VERSION: where it represents the version of the service (i.e. APACHE_VERSION) +// - SERVICE_PATH: where it represents the path to its compose file (i.e. APACHE_PATH) +func PutServiceEnvironment(env map[string]string, service string, serviceVersion string) map[string]string { + serviceUpper := strings.ToUpper(service) + + if _, exists := env[serviceUpper+"_VERSION"]; !exists { + env[serviceUpper+"_VERSION"] = serviceVersion + } + + srv, exists := Op.Services[service] + if !exists { + log.WithFields(log.Fields{ + "service": service, + }).Warn("Could not find compose file") + } else { + env[serviceUpper+"_PATH"] = filepath.Dir(srv.Path) + } + + return env +} + +// PutServiceVariantEnvironment puts the environment variables that comes in the supported-versions.yml +// file of the service, replacing "SERVICE_ with service name in uppercase. At the end, it also adds +// the version and the path for the service, calling the PutServiceEnvironment method. An example: +// - SERVICE_VARIANT: where SERVICE is the name of the service (i.e. APACHE_VARIANT) +// - SERVICE_VERSION: where it represents the version of the service (i.e. APACHE_VERSION) +func PutServiceVariantEnvironment(env map[string]string, service string, serviceVariant string, serviceVersion string) map[string]string { + type EnvVar interface{} + type supportedVersions struct { + Env []EnvVar `yaml:"variants"` } - err = os.MkdirAll(dir, 0755) + + versionsPath := path.Join( + Op.Workspace, "compose", "services", service, "_meta", "supported-versions.yml") + + bytes, err := io.ReadFile(versionsPath) + if err != nil { + return map[string]string{} + } + + sv := supportedVersions{} + err = yaml.Unmarshal(bytes, &sv) if err != nil { log.WithFields(log.Fields{ - "error": err, - "path": dir, - }).Fatal("Cannot create directory") + "supported-versions": versionsPath, + }).Error("Could not unmarshal supported versions") + + return map[string]string{} + } + + // discover variants and set them only if the variant matches + for _, e := range sv.Env { + switch i := e.(type) { + case map[interface{}]interface{}: + for k, v := range i { + srvVariant := fmt.Sprint(v) + if srvVariant == serviceVariant { + env[fmt.Sprint(k)] = srvVariant + } + } + default: + // skip + } + } + + return PutServiceEnvironment(env, service, serviceVersion) +} + +func checkConfigDirectory(dir string) { + found, err := io.Exists(dir) + if found && err == nil { + return } + _ = io.MkdirAll(dir) } func checkConfigDirs(workspace string) { @@ -208,17 +275,6 @@ func configureLogger() { } } -func exists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return true, err -} - // newConfig returns a new configuration func newConfig(workspace string) { if Op != nil { @@ -242,6 +298,10 @@ func newConfig(workspace string) { return } + // add file system services and stacks + readFilesFromFileSystem("services") + readFilesFromFileSystem("stacks") + opComposeBox = box } @@ -249,18 +309,22 @@ func packComposeFiles(op *OpConfig) *packr.Box { box := packr.New("Compose Files", "./compose") err := box.Walk(func(boxedPath string, f packr.File) error { + // there must be three tokens: i.e. 'services/aerospike/docker-compose.yml' + tokens := strings.Split(boxedPath, string(os.PathSeparator)) + composeType := tokens[0] + composeName := tokens[1] + log.WithFields(log.Fields{ - "path": boxedPath, + "service": composeName, + "path": boxedPath, }).Debug("Boxed file") - composeType, composeNameWithExtension := path.Split(boxedPath) - composeName := strings.TrimSuffix(composeNameWithExtension, filepath.Ext(composeNameWithExtension)) - if composeType == "stacks/" { + if composeType == "stacks" { op.Stacks[composeName] = Stack{ Name: composeName, Path: boxedPath, } - } else if composeType == "services/" { + } else if composeType == "services" { op.Services[composeName] = Service{ Name: composeName, Path: boxedPath, @@ -276,6 +340,47 @@ func packComposeFiles(op *OpConfig) *packr.Box { return box } +// reads the docker-compose in the workspace, merging them with what it's +// already boxed in the binary +func readFilesFromFileSystem(serviceType string) { + basePath := path.Join(Op.Workspace, "compose", serviceType) + files, err := io.ReadDir(basePath) + if err != nil { + log.WithFields(log.Fields{ + "path": basePath, + "type": serviceType, + }).Warn("Could not load file system") + return + } + + for _, f := range files { + if f.IsDir() { + name := f.Name() + composeFilePath := filepath.Join(basePath, name, "docker-compose.yml") + found, err := io.Exists(composeFilePath) + if found && err == nil { + log.WithFields(log.Fields{ + "service": name, + "path": composeFilePath, + }).Debug("Workspace file") + + if serviceType == "services" { + // add a service or a stack + Op.Services[name] = Service{ + Name: f.Name(), + Path: composeFilePath, + } + } else if serviceType == "stacks" { + Op.Stacks[name] = Stack{ + Name: f.Name(), + Path: composeFilePath, + } + } + } + } + } +} + // which checks if software is installed func which(software string) { path, err := exec.LookPath(software) diff --git a/cli/config/config_test.go b/cli/config/config_test.go index ae4413d087..c0905d6ec9 100644 --- a/cli/config/config_test.go +++ b/cli/config/config_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + io "github.com/elastic/metricbeat-tests-poc/cli/internal" + "github.com/Flaque/filet" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -19,16 +21,16 @@ func TestCheckConfigDirsCreatesWorkspaceAtHome(t *testing.T) { workspace := path.Join(tmpDir, ".op") - e, _ := exists(workspace) + e, _ := io.Exists(workspace) assert.False(t, e) checkConfigDirs(workspace) - e, _ = exists(workspace) + e, _ = io.Exists(workspace) assert.True(t, e) - e, _ = exists(path.Join(workspace, "compose", "services")) + e, _ = io.Exists(path.Join(workspace, "compose", "services")) assert.True(t, e) - e, _ = exists(path.Join(workspace, "compose", "stacks")) + e, _ = io.Exists(path.Join(workspace, "compose", "stacks")) assert.True(t, e) } @@ -106,7 +108,6 @@ func checkLoggerWithLogLevel(t *testing.T, level string) { } func cleanUpEnv() { - os.Unsetenv("OP_CONFIG_PATH") os.Unsetenv("OP_LOG_LEVEL") os.Unsetenv("OP_LOG_INCLUDE_TIMESTAMP") } diff --git a/cli/go.mod b/cli/go.mod index 13942c0139..0390c72243 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -13,6 +13,8 @@ require ( github.com/gogo/protobuf v1.3.1 // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/gorilla/mux v1.7.3 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/rogpeppe/go-internal v1.5.0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/spf13/afero v1.2.2 // indirect @@ -28,7 +30,8 @@ require ( google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 // indirect google.golang.org/grpc v1.24.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.2.4 // indirect + gopkg.in/src-d/go-git.v4 v4.13.1 + gopkg.in/yaml.v2 v2.2.4 ) replace github.com/testcontainers/testcontainers-go v0.0.8 => github.com/mdelapenya/testcontainers-go v0.0.9-compose-1 diff --git a/cli/go.sum b/cli/go.sum index e3ac3fae4a..6f668b1121 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -10,7 +10,13 @@ github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiU github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -20,6 +26,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -33,8 +40,14 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-redis/redis v6.15.5+incompatible h1:pLky8I0rgiblWfa8C1EV7fPEUv0aH6vKRaYHc/YRHVk= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= @@ -59,6 +72,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -71,8 +85,13 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -81,11 +100,17 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mdelapenya/testcontainers-go v0.0.9-compose-1 h1:KsiEki4MDtb5YEjEqoT90X9uVuIBBCdAzuCSloVt8ms= github.com/mdelapenya/testcontainers-go v0.0.9-compose-1/go.mod h1:8vQSG3E0hR0tTCU89Gq6WZ+QWGa07tjOQyecyA7/U2o= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -102,6 +127,7 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -119,6 +145,8 @@ github.com/rogpeppe/go-internal v1.5.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -137,18 +165,25 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= +github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf h1:fnPsqIDRbCSgumaMCRpoIoF2s4qxv0xSSS0BVZUE/ss= golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -163,6 +198,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 h1:N66aaryRB3Ax92gH0v3hp1QYZ3zWWCCUR/j8Ifh45Ss= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -177,9 +213,11 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c h1:S/FtSvpNLtFBgjTqcKsRpsa6aVsI6iztaz1bQd9BJwE= golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -197,6 +235,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3 h1:2AmBLzhAfXj+2HCW09VCkJtHIYgHTIPcTeYqgP7Bwt0= golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -219,8 +258,16 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= +gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= +gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/cli/internal/git.go b/cli/internal/git.go new file mode 100644 index 0000000000..bf53f5619a --- /dev/null +++ b/cli/internal/git.go @@ -0,0 +1,201 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/lann/builder" + log "github.com/sirupsen/logrus" + + git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + ssh "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" +) + +// GitProtocol the git protocol string representation +const GitProtocol = "git@" + +// Project representes a git project +type Project struct { + BaseWorkspace string + Branch string + Domain string + Name string + Protocol string + User string +} + +// GetURL Returns the workspace of a Project +func (d *Project) GetURL() string { + return d.Protocol + d.Domain + ":" + d.User + "/" + d.Name +} + +// GetWorkspace Returns the workspace of a Project +func (d *Project) GetWorkspace() string { + return filepath.Join(d.BaseWorkspace, d.Name) +} + +type projectBuilder builder.Builder + +func (b projectBuilder) Build() Project { + return builder.GetStruct(b).(Project) +} + +func (b projectBuilder) CheckRemoteAndBranch() projectBuilder { + workspace := filepath.Join(b.getField("BaseWorkspace"), b.getField("Name")) + + branch := GetBranch(workspace) + remote := GetRemote(workspace, b.getField("Domain")) + + return b.withBranch(branch).withUser(remote) +} + +func (b projectBuilder) WithBaseWorkspace(baseWorkspace string) projectBuilder { + return builder.Set(b, "BaseWorkspace", baseWorkspace).(projectBuilder) +} + +func (b projectBuilder) WithDomain(domain string) projectBuilder { + return builder.Set(b, "Domain", domain).(projectBuilder) +} + +func (b projectBuilder) WithGitProtocol() projectBuilder { + return builder.Set(b, "Protocol", GitProtocol).(projectBuilder) +} + +func (b projectBuilder) WithName(name string) projectBuilder { + return builder.Set(b, "Name", name).(projectBuilder) +} + +func (b projectBuilder) WithRemote(remote string) projectBuilder { + coordinates := strings.Split(remote, ":") + + return b.withUser(coordinates[0]).withBranch(coordinates[1]) +} + +func (b projectBuilder) withBranch(branch string) projectBuilder { + return builder.Set(b, "Branch", branch).(projectBuilder) +} + +func (b projectBuilder) withUser(user string) projectBuilder { + return builder.Set(b, "User", user).(projectBuilder) +} + +func (b projectBuilder) getField(fieldName string) string { + value, _ := builder.Get(b, fieldName) + + return value.(string) +} + +// ProjectBuilder builder for git projects +var ProjectBuilder = builder.Register(projectBuilder{}, Project{}).(projectBuilder) + +// Clone allows cloning an array of repositories simultaneously +func Clone(repositories ...Project) { + repositoriesChannel := make(chan Project, len(repositories)) + for i := range repositories { + repositoriesChannel <- repositories[i] + } + close(repositoriesChannel) + + workers := 5 + if len(repositoriesChannel) < workers { + workers = len(repositoriesChannel) + } + + errorChannel := make(chan error, 1) + resultChannel := make(chan bool, len(repositories)) + + for i := 0; i < workers; i++ { + // Consume work from repositoriesChannel. Loop will end when no more work. + for repository := range repositoriesChannel { + go cloneGithubRepository(repository, resultChannel, errorChannel) + } + } + + // Collect results from workers + + for i := 0; i < len(repositories); i++ { + select { + case <-resultChannel: + log.WithFields(log.Fields{ + "url": repositories[i].GetURL(), + }).Info("Git clone succeed") + case err := <-errorChannel: + if err != nil { + log.WithFields(log.Fields{ + "url": repositories[i].GetURL(), + "error": err, + }).Warn("Git clone errored") + } + } + } +} + +// GetBranch returns the current branch from a git repository +func GetBranch(gitRepositoryDir string) string { + args := []string{"rev-parse", "--abbrev-ref", "HEAD"} + + return Execute(gitRepositoryDir, "git", args[0:]...) +} + +// GetRemote returns the remote from a git repository +func GetRemote(gitRepositoryDir string, gitDomain string) string { + args := []string{"remote", "get-url", "origin"} + + remote := Execute(gitRepositoryDir, "git", args[0:]...) + + remote1 := strings.TrimPrefix(remote, GitProtocol+gitDomain+":") + remote2 := strings.Split(remote1, "/") + + return remote2[0] +} + +func cloneGithubRepository( + githubRepo Project, resultChannel chan bool, errorChannel chan error) { + + gitRepositoryDir := githubRepo.GetWorkspace() + + if _, err := os.Stat(gitRepositoryDir); os.IsExist(err) { + select { + case errorChannel <- err: + // will break parent goroutine out of loop + default: + // don't care, first error wins + } + return + } + + githubRepositoryURL := githubRepo.GetURL() + + log.WithFields(log.Fields{ + "url": githubRepositoryURL, + "directory": gitRepositoryDir, + }).Info("Cloning project. This process could take long depending on its size") + + auth, err1 := ssh.NewSSHAgentAuth("git") + if err1 != nil { + log.Fatal("Cloning using keys from SSH agent failed") + } + + _, err := git.PlainClone(gitRepositoryDir, false, &git.CloneOptions{ + Auth: auth, + URL: githubRepositoryURL, + Progress: os.Stdout, + ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", githubRepo.Branch)), + SingleBranch: true, + }) + + if err != nil { + select { + case errorChannel <- err: + // will break parent goroutine out of loop + default: + // don't care, first error wins + } + return + } + + resultChannel <- true +} diff --git a/cli/internal/io.go b/cli/internal/io.go new file mode 100644 index 0000000000..69ba9642fe --- /dev/null +++ b/cli/internal/io.go @@ -0,0 +1,207 @@ +package internal + +import ( + "errors" + "io" + "io/ioutil" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + +// CopyDir recursively copies a directory tree, attempting to preserve permissions. +// Source directory must exist, destination directory must *not* exist. +// Symlinks are ignored and skipped. +func CopyDir(src string, dst string) error { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return errors.New("source is not a directory") + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + return errors.New("destination already exists") + } + + err = MkdirAll(dst) + if err != nil { + return err + } + + entries, err := ioutil.ReadDir(src) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + err = CopyDir(srcPath, dstPath) + if err != nil { + return err + } + } else { + // Skip symlinks. + if entry.Mode()&os.ModeSymlink != 0 { + continue + } + + err = CopyFile(srcPath, dstPath, 10000) + if err != nil { + return err + } + } + } + + return nil +} + +// CopyFile copies a file from a source to a destiny +// Optimising the copy of files in Go: +// https://opensource.com/article/18/6/copying-files-go +func CopyFile(src string, dst string, bufferSize int64) error { + sourceFileStat, err := os.Stat(src) + if err != nil { + return err + } + + if !sourceFileStat.Mode().IsRegular() { + return errors.New(src + " is not a regular file") + } + + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + _, err = os.Stat(dst) + if err == nil { + return errors.New("File " + dst + " already exists") + } + + err = MkdirAll(dst) + if err != nil { + return err + } + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + buf := make([]byte, bufferSize) + for { + n, err := source.Read(buf) + if err != nil && err != io.EOF { + return err + } + if n == 0 { + break + } + + if _, err := destination.Write(buf[:n]); err != nil { + return err + } + } + + return err +} + +// Exists checks if a path exists in the file system +func Exists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return true, err +} + +// MkdirAll creates all directories for a directory path +func MkdirAll(path string) error { + if _, err := os.Stat(path); os.IsNotExist(err) { + err = os.MkdirAll(path, 0755) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "path": path, + }).Fatal("Directory cannot be created") + + return err + } + } + + return nil +} + +// FindFiles finds files recursively using a Glob pattern for the matching +func FindFiles(pattern string) []string { + matches, err := filepath.Glob(pattern) + + if err != nil { + log.WithFields(log.Fields{ + "pattern": pattern, + }).Warn("pattern is not a Glob") + + return []string{} + } + + return matches +} + +// ReadDir lists the contents of a directory +func ReadDir(path string) ([]os.FileInfo, error) { + files, err := ioutil.ReadDir(path) + if err != nil { + log.WithFields(log.Fields{ + "path": path, + }).Warn("Could not read file system") + return []os.FileInfo{}, err + } + + return files, nil +} + +// ReadFile returns the byte array representing a file +func ReadFile(path string) ([]byte, error) { + bytes, err := ioutil.ReadFile(path) + if err != nil { + log.WithFields(log.Fields{ + "path": path, + }).Warn("Could not read file") + return []byte{}, err + } + + return bytes, nil +} + +// WriteFile writes bytes into target +func WriteFile(bytes []byte, target string) error { + err := ioutil.WriteFile(target, bytes, 0755) + if err != nil { + log.WithFields(log.Fields{ + "target": target, + "error": err, + }).Error("Cannot write file at workdir.") + + return err + } + + return nil +} diff --git a/cli/internal/io_test.go b/cli/internal/io_test.go new file mode 100644 index 0000000000..227aa104b2 --- /dev/null +++ b/cli/internal/io_test.go @@ -0,0 +1,23 @@ +package internal + +import ( + "path" + "testing" + + "github.com/Flaque/filet" + "github.com/stretchr/testify/assert" +) + +func TestMkdirAll(t *testing.T) { + defer filet.CleanUp(t) + + tmpDir := filet.TmpDir(t, "") + + dir := path.Join(tmpDir, ".op", "compose", "services") + + err := MkdirAll(dir) + assert.Nil(t, err) + + e, _ := Exists(dir) + assert.True(t, e) +} diff --git a/cli/internal/shell.go b/cli/internal/shell.go new file mode 100644 index 0000000000..bfcc76be5d --- /dev/null +++ b/cli/internal/shell.go @@ -0,0 +1,33 @@ +package internal + +import ( + "bytes" + "os/exec" + "strings" + + log "github.com/sirupsen/logrus" +) + +// Execute executes a command in the machine the program is running +// - workspace: represents the location where to execute the command +// - command: represents the name of the binary to execute +// - args: represents the arguments to be passed to the command +func Execute(workspace string, command string, args ...string) string { + cmd := exec.Command(command, args[0:]...) + + cmd.Dir = workspace + + var out bytes.Buffer + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + log.WithFields(log.Fields{ + "baseDir": workspace, + "command": command, + "args": args, + }).Fatal("Error executing command") + } + + return strings.Trim(out.String(), "\n") +} diff --git a/cli/internal/state.go b/cli/internal/state.go new file mode 100644 index 0000000000..0e79ad3a67 --- /dev/null +++ b/cli/internal/state.go @@ -0,0 +1,109 @@ +package internal + +import ( + "os" + "path/filepath" + "strings" + + log "github.com/sirupsen/logrus" + + "gopkg.in/yaml.v2" +) + +// stateRun represents a Run +type stateRun struct { + ID string // ID of the run + Stack stateService // stack of the run (Optional) + Env map[string]string // environment for the run + Services []stateService // services in the run +} + +// stateService represents a service in a Run +type stateService struct { + Name string +} + +// Recover recovers the state for a run +func Recover(id string, workdir string) map[string]string { + run := stateRun{ + Env: map[string]string{}, + } + + stateFile := filepath.Join(workdir, id+".run") + bytes, err := ReadFile(stateFile) //nolint + if err != nil { + return run.Env + } + + err = yaml.Unmarshal(bytes, &run) + if err != nil { + log.WithFields(log.Fields{ + "stateFile": stateFile, + }).Error("Could not unmarshal state") + } + + return run.Env +} + +// Destroy destroys the state for a run +func Destroy(id string, workdir string) { + stateFile := filepath.Join(workdir, id+".run") + err := os.Remove(stateFile) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "stateFile": stateFile, + }).Warn("Could not destroy state") + + return + } + + log.WithFields(log.Fields{ + "stateFile": stateFile, + }).Debug("State destroyed") +} + +// Update updates the state of en execution, using ID as the file name for the run. +// The state file will be located under 'workdir', which by default will be the tool's +// workspace. +func Update(id string, workdir string, composeFilePaths []string, env map[string]string) { + run := stateRun{ + ID: id, + Env: env, + Services: []stateService{}, + } + + if strings.HasSuffix(id, "-stack") { + run.Stack = stateService{ + Name: filepath.Base(filepath.Dir(composeFilePaths[0])), + } + } + + args := []string{} + for i, f := range composeFilePaths { + args = append(args, "-f", f) + + if i > 0 { + run.Services = append(run.Services, stateService{ + Name: filepath.Base(filepath.Dir(f)), + }) + } + } + args = append(args, "config") + + stateFile := filepath.Join(workdir, id+".run") + + bytes, err := yaml.Marshal(&run) + if err != nil { + log.WithFields(log.Fields{ + "stateFile": stateFile, + }).Error("Could not marshal state") + } + + WriteFile(bytes, stateFile) //nolint + + log.WithFields(log.Fields{ + "dir": workdir, + "stateFile": stateFile, + }).Debug("Updating state") +} diff --git a/cli/internal/state_test.go b/cli/internal/state_test.go new file mode 100644 index 0000000000..c5cd65bbdf --- /dev/null +++ b/cli/internal/state_test.go @@ -0,0 +1,65 @@ +package internal + +import ( + "path/filepath" + "testing" + + "github.com/Flaque/filet" + "github.com/stretchr/testify/assert" +) + +func TestRecover(t *testing.T) { + defer filet.CleanUp(t) + + tmpDir := filet.TmpDir(t, "") + + workspace := filepath.Join(tmpDir, ".op") + + ID := "mystack-stack" + composeFiles := []string{ + filepath.Join(workspace, "compose/services/a/1.yml"), + filepath.Join(workspace, "compose/services/b/2.yml"), + filepath.Join(workspace, "compose/services/c/3.yml"), + filepath.Join(workspace, "compose/services/d/4.yml"), + } + initialEnv := map[string]string{ + "foo": "bar", + } + + _ = MkdirAll(workspace) + + Update(ID, workspace, composeFiles, initialEnv) + + runFile := filepath.Join(workspace, ID+".run") + e, _ := Exists(runFile) + assert.True(t, e) + + env := Recover(ID, workspace) + + value, e := env["foo"] + assert.True(t, e) + assert.Equal(t, "bar", value) +} + +func TestUpdateCreatesStateFile(t *testing.T) { + defer filet.CleanUp(t) + + tmpDir := filet.TmpDir(t, "") + + workspace := filepath.Join(tmpDir, ".op") + + ID := "mystack-stack" + composeFiles := []string{ + filepath.Join(workspace, "compose/services/a/1.yml"), + filepath.Join(workspace, "compose/services/b/2.yml"), + filepath.Join(workspace, "compose/services/c/3.yml"), + filepath.Join(workspace, "compose/services/d/4.yml"), + } + runFile := filepath.Join(workspace, ID+".run") + _ = MkdirAll(runFile) + + Update(ID, workspace, composeFiles, map[string]string{}) + + e, _ := Exists(runFile) + assert.True(t, e) +} diff --git a/cli/services/manager.go b/cli/services/manager.go index 0d38f82680..900e7f9f82 100644 --- a/cli/services/manager.go +++ b/cli/services/manager.go @@ -2,8 +2,10 @@ package services import ( "fmt" + "path/filepath" "github.com/elastic/metricbeat-tests-poc/cli/config" + state "github.com/elastic/metricbeat-tests-poc/cli/internal" log "github.com/sirupsen/logrus" tc "github.com/testcontainers/testcontainers-go" @@ -12,7 +14,7 @@ import ( // ServiceManager manages lifecycle of a service type ServiceManager interface { AddServicesToCompose(stack string, composeNames []string, env map[string]string) error - RemoveServicesFromCompose(stack string, composeNames []string) error + RemoveServicesFromCompose(stack string, composeNames []string, env map[string]string) error RunCompose(isStack bool, composeNames []string, env map[string]string) error StopCompose(isStack bool, composeNames []string) error } @@ -36,11 +38,21 @@ func (sm *DockerServiceManager) AddServicesToCompose(stack string, composeNames newComposeNames := []string{stack} newComposeNames = append(newComposeNames, composeNames...) - return executeCompose(sm, true, newComposeNames, []string{"up", "-d"}, env) + persistedEnv := state.Recover(stack+"-stack", config.Op.Workspace) + for k, v := range env { + persistedEnv[k] = v + } + + err := executeCompose(sm, true, newComposeNames, []string{"up", "-d"}, persistedEnv) + if err != nil { + return err + } + + return nil } // RemoveServicesFromCompose removes services from a running docker compose -func (sm *DockerServiceManager) RemoveServicesFromCompose(stack string, composeNames []string) error { +func (sm *DockerServiceManager) RemoveServicesFromCompose(stack string, composeNames []string, env map[string]string) error { log.WithFields(log.Fields{ "stack": stack, "services": composeNames, @@ -49,11 +61,16 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(stack string, composeN newComposeNames := []string{stack} newComposeNames = append(newComposeNames, composeNames...) + persistedEnv := state.Recover(stack+"-stack", config.Op.Workspace) + for k, v := range env { + persistedEnv[k] = v + } + for _, composeName := range composeNames { command := []string{"rm", "-fvs"} command = append(command, composeName) - err := executeCompose(sm, true, newComposeNames, command, map[string]string{}) + err := executeCompose(sm, true, newComposeNames, command, persistedEnv) if err != nil { log.WithFields(log.Fields{ "command": command, @@ -81,19 +98,24 @@ func (sm *DockerServiceManager) StopCompose(isStack bool, composeNames []string) b = true } - composeFilePath, err := config.GetPackedCompose(b, composeName) + composeFilePath, err := config.GetComposeFile(b, composeName) if err != nil { return fmt.Errorf("Could not get compose file: %s - %v", composeFilePath, err) } composeFilePaths[i] = composeFilePath } - compose := tc.NewLocalDockerCompose(composeFilePaths, composeNames[0]) - execError := compose.Down() - err := execError.Error + ID := composeNames[0] + "-service" + if isStack { + ID = composeNames[0] + "-stack" + } + persistedEnv := state.Recover(ID, config.Op.Workspace) + + err := executeCompose(sm, isStack, composeNames, []string{"down"}, persistedEnv) if err != nil { return fmt.Errorf("Could not stop compose file: %v - %v", composeFilePaths, err) } + defer state.Destroy(ID, config.Op.Workspace) log.WithFields(log.Fields{ "composeFilePath": composeFilePaths, @@ -111,7 +133,7 @@ func executeCompose(sm *DockerServiceManager, isStack bool, composeNames []strin b = true } - composeFilePath, err := config.GetPackedCompose(b, composeName) + composeFilePath, err := config.GetComposeFile(b, composeName) if err != nil { return fmt.Errorf("Could not get compose file: %s - %v", composeFilePath, err) } @@ -128,6 +150,13 @@ func executeCompose(sm *DockerServiceManager, isStack bool, composeNames []strin return fmt.Errorf("Could not run compose file: %v - %v", composeFilePaths, err) } + suffix := "-service" + if isStack { + suffix = "-stack" + } + ID := filepath.Base(filepath.Dir(composeFilePaths[0])) + suffix + defer state.Update(ID, config.Op.Workspace, composeFilePaths, env) + log.WithFields(log.Fields{ "cmd": command, "composeFilePaths": composeFilePaths, diff --git a/metricbeat-tests/Makefile b/metricbeat-tests/Makefile index d06f1e8960..bc10bd1d6f 100644 --- a/metricbeat-tests/Makefile +++ b/metricbeat-tests/Makefile @@ -43,3 +43,7 @@ run-elastic-stack: .PHONY: shutdown-elastic-stack shutdown-elastic-stack: OP_LOG_LEVEL=${LOG_LEVEL} ./op stop stack metricbeat + +.PHONY: sync-integrations +sync-integrations: + OP_LOG_LEVEL=${LOG_LEVEL} ./op sync integrations --remove diff --git a/metricbeat-tests/configurations/mysql.yml b/metricbeat-tests/configurations/mysql.yml index ccf27a6e25..7cc9d34917 100644 --- a/metricbeat-tests/configurations/mysql.yml +++ b/metricbeat-tests/configurations/mysql.yml @@ -1,5 +1,5 @@ metricbeat.modules: - module: mysql - hosts: ["root:secret@tcp(mysql:3306)/"] + hosts: ["root:test@tcp(mysql:3306)/"] #username: root #password: secret diff --git a/metricbeat-tests/features/mysql.feature b/metricbeat-tests/features/mysql.feature index e0a52e4af0..754bf285e9 100644 --- a/metricbeat-tests/features/mysql.feature +++ b/metricbeat-tests/features/mysql.feature @@ -2,18 +2,18 @@ Feature: As a Metricbeat developer I want to check that the MySQL module works as expected Scenario Outline: Check module is sending metrics to Elasticsearch without errors - Given MySQL "" is running for metricbeat - And metricbeat is installed and configured for MySQL module + Given "" v, variant of "MySQL", is running for metricbeat + And metricbeat is installed and configured for "", variant of the "MySQL" module And metricbeat waits "20" seconds for the service When metricbeat runs for "20" seconds Then there are "" events in the index And there are no errors in the index Examples: -| mysql_version | variant | -| mariadb-10.2.23 | MariaDB | -| mariadb-10.3.14 | MariaDB | -| mariadb-10.4.4 | MariaDB | -| mysql-5.7.12 | MySQL | -| mysql-8.0.13 | MySQL | -| percona-5.7.24 | Percona | -| percona-8.0.13-4 | Percona | +| variant | version | +| MariaDB | 10.2.23 | +| MariaDB | 10.3.14 | +| MariaDB | 10.4.4 | +| MySQL | 5.7.12 | +| MySQL | 8.0.13 | +| Percona | 5.7.24 | +| Percona | 8.0.13-4 | diff --git a/metricbeat-tests/go.sum b/metricbeat-tests/go.sum index 1a6069f048..0c5b350d8c 100644 --- a/metricbeat-tests/go.sum +++ b/metricbeat-tests/go.sum @@ -12,7 +12,13 @@ github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiU github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -22,6 +28,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -37,8 +44,14 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/elastic/go-elasticsearch/v8 v8.0.0-20190731061900-ea052088db25 h1:7jd4dZ3/qtoQL7FEg6XXn1/nopYTz9HJSTPcZsj6h0o= github.com/elastic/go-elasticsearch/v8 v8.0.0-20190731061900-ea052088db25/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-redis/redis v6.15.5+incompatible h1:pLky8I0rgiblWfa8C1EV7fPEUv0aH6vKRaYHc/YRHVk= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= @@ -64,6 +77,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -75,8 +89,13 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -86,8 +105,13 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mdelapenya/testcontainers-go v0.0.9-compose-1 h1:KsiEki4MDtb5YEjEqoT90X9uVuIBBCdAzuCSloVt8ms= @@ -111,6 +135,7 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -128,6 +153,8 @@ github.com/rogpeppe/go-internal v1.5.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -147,19 +174,26 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= +github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf h1:fnPsqIDRbCSgumaMCRpoIoF2s4qxv0xSSS0BVZUE/ss= golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -174,6 +208,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 h1:N66aaryRB3Ax92gH0v3hp1QYZ3zWWCCUR/j8Ifh45Ss= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -188,10 +223,12 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c h1:S/FtSvpNLtFBgjTqcKsRpsa6aVsI6iztaz1bQd9BJwE= golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -209,6 +246,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -230,8 +268,16 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= +gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= +gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/metricbeat-tests/metricbeat_test.go b/metricbeat-tests/metricbeat_test.go index 2ba8c5f3b6..5b299ff700 100644 --- a/metricbeat-tests/metricbeat_test.go +++ b/metricbeat-tests/metricbeat_test.go @@ -8,13 +8,14 @@ import ( "strings" "github.com/DATA-DOG/godog" + "github.com/elastic/metricbeat-tests-poc/cli/config" "github.com/elastic/metricbeat-tests-poc/cli/services" log "github.com/sirupsen/logrus" ) // metricbeatVersion is the version of the metricbeat to use // It can be overriden by OP_METRICBEAT_VERSION env var -var metricbeatVersion = "7.4.0" +var metricbeatVersion = "7.6.0" //nolint:unused var query ElasticsearchQuery @@ -35,6 +36,7 @@ type MetricbeatTestSuite struct { IndexName string // the unique name for the index to be used in this test suite ServiceName string // the service to be monitored by metricbeat ServiceType string // the type of the service to be monitored by metricbeat + ServiceVariant string // the variant of the service to be monitored by metricbeat ServiceVersion string // the version of the service to be monitored by metricbeat Version string // the metricbeat version for the test } @@ -72,12 +74,16 @@ func (mts *MetricbeatTestSuite) CleanUp() error { } defer fn(context.Background()) + env := map[string]string{ + "stackVersion": stackVersion, + } + services := []string{"metricbeat"} if mts.ServiceName != "" { services = append(services, mts.ServiceName) } - err := serviceManager.RemoveServicesFromCompose("metricbeat", services) + err := serviceManager.RemoveServicesFromCompose("metricbeat", services, env) if err != nil { log.WithFields(log.Fields{ "service": mts.ServiceName, @@ -106,7 +112,9 @@ func MetricbeatFeatureContext(s *godog.Suite) { testSuite := MetricbeatTestSuite{} s.Step(`^([^"]*) "([^"]*)" is running for metricbeat$`, testSuite.serviceIsRunningForMetricbeat) + s.Step(`^"([^"]*)" v([^"]*), variant of "([^"]*)", is running for metricbeat$`, testSuite.serviceVariantIsRunningForMetricbeat) s.Step(`^metricbeat is installed and configured for ([^"]*) module$`, testSuite.installedAndConfiguredForModule) + s.Step(`^metricbeat is installed and configured for "([^"]*)", variant of the "([^"]*)" module$`, testSuite.installedAndConfiguredForVariantModule) s.Step(`^metricbeat waits "([^"]*)" seconds for the service$`, testSuite.waitsSeconds) s.Step(`^metricbeat runs for "([^"]*)" seconds$`, testSuite.runsForSeconds) s.Step(`^there are no errors in the index$`, testSuite.thereAreNoErrorsInTheIndex) @@ -141,6 +149,22 @@ func (mts *MetricbeatTestSuite) installedAndConfiguredForModule(serviceType stri return nil } +func (mts *MetricbeatTestSuite) installedAndConfiguredForVariantModule(serviceVariant string, serviceType string) error { + serviceType = strings.ToLower(serviceType) + + // at this point we have everything to define the index name + mts.Version = metricbeatVersion + mts.setIndexName() + mts.ServiceVariant = serviceVariant + mts.ServiceType = serviceType + + // look up configurations under workspace's configurations directory + dir, _ := os.Getwd() + mts.configurationFile = path.Join(dir, "configurations", mts.ServiceName+".yml") + + return nil +} + func (mts *MetricbeatTestSuite) installedUsingConfiguration(configuration string) error { // at this point we have everything to define the index name mts.Version = metricbeatVersion @@ -231,14 +255,40 @@ func (mts *MetricbeatTestSuite) serviceIsRunningForMetricbeat(serviceType string serviceType = strings.ToLower(serviceType) env := map[string]string{ - serviceType + "Tag": serviceVersion, - "stackVersion": stackVersion, + "stackVersion": stackVersion, + } + env = config.PutServiceEnvironment(env, serviceType, serviceVersion) + + err := serviceManager.AddServicesToCompose("metricbeat", []string{serviceType}, env) + if err != nil { + log.WithFields(log.Fields{ + "service": serviceType, + "version": serviceVersion, + }).Error("Could not run the service.") + } + + mts.ServiceName = serviceType + mts.ServiceVersion = serviceVersion + + return err +} + +func (mts *MetricbeatTestSuite) serviceVariantIsRunningForMetricbeat( + serviceVariant string, serviceVersion string, serviceType string) error { + + serviceVariant = strings.ToLower(serviceVariant) + serviceType = strings.ToLower(serviceType) + + env := map[string]string{ + "stackVersion": stackVersion, } + env = config.PutServiceVariantEnvironment(env, serviceType, serviceVariant, serviceVersion) err := serviceManager.AddServicesToCompose("metricbeat", []string{serviceType}, env) if err != nil { log.WithFields(log.Fields{ "service": serviceType, + "variant": serviceVariant, "version": serviceVersion, }).Error("Could not run the service.") } diff --git a/metricbeat-tests/runner_test.go b/metricbeat-tests/runner_test.go index 1abe448c5e..8a1d518b7d 100644 --- a/metricbeat-tests/runner_test.go +++ b/metricbeat-tests/runner_test.go @@ -14,7 +14,7 @@ import ( // stackVersion is the version of the stack to use // It can be overriden by OP_STACK_VERSION env var -var stackVersion = "7.4.0" +var stackVersion = "7.6.0" var opt = godog.Options{Output: colors.Colored(os.Stdout)}