Merge pull request #132 from mdelapenya/ingest-management-impl

[Ingest Manager] Add basic implementation for stand-alone mode scenarios

.ci/.e2e-tests.yaml

@@ -6,6 +6,8 @@ SUITES:
     feature: "filebeat"
   - suite: "helm"
     feature: "metricbeat"
+  - suite: "ingest-manager"
+    feature: "stand_alone_mode"
   - suite: "metricbeat"
     feature: "apache"
   - suite: "metricbeat"

.ci/Jenkinsfile

@@ -24,7 +24,7 @@ pipeline {
     issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
   }
   parameters {
-    choice(name: 'runTestsSuite', choices: ['all', 'helm', 'metricbeat'], description: 'Choose which test suite to run (default: all)')
+    choice(name: 'runTestsSuite', choices: ['all', 'helm', 'ingest-manager', 'metricbeat'], description: 'Choose which test suite to run (default: all)')
     choice(name: 'LOG_LEVEL', choices: ['INFO', 'DEBUG'], description: 'Log level to be used')
     choice(name: 'QUERY_MAX_ATTEMPTS', choices: ['5', '10', '20'], description: 'Number of attempts to create the connection to Elasticsearch')
     choice(name: 'RETRY_TIMEOUT', choices: ['3', '5', '7', '11'], description: 'Number of seconds between retry')

cli/config/compose/profiles/ingest-manager/docker-compose.yml

@@ -0,0 +1,42 @@
+version: '2.3'
+services:
+  elasticsearch:
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://127.0.0.1:9200/"]
+      retries: 300
+      interval: 1s
+    environment:
+      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+      - network.host=
+      - transport.host=127.0.0.1
+      - http.host=0.0.0.0
+      - indices.id_field_data.enabled=true
+      - xpack.license.self_generated.type=trial
+      - xpack.security.enabled=true
+      - xpack.security.authc.api_key.enabled=true
+      - ELASTIC_USERNAME=elastic
+      - ELASTIC_PASSWORD=changeme
+    image: "docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT"
+    ports:
+      - "9200:9200"
+  kibana:
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      package-registry:
+        condition: service_healthy
+    healthcheck:
+      test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null"
+      retries: 600
+      interval: 1s
+    image: "docker.elastic.co/kibana/kibana:8.0.0-SNAPSHOT"
+    ports:
+      - "5601:5601"
+    volumes:
+      - ${kibanaConfigPath}:/usr/share/kibana/config/kibana.yml
+  package-registry:
+    image: docker.elastic.co/package-registry/package-registry:master
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080"]
+      retries: 300
+      interval: 1s

cli/config/compose/profiles/metricbeat/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - xpack.security.enabled=false
       - xpack.monitoring.collection.enabled=true
       - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=p4ssw0rd
+      - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion}"
     ports:
       - "9200:9200"

cli/config/compose/services/elastic-agent/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '2.3'
+services:
+  elastic-agent:
+    image: docker.elastic.co/beats/elastic-agent:${elasticAgentTag:-8.0.0-SNAPSHOT}
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    environment:
+      - "KIBANA_HOST=http://${kibanaHost:-kibana}:${kibanaPort:-5601}"
+    volumes:
+      - "${elasticAgentConfigFile}:/usr/share/elastic-agent/elastic-agent.yml"

cli/config/compose/services/elasticsearch/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - xpack.security=false
       - xpack.monitoring.collection.enabled=true
       - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=p4ssw0rd
+      - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${elasticsearchTag}"
     ports:
       - "9200:9200"

cli/config/compose/services/kibana/docker-compose.yml

@@ -4,7 +4,7 @@ services:
     environment:
       - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
       - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=p4ssw0rd
+      - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/kibana/kibana:${kibanaTag}"
     ports:
       - "5601:5601"

cli/config/compose/services/metricbeat/docker-compose.yml

@@ -6,10 +6,10 @@ services:
       "-E", "logging.level=${logLevel}",
       "-E", "setup.ilm.rollover_alias=${indexName}",
       "-E", "output.elasticsearch.hosts=http://elasticsearch:9200",
-      "-E", "output.elasticsearch.password=p4ssw0rd",
+      "-E", "output.elasticsearch.password=changeme",
       "-E", "output.elasticsearch.username=elastic",
       "-E", "setup.kibana.host=http://kibana:5601",
-      "-E", "setup.kibana.password=p4ssw0rd",
+      "-E", "setup.kibana.password=changeme",
       "-E", "setup.kibana.username=elastic",
     ]
     environment:

cli/docker/docker.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"bytes"
 	"context"
 
 	"github.com/docker/docker/api/types"
@@ -14,24 +15,27 @@ var instance *client.Client
 // OPNetworkName name of the network used by the tool
 const OPNetworkName = "elastic-dev-network"
 
-// ExecCommandIntoContainer executes a command, as a user, into a container, in a detach state
-func ExecCommandIntoContainer(ctx context.Context, containerName string, user string, cmd []string, detach bool) error {
+// ExecCommandIntoContainer executes a command, as a user, into a container
+func ExecCommandIntoContainer(ctx context.Context, containerName string, user string, cmd []string) (string, error) {
 	dockerClient := getDockerClient()
 
+	detach := false
+	tty := false
+
 	log.WithFields(log.Fields{
 		"container": containerName,
 		"command":   cmd,
 		"detach":    detach,
-		"tty":       false,
+		"tty":       tty,
 	}).Debug("Creating command to be executed in container")
 
 	response, err := dockerClient.ContainerExecCreate(
 		ctx, containerName, types.ExecConfig{
 			User:         user,
-			Tty:          false,
+			Tty:          tty,
 			AttachStdin:  false,
-			AttachStderr: false,
-			AttachStdout: false,
+			AttachStderr: true,
+			AttachStdout: true,
 			Detach:       detach,
 			Cmd:          cmd,
 		})
@@ -42,31 +46,57 @@ func ExecCommandIntoContainer(ctx context.Context, containerName string, user st
 			"command":   cmd,
 			"error":     err,
 			"detach":    detach,
-			"tty":       false,
+			"tty":       tty,
 		}).Warn("Could not create command in container")
-		return err
+		return "", err
 	}
 
 	log.WithFields(log.Fields{
 		"container": containerName,
 		"command":   cmd,
 		"detach":    detach,
-		"tty":       false,
+		"tty":       tty,
 	}).Debug("Command to be executed in container created")
 
-	err = dockerClient.ContainerExecStart(ctx, response.ID, types.ExecStartCheck{
+	resp, err := dockerClient.ContainerExecAttach(ctx, response.ID, types.ExecStartCheck{
 		Detach: detach,
-		Tty:    false,
+		Tty:    tty,
 	})
+	if err != nil {
+		log.WithFields(log.Fields{
+			"container": containerName,
+			"command":   cmd,
+			"detach":    detach,
+			"error":     err,
+			"tty":       tty,
+		}).Error("Could not execute command in container")
+		return "", err
+	}
+	defer resp.Close()
+
+	buf := new(bytes.Buffer)
+	_, err = buf.ReadFrom(resp.Reader)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"container": containerName,
+			"command":   cmd,
+			"detach":    detach,
+			"error":     err,
+			"tty":       tty,
+		}).Error("Could not parse command output from container")
+		return "", err
+	}
+	output := buf.String()
 
 	log.WithFields(log.Fields{
 		"container": containerName,
 		"command":   cmd,
 		"detach":    detach,
-		"tty":       false,
+		"output":    output,
+		"tty":       tty,
 	}).Debug("Command sucessfully executed in container")
 
-	return err
+	return output, nil
 }
 
 // InspectContainer returns the JSON representation of the inspection of a

cli/shell/curl.go

@@ -0,0 +1,99 @@
+package shell
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// HTTPRequest configures an HTTP request
+type HTTPRequest struct {
+	BasicAuthUser     string
+	BasicAuthPassword string
+	Headers           map[string]string
+	method            string
+	Payload           []byte
+	URL               string
+}
+
+// Get executes a GET request
+func Get(r HTTPRequest) (string, error) {
+	r.method = "GET"
+
+	return request(r)
+}
+
+// Post executes a POST request
+func Post(r HTTPRequest) (string, error) {
+	r.method = "POST"
+
+	return request(r)
+}
+
+// Post executes a request
+func request(r HTTPRequest) (string, error) {
+	log.WithFields(log.Fields{
+		"method": r.method,
+		"url":    r.URL,
+	}).Debug("Executing request")
+
+	var body io.Reader
+	if r.Payload != nil {
+		body = bytes.NewReader(r.Payload)
+	} else {
+		body = nil
+	}
+
+	req, err := http.NewRequest(r.method, r.URL, body)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"method": r.method,
+			"url":    r.URL,
+		}).Warn("Error creating request")
+		return "", err
+	}
+
+	if r.Headers != nil {
+		for k, v := range r.Headers {
+			req.Header.Set(k, v)
+		}
+	}
+
+	if r.BasicAuthUser != "" {
+		req.SetBasicAuth(r.BasicAuthUser, r.BasicAuthPassword)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"method": r.method,
+			"url":    r.URL,
+		}).Warn("Error executing request")
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"method": r.method,
+			"url":    r.URL,
+		}).Warn("Could not read response body")
+		return "", err
+	}
+	bodyString := string(bodyBytes)
+
+	// http.Status ==> [2xx, 4xx)
+	if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusBadRequest {
+		return bodyString, nil
+	}
+
+	return bodyString, fmt.Errorf("%s request failed with %d", r.method, resp.StatusCode)
+}