Cloud deployment with local agent (#2143) (#2336)

Cloud deployment with local agent (#2143)

(cherry picked from commit 06a8da1)

Co-authored-by: Michal Pristas <[email protected]>

README.md

@@ -144,6 +144,34 @@ kubectl apply -f elastic-agent-${ELASTIC_AGENT_MODE}-kubernetes.yaml
 kubectl -n kube-system get pods -l app=elastic-agent
 ```
 
+## Testing on Elastic Cloud
+
+Elastic employees can create an Elastic Cloud deployment with a locally
+built Elastic Agent, by pushing images to an internal Docker repository. The images will be 
+based on the SNAPSHOT images with the version defined in `version/version.go`.
+
+Prerequisite to running following commands is having `terraform` installed and running `terraform init` from within `testing/environments/cloud`.
+
+Running a shorthand `make deploy_local` in `testing/environments/cloud` will build Agent, tag the docker image correctly, push it to the repository and deploy to Elastic Cloud.
+
+For more advanced scenarios:
+Running `make build_elastic_agent_docker_image` in `testing/environments/cloud` will build and push the images. 
+Running `make push_elastic_agent_docker_image` in `testing/environments/cloud` will publish built docker image to CI docker repository.
+
+Once docker images are published you can run `EC_API_KEY=your_api_key make apply` from `testing/environments/cloud` directory to deploy them to Elastic Cloud. 
+To get `EC_API_KEY` follow [this guide](https://www.elastic.co/guide/en/cloud/current/ec-api-authentication.html)
+
+The custom images are tagged with the current version, commit and timestamp. The
+timestamp is included to force a new Docker image to be used, which enables pushing new
+binaries without recreating the deployment.
+
+To specify custom images create your `docker_image.auto.tfvars` file similar to `docker_image.auto.tfvars.sample`. 
+
+You can also use `mage cloud:image` and `mage cloud:push` respectively from repo root directory. 
+To deploy your changes use `make apply` (from `testing/environments/cloud`) with `EC_API_KEY` instead of `make deploy_local` described above.
+
+SNAPSHOT images are used by default. To use non-snapshot image specify `SNAPSHOT=false` explicitly.
+
 ## Updating dependencies/PRs
 Even though we prefer `mage` to our automation, we still have some
 rules implemented on our `Makefile` as well as CI will use the

magefile.go

@@ -50,10 +50,15 @@ const (
 	snapshotEnv       = "SNAPSHOT"
 	devEnv            = "DEV"
 	externalArtifacts = "EXTERNAL"
+	platformsEnv      = "PLATFORMS"
+	packagesEnv       = "PACKAGES"
 	configFile        = "elastic-agent.yml"
 	agentDropPath     = "AGENT_DROP_PATH"
 	specSuffix        = ".spec.yml"
 	checksumFilename  = "checksum.yml"
+	commitLen         = 7
+
+	cloudImageTmpl = "docker.elastic.co/observability-ci/elastic-agent:%s"
 )
 
 // Aliases for commands required by master makefile
@@ -96,6 +101,9 @@ type Demo mg.Namespace
 // Dev runs package and build for dev purposes.
 type Dev mg.Namespace
 
+// Cloud produces or pushes cloud image for cloud testing.
+type Cloud mg.Namespace
+
 func CheckNoChanges() error {
 	fmt.Println(">> fmt - go run")
 	err := sh.RunV("go", "mod", "tidy", "-v")
@@ -547,6 +555,111 @@ func (Demo) NoEnroll() error {
 	return runAgent(env)
 }
 
+// Image builds a cloud image
+func (Cloud) Image() {
+	platforms := os.Getenv(platformsEnv)
+	defer os.Setenv(platformsEnv, platforms)
+
+	packages := os.Getenv(packagesEnv)
+	defer os.Setenv(packagesEnv, packages)
+
+	snapshot := os.Getenv(snapshotEnv)
+	defer os.Setenv(snapshotEnv, snapshot)
+
+	dev := os.Getenv(devEnv)
+	defer os.Setenv(devEnv, dev)
+
+	os.Setenv(platformsEnv, "linux/amd64")
+	os.Setenv(packagesEnv, "docker")
+	os.Setenv(devEnv, "true")
+
+	if s, err := strconv.ParseBool(snapshot); err == nil && !s {
+		// only disable SNAPSHOT build when explicitely defined
+		os.Setenv(snapshotEnv, "false")
+		devtools.Snapshot = false
+	} else {
+		os.Setenv(snapshotEnv, "true")
+		devtools.Snapshot = true
+	}
+
+	devtools.DevBuild = true
+	devtools.Platforms = devtools.Platforms.Filter("linux/amd64")
+	devtools.SelectedPackageTypes = []devtools.PackageType{devtools.Docker}
+
+	if _, hasExternal := os.LookupEnv(externalArtifacts); !hasExternal {
+		devtools.ExternalBuild = true
+	}
+
+	Package()
+}
+
+// Push builds a cloud image tags it correctly and pushes to remote image repo.
+// Previous login to elastic registry is required!
+func (Cloud) Push() error {
+	snapshot := os.Getenv(snapshotEnv)
+	defer os.Setenv(snapshotEnv, snapshot)
+
+	os.Setenv(snapshotEnv, "true")
+
+	version := getVersion()
+	var tag string
+	if envTag, isPresent := os.LookupEnv("CUSTOM_IMAGE_TAG"); isPresent && len(envTag) > 0 {
+		tag = envTag
+	} else {
+		commit := dockerCommitHash()
+		time := time.Now().Unix()
+
+		tag = fmt.Sprintf("%s-%s-%d", version, commit, time)
+	}
+
+	sourceCloudImageName := fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud:%s", version)
+	var targetCloudImageName string
+	if customImage, isPresent := os.LookupEnv("CI_ELASTIC_AGENT_DOCKER_IMAGE"); isPresent && len(customImage) > 0 {
+		targetCloudImageName = fmt.Sprintf("%s:%s", customImage, tag)
+	} else {
+		targetCloudImageName = fmt.Sprintf(cloudImageTmpl, tag)
+	}
+
+	fmt.Printf(">> Setting a docker image tag to %s\n", targetCloudImageName)
+	err := sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName)
+	if err != nil {
+		return fmt.Errorf("Failed setting a docker image tag: %w", err)
+	}
+	fmt.Println(">> Docker image tag updated successfully")
+
+	fmt.Println(">> Pushing a docker image to remote registry")
+	err = sh.RunV("docker", "image", "push", targetCloudImageName)
+	if err != nil {
+		return fmt.Errorf("Failed pushing docker image: %w", err)
+	}
+	fmt.Printf(">> Docker image pushed to remote registry successfully: %s\n", targetCloudImageName)
+
+	return nil
+}
+
+func dockerCommitHash() string {
+	commit, err := devtools.CommitHash()
+	if err == nil && len(commit) > commitLen {
+		return commit[:commitLen]
+	}
+
+	return ""
+}
+
+func getVersion() string {
+	version, found := os.LookupEnv("BEAT_VERSION")
+	if !found {
+		version = release.Version()
+	}
+	if !strings.Contains(version, "SNAPSHOT") {
+		if _, ok := os.LookupEnv(snapshotEnv); ok {
+			version += "-SNAPSHOT"
+		}
+	}
+
+	return version
+}
+
 func runAgent(env map[string]string) error {
 	prevPlatforms := os.Getenv("PLATFORMS")
 	defer os.Setenv("PLATFORMS", prevPlatforms)
@@ -964,12 +1077,11 @@ func dockerBuild(tag string) error {
 }
 
 func dockerTag() string {
-	const commitLen = 7
 	tagBase := "elastic-agent"
 
-	commit, err := devtools.CommitHash()
-	if err == nil && len(commit) > commitLen {
-		return fmt.Sprintf("%s-%s", tagBase, commit[:commitLen])
+	commit := dockerCommitHash()
+	if len(commit) > 0 {
+		return fmt.Sprintf("%s-%s", tagBase, commit)
 	}
 
 	return tagBase

testing/environments/cloud/.gitignore

@@ -0,0 +1 @@
+*.auto.tfvars

testing/environments/cloud/Makefile

@@ -0,0 +1,93 @@
+MAKEFILE_PATH:=$(abspath $(lastword ${MAKEFILE_LIST}))
+MAKEFILE_DIR:=$(dir ${MAKEFILE_PATH})
+REPO_ROOT:=$(abspath ${MAKEFILE_DIR}/../../../)
+SNAPSHOT_SUFFIX:=-SNAPSHOT
+ifeq (${SNAPSHOT}, false)
+SNAPSHOT_SUFFIX:=""
+endif
+ifeq (${SNAPSHOT}, 0)
+SNAPSHOT_SUFFIX:=""
+endif
+VERSION:=$(shell grep defaultBeatVersion ${REPO_ROOT}/version/version.go | cut -d\" -f2)${SNAPSHOT_SUFFIX}
+
+export DOCKER_BUILDKIT=1
+
+ELASTIC_AGENT_DOCKER_IMAGE=docker.elastic.co/cloud-release/elastic-agent-cloud
+ELASTICSEARCH_DOCKER_IMAGE=docker.elastic.co/cloud-release/elasticsearch-cloud-ess
+KIBANA_DOCKER_IMAGE=docker.elastic.co/cloud-release/kibana-cloud
+
+CI_ELASTIC_AGENT_DOCKER_IMAGE=docker.elastic.co/observability-ci/elastic-agent
+CI_KIBANA_DOCKER_IMAGE=docker.elastic.co/observability-ci/kibana
+
+# Tag custom images with the username and current timestamp.
+# The timestamp must be included to force images to be pulled.
+USER_NAME?=${USER}
+CUSTOM_IMAGE_TAG:=${VERSION}-${USER_NAME}-$(shell date +%s)
+
+USER_ELASTIC_AGENT_DOCKER_IMAGE=docker.elastic.co/observability-ci/${USER_NAME}-elastic-agent
+USER_KIBANA_DOCKER_IMAGE=docker.elastic.co/observability-ci/${USER_NAME}-kibana
+
+.DEFAULT_GOAL := docker_image.auto.tfvars
+
+##############################################################################
+# Target for creating a .tfvars file, defining the custom Docker images to
+# use in the deployment.
+##############################################################################
+
+docker_image.auto.tfvars: kibana_docker_image elastic_agent_docker_image
+	@echo 'docker_image_override={"elasticsearch":"${ELASTICSEARCH_DOCKER_IMAGE}","kibana":"${CI_KIBANA_DOCKER_IMAGE}","agent":"${CI_ELASTIC_AGENT_DOCKER_IMAGE}"}' > $@
+	@echo 'docker_image_tag_override={"elasticsearch":"${VERSION}","kibana":"${VERSION}","agent":"${CUSTOM_IMAGE_TAG}"}' >> $@
+
+##############################################################################
+# Terraform shortcut rules.
+##############################################################################
+
+.PHONY: apply
+apply:
+	@terraform apply -auto-approve
+
+
+.PHONY: snap
+snap:
+	@echo ${VERSION}
+
+.PHONY: destroy
+destroy:
+	@terraform destroy -auto-approve
+
+
+.PHONY: deploy_local
+deploy_local: build_elastic_agent_docker_image push_elastic_agent_docker_image
+	@echo 'docker_image_override={"elasticsearch":"${ELASTICSEARCH_DOCKER_IMAGE}","kibana":"${KIBANA_DOCKER_IMAGE}","agent":"${CI_ELASTIC_AGENT_DOCKER_IMAGE}"}' > docker_image.auto.tfvars
+	@echo 'docker_image_tag_override={"elasticsearch":"${VERSION}","kibana":"${VERSION}","agent":"${CUSTOM_IMAGE_TAG}"}' >> docker_image.auto.tfvars
+	@terraform apply -auto-approve
+
+##############################################################################
+# Targets for building and pushing custom Kibana and Elastic Agent images.
+##############################################################################
+
+# kibana_docker_image builds the Cloud Kibana image.
+# The image will be based off the
+# stack version defined in ${REPO_ROOT}/.env, unless overridden.
+.PHONY: build_kibana_docker_image
+kibana_docker_image: build_kibana_docker_image
+	docker push ${CI_KIBANA_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG}
+build_kibana_docker_image:
+	$(MAKE) -C ${REPO_ROOT} build-package
+	docker build -t ${CI_KIBANA_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG} \
+		-f "${REPO_ROOT}/testing/docker/kibana/Dockerfile-apmpackage" \
+		--build-arg KIBANA_IMAGE=${KIBANA_DOCKER_IMAGE}:${IMAGE_TAG} \
+		--platform linux/amd64 \
+		"${REPO_ROOT}/build/packages"
+
+# build_elastic_agent_docker_image builds the Cloud Elastic Agent image.
+# The image will be based
+# off the version defined in version.go,
+# CUSTOM_IMAGE_TAG is appended to the version resulting in e.g:
+# 8.7.0-SNAPSHOT-doe-1673961915
+# unless overridden.
+.PHONY: build_elastic_agent_docker_image
+push_elastic_agent_docker_image: 
+	cd ${REPO_ROOT}; CI_ELASTIC_AGENT_DOCKER_IMAGE=${CI_ELASTIC_AGENT_DOCKER_IMAGE} CUSTOM_IMAGE_TAG=${CUSTOM_IMAGE_TAG} mage cloud:push
+build_elastic_agent_docker_image:
+	cd ${REPO_ROOT}; DOCKER_BUILDKIT=0 COMPOSE_DOCKER_CLI_BUILD=0 DOCKER_DEFAULT_PLATFORM=linux/amd64 mage cloud:image

testing/environments/cloud/README.md

@@ -0,0 +1,27 @@
+# Cloud-first testing
+
+Elastic employees can create an Elastic Cloud deployment with a locally
+built Elastic Agent, by pushing images to an internal Docker repository. The images will be 
+based on the SNAPSHOT images with the version defined in `version/version.go`.
+
+Prerequisite to running following commands is having `terraform` installed and running `terraform init` from within `testing/environments/cloud`.
+
+Running a shorthand `make deploy_local` in `testing/environments/cloud` will build Agent, tag the docker image correctly, push it to the repository and deploy to Elastic Cloud.
+
+For more advanced scenarios:
+Running `make build_elastic_agent_docker_image` in `testing/environments/cloud` will build and push the images. 
+Running `make push_elastic_agent_docker_image` in `testing/environments/cloud` will publish built docker image to CI docker repository.
+
+Once docker images are published you can run `EC_API_KEY=your_api_key make apply` from `testing/environments/cloud` directory to deploy them to Elastic Cloud. 
+To get `EC_API_KEY` follow [this guide](https://www.elastic.co/guide/en/cloud/current/ec-api-authentication.html)
+
+The custom images are tagged with the current version, commit and timestamp. The
+timestamp is included to force a new Docker image to be used, which enables pushing new
+binaries without recreating the deployment.
+
+To specify custom images create your `docker_image.auto.tfvars` file similar to `docker_image.auto.tfvars.sample`. 
+
+You can also use `mage cloud:image` and `mage cloud:push` respectively from repo root directory. 
+To deploy your changes use `make apply` (from `testing/environments/cloud`) with `EC_API_KEY` instead of `make deploy_local` described above.
+
+SNAPSHOT images are used by default. To use non-snapshot image specify `SNAPSHOT=false` explicitly.

testing/environments/cloud/docker_image.auto.tfvars.sample

@@ -0,0 +1,2 @@
+docker_image_override={"elasticsearch":"docker.elastic.co/cloud-release/elasticsearch-cloud-ess","kibana":"docker.elastic.co/cloud-release/kibana-cloud","agent":"docker.elastic.co/observability-ci/elastic-agent"}
+docker_image_tag_override={"elasticsearch":"8.6.0-f96862e0-SNAPSHOT","kibana":"8.6.0-f96862e0-SNAPSHOT","agent":"mysampletag"}