Merge pull request #199 from Callisto13/metal-tests

Add python tool for testing against Equinix

Makefile

@@ -100,7 +100,11 @@ test-with-cov: ## Run unit tests with coverage
 	go test -v -race -timeout 2m -p 1 -covermode=atomic -coverprofile=coverage.txt ./...
 
 .PHONY: test-e2e
-test-e2e: compile-e2e ## Run e2e tests locally in a container
+test-e2e: compile-e2e ## Run e2e tests locally
+		go test -timeout 30m -p 1 -v -tags=e2e ./test/e2e/...
+
+.PHONY: test-e2e-docker
+test-e2e-docker: compile-e2e ## Run e2e tests locally in a container
 	docker run --rm -it \
 		--privileged \
 		--volume /dev:/dev \
@@ -109,11 +113,11 @@ test-e2e: compile-e2e ## Run e2e tests locally in a container
 		--ipc=host \
 		--workdir=/src/flintlock \
 		$(test_image):latest \
-		"go test -timeout 30m -p 1 -v -tags=e2e ./test/e2e/..."
+		"make test-e2e"
 
 .PHONY: test-e2e-metal
 test-e2e-metal: ## Run e2e tests in Equinix
-	echo "coming soon to some hardware near you"
+	./test/tools/run.py run-e2e -o $(EQUINIX_ORG_ID)
 
 .PHONY: compile-e2e
 compile-e2e: ## Test e2e compilation

hack/scripts/bootstrap.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# WARNING: THIS SCRIPT HAS MUTLIPLE PURPOSES.
+# TAKE CARE WHEN EDITING.
+
 set -euxo pipefail
 
 if [[ $(id -u) != 0 ]]; then
@@ -28,7 +31,8 @@ apt install -y \
     git
 
 # install go
-export PATH="$PATH:$INSTALL_ROOT/go/bin"
+export GOPATH=/root/go
+export PATH="$PATH:$INSTALL_ROOT/go/bin:$GOPATH/bin"
 curl -sL "https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz" | tar xz -C "$INSTALL_ROOT" && \
   go version
 

hack/scripts/devpool.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# WARNING: THIS SCRIPT HAS MUTLIPLE PURPOSES.
+# TAKE CARE WHEN EDITING.
+
 set -ex
 
 if [[ $(id -u) != 0 ]]; then
@@ -11,11 +14,13 @@ fi
 CROOT=/var/lib/containerd-dev
 # This is the name of the thinpool.
 POOL="${1:-dev-thinpool}"
+set +u
 # This is the tag which will be appended to the loop device volumes
 VOL_TAG=""
 if [[ -n "$2" ]]; then
   VOL_TAG="-$2"
 fi
+set -u
 # These are some useful vars for useful things
 DIR="${CROOT}/snapshotter/devmapper"
 META="${CROOT}/snapshotter/devmapper/metadata$VOL_TAG"

hack/tools/.gitignore

@@ -1,2 +1,2 @@
 share/
-bin/
+bin/

test/tools/.gitignore

@@ -0,0 +1,2 @@
+*.key
+__pycache__

test/tools/README.md

@@ -0,0 +1,73 @@
+# run.py
+
+A handy tool to run tests in and interact with Equinix devices.
+
+Install dependencies:
+
+```bash
+pip3 install -r test/tools/requirements.txt
+```
+
+Run the tool:
+
+```bash
+./test/tools/run.py
+Usage: run.py [OPTIONS] COMMAND [ARGS]...
+
+  General thing doer for flintlock
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  create-device
+  delete-device
+  run-e2e
+```
+
+Run `--help` on all those subcommands to see the run options.
+
+For all commands you will need your Organisation ID and to set your API token
+in the environment as `METAL_AUTH_TOKEN`.
+
+## Running the e2es
+
+```bash
+export METAL_AUTH_TOKEN=<your token>
+./test/tools/run.py run-e2e --org-id <your org id>
+```
+
+This will:
+- Create a new project in your org
+- Create a new ssh key, saving the files locally for debugging
+- Create a new device
+- Bootstrap the device with userdata
+- Wait for the device to be running
+- Wait for the userdata to complete bootstrapping
+- Run the e2e tests, streaming the output
+- Delete the device, key, and project
+
+To keep the device around for debugging, add `--skip-delete`.
+
+To not creating anything new and run the tests in an existing device, set `--existing-device-id <id>`.
+
+## Creating a device
+
+```bash
+export METAL_AUTH_TOKEN=<your token>
+./test/tools/run.py create-device --org-id <your org id> --project-id <existing project-id>
+```
+
+This will create a device in an existing project bootstrapped with the default userdata.
+You can use `--userdata` to override this.
+
+Nothing will be cleaned up afterwards.
+
+## Deleting a device
+
+```bash
+export METAL_AUTH_TOKEN=<your token>
+./test/tools/run.py delete-device --org-id <your org id> --device-id <existing project-id>
+```
+
+This will delete the given device. The project will not be deleted.

test/tools/metal.py

@@ -0,0 +1,171 @@
+import packet
+import time
+import spur
+import logging
+import os
+import sys
+from Crypto.PublicKey import RSA
+from os.path import dirname, abspath
+
+class Welder():
+   def __init__(self, auth_token, org_id, level=logging.INFO):
+      self.org_id = org_id
+      self.base = dirname(dirname(dirname(abspath(__file__))))
+      self.packet_manager = packet.Manager(auth_token=auth_token)
+      self.private_key_path = dirname(abspath(__file__))+"/private.key"
+      self.public_key_path = dirname(abspath(__file__))+"/public.key"
+      self.logger = self.set_logger(level)
+      self.ip = None
+
+   def set_logger(self, level):
+      logger = logging.getLogger('welder')
+      logger.setLevel(level)
+      c_handler = logging.StreamHandler()
+      c_handler.setLevel(level)
+      c_format = logging.Formatter('%(asctime)s %(levelname)s %(name)s - %(message)s', "%Y-%m-%d.%H:%M:%S")
+      c_handler.setFormatter(c_format)
+      logger.addHandler(c_handler)
+
+      return logger
+
+   def create_all(self, prj_name_or_id, dev_name, key_name, userdata=None):
+      project = self.create_new_project_if_not_exist(prj_name_or_id)
+      self.project = project
+      key = self.create_new_key(project.id, key_name)
+      self.key = key
+      if userdata == None:
+         userdata = self.create_user_data()
+      device = self.create_device(project.id, dev_name, key.id, userdata)
+      self.device = device
+      self.dev_id = device.id
+      self.wait_until_device_ready(device)
+
+      return self.ip
+
+   def create_new_project_if_not_exist(self, name_or_id):
+      project = None
+      try:
+         project = self.packet_manager.get_project(name_or_id)
+         self.logger.info(f"using found project {name_or_id}")
+         return project
+      except Exception:
+         pass
+
+      project = self.packet_manager.create_organization_project(
+          org_id=self.org_id,
+          name=name_or_id
+      )
+      self.logger.info(f"created project {name_or_id}")
+
+      return project
+
+   def create_new_key(self, project_id, name):
+      key = RSA.generate(2048)
+      with open(self.private_key_path, 'wb') as priv_file:
+          os.chmod(self.private_key_path, 0o600)
+          priv_file.write(key.exportKey('PEM'))
+
+      pubkey = key.publickey().exportKey('OpenSSH')
+      with open(self.public_key_path, 'wb') as pub_file:
+          pub_file.write(pubkey)
+
+      key = self.packet_manager.create_project_ssh_key(project_id, name, pubkey.decode("utf-8"))
+      self.logger.info(f"created key {key.label}")
+
+      return key
+
+   def create_user_data(self):
+      files = ["hack/scripts/bootstrap.sh", "hack/scripts/devpool.sh", "test/tools/userdata.sh"]
+      userdata = ""
+      for file in files:
+         with open(self.base+"/"+file) as f:
+             userdata += f.read()
+             userdata += "\n"
+
+      return userdata
+
+   def create_device(self, project_id, name, key_id, userdata):
+      device = self.packet_manager.create_device(project_id=project_id,
+                                     hostname=name,
+                                     plan='c1.small.x86', metro='sv',
+                                     operating_system='ubuntu_18_04',
+                                     facility="ewr1", billing_cycle='hourly',
+                                     project_ssh_keys=[key_id],
+                                     userdata=userdata)
+      self.logger.info(f"created device {device.hostname}")
+
+      return device
+
+   # TODO timeouts
+   def wait_until_device_ready(self, device):
+      while True:
+         d = self.packet_manager.get_device(device_id=device.id)
+         self.logger.info(f"checking state of device {d.hostname} ...")
+         if d.state == "active":
+             self.logger.info(f"{d.hostname} running")
+             break
+         self.logger.info(f"state '{d.state}' != 'active', sleeping for 10s")
+         time.sleep(10)
+
+      ips = device.ips()
+      self.ip = ips[0].address
+
+      while True:
+         shell = self.new_shell(self.ip)
+         self.logger.info("checking userdata has completed...")
+         with shell:
+            result = shell.run(["ls", "/flintlock_ready"], allow_error=True)
+         if result.return_code == 0:
+            self.logger.info("userdata ran successfully")
+            break
+         self.logger.info("userdata still running, sleeping for 10s")
+         time.sleep(10)
+
+      self.logger.info(f"device {device.hostname} (id: {device.id}) ready")
+
+   def run_ssh_command(self, cmd, cwd):
+      shell = self.new_shell(self.ip)
+      with shell:
+         result = shell.run(command=cmd, cwd=cwd, stdout=sys.stdout.buffer, stderr=sys.stderr.buffer, allow_error=True)
+      self.logger.info("command exited with code %d", result.return_code)
+
+   def delete_all(self, project, device, key):
+      if device != None:
+         device.delete()
+         self.logger.info(f"deleted device {device.hostname}")
+
+      if key != None:
+         key.delete()
+         os.remove(self.private_key_path)
+         os.remove(self.public_key_path)
+         self.logger.info(f"deleted key {key.label}")
+
+      if project != None:
+         project.delete()
+         self.logger.info(f"deleted project {project.name}")
+
+   def new_shell(self, ip):
+      shell = spur.SshShell(
+         hostname=ip,
+         username="root",
+         load_system_host_keys=False,
+         missing_host_key=spur.ssh.MissingHostKey.accept,
+         private_key_file=self.private_key_path
+      )
+
+      return shell
+
+   def get_device(self, device_id):
+      return self.packet_manager.get_device(device_id=device_id)
+
+   def get_device_ip(self, device_id):
+      d = self.get_device(device_id)
+      ips = d.ips()
+      self.ip = ips[0].address
+
+      return self.ip
+
+   def delete_device(self, device_id):
+      device = self.get_device(device_id)
+      device.delete()
+      self.logger.info(f"deleted device {device.hostname}")

test/tools/requirements.txt

@@ -0,0 +1,4 @@
+click==7.1.2
+packet-python==1.44.1
+pycryptodome==3.11.0
+spur==0.3.22