diff --git a/Makefile b/Makefile index b64fc189..874fba52 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,11 @@ test-with-cov: ## Run unit tests with coverage go test -v -race -timeout 2m -p 1 -covermode=atomic -coverprofile=coverage.txt ./... .PHONY: test-e2e -test-e2e: compile-e2e ## Run e2e tests locally in a container +test-e2e: compile-e2e ## Run e2e tests locally + go test -timeout 30m -p 1 -v -tags=e2e ./test/e2e/... + +.PHONY: test-e2e-docker +test-e2e-docker: compile-e2e ## Run e2e tests locally in a container docker run --rm -it \ --privileged \ --volume /dev:/dev \ @@ -109,11 +113,11 @@ test-e2e: compile-e2e ## Run e2e tests locally in a container --ipc=host \ --workdir=/src/flintlock \ $(test_image):latest \ - "go test -timeout 30m -p 1 -v -tags=e2e ./test/e2e/..." + "make test-e2e" .PHONY: test-e2e-metal test-e2e-metal: ## Run e2e tests in Equinix - echo "coming soon to some hardware near you" + ./test/tools/run.py run-e2e -o $(EQUINIX_ORG_ID) .PHONY: compile-e2e compile-e2e: ## Test e2e compilation diff --git a/hack/scripts/bootstrap.sh b/hack/scripts/bootstrap.sh index 07527e7a..89e3c078 100755 --- a/hack/scripts/bootstrap.sh +++ b/hack/scripts/bootstrap.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# WARNING: THIS SCRIPT HAS MUTLIPLE PURPOSES. +# TAKE CARE WHEN EDITING. + set -euxo pipefail if [[ $(id -u) != 0 ]]; then @@ -28,7 +31,8 @@ apt install -y \ git # install go -export PATH="$PATH:$INSTALL_ROOT/go/bin" +export GOPATH=/root/go +export PATH="$PATH:$INSTALL_ROOT/go/bin:$GOPATH/bin" curl -sL "https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz" | tar xz -C "$INSTALL_ROOT" && \ go version diff --git a/hack/scripts/devpool.sh b/hack/scripts/devpool.sh index 4706432e..4845dce6 100755 --- a/hack/scripts/devpool.sh +++ b/hack/scripts/devpool.sh @@ -1,5 +1,8 @@ #!/bin/bash +# WARNING: THIS SCRIPT HAS MUTLIPLE PURPOSES. +# TAKE CARE WHEN EDITING. + set -ex if [[ $(id -u) != 0 ]]; then @@ -11,11 +14,13 @@ fi CROOT=/var/lib/containerd-dev # This is the name of the thinpool. POOL="${1:-dev-thinpool}" +set +u # This is the tag which will be appended to the loop device volumes VOL_TAG="" if [[ -n "$2" ]]; then VOL_TAG="-$2" fi +set -u # These are some useful vars for useful things DIR="${CROOT}/snapshotter/devmapper" META="${CROOT}/snapshotter/devmapper/metadata$VOL_TAG" diff --git a/hack/tools/.gitignore b/hack/tools/.gitignore index 1a22ccd3..2a7e6ccb 100644 --- a/hack/tools/.gitignore +++ b/hack/tools/.gitignore @@ -1,2 +1,2 @@ share/ -bin/ \ No newline at end of file +bin/ diff --git a/test/e2e/infra/metal.py b/test/e2e/infra/metal.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/tools/.gitignore b/test/tools/.gitignore new file mode 100644 index 00000000..62cce0e6 --- /dev/null +++ b/test/tools/.gitignore @@ -0,0 +1,2 @@ +*.key +__pycache__ diff --git a/test/tools/README.md b/test/tools/README.md new file mode 100644 index 00000000..657caeeb --- /dev/null +++ b/test/tools/README.md @@ -0,0 +1,73 @@ +# run.py + +A handy tool to run tests in and interact with Equinix devices. + +Install dependencies: + +```bash +pip3 install -r test/tools/requirements.txt +``` + +Run the tool: + +```bash +./test/tools/run.py +Usage: run.py [OPTIONS] COMMAND [ARGS]... + + General thing doer for flintlock + +Options: + --help Show this message and exit. + +Commands: + create-device + delete-device + run-e2e +``` + +Run `--help` on all those subcommands to see the run options. + +For all commands you will need your Organisation ID and to set your API token +in the environment as `METAL_AUTH_TOKEN`. + +## Running the e2es + +```bash +export METAL_AUTH_TOKEN= +./test/tools/run.py run-e2e --org-id +``` + +This will: +- Create a new project in your org +- Create a new ssh key, saving the files locally for debugging +- Create a new device +- Bootstrap the device with userdata +- Wait for the device to be running +- Wait for the userdata to complete bootstrapping +- Run the e2e tests, streaming the output +- Delete the device, key, and project + +To keep the device around for debugging, add `--skip-delete`. + +To not creating anything new and run the tests in an existing device, set `--existing-device-id `. + +## Creating a device + +```bash +export METAL_AUTH_TOKEN= +./test/tools/run.py create-device --org-id --project-id +``` + +This will create a device in an existing project bootstrapped with the default userdata. +You can use `--userdata` to override this. + +Nothing will be cleaned up afterwards. + +## Deleting a device + +```bash +export METAL_AUTH_TOKEN= +./test/tools/run.py delete-device --org-id --device-id +``` + +This will delete the given device. The project will not be deleted. diff --git a/test/tools/metal.py b/test/tools/metal.py new file mode 100644 index 00000000..caf967f0 --- /dev/null +++ b/test/tools/metal.py @@ -0,0 +1,171 @@ +import packet +import time +import spur +import logging +import os +import sys +from Crypto.PublicKey import RSA +from os.path import dirname, abspath + +class Welder(): + def __init__(self, auth_token, org_id, level=logging.INFO): + self.org_id = org_id + self.base = dirname(dirname(dirname(abspath(__file__)))) + self.packet_manager = packet.Manager(auth_token=auth_token) + self.private_key_path = dirname(abspath(__file__))+"/private.key" + self.public_key_path = dirname(abspath(__file__))+"/public.key" + self.logger = self.set_logger(level) + self.ip = None + + def set_logger(self, level): + logger = logging.getLogger('welder') + logger.setLevel(level) + c_handler = logging.StreamHandler() + c_handler.setLevel(level) + c_format = logging.Formatter('%(asctime)s %(levelname)s %(name)s - %(message)s', "%Y-%m-%d.%H:%M:%S") + c_handler.setFormatter(c_format) + logger.addHandler(c_handler) + + return logger + + def create_all(self, prj_name_or_id, dev_name, key_name, userdata=None): + project = self.create_new_project_if_not_exist(prj_name_or_id) + self.project = project + key = self.create_new_key(project.id, key_name) + self.key = key + if userdata == None: + userdata = self.create_user_data() + device = self.create_device(project.id, dev_name, key.id, userdata) + self.device = device + self.dev_id = device.id + self.wait_until_device_ready(device) + + return self.ip + + def create_new_project_if_not_exist(self, name_or_id): + project = None + try: + project = self.packet_manager.get_project(name_or_id) + self.logger.info(f"using found project {name_or_id}") + return project + except Exception: + pass + + project = self.packet_manager.create_organization_project( + org_id=self.org_id, + name=name_or_id + ) + self.logger.info(f"created project {name_or_id}") + + return project + + def create_new_key(self, project_id, name): + key = RSA.generate(2048) + with open(self.private_key_path, 'wb') as priv_file: + os.chmod(self.private_key_path, 0o600) + priv_file.write(key.exportKey('PEM')) + + pubkey = key.publickey().exportKey('OpenSSH') + with open(self.public_key_path, 'wb') as pub_file: + pub_file.write(pubkey) + + key = self.packet_manager.create_project_ssh_key(project_id, name, pubkey.decode("utf-8")) + self.logger.info(f"created key {key.label}") + + return key + + def create_user_data(self): + files = ["hack/scripts/bootstrap.sh", "hack/scripts/devpool.sh", "test/tools/userdata.sh"] + userdata = "" + for file in files: + with open(self.base+"/"+file) as f: + userdata += f.read() + userdata += "\n" + + return userdata + + def create_device(self, project_id, name, key_id, userdata): + device = self.packet_manager.create_device(project_id=project_id, + hostname=name, + plan='c1.small.x86', metro='sv', + operating_system='ubuntu_18_04', + facility="ewr1", billing_cycle='hourly', + project_ssh_keys=[key_id], + userdata=userdata) + self.logger.info(f"created device {device.hostname}") + + return device + + # TODO timeouts + def wait_until_device_ready(self, device): + while True: + d = self.packet_manager.get_device(device_id=device.id) + self.logger.info(f"checking state of device {d.hostname} ...") + if d.state == "active": + self.logger.info(f"{d.hostname} running") + break + self.logger.info(f"state '{d.state}' != 'active', sleeping for 10s") + time.sleep(10) + + ips = device.ips() + self.ip = ips[0].address + + while True: + shell = self.new_shell(self.ip) + self.logger.info("checking userdata has completed...") + with shell: + result = shell.run(["ls", "/flintlock_ready"], allow_error=True) + if result.return_code == 0: + self.logger.info("userdata ran successfully") + break + self.logger.info("userdata still running, sleeping for 10s") + time.sleep(10) + + self.logger.info(f"device {device.hostname} (id: {device.id}) ready") + + def run_ssh_command(self, cmd, cwd): + shell = self.new_shell(self.ip) + with shell: + result = shell.run(command=cmd, cwd=cwd, stdout=sys.stdout.buffer, stderr=sys.stderr.buffer, allow_error=True) + self.logger.info("command exited with code %d", result.return_code) + + def delete_all(self, project, device, key): + if device != None: + device.delete() + self.logger.info(f"deleted device {device.hostname}") + + if key != None: + key.delete() + os.remove(self.private_key_path) + os.remove(self.public_key_path) + self.logger.info(f"deleted key {key.label}") + + if project != None: + project.delete() + self.logger.info(f"deleted project {project.name}") + + def new_shell(self, ip): + shell = spur.SshShell( + hostname=ip, + username="root", + load_system_host_keys=False, + missing_host_key=spur.ssh.MissingHostKey.accept, + private_key_file=self.private_key_path + ) + + return shell + + def get_device(self, device_id): + return self.packet_manager.get_device(device_id=device_id) + + def get_device_ip(self, device_id): + d = self.get_device(device_id) + ips = d.ips() + self.ip = ips[0].address + + return self.ip + + def delete_device(self, device_id): + device = self.get_device(device_id) + device.delete() + self.logger.info(f"deleted device {device.hostname}") diff --git a/test/tools/requirements.txt b/test/tools/requirements.txt new file mode 100644 index 00000000..caa004f9 --- /dev/null +++ b/test/tools/requirements.txt @@ -0,0 +1,4 @@ +click==7.1.2 +packet-python==1.44.1 +pycryptodome==3.11.0 +spur==0.3.22 diff --git a/test/tools/run.py b/test/tools/run.py new file mode 100755 index 00000000..c12bee82 --- /dev/null +++ b/test/tools/run.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from test import Test +from metal import Welder +import click +import os +import sys +import random +import string +from os.path import dirname, abspath + +def generate_string(): + letters = string.ascii_letters + return ( ''.join(random.choice(letters) for i in range(10)) ) + +def generated_project_name(): + return "flintlock_prj_"+generate_string() + +def generated_key_name(): + return "flintlock_key_"+generate_string() + +@click.group() +def cli(): + """ + General thing doer for flintlock + """ + pass + +@cli.command() +@click.option('-o', '--org-id', type=str, help='Equinix organisation id (required)') +@click.option('-p', '--project-name', type=str, help='Name of the project to create (default: randomly generated)', default=generated_project_name()) +@click.option('-k', '--ssh-key-name', type=str, help='Name of the ssh key to create and attach to the device (default: randomly generated)', default=generated_key_name()) +@click.option('-d', '--device-name', type=str, help='Name of the device to create (default: randomly generated)', default='T-800') +@click.option('-e', '--existing-device-id', type=str, help='Skip create and set the UUID of an existing device to run tests against', default=None) +@click.option('-s', '--skip-delete', is_flag=True, help='Skip cleanup of Equinix infrastructure for debugging after run', default=False) +def run_e2e(org_id, project_name, ssh_key_name, device_name, skip_delete, existing_device_id): + token = os.environ.get("METAL_AUTH_TOKEN") + if token is None: + click.echo("must set METAL_AUTH_TOKEN") + sys.exit() + if org_id is None: + click.echo("must set --org-id") + sys.exit() + + if existing_device_id == None: + click.echo(f"Running e2e tests. Will create project '{project_name}', ssh_key '{ssh_key_name}' and device '{device_name}'") + click.echo("Note: this will create and bootstrap a new device in Equinix and may take some time") + else: + skip_delete = True + tool_dir = dirname(abspath(__file__)) + if os.path.exists(tool_dir+"/private.key") != True: + click.echo(f"`private.key` file must be saved at `{tool_dir}` when `--existing-device-id` flag set") + sys.exit() + click.echo("running e2e tests using device '{existing_device_id}`") + + runner = Test(token, org_id, project_name, ssh_key_name, device_name, skip_delete, existing_device_id) + with runner: + runner.setup() + runner.run_tests() + + if skip_delete: + dev_id, dev_ip = runner.device_details() + click.echo(f"Device `{dev_id}` left alive for debugging. Use with `--existing-device-id` to re-run tests. SSH command `ssh -i hack/tools/private.key root@{dev_ip}`. Delete device with `delete-device` command.") + +@cli.command() +@click.option('-o', '--org-id', type=str, help='Equinix organisation id (required)') +@click.option('-p', '--project-id', type=str, help='ID of the project to create the device in (required)') +@click.option('-k', '--ssh-key-name', type=str, help='Name of the ssh key to create and attach to the device (default: randomly generated)', default=generated_key_name()) +@click.option('-d', '--device-name', type=str, help='Name of the device to create (default: randomly generated)', default='T-800') +@click.option('-u', '--userdata', type=str, help='String containing shell bootstrap userdata (default: standard flintlockd bootstrapping, see readme for details)') +def create_device(org_id, project_id, ssh_key_name, device_name, userdata): + token = os.environ.get("METAL_AUTH_TOKEN") + if token is None: + click.echo("must set METAL_AUTH_TOKEN") + sys.exit() + if org_id is None: + click.echo("must set --org-id") + sys.exit() + if project_id is None: + click.echo("must set --project-id") + sys.exit() + + click.echo(f"Creating device {device_name}") + + welder = Welder(token, org_id) + ip = welder.create_all(project_id, device_name, ssh_key_name, userdata) + + click.echo(f"Device {device_name} created. SSH command `ssh -i hack/tools/private.key root@{ip}`. Run tests with `run-e2e`. Delete with `delete-device`.") + +@cli.command() +@click.option('-o', '--org-id', type=str, help='Equinix organisation id (required)') +@click.option('-d', '--device-id', type=str, help='Name of the device to delete (required)') +def delete_device(org_id, device_id): + token = os.environ.get("METAL_AUTH_TOKEN") + if token is None: + click.echo("must set METAL_AUTH_TOKEN") + sys.exit() + if org_id is None: + click.echo("must set --org-id") + sys.exit() + if device_id is None: + click.echo("must set --device-id") + sys.exit() + + click.echo(f"Deleting device {device_id}") + + welder = Welder(token, org_id) + welder.delete_device(device_id) + +if __name__ == "__main__": + cli() diff --git a/test/tools/test.py b/test/tools/test.py new file mode 100644 index 00000000..f35a8b3a --- /dev/null +++ b/test/tools/test.py @@ -0,0 +1,47 @@ +from metal import Welder + +class Test: + def __init__(self, auth_token, org_id, prj_name, key_name, dev_name, skip_delete, dev_id=None): + self.welder = Welder(auth_token, org_id) + self.prj_name = prj_name + self.key_name = key_name + self.dev_name = dev_name + self.skip_delete = skip_delete + self.dev_id = dev_id + self.dev_ip = None + self.project = None + self.key = None + self.device = None + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + if self.skip_delete == False: + self.teardown() + + def setup(self): + if self.dev_id != None: + self.fetch_infra() + else: + self.create_infra() + + def run_tests(self): + cmd = ['make', 'test-e2e'] + self.welder.run_ssh_command(cmd, "/root/work/flintlock") + + def teardown(self): + self.welder.delete_all(self.project, self.device, self.key) + + def create_infra(self): + self.dev_ip = self.welder.create_all(self.prj_name, self.dev_name, self.key_name) + + def fetch_infra(self): + try: + ip = self.welder.get_device_ip(self.dev_id) + except: + raise + self.ip = ip + + def device_details(self): + return self.dev_id, self.dev_ip diff --git a/test/tools/userdata.sh b/test/tools/userdata.sh new file mode 100644 index 00000000..6bc6559a --- /dev/null +++ b/test/tools/userdata.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +CTRD_ROOT="/var/lib/containerd-dev" +CTRD_STATE="/run/containerd-dev" +CTRD_CFG="/etc/containerd" +DM_ROOT="$CTRD_ROOT/snapshotter/devmapper" + +mkdir -p "$DM_ROOT" "$CTRD_STATE" "$CTRD_CFG" +cat > "$CTRD_CFG/config-dev.toml" <