From b741209dd85a9a473c86de1a92d98fd0e778a624 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 11 Apr 2020 17:22:15 +0530 Subject: [PATCH 001/134] add gitignore file --- .gitignore | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..db1f598 --- /dev/null +++ b/.gitignore @@ -0,0 +1,100 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + From 17daac93efe0abfe657d8c0b8294365f31b77609 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 11 Apr 2020 17:24:12 +0530 Subject: [PATCH 002/134] added requirements file --- requirements.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e770cbf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +awscli==1.18.39 +botocore==1.15.39 +colorama==0.4.3 +docutils==0.15.2 +jmespath==0.9.5 +pyasn1==0.4.8 +python-dateutil==2.8.1 +PyYAML==5.3.1 +rsa==3.4.2 +s3transfer==0.3.3 +six==1.14.0 +urllib3==1.25.8 From ce121763d0236a57fd678cb332286e47d95fd649 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 11 Apr 2020 17:27:45 +0530 Subject: [PATCH 003/134] add init main file --- sebs/main.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 sebs/main.py diff --git a/sebs/main.py b/sebs/main.py new file mode 100644 index 0000000..6f40356 --- /dev/null +++ b/sebs/main.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +""" +Module Docstring +""" + +__author__ = "Your Name" +__version__ = "0.1.0" +__license__ = "MIT" + +import argparse + + +def main(args): + """ Main entry point of the app """ + print("hello world") + print(args) + + +if __name__ == "__main__": + """ This is executed when run from the command line """ + parser = argparse.ArgumentParser() + + # Required positional argument + parser.add_argument("arg", help="Required positional argument") + + # Optional argument flag which defaults to False + parser.add_argument("-f", "--flag", action="store_true", default=False) + + # Optional argument which requires a parameter (eg. -d test) + parser.add_argument("-n", "--name", action="store", dest="name") + + # Optional verbosity counter (eg. -v, -vv, -vvv, etc.) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Verbosity (-v, -vv, etc)") + + # Specify output of "--version" + parser.add_argument( + "--version", + action="version", + version="%(prog)s (version {version})".format(version=__version__)) + + args = parser.parse_args() + main(args) + From 58c936f69e69432c0a27845bfb8db5b2027332f6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 11 Apr 2020 17:28:05 +0530 Subject: [PATCH 004/134] add vscode directory to ignore file --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index db1f598..c5ffdc7 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,5 @@ ENV/ # mypy .mypy_cache/ +# vscode +.vscode From 1fe6618149399d1d38278364fbeb9988b04e995b Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 11 Apr 2020 17:33:30 +0530 Subject: [PATCH 005/134] updated requirements with pylint --- requirements.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/requirements.txt b/requirements.txt index e770cbf..e7956ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,19 @@ +astroid==2.3.3 awscli==1.18.39 botocore==1.15.39 colorama==0.4.3 docutils==0.15.2 +isort==4.3.21 jmespath==0.9.5 +lazy-object-proxy==1.4.3 +mccabe==0.6.1 pyasn1==0.4.8 +pylint==2.4.4 python-dateutil==2.8.1 PyYAML==5.3.1 rsa==3.4.2 s3transfer==0.3.3 six==1.14.0 +typed-ast==1.4.1 urllib3==1.25.8 +wrapt==1.11.2 From 33055ceb4780376f0ce4158cd8cb1a54a254125c Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 11 Apr 2020 19:19:25 +0530 Subject: [PATCH 006/134] removed awscli but added boto3 to requirements --- requirements.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e7956ee..6609a94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,13 @@ astroid==2.3.3 -awscli==1.18.39 +boto3==1.12.39 botocore==1.15.39 +cached-property==1.5.1 +certifi==2020.4.5.1 +chardet==3.0.4 colorama==0.4.3 docutils==0.15.2 +ec2-metadata==2.2.0 +idna==2.9 isort==4.3.21 jmespath==0.9.5 lazy-object-proxy==1.4.3 @@ -11,6 +16,7 @@ pyasn1==0.4.8 pylint==2.4.4 python-dateutil==2.8.1 PyYAML==5.3.1 +requests==2.23.0 rsa==3.4.2 s3transfer==0.3.3 six==1.14.0 From a69a5b78f4190b13995d42fd2e882a3abb064aa1 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 11 Apr 2020 20:32:49 +0530 Subject: [PATCH 007/134] update license, authors and args --- sebs/main.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sebs/main.py b/sebs/main.py index 6f40356..6c27c6e 100644 --- a/sebs/main.py +++ b/sebs/main.py @@ -3,9 +3,9 @@ Module Docstring """ -__author__ = "Your Name" +__authors__ = ["Levi Blaney", "Neha Singh"] __version__ = "0.1.0" -__license__ = "MIT" +__license__ = "GPLv3" import argparse @@ -20,11 +20,7 @@ def main(args): """ This is executed when run from the command line """ parser = argparse.ArgumentParser() - # Required positional argument - parser.add_argument("arg", help="Required positional argument") - - # Optional argument flag which defaults to False - parser.add_argument("-f", "--flag", action="store_true", default=False) + parser.add_argument('-b','--backup', action='append', help=' List of Devices to Backup', required=True) # Optional argument which requires a parameter (eg. -d test) parser.add_argument("-n", "--name", action="store", dest="name") From 76c1700bf2ffc5e09663af90e5ccd492c1972513 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 00:47:49 +0530 Subject: [PATCH 008/134] init commit of ec2 helper classes --- sebs/ec2.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 sebs/ec2.py diff --git a/sebs/ec2.py b/sebs/ec2.py new file mode 100644 index 0000000..f831a8e --- /dev/null +++ b/sebs/ec2.py @@ -0,0 +1,116 @@ +import sys +import boto3 +import requests +from ec2_metadata import ec2_metadata + + + +class Instance: + def __init__(self): + self.instance = self.get_instance() + self.backup = [] + + def get_instance(self): + + try: + instance_id = ec2_metadata.instance_id + except requests.exceptions.ConnectTimeout: + print( + 'Failled to get instance metadata, are you sure you are running on an EC2 instance?') + sys.exit(1) + except: + t, v, _tb = sys.exc_info() + print("Unexpected error {}: {}".format(t, v)) + sys.exit(1) + + try: + ec2 = boto3.resource('ec2') + instance = ec2.Instance(instance_id) + # We have to call load to see if we are really connected + instance.load() + except: + t, v, _tb = sys.exc_info() + print("Unexpected error {}: {}".format(t, v)) + sys.exit(2) + + return instance + + def add_stateful_device(self, device_name): + + sv = StatefulVolume(device_name) + + sv.get_status() + + self.backup.append(sv) + + def tag_stateful_volumes(self): + # Loop thrugh the volumes and call "tag_volume" on them. + pass + + def attach_stateful_volumes(self): + + for sv in self.backup: + # call copy on the volume + + # call attach on the volume + pass + + +class StatefulVolume: + def __init__(self, deviceName): + self.deviceName = deviceName + self.ready = False + self.status = 'Unknown' + self.volume + + def get_status(self): + client = boto3.client('ec2') + response = client.describe_volumes( + Filters=[ + { + 'Name': 'tag:{}'.format(args.name), + 'Values': [ + self.deviceName, + ] + }, + ] + ) + + if not response['Volumes']: + # No previous volume found + self.status = 'New' + # Might have to do other stuffs + elif response['Volumes'].size() != 1: + # too many volumes found + self.status = 'Duplicate' + # Might have to do other stuffs + else: + # Previous backup volume found + volumeId = response['Volumes'][0]['VolumeId'] + self.status = 'Not Attached' + + ec2 = boto3.resource('ec2') + self.volume = ec2.Volume(volumeId) + # Might have to do other stuffs + return self.status + + def tag_volume(self): + # If status is 'New' Find the new volume and tag it + + if self.status != 'New': + return self.status + + # Find the volumeID of the attached Device and tag it + pass + + def copy(self, target_az): + # If the current volume az is in the target AZ do nothing + if target_az == self.volume.availability_zone: + return + + # else make a snapshot and get a new device created in the correct AZ + pass + + def attach(self, instance): + # if not attached to the current instance then attach it and set the status to attached + pass From 696ab527fb4ce00f154a9e456c55f0303cb6d4b8 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 00:48:19 +0530 Subject: [PATCH 009/134] add rough app outline --- sebs/main.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/sebs/main.py b/sebs/main.py index 6c27c6e..601cd27 100644 --- a/sebs/main.py +++ b/sebs/main.py @@ -7,7 +7,9 @@ __version__ = "0.1.0" __license__ = "GPLv3" +import sys import argparse +from ec2 import Instance def main(args): @@ -15,15 +17,34 @@ def main(args): print("hello world") print(args) + # Get a handler for the current EC2 instance + server = Instance() + + # Add the requested Stateful Devices to the server + for device in args.backup: + server.add_stateful_device(device) + + # Make sure the Stateful Volumes are attached to this server + server.attach_stateful_volumes() + + # Tag the Stateful Volumes so they can be found on next boot + server.tag_stateful_volumes() + + # I think we done? + + sys.exit() + if __name__ == "__main__": """ This is executed when run from the command line """ parser = argparse.ArgumentParser() - parser.add_argument('-b','--backup', action='append', help=' List of Devices to Backup', required=True) + parser.add_argument('-b', '--backup', action='append', + help=' List of Devices to Backup', required=True) # Optional argument which requires a parameter (eg. -d test) - parser.add_argument("-n", "--name", action="store", dest="name") + parser.add_argument("-n", "--name", default='sebs', + help=' specify a unique name.') # Optional verbosity counter (eg. -v, -vv, -vvv, etc.) parser.add_argument( @@ -41,4 +62,3 @@ def main(args): args = parser.parse_args() main(args) - From 12a2f017b634669bd28667297edca2d9e96aa8e3 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 00:50:11 +0530 Subject: [PATCH 010/134] update requirments with pep formater --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 6609a94..8e36d23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ astroid==2.3.3 +autopep8==1.5.1 boto3==1.12.39 botocore==1.15.39 cached-property==1.5.1 @@ -13,6 +14,7 @@ jmespath==0.9.5 lazy-object-proxy==1.4.3 mccabe==0.6.1 pyasn1==0.4.8 +pycodestyle==2.5.0 pylint==2.4.4 python-dateutil==2.8.1 PyYAML==5.3.1 From 891d03e17f4e58ef83a61c83d914530167781997 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 15:55:30 +0530 Subject: [PATCH 011/134] add init files --- sebs/__init__.py | 0 tests/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 sebs/__init__.py create mode 100644 tests/__init__.py diff --git a/sebs/__init__.py b/sebs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 1f41d42993591e8cc0a812ababf9528ddaa2a334 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 16:43:00 +0530 Subject: [PATCH 012/134] add example unit test file --- tests/test_main.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/test_main.py diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..166e409 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,39 @@ +""" +This file demonstrates common uses for the Python unittest module +https://docs.python.org/3/library/unittest.html +""" +import random +import unittest +import sebs + + +class TestSequenceFunctions(unittest.TestCase): + """ This is one of potentially many TestCases """ + + def setUp(self): + self.seq = list(range(10)) + + def test_shuffle(self): + """ make sure the shuffled sequence does not lose any elements """ + random.shuffle(self.seq) + self.seq.sort() + self.assertEqual(self.seq, list(range(10))) + + # should raise an exception for an immutable sequence + self.assertRaises(TypeError, random.shuffle, (1, 2, 3)) + + def test_choice(self): + """ test a choice """ + element = random.choice(self.seq) + self.assertTrue(element in self.seq) + + def test_sample(self): + """ test that an exception is raised """ + with self.assertRaises(ValueError): + random.sample(self.seq, 20) + for element in random.sample(self.seq, 5): + self.assertTrue(element in self.seq) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 1bfcf2d4fe5c30969109fa024202039e0e59b4a6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 16:43:58 +0530 Subject: [PATCH 013/134] add precommit hook files --- scripts/install-hooks.bash | 8 ++++++++ scripts/pre-commit.bash | 29 +++++++++++++++++++++++++++++ scripts/run-lint.bash | 25 +++++++++++++++++++++++++ scripts/run-tests.bash | 16 ++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100755 scripts/install-hooks.bash create mode 100755 scripts/pre-commit.bash create mode 100755 scripts/run-lint.bash create mode 100755 scripts/run-tests.bash diff --git a/scripts/install-hooks.bash b/scripts/install-hooks.bash new file mode 100755 index 0000000..c771dc9 --- /dev/null +++ b/scripts/install-hooks.bash @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +GIT_DIR=$(git rev-parse --git-dir) + +echo "Installing hooks..." +# this command creates symlink to our pre-commit script +ln -s ../../scripts/pre-commit.bash $GIT_DIR/hooks/pre-commit +echo "Done!" diff --git a/scripts/pre-commit.bash b/scripts/pre-commit.bash new file mode 100755 index 0000000..884d6ba --- /dev/null +++ b/scripts/pre-commit.bash @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +cd "${0%/*}/.." + +cd .. + +if [ -z $VIRTUAL_ENV ]; then + source ./venv/bin/activate +fi + +echo "Running pre-commit hook" +echo +echo "Linting changed python files" +echo +./scripts/run-lint.bash +echo +# $? stores exit value of the last command +if [ $? -ne 0 ]; then + echo "Please lint before commiting." + exit 1 +fi + +./scripts/run-tests.bash + +# $? stores exit value of the last command +if [ $? -ne 0 ]; then + echo "Tests must pass before commiting!" + exit 1 +fi diff --git a/scripts/run-lint.bash b/scripts/run-lint.bash new file mode 100755 index 0000000..c6fac49 --- /dev/null +++ b/scripts/run-lint.bash @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# if any command inside script returns error, exit and return that error +set -e + +# magic line to ensure that we're always inside the root of our application, +# no matter from which directory we'll run script +# thanks to it we can just enter `./scripts/run-tests.bash` +cd "${0%/*}/.." + +if(git diff --cached --name-only --diff-filter=AM HEAD | grep 'py') +then + if !(git diff --cached --name-only --diff-filter=AM HEAD | grep 'py' | xargs -P 10 -n1 autopep8 --in-place --exit-code) + then + echo + echo "Error: You attempted to commit one or more python files with format errors." + echo + echo "Please fix them and retry the commit." + echo + exit 1 + fi + exit 0 +fi + +echo "No modified python files to lint" diff --git a/scripts/run-tests.bash b/scripts/run-tests.bash new file mode 100755 index 0000000..d1d1337 --- /dev/null +++ b/scripts/run-tests.bash @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# if any command inside script returns error, exit and return that error +set -e + +# magic line to ensure that we're always inside the root of our application, +# no matter from which directory we'll run script +# thanks to it we can just enter `./scripts/run-tests.bash` +cd "${0%/*}/.." + +# let's fake failing test for now +echo -e "Running tests \n" +coverage run --source sebs -m unittest discover +echo -e "\n" +coverage report -m +echo -e "\n" From bc17dc9cb550e43606e51b91b61d55838a336edc Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 17:04:13 +0530 Subject: [PATCH 014/134] move arg parsing to its own module for easier testing --- sebs/cli.py | 28 ++++++++++++++++++++++++++++ sebs/main.py | 26 ++------------------------ 2 files changed, 30 insertions(+), 24 deletions(-) create mode 100644 sebs/cli.py diff --git a/sebs/cli.py b/sebs/cli.py new file mode 100644 index 0000000..9cd0f21 --- /dev/null +++ b/sebs/cli.py @@ -0,0 +1,28 @@ +import argparse + + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument('-b', '--backup', action='append', + help=' List of Devices to Backup', required=True) + + # Optional argument which requires a parameter (eg. -d test) + parser.add_argument("-n", "--name", default='sebs', + help=' specify a unique name.') + + # Optional verbosity counter (eg. -v, -vv, -vvv, etc.) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Verbosity (-v, -vv, etc)") + + # Specify output of "--version" + parser.add_argument( + "--version", + action="version", + version="%(prog)s (version {version})".format(version=__version__)) + + return parser.parse_args() diff --git a/sebs/main.py b/sebs/main.py index 601cd27..d66125a 100644 --- a/sebs/main.py +++ b/sebs/main.py @@ -8,7 +8,7 @@ __license__ = "GPLv3" import sys -import argparse +import cli from ec2 import Instance @@ -37,28 +37,6 @@ def main(args): if __name__ == "__main__": """ This is executed when run from the command line """ - parser = argparse.ArgumentParser() - parser.add_argument('-b', '--backup', action='append', - help=' List of Devices to Backup', required=True) - - # Optional argument which requires a parameter (eg. -d test) - parser.add_argument("-n", "--name", default='sebs', - help=' specify a unique name.') - - # Optional verbosity counter (eg. -v, -vv, -vvv, etc.) - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Verbosity (-v, -vv, etc)") - - # Specify output of "--version" - parser.add_argument( - "--version", - action="version", - version="%(prog)s (version {version})".format(version=__version__)) - - args = parser.parse_args() + args = cli.parse_args() main(args) From 22412458218ee007ac2fb68dcd63ea0106234dbf Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 19:11:07 +0530 Subject: [PATCH 015/134] rename main.py for python standards? --- sebs/{main.py => __main__.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename sebs/{main.py => __main__.py} (87%) diff --git a/sebs/main.py b/sebs/__main__.py similarity index 87% rename from sebs/main.py rename to sebs/__main__.py index d66125a..d14451f 100644 --- a/sebs/main.py +++ b/sebs/__main__.py @@ -8,8 +8,8 @@ __license__ = "GPLv3" import sys -import cli -from ec2 import Instance +from sebs.cli import parse_args +from sebs.ec2 import Instance def main(args): @@ -38,5 +38,5 @@ def main(args): if __name__ == "__main__": """ This is executed when run from the command line """ - args = cli.parse_args() + args = parse_args(sys.argv[1:], __version__) main(args) From 4adecba988afb7b870da3464d5d76e45fc2b6673 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 19:11:30 +0530 Subject: [PATCH 016/134] create generic test layout for main.py --- tests/test_main.py | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 166e409..e39b5ed 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,39 +1,14 @@ -""" -This file demonstrates common uses for the Python unittest module -https://docs.python.org/3/library/unittest.html -""" -import random import unittest -import sebs +from sebs import __main__ -class TestSequenceFunctions(unittest.TestCase): - """ This is one of potentially many TestCases """ - +class TestApplicaton(unittest.TestCase): def setUp(self): - self.seq = list(range(10)) - - def test_shuffle(self): - """ make sure the shuffled sequence does not lose any elements """ - random.shuffle(self.seq) - self.seq.sort() - self.assertEqual(self.seq, list(range(10))) - - # should raise an exception for an immutable sequence - self.assertRaises(TypeError, random.shuffle, (1, 2, 3)) - - def test_choice(self): - """ test a choice """ - element = random.choice(self.seq) - self.assertTrue(element in self.seq) + pass - def test_sample(self): - """ test that an exception is raised """ - with self.assertRaises(ValueError): - random.sample(self.seq, 20) - for element in random.sample(self.seq, 5): - self.assertTrue(element in self.seq) + def test_main(self): + pass if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 17d476a6649630916c1d783db8b1f40aaf3978e2 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 19:12:06 +0530 Subject: [PATCH 017/134] fix version printing --- sebs/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sebs/cli.py b/sebs/cli.py index 9cd0f21..6b8c3b6 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -1,7 +1,7 @@ import argparse -def parse_args(): +def parse_args(args, ver): parser = argparse.ArgumentParser() parser.add_argument('-b', '--backup', action='append', @@ -23,6 +23,6 @@ def parse_args(): parser.add_argument( "--version", action="version", - version="%(prog)s (version {version})".format(version=__version__)) + version="%(prog)s (version {version})".format(version=ver)) - return parser.parse_args() + return parser.parse_args(args) From f8045a89cb5fefa969b8826eb378074a6bd72bcc Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 19:13:18 +0530 Subject: [PATCH 018/134] add tests for cli --- tests/test_cli.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_cli.py diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..f7ab018 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,42 @@ +import io +import sys +import unittest +import contextlib +from sebs.cli import parse_args + + +class TestArugmentParsing(unittest.TestCase): + + def setUp(self): + pass + + def test_version(self): + + try: + version = "2.1.0" + parse_args(['--version'], version) + except: + pass + + def test_single_backup(self): + args = parse_args(['-b test'], '') + self.assertEqual(len(args.backup), 1) + + def test_multiple_backup(self): + args = parse_args(['-b test1', '-b test2'], '') + + self.assertEqual(len(args.backup), 2) + + def test_default_name(self): + args = parse_args(['-b test1', '-b test2'], '') + + self.assertEqual(args.name, 'sebs') + + def test_override_name(self): + args = parse_args(['-b test1', '-b test2', '-n not-default'], '') + + self.assertEqual(args.name, ' not-default') + + +if __name__ == '__main__': + unittest.main() From 4bb42adceb2c700505885b6ba29e70b665afe14a Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 19:22:14 +0530 Subject: [PATCH 019/134] remove unused imports --- tests/test_cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index f7ab018..bcf376b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,4 @@ -import io -import sys import unittest -import contextlib from sebs.cli import parse_args From b015dbeca92909506109dafef9a231dfa11e15b6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 12 Apr 2020 19:23:17 +0530 Subject: [PATCH 020/134] modify imports --- tests/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index e39b5ed..efd7936 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,5 @@ import unittest -from sebs import __main__ +from sebs.__main__ import main class TestApplicaton(unittest.TestCase): From daf75195927435b96c820f3474256b21cd57c160 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Wed, 15 Apr 2020 04:31:19 -0400 Subject: [PATCH 021/134] add StatefulVolume tests --- tests/test_ec2_volumes.py | 137 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/test_ec2_volumes.py diff --git a/tests/test_ec2_volumes.py b/tests/test_ec2_volumes.py new file mode 100644 index 0000000..d570e3d --- /dev/null +++ b/tests/test_ec2_volumes.py @@ -0,0 +1,137 @@ +import sys +import unittest +import datetime +import botocore.session +from botocore.stub import Stubber, ANY +from unittest.mock import patch, MagicMock + + +class TestStatefulVolume(unittest.TestCase): + + def setUp(self): + # Setup our ec2 client stubb + ec2 = botocore.session.get_session().create_client('ec2') + self.stub_client = ec2 + self.stubber = Stubber(ec2) + + # Use mocks to pass out client stubb to our code + self.boto3 = MagicMock(name='module_mock') + self.mock_client = MagicMock(name='client_mock', return_value=ec2) + self.boto3.client = self.mock_client + + # Setup resource and volume mocks + self.mock_volume = MagicMock(name='volume_mock') + self.mock_volume_class = MagicMock( + name='volume_class_mock', return_value=self.mock_volume) + self.mock_volume_class.Volume = self.mock_volume + self.mock_resource = MagicMock( + name='resource_mock', return_value=self.mock_volume_class) + self.boto3.resource = self.mock_resource + + modules = { + 'boto3': self.boto3 + } + + self.default_response = { + 'Volumes': [ + { + 'Attachments': [], + 'AvailabilityZone': 'string', + 'CreateTime': datetime.datetime(2015, 1, 1), + 'Encrypted': False, + 'KmsKeyId': 'string', + 'OutpostArn': 'string', + 'Size': 123, + 'SnapshotId': 'string', + 'State': 'available', + 'VolumeId': 'string', + 'Iops': 123, + 'Tags': [ + { + 'Key': 'string', + 'Value': 'string' + }, + ], + 'VolumeType': 'gp2', + 'FastRestored': False, + 'MultiAttachEnabled': False + } + ] + } + + self.default_params = {'Filters': ANY} + + self.stubber.activate() + + self.module_patcher = patch.dict('sys.modules', modules) + self.module_patcher.start() + + from sebs.ec2 import StatefulVolume, Instance + + self.StatefulVolume = StatefulVolume + + def tearDown(self): + self.module_patcher.stop() + self.stubber.deactivate() + + def test_new_volume(self): + + response = self.default_response.copy() + response['Volumes'] = [] + + self.stubber.add_response( + 'describe_volumes', response, self.default_params) + + sv = self.StatefulVolume('xdf', 'test') + + status = sv.get_status() + + self.stubber.assert_no_pending_responses() + self.assertEqual(sv.status, 'New', 'Should be a new Volume') + + def test_duplicate_volumes(self): + + response = self.default_response.copy() + response['Volumes'] = [{}, {}] + + self.stubber.add_response( + 'describe_volumes', response, self.default_params) + + sv = self.StatefulVolume('xdf', 'test') + + status = sv.get_status() + + self.stubber.assert_no_pending_responses() + self.assertEqual(sv.status, 'Duplicate', + 'Should be a duplicate volume') + + def test_existing_volume(self): + + self.stubber.add_response( + 'describe_volumes', self.default_response, self.default_params) + + sv = self.StatefulVolume('xdf', 'test') + + status = sv.get_status() + + self.stubber.assert_no_pending_responses() + self.assertEqual(sv.status, 'Not Attached', + 'Should find an existing volume') + self.assertIsInstance(sv.volume, MagicMock, + 'Should be our volume mock') + + def test_class_properties(self): + sv = self.StatefulVolume('xdf', 'test') + + self.stubber.assert_no_pending_responses() + + self.assertEqual(sv.deviceName, 'xdf', 'Should set the deviceName') + self.assertEqual( + sv.ready, False, 'Should set the volume to not ready.') + self.assertEqual(sv.status, 'Unknown', 'Should set the status') + self.assertEqual(sv.volume, None, 'Should set the tag name') + self.assertEqual(sv.tag_name, 'test', 'Should set the tag name') + + +if __name__ == '__main__': + unittest.main() From 7e3d97d9060210dfdcb1fc777d2e11942077031b Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Wed, 15 Apr 2020 04:31:47 -0400 Subject: [PATCH 022/134] add changes from tests --- sebs/ec2.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index f831a8e..4306bb1 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -4,7 +4,6 @@ from ec2_metadata import ec2_metadata - class Instance: def __init__(self): self.instance = self.get_instance() @@ -57,18 +56,19 @@ def attach_stateful_volumes(self): class StatefulVolume: - def __init__(self, deviceName): + def __init__(self, deviceName, tag_name): self.deviceName = deviceName self.ready = False self.status = 'Unknown' - self.volume + self.volume = None + self.tag_name = tag_name def get_status(self): client = boto3.client('ec2') response = client.describe_volumes( Filters=[ { - 'Name': 'tag:{}'.format(args.name), + 'Name': 'tag:{}'.format(self.tag_name), 'Values': [ self.deviceName, ] @@ -80,7 +80,7 @@ def get_status(self): # No previous volume found self.status = 'New' # Might have to do other stuffs - elif response['Volumes'].size() != 1: + elif len(response['Volumes']) != 1: # too many volumes found self.status = 'Duplicate' # Might have to do other stuffs From c3e892a079a4f7071150fcda9a64bc0e752bc07e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Wed, 15 Apr 2020 04:31:59 -0400 Subject: [PATCH 023/134] remove problem import for now --- tests/test_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index efd7936..d1158b8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,6 @@ import unittest -from sebs.__main__ import main +# Import causes problems with other test files =/ +# from sebs.__main__ import main class TestApplicaton(unittest.TestCase): From 25776633752ede127a076387d477b4d0a410704a Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Wed, 15 Apr 2020 04:35:55 -0400 Subject: [PATCH 024/134] updated test name --- tests/{test_ec2_volumes.py => test_stateful_volume.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/{test_ec2_volumes.py => test_stateful_volume.py} (98%) diff --git a/tests/test_ec2_volumes.py b/tests/test_stateful_volume.py similarity index 98% rename from tests/test_ec2_volumes.py rename to tests/test_stateful_volume.py index d570e3d..041b695 100644 --- a/tests/test_ec2_volumes.py +++ b/tests/test_stateful_volume.py @@ -66,7 +66,7 @@ def setUp(self): self.module_patcher = patch.dict('sys.modules', modules) self.module_patcher.start() - from sebs.ec2 import StatefulVolume, Instance + from sebs.ec2 import StatefulVolume self.StatefulVolume = StatefulVolume From 57072090d96b1a58fc02c315379321409d769a2d Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 08:42:10 -0400 Subject: [PATCH 025/134] add method to tag sebs volumes --- sebs/ec2.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 4306bb1..ab0ab31 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -56,8 +56,9 @@ def attach_stateful_volumes(self): class StatefulVolume: - def __init__(self, deviceName, tag_name): - self.deviceName = deviceName + def __init__(self, instance_id, device_name, tag_name): + self.instance_id = instance_id + self.device_name = device_name self.ready = False self.status = 'Unknown' self.volume = None @@ -65,12 +66,14 @@ def __init__(self, deviceName, tag_name): def get_status(self): client = boto3.client('ec2') + ec2 = boto3.resource('ec2') + response = client.describe_volumes( Filters=[ { 'Name': 'tag:{}'.format(self.tag_name), 'Values': [ - self.deviceName, + self.device_name, ] }, ] @@ -79,7 +82,27 @@ def get_status(self): if not response['Volumes']: # No previous volume found self.status = 'New' - # Might have to do other stuffs + + response = client.describe_volumes( + Filters=[ + { + 'Name': 'attachment.instance-id', + 'Values': [ + self.instance_id, + ] + }, + ] + ) + + if not response['Volumes']: + print( + f"Could not find {self.device_name} for {self.instance_id}") + sys.exit(2) + + volumeId = response['Volumes'][0]['VolumeId'] + + self.volume = ec2.Volume(volumeId) + elif len(response['Volumes']) != 1: # too many volumes found self.status = 'Duplicate' @@ -89,7 +112,6 @@ def get_status(self): volumeId = response['Volumes'][0]['VolumeId'] self.status = 'Not Attached' - ec2 = boto3.resource('ec2') self.volume = ec2.Volume(volumeId) # Might have to do other stuffs return self.status @@ -100,8 +122,13 @@ def tag_volume(self): if self.status != 'New': return self.status - # Find the volumeID of the attached Device and tag it - pass + self.volume.create_tags(Tags=[ + { + 'Key': self.tag_name, + 'Value': self.device_name + }, + ] + ) def copy(self, target_az): # If the current volume az is in the target AZ do nothing From a91d25ea29640c75f4415566ef47f9280f15334b Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 08:49:30 -0400 Subject: [PATCH 026/134] update tests --- tests/test_stateful_volume.py | 70 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index 041b695..80eb09b 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -3,7 +3,7 @@ import datetime import botocore.session from botocore.stub import Stubber, ANY -from unittest.mock import patch, MagicMock +from unittest.mock import patch, MagicMock, Mock class TestStatefulVolume(unittest.TestCase): @@ -20,7 +20,9 @@ def setUp(self): self.boto3.client = self.mock_client # Setup resource and volume mocks - self.mock_volume = MagicMock(name='volume_mock') + self.actual_volume = Mock(name='actual_volume') + self.mock_volume = MagicMock( + name='volume_mock', return_value=self.actual_volume) self.mock_volume_class = MagicMock( name='volume_class_mock', return_value=self.mock_volume) self.mock_volume_class.Volume = self.mock_volume @@ -32,6 +34,10 @@ def setUp(self): 'boto3': self.boto3 } + self.instance_id = 'i-1234567890abcdef0' + self.tag_name = 'sebs' + self.device_name = 'xdf' + self.default_response = { 'Volumes': [ { @@ -44,7 +50,7 @@ def setUp(self): 'Size': 123, 'SnapshotId': 'string', 'State': 'available', - 'VolumeId': 'string', + 'VolumeId': 'vol-XXXXXX', 'Iops': 123, 'Tags': [ { @@ -82,12 +88,22 @@ def test_new_volume(self): self.stubber.add_response( 'describe_volumes', response, self.default_params) - sv = self.StatefulVolume('xdf', 'test') + self.stubber.add_response( + 'describe_volumes', self.default_response, {'Filters': [ + { + 'Name': 'attachment.instance-id', + 'Values': [self.instance_id] + } + ]}) + + sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') - status = sv.get_status() + sv.get_status() self.stubber.assert_no_pending_responses() self.assertEqual(sv.status, 'New', 'Should be a new Volume') + self.assertIsInstance(sv.volume, Mock, + 'Should be our volume mock') def test_duplicate_volumes(self): @@ -97,9 +113,9 @@ def test_duplicate_volumes(self): self.stubber.add_response( 'describe_volumes', response, self.default_params) - sv = self.StatefulVolume('xdf', 'test') + sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') - status = sv.get_status() + sv.get_status() self.stubber.assert_no_pending_responses() self.assertEqual(sv.status, 'Duplicate', @@ -110,28 +126,58 @@ def test_existing_volume(self): self.stubber.add_response( 'describe_volumes', self.default_response, self.default_params) - sv = self.StatefulVolume('xdf', 'test') + sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') - status = sv.get_status() + sv.get_status() self.stubber.assert_no_pending_responses() self.assertEqual(sv.status, 'Not Attached', 'Should find an existing volume') - self.assertIsInstance(sv.volume, MagicMock, + self.assertIsInstance(sv.volume, Mock, 'Should be our volume mock') def test_class_properties(self): - sv = self.StatefulVolume('xdf', 'test') + sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') self.stubber.assert_no_pending_responses() - self.assertEqual(sv.deviceName, 'xdf', 'Should set the deviceName') + self.assertEqual(sv.device_name, 'xdf', 'Should set the deviceName') self.assertEqual( sv.ready, False, 'Should set the volume to not ready.') self.assertEqual(sv.status, 'Unknown', 'Should set the status') self.assertEqual(sv.volume, None, 'Should set the tag name') self.assertEqual(sv.tag_name, 'test', 'Should set the tag name') + def test_volume_tagging(self): + response = self.default_response.copy() + response['Volumes'] = [] + + self.stubber.add_response( + 'describe_volumes', response, {'Filters': [ + { + 'Name': f'tag:{self.tag_name}', + 'Values': [self.device_name] + } + ]}) + + self.stubber.add_response( + 'describe_volumes', self.default_response, {'Filters': [ + { + 'Name': 'attachment.instance-id', + 'Values': [self.instance_id] + } + ]}) + + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) + + sv.get_status() + + sv.tag_volume() + + self.actual_volume.create_tags.assert_called_with( + Tags=[{'Key': self.tag_name, 'Value': self.device_name}]) + if __name__ == '__main__': unittest.main() From 307b2bfeab1df6750a1fee99a6440117e921b8b0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 09:21:18 -0400 Subject: [PATCH 027/134] pass tag to instance and then volume --- sebs/ec2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index ab0ab31..353706c 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -5,8 +5,9 @@ class Instance: - def __init__(self): + def __init__(self, volume_tag): self.instance = self.get_instance() + self.volume_tag = volume_tag self.backup = [] def get_instance(self): @@ -36,7 +37,7 @@ def get_instance(self): def add_stateful_device(self, device_name): - sv = StatefulVolume(device_name) + sv = StatefulVolume(self.instance.id, device_name, self.volume_tag) sv.get_status() From 64a0887b43ecbca93131218e99ef374524a5298a Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 12:01:41 -0400 Subject: [PATCH 028/134] added functions for copy and attach --- sebs/ec2.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 10 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 353706c..af09645 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -64,12 +64,12 @@ def __init__(self, instance_id, device_name, tag_name): self.status = 'Unknown' self.volume = None self.tag_name = tag_name + self.ec2_client = boto3.client('ec2') + self.ec2_resource = boto3.resource('ec2') def get_status(self): - client = boto3.client('ec2') - ec2 = boto3.resource('ec2') - response = client.describe_volumes( + response = self.ec2_client.describe_volumes( Filters=[ { 'Name': 'tag:{}'.format(self.tag_name), @@ -84,7 +84,7 @@ def get_status(self): # No previous volume found self.status = 'New' - response = client.describe_volumes( + response = self.ec2_client.describe_volumes( Filters=[ { 'Name': 'attachment.instance-id', @@ -102,7 +102,7 @@ def get_status(self): volumeId = response['Volumes'][0]['VolumeId'] - self.volume = ec2.Volume(volumeId) + self.volume = self.ec2_resource.Volume(volumeId) elif len(response['Volumes']) != 1: # too many volumes found @@ -113,7 +113,7 @@ def get_status(self): volumeId = response['Volumes'][0]['VolumeId'] self.status = 'Not Attached' - self.volume = ec2.Volume(volumeId) + self.volume = self.ec2_resource.Volume(volumeId) # Might have to do other stuffs return self.status @@ -136,9 +136,95 @@ def copy(self, target_az): if target_az == self.volume.availability_zone: return - # else make a snapshot and get a new device created in the correct AZ - pass + snapshot = self.volume.create_snapshot( + Description='Intermediate snapshot for SEBS.', + TagSpecifications=[ + { + 'ResourceType': 'snapshot', + 'Tags': [ + { + 'Key': self.tag_name, + 'Value': self.device_name + }, + ] + }, + ] + ) + + # Not sure but we probably have to wait until its completed + snapshot.wait_until_completed() + + response = self.ec2_client.create_volume( + AvailabilityZone=target_az, + Encrypted=snapshot.encrypted, + Iops=self.volume.iops, + KmsKeyId=self.volume.kms_key_id, + OutpostArn=self.volume.outpost_arn, + Size=self.volume.size, + SnapshotId=snapshot.snapshot_id, + VolumeType=self.volume.volume_type, + TagSpecifications=[ + { + 'ResourceType': 'volume', + 'Tags': [ + { + 'Key': self.tag_name, + 'Value': self.device_name, + }, + ] + }, + ] + ) + + # Cleanup this temporary snapshot + snapshot.delete() + + self.volume = self.ec2_resource.Volume(response['VolumeId']) + + waiter = self.ec2_client.get_waiter('volume_available') + + waiter.wait(self.volume.volume_id) def attach(self, instance): - # if not attached to the current instance then attach it and set the status to attached - pass + + # Need to find and delete any current volumes + response = self.ec2_client.describe_volumes( + Filters=[ + { + 'Name': 'attachment.instance-id', + 'Values': [ + self.instance_id, + ] + }, + { + 'Name': 'attachment.device', + 'Values': [ + self.device_name, + ] + }, + ] + ) + + if response['Volumes']: + prev_volume = self.ec2_resource.Volume( + response['Volumes']['VolumeId']) + + pre_volume.detach_from_instance( + Device=self.device_name, + InstanceId=self.instance_id + ) + + waiter = self.ec2_client.get_waiter('volume_available') + + waiter.wait(prev_volume.volume_id) + + prev_volume.delete() + + self.volume.attach_to_instance( + Device=self.device_name, + InstanceId=self.instance_id + ) + + waiter = self.ec2_client.get_waiter('volume_in_use') + + waiter.wait(self.volume.volume_id) From 218e5e11b949096629f7223cdd69b71540051928 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 12:02:25 -0400 Subject: [PATCH 029/134] add setup.py --- setup.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b4e85b6 --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="sebs-test", + version="0.0.1", + author="Levi Blaney", + author_email="shadycuz", + description="Create Stateful Elastic Block Device", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/DontShaveTheYak/sebs", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=[ + 'boto3', + 'ec2-metadata' + ], + python_requires='>=3.6', +) From c60323d3113b793c6d9f46ffb8beed40906f9ab5 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 13:50:42 -0400 Subject: [PATCH 030/134] remove name == main because python is hard --- sebs/__main__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sebs/__main__.py b/sebs/__main__.py index d14451f..cb47fb2 100644 --- a/sebs/__main__.py +++ b/sebs/__main__.py @@ -35,8 +35,5 @@ def main(args): sys.exit() -if __name__ == "__main__": - """ This is executed when run from the command line """ - - args = parse_args(sys.argv[1:], __version__) - main(args) +args = parse_args(sys.argv[1:], __version__) +main(args) From 16f7b75ad6da5c61bf7e6fd8244aaa9bd1c04895 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 13:50:58 -0400 Subject: [PATCH 031/134] add script --- bin/sebs | 11 +++++++++++ setup.py | 1 + 2 files changed, 12 insertions(+) create mode 100644 bin/sebs diff --git a/bin/sebs b/bin/sebs new file mode 100644 index 0000000..4414820 --- /dev/null +++ b/bin/sebs @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +import sys + +from sebs import __main__ + +def main(): + __main__.main() + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/setup.py b/setup.py index b4e85b6..b095fb4 100644 --- a/setup.py +++ b/setup.py @@ -22,5 +22,6 @@ 'boto3', 'ec2-metadata' ], + scripts=['bin/sebs'], python_requires='>=3.6', ) From ef40a60dd6bc4fa7113df75c28171f289aeb188a Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 13:56:35 -0400 Subject: [PATCH 032/134] pass tag to instance --- sebs/__main__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sebs/__main__.py b/sebs/__main__.py index cb47fb2..ec0d5bd 100644 --- a/sebs/__main__.py +++ b/sebs/__main__.py @@ -13,12 +13,9 @@ def main(args): - """ Main entry point of the app """ - print("hello world") - print(args) # Get a handler for the current EC2 instance - server = Instance() + server = Instance(args.name) # Add the requested Stateful Devices to the server for device in args.backup: From 67d81162fb71c860a17e9953153d615600fd682d Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 14:06:55 -0400 Subject: [PATCH 033/134] set region using ec2 metadata --- sebs/ec2.py | 7 ++++--- tests/test_stateful_volume.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index af09645..3903161 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -24,7 +24,7 @@ def get_instance(self): sys.exit(1) try: - ec2 = boto3.resource('ec2') + ec2 = boto3.resource('ec2', region_name=ec2_metadata.region) instance = ec2.Instance(instance_id) # We have to call load to see if we are really connected instance.load() @@ -64,8 +64,9 @@ def __init__(self, instance_id, device_name, tag_name): self.status = 'Unknown' self.volume = None self.tag_name = tag_name - self.ec2_client = boto3.client('ec2') - self.ec2_resource = boto3.resource('ec2') + self.ec2_client = boto3.client('ec2', region_name=ec2_metadata.region) + self.ec2_resource = boto3.resource( + 'ec2', region_name=ec2_metadata.region) def get_status(self): diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index 80eb09b..fdae259 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -31,7 +31,8 @@ def setUp(self): self.boto3.resource = self.mock_resource modules = { - 'boto3': self.boto3 + 'boto3': self.boto3, + 'ec2_metadata': MagicMock() } self.instance_id = 'i-1234567890abcdef0' From 7596c8976416105cb9672f996c25527f113dface Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 14:33:27 -0400 Subject: [PATCH 034/134] added logging --- sebs/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sebs/__main__.py b/sebs/__main__.py index ec0d5bd..4687ade 100644 --- a/sebs/__main__.py +++ b/sebs/__main__.py @@ -17,6 +17,8 @@ def main(args): # Get a handler for the current EC2 instance server = Instance(args.name) + print(f'Running on {server.instance.instance_id}') + # Add the requested Stateful Devices to the server for device in args.backup: server.add_stateful_device(device) @@ -28,7 +30,7 @@ def main(args): server.tag_stateful_volumes() # I think we done? - + print('All done') sys.exit() From 42de2696c0979abeef14478db2922e6a0a8a6178 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 14:33:38 -0400 Subject: [PATCH 035/134] called methods and added logic --- sebs/ec2.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 3903161..be159f7 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -36,7 +36,7 @@ def get_instance(self): return instance def add_stateful_device(self, device_name): - + print(f'Handling {device_name}') sv = StatefulVolume(self.instance.id, device_name, self.volume_tag) sv.get_status() @@ -44,16 +44,15 @@ def add_stateful_device(self, device_name): self.backup.append(sv) def tag_stateful_volumes(self): - # Loop thrugh the volumes and call "tag_volume" on them. - pass + + for sv in self.backup: + sv.tag_volume() def attach_stateful_volumes(self): for sv in self.backup: - # call copy on the volume - - # call attach on the volume - pass + sv.copy(ec2_metadata.availability_zone) + sv.attach() class StatefulVolume: @@ -116,11 +115,12 @@ def get_status(self): self.volume = self.ec2_resource.Volume(volumeId) # Might have to do other stuffs + + print(f'{self.device_name} is {self.status}') return self.status def tag_volume(self): - # If status is 'New' Find the new volume and tag it - + print(f'Tagging {self.volume.volume_id} with control tag.') if self.status != 'New': return self.status @@ -137,6 +137,8 @@ def copy(self, target_az): if target_az == self.volume.availability_zone: return + print(f'Copying {self.volume.volume_id} to {target_az}') + snapshot = self.volume.create_snapshot( Description='Intermediate snapshot for SEBS.', TagSpecifications=[ @@ -186,8 +188,8 @@ def copy(self, target_az): waiter.wait(self.volume.volume_id) - def attach(self, instance): - + def attach(self): + print(f'Attaching {self.volume.volume_id} to {self.instance_id}') # Need to find and delete any current volumes response = self.ec2_client.describe_volumes( Filters=[ From 8c7cbf32b18b0c311eb5b47c37f084ecbb6b6673 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 14:39:19 -0400 Subject: [PATCH 036/134] added logic for handling new volumes --- sebs/ec2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sebs/ec2.py b/sebs/ec2.py index be159f7..8f2b8b6 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -133,6 +133,8 @@ def tag_volume(self): ) def copy(self, target_az): + if self.status == 'New': + return self.status # If the current volume az is in the target AZ do nothing if target_az == self.volume.availability_zone: return @@ -189,6 +191,9 @@ def copy(self, target_az): waiter.wait(self.volume.volume_id) def attach(self): + if self.status == 'New': + return self.status + print(f'Attaching {self.volume.volume_id} to {self.instance_id}') # Need to find and delete any current volumes response = self.ec2_client.describe_volumes( From 654c046532817535a2783c56133e7b2289da8a22 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 9 May 2020 14:40:11 -0400 Subject: [PATCH 037/134] fixed issue with list slice --- sebs/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 8f2b8b6..72861fa 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -215,7 +215,7 @@ def attach(self): if response['Volumes']: prev_volume = self.ec2_resource.Volume( - response['Volumes']['VolumeId']) + response['Volumes'][0]['VolumeId']) pre_volume.detach_from_instance( Device=self.device_name, From 618e62046a46edd1747eb07f645ab2d5acef7628 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 05:11:10 -0400 Subject: [PATCH 038/134] fix issue where it accepts not existing volumes --- sebs/ec2.py | 8 +++++++- tests/test_stateful_volume.py | 31 ++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 72861fa..c365506 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -92,12 +92,18 @@ def get_status(self): self.instance_id, ] }, + { + 'Name': 'attachment.device', + 'Values': [ + self.device_name, + ] + } ] ) if not response['Volumes']: print( - f"Could not find {self.device_name} for {self.instance_id}") + f"Could not find EBS volume mounted at {self.device_name} for {self.instance_id}") sys.exit(2) volumeId = response['Volumes'][0]['VolumeId'] diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index fdae259..cc25e5f 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -37,7 +37,7 @@ def setUp(self): self.instance_id = 'i-1234567890abcdef0' self.tag_name = 'sebs' - self.device_name = 'xdf' + self.device_name = '/dev/xdf' self.default_response = { 'Volumes': [ @@ -94,10 +94,17 @@ def test_new_volume(self): { 'Name': 'attachment.instance-id', 'Values': [self.instance_id] + }, + { + 'Name': 'attachment.device', + 'Values': [ + self.device_name, + ] } ]}) - sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) sv.get_status() @@ -114,7 +121,8 @@ def test_duplicate_volumes(self): self.stubber.add_response( 'describe_volumes', response, self.default_params) - sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) sv.get_status() @@ -127,7 +135,8 @@ def test_existing_volume(self): self.stubber.add_response( 'describe_volumes', self.default_response, self.default_params) - sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) sv.get_status() @@ -138,16 +147,18 @@ def test_existing_volume(self): 'Should be our volume mock') def test_class_properties(self): - sv = self.StatefulVolume(self.instance_id, 'xdf', 'test') + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) self.stubber.assert_no_pending_responses() - self.assertEqual(sv.device_name, 'xdf', 'Should set the deviceName') + self.assertEqual(sv.device_name, self.device_name, + 'Should set the deviceName') self.assertEqual( sv.ready, False, 'Should set the volume to not ready.') self.assertEqual(sv.status, 'Unknown', 'Should set the status') self.assertEqual(sv.volume, None, 'Should set the tag name') - self.assertEqual(sv.tag_name, 'test', 'Should set the tag name') + self.assertEqual(sv.tag_name, self.tag_name, 'Should set the tag name') def test_volume_tagging(self): response = self.default_response.copy() @@ -166,6 +177,12 @@ def test_volume_tagging(self): { 'Name': 'attachment.instance-id', 'Values': [self.instance_id] + }, + { + 'Name': 'attachment.device', + 'Values': [ + self.device_name, + ] } ]}) From 9eee234ef662feed7024da14f908e0605da4a4f3 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 05:28:09 -0400 Subject: [PATCH 039/134] update logging and status checks --- sebs/ec2.py | 16 ++++++++-------- tests/test_stateful_volume.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index c365506..217fb28 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -104,25 +104,25 @@ def get_status(self): if not response['Volumes']: print( f"Could not find EBS volume mounted at {self.device_name} for {self.instance_id}") - sys.exit(2) + self.status = 'Failed' + return self.status volumeId = response['Volumes'][0]['VolumeId'] + print(f'No pre-existing volume for {self.device_name}') + self.volume = self.ec2_resource.Volume(volumeId) elif len(response['Volumes']) != 1: - # too many volumes found - self.status = 'Duplicate' - # Might have to do other stuffs + print( + f"Found duplicate EBS volumes with tag {self.tag_name} for device {self.device_name}") + self.status = 'Failed' else: - # Previous backup volume found volumeId = response['Volumes'][0]['VolumeId'] + print(f'Found existing Volume {volumeId} for {self.device_name}') self.status = 'Not Attached' - self.volume = self.ec2_resource.Volume(volumeId) - # Might have to do other stuffs - print(f'{self.device_name} is {self.status}') return self.status def tag_volume(self): diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index cc25e5f..a26931b 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -127,7 +127,7 @@ def test_duplicate_volumes(self): sv.get_status() self.stubber.assert_no_pending_responses() - self.assertEqual(sv.status, 'Duplicate', + self.assertEqual(sv.status, 'Failed', 'Should be a duplicate volume') def test_existing_volume(self): From 7de5ef1dba3882bfda41570bd7abd82b23d1c641 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 05:52:16 -0400 Subject: [PATCH 040/134] adjust status --- sebs/ec2.py | 23 +++++++++++++++-------- tests/test_stateful_volume.py | 10 +++++++--- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 217fb28..740f5b9 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -104,19 +104,23 @@ def get_status(self): if not response['Volumes']: print( f"Could not find EBS volume mounted at {self.device_name} for {self.instance_id}") - self.status = 'Failed' + self.status = 'Missing' + self.ready = False return self.status volumeId = response['Volumes'][0]['VolumeId'] print(f'No pre-existing volume for {self.device_name}') - + self.status = 'Mounted' self.volume = self.ec2_resource.Volume(volumeId) + self.tag_volume() + elif len(response['Volumes']) != 1: print( f"Found duplicate EBS volumes with tag {self.tag_name} for device {self.device_name}") - self.status = 'Failed' + self.status = 'Duplicate' + self.ready = False else: volumeId = response['Volumes'][0]['VolumeId'] print(f'Found existing Volume {volumeId} for {self.device_name}') @@ -126,9 +130,8 @@ def get_status(self): return self.status def tag_volume(self): - print(f'Tagging {self.volume.volume_id} with control tag.') - if self.status != 'New': - return self.status + print( + f'Tagging {self.volume.volume_id} with control tag {self.tag_name}.') self.volume.create_tags(Tags=[ { @@ -138,8 +141,10 @@ def tag_volume(self): ] ) + self.ready = True + def copy(self, target_az): - if self.status == 'New': + if self.status != 'Not Attached': return self.status # If the current volume az is in the target AZ do nothing if target_az == self.volume.availability_zone: @@ -197,7 +202,7 @@ def copy(self, target_az): waiter.wait(self.volume.volume_id) def attach(self): - if self.status == 'New': + if self.status != 'Not Attached': return self.status print(f'Attaching {self.volume.volume_id} to {self.instance_id}') @@ -242,3 +247,5 @@ def attach(self): waiter = self.ec2_client.get_waiter('volume_in_use') waiter.wait(self.volume.volume_id) + + self.status = 'Mounted' diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index a26931b..47d63e7 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -109,9 +109,11 @@ def test_new_volume(self): sv.get_status() self.stubber.assert_no_pending_responses() - self.assertEqual(sv.status, 'New', 'Should be a new Volume') + self.assertEqual(sv.status, 'Mounted', + 'Volume should be mounted already.') self.assertIsInstance(sv.volume, Mock, - 'Should be our volume mock') + 'Should have a boto3 Volume resource') + self.assertEqual(sv.ready, True, 'Should be ready.') def test_duplicate_volumes(self): @@ -127,8 +129,9 @@ def test_duplicate_volumes(self): sv.get_status() self.stubber.assert_no_pending_responses() - self.assertEqual(sv.status, 'Failed', + self.assertEqual(sv.status, 'Duplicate', 'Should be a duplicate volume') + self.assertEqual(sv.ready, False, 'Volume should not be Ready') def test_existing_volume(self): @@ -145,6 +148,7 @@ def test_existing_volume(self): 'Should find an existing volume') self.assertIsInstance(sv.volume, Mock, 'Should be our volume mock') + self.assertEqual(sv.ready, False, 'Volume should not be Ready') def test_class_properties(self): sv = self.StatefulVolume( From 421f67b4a66a4af49359b640d2263e206e149517 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 06:31:39 -0400 Subject: [PATCH 041/134] fix issue with running twice in a row --- sebs/ec2.py | 17 ++++++++++++----- tests/test_stateful_volume.py | 14 ++++++++------ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 740f5b9..c1bfcd1 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -41,7 +41,8 @@ def add_stateful_device(self, device_name): sv.get_status() - self.backup.append(sv) + if sv.status != 'Failed': + self.backup.append(sv) def tag_stateful_volumes(self): @@ -111,7 +112,7 @@ def get_status(self): volumeId = response['Volumes'][0]['VolumeId'] print(f'No pre-existing volume for {self.device_name}') - self.status = 'Mounted' + self.status = 'Attached' self.volume = self.ec2_resource.Volume(volumeId) self.tag_volume() @@ -122,11 +123,17 @@ def get_status(self): self.status = 'Duplicate' self.ready = False else: - volumeId = response['Volumes'][0]['VolumeId'] + volume = response['Volumes'][0] + volumeId = volume['VolumeId'] print(f'Found existing Volume {volumeId} for {self.device_name}') self.status = 'Not Attached' self.volume = self.ec2_resource.Volume(volumeId) + print(self.volume) + for attachment in self.volume.attachments: + if attachment['InstanceId'] == self.instance_id: + self.status = 'Attached' + return self.status def tag_volume(self): @@ -228,7 +235,7 @@ def attach(self): prev_volume = self.ec2_resource.Volume( response['Volumes'][0]['VolumeId']) - pre_volume.detach_from_instance( + prev_volume.detach_from_instance( Device=self.device_name, InstanceId=self.instance_id ) @@ -248,4 +255,4 @@ def attach(self): waiter.wait(self.volume.volume_id) - self.status = 'Mounted' + self.status = 'Attached' diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index 47d63e7..ace2527 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -9,6 +9,11 @@ class TestStatefulVolume(unittest.TestCase): def setUp(self): + + self.instance_id = 'i-1234567890abcdef0' + self.tag_name = 'sebs' + self.device_name = '/dev/xdf' + # Setup our ec2 client stubb ec2 = botocore.session.get_session().create_client('ec2') self.stub_client = ec2 @@ -20,7 +25,8 @@ def setUp(self): self.boto3.client = self.mock_client # Setup resource and volume mocks - self.actual_volume = Mock(name='actual_volume') + self.actual_volume = Mock( + attachments=[{'InstanceId': ''}], name='actual_volume') self.mock_volume = MagicMock( name='volume_mock', return_value=self.actual_volume) self.mock_volume_class = MagicMock( @@ -35,10 +41,6 @@ def setUp(self): 'ec2_metadata': MagicMock() } - self.instance_id = 'i-1234567890abcdef0' - self.tag_name = 'sebs' - self.device_name = '/dev/xdf' - self.default_response = { 'Volumes': [ { @@ -109,7 +111,7 @@ def test_new_volume(self): sv.get_status() self.stubber.assert_no_pending_responses() - self.assertEqual(sv.status, 'Mounted', + self.assertEqual(sv.status, 'Attached', 'Volume should be mounted already.') self.assertIsInstance(sv.volume, Mock, 'Should have a boto3 Volume resource') From 9bf4b5d7976d8e6a56dbd2081f33d01feecaec38 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 06:35:02 -0400 Subject: [PATCH 042/134] remove debug print --- sebs/ec2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index c1bfcd1..76a16a2 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -129,7 +129,6 @@ def get_status(self): self.status = 'Not Attached' self.volume = self.ec2_resource.Volume(volumeId) - print(self.volume) for attachment in self.volume.attachments: if attachment['InstanceId'] == self.instance_id: self.status = 'Attached' From a30d20d446efbb09d9bf97dc1044fe0c76629269 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 06:53:16 -0400 Subject: [PATCH 043/134] change code to better handle operations order --- sebs/__main__.py | 3 +++ sebs/ec2.py | 13 ++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sebs/__main__.py b/sebs/__main__.py index 4687ade..ca03c98 100644 --- a/sebs/__main__.py +++ b/sebs/__main__.py @@ -29,6 +29,9 @@ def main(args): # Tag the Stateful Volumes so they can be found on next boot server.tag_stateful_volumes() + for sv in server.backup: + print(f"{sv.device_name} is {'Ready' if sv.ready else 'not Ready'} ") + # I think we done? print('All done') sys.exit() diff --git a/sebs/ec2.py b/sebs/ec2.py index 76a16a2..0a8149e 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -41,19 +41,20 @@ def add_stateful_device(self, device_name): sv.get_status() - if sv.status != 'Failed': - self.backup.append(sv) + self.backup.append(sv) def tag_stateful_volumes(self): for sv in self.backup: - sv.tag_volume() + if sv.status not in ['Duplicate', 'Missing']: + sv.tag_volume() def attach_stateful_volumes(self): for sv in self.backup: - sv.copy(ec2_metadata.availability_zone) - sv.attach() + if sv.status == 'Not Attached': + sv.copy(ec2_metadata.availability_zone) + sv.attach() class StatefulVolume: @@ -106,7 +107,6 @@ def get_status(self): print( f"Could not find EBS volume mounted at {self.device_name} for {self.instance_id}") self.status = 'Missing' - self.ready = False return self.status volumeId = response['Volumes'][0]['VolumeId'] @@ -121,7 +121,6 @@ def get_status(self): print( f"Found duplicate EBS volumes with tag {self.tag_name} for device {self.device_name}") self.status = 'Duplicate' - self.ready = False else: volume = response['Volumes'][0] volumeId = volume['VolumeId'] From dd2113c755d36d772777aa4d10fe180de13fdd45 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 07:12:27 -0400 Subject: [PATCH 044/134] fixed waiter commands --- sebs/ec2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 0a8149e..3255ed3 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -204,7 +204,7 @@ def copy(self, target_az): waiter = self.ec2_client.get_waiter('volume_available') - waiter.wait(self.volume.volume_id) + waiter.wait(VolumeIds=[self.volume.volume_id]) def attach(self): if self.status != 'Not Attached': @@ -240,7 +240,7 @@ def attach(self): waiter = self.ec2_client.get_waiter('volume_available') - waiter.wait(prev_volume.volume_id) + waiter.wait(VolumeIds=[prev_volume.volume_id]) prev_volume.delete() @@ -251,6 +251,6 @@ def attach(self): waiter = self.ec2_client.get_waiter('volume_in_use') - waiter.wait(self.volume.volume_id) + waiter.wait(VolumeIds=[self.volume.volume_id]) self.status = 'Attached' From 1b4359ca4b91fc782f3aaf316e07c1986ce8c1ec Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 07:44:30 -0400 Subject: [PATCH 045/134] use tenary operator to handle values of None --- sebs/ec2.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 3255ed3..f6a7b93 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -177,13 +177,12 @@ def copy(self, target_az): response = self.ec2_client.create_volume( AvailabilityZone=target_az, - Encrypted=snapshot.encrypted, - Iops=self.volume.iops, - KmsKeyId=self.volume.kms_key_id, - OutpostArn=self.volume.outpost_arn, - Size=self.volume.size, + Encrypted='' if not snapshot.encrypted else snapshot.encrypted, + Iops='' if not self.volume.iops else self.volume.iops, + KmsKeyId='' if not self.volume.kms_key_id else self.volume.kms_key_id, + OutpostArn='' if not self.volume.outpost_arn else self.volume.outpost_arn, SnapshotId=snapshot.snapshot_id, - VolumeType=self.volume.volume_type, + VolumeType='' if not self.volume.volume_type else self.volume.volume_type, TagSpecifications=[ { 'ResourceType': 'volume', From 1b51d975702f0984093bf554f268aa1280d89650 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 07:50:07 -0400 Subject: [PATCH 046/134] change encrypted to use boolean --- sebs/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index f6a7b93..989be21 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -177,7 +177,7 @@ def copy(self, target_az): response = self.ec2_client.create_volume( AvailabilityZone=target_az, - Encrypted='' if not snapshot.encrypted else snapshot.encrypted, + Encrypted=False if not snapshot.encrypted else snapshot.encrypted, Iops='' if not self.volume.iops else self.volume.iops, KmsKeyId='' if not self.volume.kms_key_id else self.volume.kms_key_id, OutpostArn='' if not self.volume.outpost_arn else self.volume.outpost_arn, From 174a8e819d5ce34ef536f4c1e621754726aa6a25 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 07:52:49 -0400 Subject: [PATCH 047/134] remove outpost as it does not accept empty string (aws bug) --- sebs/ec2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 989be21..6dd0c8b 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -180,7 +180,6 @@ def copy(self, target_az): Encrypted=False if not snapshot.encrypted else snapshot.encrypted, Iops='' if not self.volume.iops else self.volume.iops, KmsKeyId='' if not self.volume.kms_key_id else self.volume.kms_key_id, - OutpostArn='' if not self.volume.outpost_arn else self.volume.outpost_arn, SnapshotId=snapshot.snapshot_id, VolumeType='' if not self.volume.volume_type else self.volume.volume_type, TagSpecifications=[ From 55fc84369f5a83c3b9dfcb0b3f3ab537155c76f1 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 07:55:40 -0400 Subject: [PATCH 048/134] remove iops parameter as it depends on volume type --- sebs/ec2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 6dd0c8b..c816b26 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -175,10 +175,11 @@ def copy(self, target_az): # Not sure but we probably have to wait until its completed snapshot.wait_until_completed() + # If we fail to create the volume we need to remove this temp snapshot + response = self.ec2_client.create_volume( AvailabilityZone=target_az, Encrypted=False if not snapshot.encrypted else snapshot.encrypted, - Iops='' if not self.volume.iops else self.volume.iops, KmsKeyId='' if not self.volume.kms_key_id else self.volume.kms_key_id, SnapshotId=snapshot.snapshot_id, VolumeType='' if not self.volume.volume_type else self.volume.volume_type, From 7d57b906b900ff3f9155e99d4cb2ea270b4d4876 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 07:58:12 -0400 Subject: [PATCH 049/134] remove KMS key as it does not accept empty string (aws bug) --- sebs/ec2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index c816b26..5dd114a 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -180,7 +180,6 @@ def copy(self, target_az): response = self.ec2_client.create_volume( AvailabilityZone=target_az, Encrypted=False if not snapshot.encrypted else snapshot.encrypted, - KmsKeyId='' if not self.volume.kms_key_id else self.volume.kms_key_id, SnapshotId=snapshot.snapshot_id, VolumeType='' if not self.volume.volume_type else self.volume.volume_type, TagSpecifications=[ From 74d6d7b8437ddb7c069fad4a83ac321e90308ee0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 08:05:23 -0400 Subject: [PATCH 050/134] add debug statements --- sebs/ec2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sebs/ec2.py b/sebs/ec2.py index 5dd114a..9f5afa3 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -195,11 +195,14 @@ def copy(self, target_az): ] ) + print(response) + # Cleanup this temporary snapshot snapshot.delete() self.volume = self.ec2_resource.Volume(response['VolumeId']) + print(f'Waiting on volume {self.volume.volume_id} to be avaliable.') waiter = self.ec2_client.get_waiter('volume_available') waiter.wait(VolumeIds=[self.volume.volume_id]) From 8a78251989db9e3ebbbcfd514f30540d10932cf1 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 08:15:49 -0400 Subject: [PATCH 051/134] remove the encryption because volume is never created aws bug? --- sebs/ec2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 9f5afa3..219f628 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -179,7 +179,6 @@ def copy(self, target_az): response = self.ec2_client.create_volume( AvailabilityZone=target_az, - Encrypted=False if not snapshot.encrypted else snapshot.encrypted, SnapshotId=snapshot.snapshot_id, VolumeType='' if not self.volume.volume_type else self.volume.volume_type, TagSpecifications=[ From 15fedf2419365a9f8a4a2e4d973d3b781ff304d0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 08:23:11 -0400 Subject: [PATCH 052/134] dont delete snapshot while volume is being created --- sebs/ec2.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 219f628..d8aa24f 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -194,11 +194,6 @@ def copy(self, target_az): ] ) - print(response) - - # Cleanup this temporary snapshot - snapshot.delete() - self.volume = self.ec2_resource.Volume(response['VolumeId']) print(f'Waiting on volume {self.volume.volume_id} to be avaliable.') @@ -206,6 +201,9 @@ def copy(self, target_az): waiter.wait(VolumeIds=[self.volume.volume_id]) + # Cleanup this temporary snapshot + snapshot.delete() + def attach(self): if self.status != 'Not Attached': return self.status From e8125971d288e1cd837d5f0a623939fdbd2c539b Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 08:37:29 -0400 Subject: [PATCH 053/134] adding debug values --- sebs/ec2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sebs/ec2.py b/sebs/ec2.py index d8aa24f..28ded52 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -231,6 +231,9 @@ def attach(self): prev_volume = self.ec2_resource.Volume( response['Volumes'][0]['VolumeId']) + print( + f'Detaching curent Volume {prev_volume.volume_id} attached to {self.instance_id}') + prev_volume.detach_from_instance( Device=self.device_name, InstanceId=self.instance_id @@ -240,8 +243,12 @@ def attach(self): waiter.wait(VolumeIds=[prev_volume.volume_id]) + print('Waiting on detachment and then deleting.') + prev_volume.delete() + print(f'Attaching sebs {self.volume.volume_id} to {self.instance_id}') + self.volume.attach_to_instance( Device=self.device_name, InstanceId=self.instance_id From 07cfe20a73ea052e9e66ad7f8af300d1fc33c9b3 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 10 May 2020 09:13:50 -0400 Subject: [PATCH 054/134] cleanup volume thats in the wrong az --- sebs/ec2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 28ded52..9a2fabd 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -194,6 +194,9 @@ def copy(self, target_az): ] ) + # This should be the existing volume thats in the wrong AZ + prev_volume = self.volume + self.volume = self.ec2_resource.Volume(response['VolumeId']) print(f'Waiting on volume {self.volume.volume_id} to be avaliable.') @@ -201,7 +204,8 @@ def copy(self, target_az): waiter.wait(VolumeIds=[self.volume.volume_id]) - # Cleanup this temporary snapshot + # Cleanup this temporary resources + prev_volume.delete() snapshot.delete() def attach(self): From c83f8f59f1e61841cf25f135d1f2548c8179adaf Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Wed, 13 May 2020 04:29:29 -0400 Subject: [PATCH 055/134] update test names --- tests/test_stateful_volume.py | 110 ++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 30 deletions(-) diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index ace2527..ce16f03 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -1,4 +1,5 @@ import sys +import boto3 import unittest import datetime import botocore.session @@ -16,13 +17,20 @@ def setUp(self): # Setup our ec2 client stubb ec2 = botocore.session.get_session().create_client('ec2') - self.stub_client = ec2 - self.stubber = Stubber(ec2) + self.ec2_client = ec2 + self.stub_client = Stubber(ec2) + + # Setup our ec2 resource stub + ec2_resource = boto3.resource('ec2') + self.stub_resource = Stubber(ec2_resource.meta.client) # Use mocks to pass out client stubb to our code self.boto3 = MagicMock(name='module_mock') self.mock_client = MagicMock(name='client_mock', return_value=ec2) + self.mock_resource = MagicMock( + name='resource_mock', return_value=ec2_resource) self.boto3.client = self.mock_client + self.boto3.resource = self.mock_resource # Setup resource and volume mocks self.actual_volume = Mock( @@ -70,7 +78,7 @@ def setUp(self): self.default_params = {'Filters': ANY} - self.stubber.activate() + self.stub_client.activate() self.module_patcher = patch.dict('sys.modules', modules) self.module_patcher.start() @@ -81,17 +89,37 @@ def setUp(self): def tearDown(self): self.module_patcher.stop() - self.stubber.deactivate() + self.stub_client.deactivate() + + def test_class_properties(self): + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) - def test_new_volume(self): + self.stub_client.assert_no_pending_responses() + + self.assertEqual(sv.instance_id, self.instance_id, + 'Should set the instance_id we pass in.') + self.assertEqual(sv.device_name, self.device_name, + 'Should set the deviceName') + self.assertEqual( + sv.ready, False, 'Should set the volume to not ready.') + self.assertEqual(sv.status, 'Unknown', 'Should set the status') + self.assertEqual(sv.volume, None, 'Should set the tag name') + self.assertEqual(sv.tag_name, self.tag_name, 'Should set the tag name') + self.assertEqual(sv.ec2_client, self.ec2_client, + 'Should set an ec2 client') + self.assertEqual(sv.ec2_resource, self.mock_volume_class, + 'Should set our mock volume.') + + def test_status_new(self): response = self.default_response.copy() response['Volumes'] = [] - self.stubber.add_response( + self.stub_client.add_response( 'describe_volumes', response, self.default_params) - self.stubber.add_response( + self.stub_client.add_response( 'describe_volumes', self.default_response, {'Filters': [ { 'Name': 'attachment.instance-id', @@ -110,19 +138,53 @@ def test_new_volume(self): sv.get_status() - self.stubber.assert_no_pending_responses() + self.stub_client.assert_no_pending_responses() self.assertEqual(sv.status, 'Attached', 'Volume should be mounted already.') self.assertIsInstance(sv.volume, Mock, 'Should have a boto3 Volume resource') self.assertEqual(sv.ready, True, 'Should be ready.') - def test_duplicate_volumes(self): + def test_status_missing(self): + + response = self.default_response.copy() + response['Volumes'] = [] + + self.stub_client.add_response( + 'describe_volumes', response, self.default_params) + + self.stub_client.add_response( + 'describe_volumes', response, {'Filters': [ + { + 'Name': 'attachment.instance-id', + 'Values': [self.instance_id] + }, + { + 'Name': 'attachment.device', + 'Values': [ + self.device_name, + ] + } + ]}) + + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) + + sv.get_status() + + self.stub_client.assert_no_pending_responses() + self.assertEqual(sv.status, 'Missing', + 'We should not find a tagged volume.') + self.assertEqual(sv.volume, None, + 'Should not have set a volume resourse.') + self.assertEqual(sv.ready, False, 'Should not be ready.') + + def test_status_duplicate(self): response = self.default_response.copy() response['Volumes'] = [{}, {}] - self.stubber.add_response( + self.stub_client.add_response( 'describe_volumes', response, self.default_params) sv = self.StatefulVolume( @@ -130,14 +192,16 @@ def test_duplicate_volumes(self): sv.get_status() - self.stubber.assert_no_pending_responses() + self.stub_client.assert_no_pending_responses() self.assertEqual(sv.status, 'Duplicate', 'Should be a duplicate volume') + self.assertEqual(sv.volume, None, + 'Should not have set a volume resourse.') self.assertEqual(sv.ready, False, 'Volume should not be Ready') - def test_existing_volume(self): + def test_status_not_attached(self): - self.stubber.add_response( + self.stub_client.add_response( 'describe_volumes', self.default_response, self.default_params) sv = self.StatefulVolume( @@ -145,32 +209,18 @@ def test_existing_volume(self): sv.get_status() - self.stubber.assert_no_pending_responses() + self.stub_client.assert_no_pending_responses() self.assertEqual(sv.status, 'Not Attached', 'Should find an existing volume') self.assertIsInstance(sv.volume, Mock, 'Should be our volume mock') self.assertEqual(sv.ready, False, 'Volume should not be Ready') - def test_class_properties(self): - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) - - self.stubber.assert_no_pending_responses() - - self.assertEqual(sv.device_name, self.device_name, - 'Should set the deviceName') - self.assertEqual( - sv.ready, False, 'Should set the volume to not ready.') - self.assertEqual(sv.status, 'Unknown', 'Should set the status') - self.assertEqual(sv.volume, None, 'Should set the tag name') - self.assertEqual(sv.tag_name, self.tag_name, 'Should set the tag name') - def test_volume_tagging(self): response = self.default_response.copy() response['Volumes'] = [] - self.stubber.add_response( + self.stub_client.add_response( 'describe_volumes', response, {'Filters': [ { 'Name': f'tag:{self.tag_name}', @@ -178,7 +228,7 @@ def test_volume_tagging(self): } ]}) - self.stubber.add_response( + self.stub_client.add_response( 'describe_volumes', self.default_response, {'Filters': [ { 'Name': 'attachment.instance-id', From 96408515db4769768947790e5b04e6c953db137e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Wed, 13 May 2020 05:43:42 -0400 Subject: [PATCH 056/134] make sure we consitently return a status --- sebs/ec2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 9a2fabd..39a696d 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -153,7 +153,7 @@ def copy(self, target_az): return self.status # If the current volume az is in the target AZ do nothing if target_az == self.volume.availability_zone: - return + return self.status print(f'Copying {self.volume.volume_id} to {target_az}') @@ -208,6 +208,8 @@ def copy(self, target_az): prev_volume.delete() snapshot.delete() + return self.status + def attach(self): if self.status != 'Not Attached': return self.status From 9f199b17458bc707190b86719b96cc27adba4bab Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Wed, 13 May 2020 05:45:20 -0400 Subject: [PATCH 057/134] add partial coverage for copy function --- tests/test_stateful_volume.py | 76 ++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 6 deletions(-) diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index ce16f03..211edb0 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -33,16 +33,21 @@ def setUp(self): self.boto3.resource = self.mock_resource # Setup resource and volume mocks - self.actual_volume = Mock( - attachments=[{'InstanceId': ''}], name='actual_volume') + self.mock_snapshot = MagicMock( + name='snapshot_mock', snapshot_id='sn-12345') + self.actual_volume = Mock(name='actual_volume', + attachments=[{'InstanceId': ''}], volume_type='GP2', volume_id='vol-1111') + + self.actual_volume.create_snapshot = Mock( + return_value=self.mock_snapshot) self.mock_volume = MagicMock( name='volume_mock', return_value=self.actual_volume) self.mock_volume_class = MagicMock( name='volume_class_mock', return_value=self.mock_volume) self.mock_volume_class.Volume = self.mock_volume - self.mock_resource = MagicMock( + + self.boto3.resource = MagicMock( name='resource_mock', return_value=self.mock_volume_class) - self.boto3.resource = self.mock_resource modules = { 'boto3': self.boto3, @@ -201,6 +206,8 @@ def test_status_duplicate(self): def test_status_not_attached(self): + self.mock_volume + self.stub_client.add_response( 'describe_volumes', self.default_response, self.default_params) @@ -212,8 +219,8 @@ def test_status_not_attached(self): self.stub_client.assert_no_pending_responses() self.assertEqual(sv.status, 'Not Attached', 'Should find an existing volume') - self.assertIsInstance(sv.volume, Mock, - 'Should be our volume mock') + self.assertEqual(sv.volume, self.actual_volume, + 'Should be our volume mock') self.assertEqual(sv.ready, False, 'Volume should not be Ready') def test_volume_tagging(self): @@ -252,6 +259,63 @@ def test_volume_tagging(self): self.actual_volume.create_tags.assert_called_with( Tags=[{'Key': self.tag_name, 'Value': self.device_name}]) + def test_copy_new(self): + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) + sv.status = 'New' + + response = sv.copy('fakeAZ') + + self.assertEqual(response, sv.status, + "Should do nothing if status in not 'Not Attached'.") + + def test_copy_same_az(self): + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) + + self.actual_volume.availability_zone = 'fakeAZ' + sv.status = 'Not Attached' + sv.volume = self.actual_volume + + response = sv.copy('fakeAZ') + + self.assertEqual(response, sv.status, + "Should not change the status if in the same AZ.") + self.actual_volume.copy.assert_not_called() + + def test_copy_different_az(self): + + self.stub_client.add_response( + 'create_volume', {'VolumeId': 'vol-2222', + 'AvailabilityZone': 'newAZ', + 'Encrypted': True, + 'Size': 50, + 'SnapshotId': 'sn-2323', + 'VolumeType': 'gp2'}, {'AvailabilityZone': 'newAZ', + 'SnapshotId': ANY, + 'VolumeType': ANY, + 'TagSpecifications': ANY}) + + # Volume ID's dont match here because we are not creating a second mock volume correctly + self.stub_client.add_response('describe_volumes', {'Volumes': [ + {'VolumeId': 'vol-2222', 'State': 'available'}]}, {'VolumeIds': ['vol-1111']}) + + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) + + self.actual_volume.availability_zone = 'fakeAZ' + sv.status = 'Not Attached' + sv.volume = self.actual_volume + + response = sv.copy('newAZ') + + # Last test we need here is that we are actually creating a second volume + self.assertEqual(response, sv.status, + 'Should not change the status after a copy') + self.assertFalse(sv.ready, 'We should not be ready after copying.') + self.actual_volume.delete.assert_called_once() + self.mock_snapshot.delete.assert_called_once() + if __name__ == '__main__': unittest.main() From 8e55f669139074019525849cb9e05d45ec563100 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 14 May 2020 03:56:06 -0400 Subject: [PATCH 058/134] finish tests for copy --- tests/test_stateful_volume.py | 53 ++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index 211edb0..ec81a9c 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -27,27 +27,31 @@ def setUp(self): # Use mocks to pass out client stubb to our code self.boto3 = MagicMock(name='module_mock') self.mock_client = MagicMock(name='client_mock', return_value=ec2) - self.mock_resource = MagicMock( - name='resource_mock', return_value=ec2_resource) self.boto3.client = self.mock_client - self.boto3.resource = self.mock_resource # Setup resource and volume mocks self.mock_snapshot = MagicMock( name='snapshot_mock', snapshot_id='sn-12345') - self.actual_volume = Mock(name='actual_volume', - attachments=[{'InstanceId': ''}], volume_type='GP2', volume_id='vol-1111') + self.first_volume = Mock(name='first_volume', + attachments=[{'InstanceId': ''}], volume_type='GP2', volume_id='vol-1111') + + self.second_volume = Mock(name='second_volume', + attachments=[{'InstanceId': ''}], volume_type='GP2', volume_id='vol-2222') - self.actual_volume.create_snapshot = Mock( + self.first_volume.create_snapshot = Mock( return_value=self.mock_snapshot) - self.mock_volume = MagicMock( - name='volume_mock', return_value=self.actual_volume) - self.mock_volume_class = MagicMock( - name='volume_class_mock', return_value=self.mock_volume) - self.mock_volume_class.Volume = self.mock_volume + + self.mock_resource = MagicMock( + name='mock_resource_object') + + self.mock_resource.Volume = MagicMock( + name='mock_volume_constructor') + + self.mock_resource.Volume.side_effect = [ + self.first_volume, self.second_volume] self.boto3.resource = MagicMock( - name='resource_mock', return_value=self.mock_volume_class) + name='mock_resource_contructor', return_value=self.mock_resource) modules = { 'boto3': self.boto3, @@ -113,7 +117,7 @@ def test_class_properties(self): self.assertEqual(sv.tag_name, self.tag_name, 'Should set the tag name') self.assertEqual(sv.ec2_client, self.ec2_client, 'Should set an ec2 client') - self.assertEqual(sv.ec2_resource, self.mock_volume_class, + self.assertEqual(sv.ec2_resource, self.mock_resource, 'Should set our mock volume.') def test_status_new(self): @@ -206,8 +210,6 @@ def test_status_duplicate(self): def test_status_not_attached(self): - self.mock_volume - self.stub_client.add_response( 'describe_volumes', self.default_response, self.default_params) @@ -219,7 +221,7 @@ def test_status_not_attached(self): self.stub_client.assert_no_pending_responses() self.assertEqual(sv.status, 'Not Attached', 'Should find an existing volume') - self.assertEqual(sv.volume, self.actual_volume, + self.assertEqual(sv.volume, self.first_volume, 'Should be our volume mock') self.assertEqual(sv.ready, False, 'Volume should not be Ready') @@ -256,7 +258,7 @@ def test_volume_tagging(self): sv.tag_volume() - self.actual_volume.create_tags.assert_called_with( + self.first_volume.create_tags.assert_called_with( Tags=[{'Key': self.tag_name, 'Value': self.device_name}]) def test_copy_new(self): @@ -273,15 +275,15 @@ def test_copy_same_az(self): sv = self.StatefulVolume( self.instance_id, self.device_name, self.tag_name) - self.actual_volume.availability_zone = 'fakeAZ' + self.first_volume.availability_zone = 'fakeAZ' sv.status = 'Not Attached' - sv.volume = self.actual_volume + sv.volume = self.first_volume response = sv.copy('fakeAZ') self.assertEqual(response, sv.status, "Should not change the status if in the same AZ.") - self.actual_volume.copy.assert_not_called() + self.first_volume.copy.assert_not_called() def test_copy_different_az(self): @@ -296,16 +298,15 @@ def test_copy_different_az(self): 'VolumeType': ANY, 'TagSpecifications': ANY}) - # Volume ID's dont match here because we are not creating a second mock volume correctly self.stub_client.add_response('describe_volumes', {'Volumes': [ - {'VolumeId': 'vol-2222', 'State': 'available'}]}, {'VolumeIds': ['vol-1111']}) + {'VolumeId': 'vol-2222', 'State': 'available'}]}, {'VolumeIds': ['vol-2222']}) sv = self.StatefulVolume( self.instance_id, self.device_name, self.tag_name) - self.actual_volume.availability_zone = 'fakeAZ' + self.first_volume.availability_zone = 'fakeAZ' sv.status = 'Not Attached' - sv.volume = self.actual_volume + sv.volume = self.mock_resource.Volume() response = sv.copy('newAZ') @@ -313,7 +314,9 @@ def test_copy_different_az(self): self.assertEqual(response, sv.status, 'Should not change the status after a copy') self.assertFalse(sv.ready, 'We should not be ready after copying.') - self.actual_volume.delete.assert_called_once() + self.assertEqual(sv.volume, self.second_volume, + 'Should have our second volume.') + self.first_volume.delete.assert_called_once() self.mock_snapshot.delete.assert_called_once() From c6a7482b3bae327d88f329104940c876af68f2c6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 14 May 2020 04:17:57 -0400 Subject: [PATCH 059/134] add tests for attachment --- sebs/ec2.py | 2 ++ tests/test_stateful_volume.py | 53 +++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 39a696d..666d108 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -265,3 +265,5 @@ def attach(self): waiter.wait(VolumeIds=[self.volume.volume_id]) self.status = 'Attached' + + return self.status diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index ec81a9c..67ae939 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -268,7 +268,7 @@ def test_copy_new(self): response = sv.copy('fakeAZ') - self.assertEqual(response, sv.status, + self.assertEqual(response, 'New', "Should do nothing if status in not 'Not Attached'.") def test_copy_same_az(self): @@ -281,7 +281,7 @@ def test_copy_same_az(self): response = sv.copy('fakeAZ') - self.assertEqual(response, sv.status, + self.assertEqual(response, 'Not Attached', "Should not change the status if in the same AZ.") self.first_volume.copy.assert_not_called() @@ -311,7 +311,7 @@ def test_copy_different_az(self): response = sv.copy('newAZ') # Last test we need here is that we are actually creating a second volume - self.assertEqual(response, sv.status, + self.assertEqual(response, 'Not Attached', 'Should not change the status after a copy') self.assertFalse(sv.ready, 'We should not be ready after copying.') self.assertEqual(sv.volume, self.second_volume, @@ -319,6 +319,53 @@ def test_copy_different_az(self): self.first_volume.delete.assert_called_once() self.mock_snapshot.delete.assert_called_once() + def test_attach_new(self): + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) + sv.status = 'New' + + response = sv.attach() + + self.assertEqual(response, 'New', + "Should do nothing if status in not 'Not Attached'.") + + def test_attach(self): + + self.stub_client.add_response('describe_volumes', {'Volumes': [ + {'VolumeId': 'vol-1111', 'State': 'in-use'}]}, {'Filters': [ + { + 'Name': 'attachment.instance-id', + 'Values': [ + self.instance_id, + ] + }, + { + 'Name': 'attachment.device', + 'Values': [ + self.device_name, + ] + }, + ]}) + + self.stub_client.add_response('describe_volumes', {'Volumes': [ + {'VolumeId': 'vol-2222', 'State': 'available'}]}, {'VolumeIds': ['vol-1111']}) + + self.stub_client.add_response('describe_volumes', {'Volumes': [ + {'VolumeId': 'vol-2222', 'State': 'in-use'}]}, {'VolumeIds': ['vol-2222']}) + + sv = self.StatefulVolume( + self.instance_id, self.device_name, self.tag_name) + sv.status = 'Not Attached' + sv.volume = self.second_volume + + response = sv.attach() + + self.assertEqual(response, 'Attached', + 'Should be Attached to the instance.') + self.assertFalse(sv.ready, 'Should not be Ready') + # We shoud delete the previous volume + self.first_volume.delete.assert_called_once() + if __name__ == '__main__': unittest.main() From c2b5f56113307300d38408b8bcdd014ec01f5814 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 14 May 2020 06:17:36 -0400 Subject: [PATCH 060/134] add some tests for instance --- tests/test_instance.py | 76 +++++++++++++++++++++++++++++++++++ tests/test_stateful_volume.py | 5 ++- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tests/test_instance.py diff --git a/tests/test_instance.py b/tests/test_instance.py new file mode 100644 index 0000000..ae12775 --- /dev/null +++ b/tests/test_instance.py @@ -0,0 +1,76 @@ +import unittest +from sebs.ec2 import Instance +from unittest.mock import patch, MagicMock, call + + +class TestInstance(unittest.TestCase): + + def setUp(self): + self.default_tag = 'sebs' + self.device_name = '/dev/xdf' + self.mock_instance = MagicMock(name='mock_instance', id='in-1111') + + def tearDown(self): + pass + + @patch('sebs.ec2.Instance.get_instance') + def test_class_properites(self, mock_method): + mock_method.return_value = self.mock_instance + + server = Instance(self.default_tag) + + self.assertEqual(server.instance, self.mock_instance, + 'Should set an instance field.') + self.assertEqual(server.volume_tag, self.default_tag, + 'Should set volume_tag field.') + self.assertEqual(server.backup, [], 'Should have empty backup list.') + + mock_method.assert_called_once() + + @patch('sebs.ec2.StatefulVolume') + @patch('sebs.ec2.Instance.get_instance') + def test_add_device(self, mock_method, mock_volume_class): + mock_method.return_value = self.mock_instance + mock_volume = MagicMock(name='mock_volume') + mock_volume_class.return_value = mock_volume + + server = Instance(self.default_tag) + + server.add_stateful_device(self.device_name) + + mock_method.assert_called_once() + mock_volume_class.assert_called_once() + mock_volume_class.assert_called_once_with( + self.mock_instance.id, self.device_name, self.default_tag) + self.assertIn(mock_volume, server.backup, + 'Should put our device in the backup list.') + mock_volume.get_status.assert_called_once() + + @patch('sebs.ec2.StatefulVolume') + @patch('sebs.ec2.Instance.get_instance') + def test_add_multiple_devices(self, mock_method, mock_volume_class): + mock_method.return_value = self.mock_instance + mock_volume = MagicMock(name='mock_volume_1') + mock_volume2 = MagicMock(name='mock_volume_1') + mock_volume_class.return_value = mock_volume + + server = Instance(self.default_tag) + + server.add_stateful_device(self.device_name) + server.add_stateful_device('/dev/2') + + mock_method.assert_called_once() + self.assertEqual(mock_volume_class.call_count, 2, + 'Should create two volumes.') + + mock_volume_class.assert_has_calls( + [call(self.mock_instance.id, self.device_name, self.default_tag), + call(self.mock_instance.id, '/dev/2', self.default_tag)]) + + self.assertEqual(len(server.backup), 2, 'Should have two volumes.') + self.assertEqual(mock_volume.get_status.call_count, 2, + 'Should have called get_status twice') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index 67ae939..fb35fbd 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -6,11 +6,14 @@ from botocore.stub import Stubber, ANY from unittest.mock import patch, MagicMock, Mock +if('sebs.ec2' in sys.modules): + # We need to un-import it if imported already + del sys.modules["sebs.ec2"] + class TestStatefulVolume(unittest.TestCase): def setUp(self): - self.instance_id = 'i-1234567890abcdef0' self.tag_name = 'sebs' self.device_name = '/dev/xdf' From 58e8ed3735894bbe0ba1df9879ec9b5ff6d2b6bc Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 15 May 2020 05:19:53 -0400 Subject: [PATCH 061/134] add remaining tests for Instance Class --- tests/test_instance.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/test_instance.py b/tests/test_instance.py index ae12775..3654cea 100644 --- a/tests/test_instance.py +++ b/tests/test_instance.py @@ -1,6 +1,6 @@ import unittest from sebs.ec2 import Instance -from unittest.mock import patch, MagicMock, call +from unittest.mock import patch, MagicMock, call, PropertyMock class TestInstance(unittest.TestCase): @@ -51,7 +51,6 @@ def test_add_device(self, mock_method, mock_volume_class): def test_add_multiple_devices(self, mock_method, mock_volume_class): mock_method.return_value = self.mock_instance mock_volume = MagicMock(name='mock_volume_1') - mock_volume2 = MagicMock(name='mock_volume_1') mock_volume_class.return_value = mock_volume server = Instance(self.default_tag) @@ -71,6 +70,41 @@ def test_add_multiple_devices(self, mock_method, mock_volume_class): self.assertEqual(mock_volume.get_status.call_count, 2, 'Should have called get_status twice') + @patch('sebs.ec2.Instance.get_instance') + def test_tag_volumes(self, mock_method): + server = Instance(self.default_tag) + mock_volume = MagicMock(name='mock_volume_1', status='Not Attached') + mock_volume2 = MagicMock(name='mock_volume_2', status='Missing') + + server = Instance(self.default_tag) + server.backup = [mock_volume, mock_volume2] + server.tag_stateful_volumes() + + # Should tag one volume but not the other. + mock_volume.tag_volume.assert_called_once() + mock_volume2.tag_volume.assert_not_called() + + @patch('sebs.ec2.ec2_metadata') + @patch('sebs.ec2.Instance.get_instance') + def test_attach_volumes(self, mock_method, mock_metadata): + + p = PropertyMock(return_value='AZ2') + type(mock_metadata).availability_zone = p + + mock_volume = MagicMock(name='mock_volume_1', status='Not Attached') + mock_volume2 = MagicMock(name='mock_volume_2', status='Missing') + + server = Instance(self.default_tag) + server.backup = [mock_volume, mock_volume2] + server.attach_stateful_volumes() + + # Should Attach one volume but not the other. + mock_volume.copy.assert_called_once_with('AZ2') + mock_volume.attach.assert_called_once() + + mock_volume2.copy.assert_not_called() + mock_volume2.attach.assert_not_called() + if __name__ == '__main__': unittest.main() From b6adc8f942d1c82b3e5b358fb0aeb15db78e9787 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 15 May 2020 05:54:12 -0400 Subject: [PATCH 062/134] renamed main file --- sebs/{__main__.py => app.py} | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename sebs/{__main__.py => app.py} (90%) diff --git a/sebs/__main__.py b/sebs/app.py similarity index 90% rename from sebs/__main__.py rename to sebs/app.py index ca03c98..12e01c8 100644 --- a/sebs/__main__.py +++ b/sebs/app.py @@ -37,5 +37,6 @@ def main(args): sys.exit() -args = parse_args(sys.argv[1:], __version__) -main(args) +if __name__ == "__main__": + args = parse_args(sys.argv[1:], __version__) + main(args) From db954dd48b74e53f059d7391d91407a44a137194 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 15 May 2020 05:54:42 -0400 Subject: [PATCH 063/134] finally fixed name = main problem :blush: --- bin/sebs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bin/sebs b/bin/sebs index 4414820..da9fa40 100644 --- a/bin/sebs +++ b/bin/sebs @@ -1,11 +1,8 @@ #!/usr/bin/env python import sys - -from sebs import __main__ - -def main(): - __main__.main() +from sebs import app, cli if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + args = cli.parse_args(sys.argv[1:], app.__version__) + sys.exit(app.main(args)) From 2a21e06deb9f83e3ce3aa1685c1be51696eff1a6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 15 May 2020 05:55:01 -0400 Subject: [PATCH 064/134] now prints help if no args entered --- sebs/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sebs/cli.py b/sebs/cli.py index 6b8c3b6..1a38bbc 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -1,7 +1,9 @@ +import sys import argparse def parse_args(args, ver): + parser = argparse.ArgumentParser() parser.add_argument('-b', '--backup', action='append', @@ -25,4 +27,8 @@ def parse_args(args, ver): action="version", version="%(prog)s (version {version})".format(version=ver)) + if len(args) == 0: + parser.print_help(sys.stderr) + sys.exit(1) + return parser.parse_args(args) From 9c2e1ce30d607a5de905df655956e0d518d1e92f Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 15 May 2020 05:58:12 -0400 Subject: [PATCH 065/134] remove unused logic --- sebs/app.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sebs/app.py b/sebs/app.py index 12e01c8..c1085fa 100644 --- a/sebs/app.py +++ b/sebs/app.py @@ -8,7 +8,6 @@ __license__ = "GPLv3" import sys -from sebs.cli import parse_args from sebs.ec2 import Instance @@ -35,8 +34,3 @@ def main(args): # I think we done? print('All done') sys.exit() - - -if __name__ == "__main__": - args = parse_args(sys.argv[1:], __version__) - main(args) From 18f6e2f94f76671aee5b77f12ce1046d7bd7b8b6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 15 May 2020 06:55:16 -0400 Subject: [PATCH 066/134] add testing for main --- tests/test_main.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index d1158b8..bfdc224 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,14 +1,26 @@ import unittest -# Import causes problems with other test files =/ -# from sebs.__main__ import main +import argparse +from sebs.app import main +from unittest.mock import MagicMock, patch class TestApplicaton(unittest.TestCase): - def setUp(self): - pass - def test_main(self): - pass + @patch('sebs.app.Instance') + @patch('sebs.ec2.ec2_metadata') + def test_main(self, mock_metadata, mock_class): + mock_instance = MagicMock(name='mock_instance') + mock_class.return_value = mock_instance + + args = argparse.Namespace(name='sebs', backup=['/dev/xdv', '/dev/svh']) + with self.assertRaises(SystemExit): + main(args) + + mock_class.assert_called_once_with('sebs') + mock_instance.add_stateful_device.assert_any_call('/dev/xdv') + mock_instance.add_stateful_device.assert_any_call('/dev/svh') + mock_instance.attach_stateful_volumes.assert_called_once() + mock_instance.tag_stateful_volumes.assert_called_once() if __name__ == '__main__': From f3191e275e1619c8e142668cdaf157eb51f035ff Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 04:21:54 -0400 Subject: [PATCH 067/134] use logging module --- bin/sebs | 17 ++++++++++-- sebs/app.py | 11 ++++---- sebs/cli.py | 2 +- sebs/ec2.py | 78 ++++++++++++++++++++++++++++++++++------------------- 4 files changed, 73 insertions(+), 35 deletions(-) diff --git a/bin/sebs b/bin/sebs index da9fa40..be34af8 100644 --- a/bin/sebs +++ b/bin/sebs @@ -1,8 +1,21 @@ #!/usr/bin/env python import sys +import logging from sebs import app, cli +module = sys.modules['__main__'].__file__ +log = logging.getLogger(module) + if __name__ == "__main__": - args = cli.parse_args(sys.argv[1:], app.__version__) - sys.exit(app.main(args)) + """Main program. Sets up logging and do some work.""" + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, + format='%(name)s (%(levelname)s): %(message)s') + try: + args = cli.parse_args(sys.argv[1:], app.__version__) + log.setLevel(max(3 - args.verbose, 0) * 10) + sys.exit(app.main(args)) + except KeyboardInterrupt: + log.error('Program interrupted!') + finally: + logging.shutdown() diff --git a/sebs/app.py b/sebs/app.py index c1085fa..92da718 100644 --- a/sebs/app.py +++ b/sebs/app.py @@ -8,16 +8,18 @@ __license__ = "GPLv3" import sys +import logging from sebs.ec2 import Instance +log = logging.getLogger(__name__) + def main(args): + log.info(f'Starting...') # Get a handler for the current EC2 instance server = Instance(args.name) - print(f'Running on {server.instance.instance_id}') - # Add the requested Stateful Devices to the server for device in args.backup: server.add_stateful_device(device) @@ -29,8 +31,7 @@ def main(args): server.tag_stateful_volumes() for sv in server.backup: - print(f"{sv.device_name} is {'Ready' if sv.ready else 'not Ready'} ") + log.info(f"{sv.device_name} is {'Ready' if sv.ready else 'not Ready'}") - # I think we done? - print('All done') + log.info('Finished') sys.exit() diff --git a/sebs/cli.py b/sebs/cli.py index 1a38bbc..f70e5c3 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -18,7 +18,7 @@ def parse_args(args, ver): "-v", "--verbose", action="count", - default=0, + default=1, help="Verbosity (-v, -vv, etc)") # Specify output of "--version" diff --git a/sebs/ec2.py b/sebs/ec2.py index 666d108..5a6f5f2 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -1,8 +1,11 @@ import sys import boto3 +import logging import requests from ec2_metadata import ec2_metadata +log = logging.getLogger(__name__) + class Instance: def __init__(self, volume_tag): @@ -11,16 +14,17 @@ def __init__(self, volume_tag): self.backup = [] def get_instance(self): - + log.info('Getting EC2 instance metadata.') try: instance_id = ec2_metadata.instance_id + log.info(f'Running on {instance_id}') except requests.exceptions.ConnectTimeout: - print( + log.error( 'Failled to get instance metadata, are you sure you are running on an EC2 instance?') sys.exit(1) except: t, v, _tb = sys.exc_info() - print("Unexpected error {}: {}".format(t, v)) + sys.exit(1) try: @@ -30,13 +34,13 @@ def get_instance(self): instance.load() except: t, v, _tb = sys.exc_info() - print("Unexpected error {}: {}".format(t, v)) + log.error(f'Unexpected Error {t}: {v}') sys.exit(2) return instance def add_stateful_device(self, device_name): - print(f'Handling {device_name}') + log.info(f'Handling {device_name}') sv = StatefulVolume(self.instance.id, device_name, self.volume_tag) sv.get_status() @@ -44,13 +48,13 @@ def add_stateful_device(self, device_name): self.backup.append(sv) def tag_stateful_volumes(self): - + log.info(f'Tagging Volumes with control tag: {self.volume_tag}') for sv in self.backup: if sv.status not in ['Duplicate', 'Missing']: sv.tag_volume() def attach_stateful_volumes(self): - + log.info(f'Attaching Volumes to {self.instance.id}') for sv in self.backup: if sv.status == 'Not Attached': sv.copy(ec2_metadata.availability_zone) @@ -70,11 +74,11 @@ def __init__(self, instance_id, device_name, tag_name): 'ec2', region_name=ec2_metadata.region) def get_status(self): - + log.info(f'Checking for previous volume of {self.device_name}') response = self.ec2_client.describe_volumes( Filters=[ { - 'Name': 'tag:{}'.format(self.tag_name), + 'Name': f'tag:{self.tag_name}', 'Values': [ self.device_name, ] @@ -82,9 +86,13 @@ def get_status(self): ] ) + log.debug(f'Response of tag search: {response}') + if not response['Volumes']: - # No previous volume found + log.info(f'Did not find a previous volume for {self.device_name}') self.status = 'New' + log.info( + f'Checking if {self.device_name} is mounted to {self.instance_id}.') response = self.ec2_client.describe_volumes( Filters=[ @@ -103,28 +111,36 @@ def get_status(self): ] ) + log.debug(f'Reponse of local attachment search: {response}') + if not response['Volumes']: - print( + log.error( f"Could not find EBS volume mounted at {self.device_name} for {self.instance_id}") + self.status = 'Missing' return self.status volumeId = response['Volumes'][0]['VolumeId'] + log.info(f'No pre-existing volume for {self.device_name}') - print(f'No pre-existing volume for {self.device_name}') self.status = 'Attached' self.volume = self.ec2_resource.Volume(volumeId) - + log.info(f'Current volume is {volumeId} and is {self.status}') self.tag_volume() elif len(response['Volumes']) != 1: - print( - f"Found duplicate EBS volumes with tag {self.tag_name} for device {self.device_name}") + vol1 = response['Volumes'][0]['VolumeId'] + vol2 = response['Volumes'][1]['VolumeId'] + log.error( + f"Found duplicate EBS volumes with tag {self.tag_name}: {vol1} and {vol2}") + self.status = 'Duplicate' else: - volume = response['Volumes'][0] - volumeId = volume['VolumeId'] - print(f'Found existing Volume {volumeId} for {self.device_name}') + volumeId = response['Volumes'][0]['VolumeId'] + + log.info( + f'Found existing Volume {volumeId} for {self.device_name}') + self.status = 'Not Attached' self.volume = self.ec2_resource.Volume(volumeId) @@ -135,7 +151,7 @@ def get_status(self): return self.status def tag_volume(self): - print( + log.info( f'Tagging {self.volume.volume_id} with control tag {self.tag_name}.') self.volume.create_tags(Tags=[ @@ -143,8 +159,7 @@ def tag_volume(self): 'Key': self.tag_name, 'Value': self.device_name }, - ] - ) + ]) self.ready = True @@ -155,7 +170,7 @@ def copy(self, target_az): if target_az == self.volume.availability_zone: return self.status - print(f'Copying {self.volume.volume_id} to {target_az}') + log.info(f'Copying {self.volume.volume_id} to {target_az}') snapshot = self.volume.create_snapshot( Description='Intermediate snapshot for SEBS.', @@ -172,6 +187,8 @@ def copy(self, target_az): ] ) + log.debug(f'Snapshot: {snapshot.snapshot_id}') + # Not sure but we probably have to wait until its completed snapshot.wait_until_completed() @@ -194,12 +211,15 @@ def copy(self, target_az): ] ) + log.debug(f'New Volume: {response}') + # This should be the existing volume thats in the wrong AZ prev_volume = self.volume self.volume = self.ec2_resource.Volume(response['VolumeId']) - print(f'Waiting on volume {self.volume.volume_id} to be avaliable.') + log.info(f'Waiting on volume {self.volume.volume_id} to be avaliable.') + waiter = self.ec2_client.get_waiter('volume_available') waiter.wait(VolumeIds=[self.volume.volume_id]) @@ -214,7 +234,8 @@ def attach(self): if self.status != 'Not Attached': return self.status - print(f'Attaching {self.volume.volume_id} to {self.instance_id}') + log.info(f'Attaching {self.volume.volume_id} to {self.instance_id}') + # Need to find and delete any current volumes response = self.ec2_client.describe_volumes( Filters=[ @@ -233,11 +254,13 @@ def attach(self): ] ) + log.debug(f'Existing Volume: {response}') + if response['Volumes']: prev_volume = self.ec2_resource.Volume( response['Volumes'][0]['VolumeId']) - print( + log.info( f'Detaching curent Volume {prev_volume.volume_id} attached to {self.instance_id}') prev_volume.detach_from_instance( @@ -249,11 +272,12 @@ def attach(self): waiter.wait(VolumeIds=[prev_volume.volume_id]) - print('Waiting on detachment and then deleting.') + log.info('Waiting on detachment and then deleting.') prev_volume.delete() - print(f'Attaching sebs {self.volume.volume_id} to {self.instance_id}') + log.info( + f'Attaching sebs {self.volume.volume_id} to {self.instance_id}') self.volume.attach_to_instance( Device=self.device_name, From 2a00ed80d183ee8cff81b414413537974a96a808 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 04:46:20 -0400 Subject: [PATCH 068/134] update tests --- tests/test_cli.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index bcf376b..c43a87e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,19 +1,23 @@ import unittest +from io import StringIO +from unittest.mock import patch from sebs.cli import parse_args class TestArugmentParsing(unittest.TestCase): - def setUp(self): - pass - def test_version(self): - try: - version = "2.1.0" - parse_args(['--version'], version) - except: - pass + version = "2.1.0" + with patch('sys.stdout', new=StringIO()) as fakeOutput: + try: + parse_args(['--version'], version) + except: + pass + + output = fakeOutput.getvalue().strip() + self.assertTrue(version in output, + 'Should display the proper version.') def test_single_backup(self): args = parse_args(['-b test'], '') @@ -34,6 +38,11 @@ def test_override_name(self): self.assertEqual(args.name, ' not-default') + def test_verbose_level(self): + args = parse_args(['-b test1', '-b test2', '-vvv'], '') + + self.assertEqual(args.verbose, 4) + if __name__ == '__main__': unittest.main() From 45019676a37d55fb9d8376f374acf5237beebfa6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 04:46:36 -0400 Subject: [PATCH 069/134] Disable logging durring testing --- tests/test_stateful_volume.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_stateful_volume.py b/tests/test_stateful_volume.py index fb35fbd..0106e66 100644 --- a/tests/test_stateful_volume.py +++ b/tests/test_stateful_volume.py @@ -1,5 +1,6 @@ import sys import boto3 +import logging import unittest import datetime import botocore.session @@ -14,6 +15,9 @@ class TestStatefulVolume(unittest.TestCase): def setUp(self): + # Disable logging for testing + logging.disable(logging.CRITICAL) + self.instance_id = 'i-1234567890abcdef0' self.tag_name = 'sebs' self.device_name = '/dev/xdf' @@ -100,6 +104,9 @@ def setUp(self): self.StatefulVolume = StatefulVolume def tearDown(self): + # Turn logging back on + logging.disable(logging.NOTSET) + self.module_patcher.stop() self.stub_client.deactivate() @@ -194,7 +201,8 @@ def test_status_missing(self): def test_status_duplicate(self): response = self.default_response.copy() - response['Volumes'] = [{}, {}] + response['Volumes'] = [ + {'VolumeId': 'vol-1111'}, {'VolumeId': 'vol-2222'}] self.stub_client.add_response( 'describe_volumes', response, self.default_params) From c057f08904c4ea17a9b6c2ff44f1e58e9ad3fda8 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 05:15:30 -0400 Subject: [PATCH 070/134] move unit tests to its own directory --- scripts/pre-commit.bash | 2 +- scripts/run-tests.bash | 2 +- tests/{ => unit}/test_cli.py | 0 tests/{ => unit}/test_instance.py | 0 tests/{ => unit}/test_main.py | 0 tests/{ => unit}/test_stateful_volume.py | 0 6 files changed, 2 insertions(+), 2 deletions(-) rename tests/{ => unit}/test_cli.py (100%) rename tests/{ => unit}/test_instance.py (100%) rename tests/{ => unit}/test_main.py (100%) rename tests/{ => unit}/test_stateful_volume.py (100%) diff --git a/scripts/pre-commit.bash b/scripts/pre-commit.bash index 884d6ba..d308f97 100755 --- a/scripts/pre-commit.bash +++ b/scripts/pre-commit.bash @@ -13,9 +13,9 @@ echo echo "Linting changed python files" echo ./scripts/run-lint.bash -echo # $? stores exit value of the last command if [ $? -ne 0 ]; then + echo echo "Please lint before commiting." exit 1 fi diff --git a/scripts/run-tests.bash b/scripts/run-tests.bash index d1d1337..2eee195 100755 --- a/scripts/run-tests.bash +++ b/scripts/run-tests.bash @@ -10,7 +10,7 @@ cd "${0%/*}/.." # let's fake failing test for now echo -e "Running tests \n" -coverage run --source sebs -m unittest discover +coverage run --source sebs -m unittest discover -s tests/unit echo -e "\n" coverage report -m echo -e "\n" diff --git a/tests/test_cli.py b/tests/unit/test_cli.py similarity index 100% rename from tests/test_cli.py rename to tests/unit/test_cli.py diff --git a/tests/test_instance.py b/tests/unit/test_instance.py similarity index 100% rename from tests/test_instance.py rename to tests/unit/test_instance.py diff --git a/tests/test_main.py b/tests/unit/test_main.py similarity index 100% rename from tests/test_main.py rename to tests/unit/test_main.py diff --git a/tests/test_stateful_volume.py b/tests/unit/test_stateful_volume.py similarity index 100% rename from tests/test_stateful_volume.py rename to tests/unit/test_stateful_volume.py From 2ebc54e96280d56e4c6769b21c01fcc74e7c9c4b Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 11:05:08 -0400 Subject: [PATCH 071/134] add init files so tests are discoverable --- tests/functional/__init__.py | 0 tests/unit/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/functional/__init__.py create mode 100644 tests/unit/__init__.py diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 From 80203868f82059f8a5c640b24bbd920f3be6a212 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 11:06:26 -0400 Subject: [PATCH 072/134] add working functional test --- tests/functional/test_sebs.py | 195 ++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 tests/functional/test_sebs.py diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py new file mode 100644 index 0000000..b05a908 --- /dev/null +++ b/tests/functional/test_sebs.py @@ -0,0 +1,195 @@ +import json +import boto3 +import time +import unittest +import warnings +import subprocess + + +class TestSebs(unittest.TestCase): + + def setUp(self): + + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*") + + self.volume_cleanup = [] + self.instance_cleanup = [] + + self.ec2 = boto3.resource('ec2') + + self.iam = boto3.resource('iam') + + assume_role_policy_doc = {'Version': '2012-10-17'} + + assume_role_policy_doc['Statement'] = [{ + 'Action': [ + "sts:AssumeRole" + ], + 'Effect': 'Allow', + 'Principal': { + 'Service': ["ec2.amazonaws.com"] + } + }] + + self.iam_role = self.iam.create_role( + RoleName='Sebs', + AssumeRolePolicyDocument=json.dumps( + assume_role_policy_doc, indent=2), + Description='Used for functional testing.', + ) + + role_policy_doc = {'Version': '2012-10-17'} + + role_policy_doc['Statement'] = [{ + 'Action': [ + "ec2:*" + ], + 'Effect': 'Allow', + 'Resource': ["arn:aws:ec2:*:*:volume/*", "*"] + }] + + self.iam_role_policy = self.iam_role.Policy('Create-Volumes') + + self.iam_role_policy.put( + PolicyDocument=json.dumps(role_policy_doc, indent=2) + ) + + self.instance_profile = self.iam.create_instance_profile( + InstanceProfileName='Sebs-EC2-Profile', + ) + + self.instance_profile.add_role( + RoleName=self.iam_role.name + ) + + waiter = boto3.client('iam').get_waiter('instance_profile_exists') + + waiter.wait( + InstanceProfileName=self.instance_profile.name + ) + + self.git_ref = subprocess.check_output( + ["git", "rev-parse", "HEAD"]).strip().decode('ASCII') + + self.default_user_data = ( + "#!/bin/bash\n" + "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" + "yum install python3 git -y\n" + f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{self.git_ref}#egg=sebs-test --upgrade \n" + ) + + images = self.ec2.images.filter(Owners=['amazon'], Filters=[{'Name': 'name', + 'Values': ['amzn2*'] + }]) + + for image in images: + ami = image + break + + self.default_instance = dict(BlockDeviceMappings=[], + ImageId=ami.id, + InstanceType='t1.micro', + MaxCount=1, + MinCount=1, + Monitoring={'Enabled': False}, + UserData=self.default_user_data, + IamInstanceProfile={ + 'Arn': self.instance_profile.arn}, + TagSpecifications=[ + { + 'ResourceType': 'instance', + 'Tags': [ + { + 'Key': 'Name', + 'Value': 'Test' + }, + ] + }], + ) + + def tearDown(self): + + print('Running Cleanup...') + for instance in self.instance_cleanup: + print(f'Deleting {instance.id}') + instance.terminate() + + for volume in self.volume_cleanup: + print(f'Deleting {volume.id}') + volume.delete() + + if self.instance_profile: + print('Removing role from instance profile.') + self.instance_profile.remove_role( + RoleName=self.iam_role.name + ) + + if self.iam_role_policy: + print(f'Deleting policy: {self.iam_role_policy}') + self.iam_role_policy.delete() + + if self.iam_role: + print(f'Deleting Role: {self.iam_role}') + self.iam_role.delete() + + if self.instance_profile: + print(f'Deleting Instance Profile: {self.instance_profile}') + self.instance_profile.delete() + + def test_new_volume(self): + + device_name = '/dev/xvdh' + server_config = self.default_instance.copy() + server_config['BlockDeviceMappings'].append({ + 'DeviceName': device_name, + 'Ebs': { + 'DeleteOnTermination': True, + 'VolumeSize': 10, + 'VolumeType': 'gp2', + 'Encrypted': False + } + }) + + server_config['UserData'] += ( + f'/usr/local/bin/sebs -b {device_name}\n' + f'while [ ! -e {device_name} ] ; do sleep 1 ; done\n' + ) + + server_list = self.ec2.create_instances(**server_config) + + instance = server_list[0] + + self.instance_cleanup.append(instance) + + instance.wait_until_running() + instance.reload() + + for device in instance.block_device_mappings: + if device['DeviceName'] == device_name: + volume_id = device['Ebs']['VolumeId'] + break + + print(volume_id) + + waiter = boto3.client('ec2').get_waiter('volume_in_use') + + waiter.wait(VolumeIds=[volume_id]) + + volume = self.ec2.Volume(volume_id) + + # Need to add a customer waiting to look for tag creation. + time.sleep(120) + volume.reload() + print(volume.tags) + + for tag in volume.tags: + if tag['Key'] == 'sebs' or tag['Value'] == device_name: + tag_name = tag['Key'] + tag_value = tag['Value'] + break + + self.assertEqual(tag_name, 'sebs', + 'Volume should be tagged with our control tag.') + self.assertEqual(tag_value, device_name, + "Tag value should be it's device name.") From 4556fe6be83089b2330f7730a79c79c02d95377e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 11:23:10 -0400 Subject: [PATCH 073/134] change git ref to branch --- tests/functional/test_sebs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index b05a908..8a7f1ec 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -70,7 +70,7 @@ def setUp(self): ) self.git_ref = subprocess.check_output( - ["git", "rev-parse", "HEAD"]).strip().decode('ASCII') + ["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip().decode('ASCII') self.default_user_data = ( "#!/bin/bash\n" From d31a5acabb6d81f18ceda722f531ad335a126aa1 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 11:32:40 -0400 Subject: [PATCH 074/134] add test utils module --- tests/utils/__init__.py | 0 tests/utils/aws_utils.py | 62 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/aws_utils.py diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py new file mode 100644 index 0000000..e5de96b --- /dev/null +++ b/tests/utils/aws_utils.py @@ -0,0 +1,62 @@ +import json +import boto3 + + +def create_iam_resources(): + + iam = boto3.resource('iam') + + assume_role_policy_doc = {'Version': '2012-10-17'} + + assume_role_policy_doc['Statement'] = [{ + 'Action': [ + "sts:AssumeRole" + ], + 'Effect': 'Allow', + 'Principal': { + 'Service': ["ec2.amazonaws.com"] + } + }] + + iam_role = iam.create_role( + RoleName='Sebs', + AssumeRolePolicyDocument=json.dumps( + assume_role_policy_doc, indent=2), + Description='Used for functional testing.', + ) + + role_policy_doc = {'Version': '2012-10-17'} + + role_policy_doc['Statement'] = [{ + 'Action': [ + "ec2:*" + ], + 'Effect': 'Allow', + 'Resource': ["arn:aws:ec2:*:*:volume/*", "*"] + }] + + iam_role_policy = iam_role.Policy('Create-Volumes') + + iam_role_policy.put( + PolicyDocument=json.dumps(role_policy_doc, indent=2) + ) + + instance_profile = iam.create_instance_profile( + InstanceProfileName='Sebs-EC2-Profile', + ) + + instance_profile.add_role( + RoleName=iam_role.name + ) + + waiter = boto3.client('iam').get_waiter('instance_profile_exists') + + waiter.wait( + InstanceProfileName=instance_profile.name + ) + + return { + 'role': iam_role, + 'policy': iam_role_policy, + 'profile': instance_profile + } From 9a070e319e75112ac44d4ed9a1d025cad69a9213 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 12:42:02 -0400 Subject: [PATCH 075/134] moved setup code to helper utils --- tests/functional/test_sebs.py | 116 +++++++--------------------------- tests/utils/aws_utils.py | 100 ++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 96 deletions(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index 8a7f1ec..dfa2c62 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -3,13 +3,14 @@ import time import unittest import warnings -import subprocess + +from tests.utils import aws_utils class TestSebs(unittest.TestCase): def setUp(self): - + print('Setting up test environment...') warnings.filterwarnings( "ignore", category=ResourceWarning, message="unclosed.*") @@ -20,75 +21,20 @@ def setUp(self): self.iam = boto3.resource('iam') - assume_role_policy_doc = {'Version': '2012-10-17'} - - assume_role_policy_doc['Statement'] = [{ - 'Action': [ - "sts:AssumeRole" - ], - 'Effect': 'Allow', - 'Principal': { - 'Service': ["ec2.amazonaws.com"] - } - }] - - self.iam_role = self.iam.create_role( - RoleName='Sebs', - AssumeRolePolicyDocument=json.dumps( - assume_role_policy_doc, indent=2), - Description='Used for functional testing.', - ) + iam_resources = aws_utils.create_iam_resources() - role_policy_doc = {'Version': '2012-10-17'} + self.iam_role = iam_resources['role'] - role_policy_doc['Statement'] = [{ - 'Action': [ - "ec2:*" - ], - 'Effect': 'Allow', - 'Resource': ["arn:aws:ec2:*:*:volume/*", "*"] - }] + self.iam_role_policy = iam_resources['policy'] - self.iam_role_policy = self.iam_role.Policy('Create-Volumes') + self.instance_profile = iam_resources['profile'] - self.iam_role_policy.put( - PolicyDocument=json.dumps(role_policy_doc, indent=2) - ) + self.default_user_data = aws_utils.create_default_userdata() - self.instance_profile = self.iam.create_instance_profile( - InstanceProfileName='Sebs-EC2-Profile', - ) - - self.instance_profile.add_role( - RoleName=self.iam_role.name - ) - - waiter = boto3.client('iam').get_waiter('instance_profile_exists') - - waiter.wait( - InstanceProfileName=self.instance_profile.name - ) - - self.git_ref = subprocess.check_output( - ["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip().decode('ASCII') - - self.default_user_data = ( - "#!/bin/bash\n" - "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" - "yum install python3 git -y\n" - f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{self.git_ref}#egg=sebs-test --upgrade \n" - ) - - images = self.ec2.images.filter(Owners=['amazon'], Filters=[{'Name': 'name', - 'Values': ['amzn2*'] - }]) - - for image in images: - ami = image - break + ami_id = aws_utils.get_latest_ami() self.default_instance = dict(BlockDeviceMappings=[], - ImageId=ami.id, + ImageId=ami_id, InstanceType='t1.micro', MaxCount=1, MinCount=1, @@ -141,55 +87,37 @@ def test_new_volume(self): device_name = '/dev/xvdh' server_config = self.default_instance.copy() - server_config['BlockDeviceMappings'].append({ - 'DeviceName': device_name, - 'Ebs': { - 'DeleteOnTermination': True, - 'VolumeSize': 10, - 'VolumeType': 'gp2', - 'Encrypted': False - } - }) + server_config['BlockDeviceMappings'].append( + aws_utils.create_block_device(device_name)) + control_tag = 'test-new-sebs' server_config['UserData'] += ( - f'/usr/local/bin/sebs -b {device_name}\n' + f'/usr/local/bin/sebs -b {device_name} -n {control_tag}\n' f'while [ ! -e {device_name} ] ; do sleep 1 ; done\n' ) - server_list = self.ec2.create_instances(**server_config) - - instance = server_list[0] + instance = aws_utils.create_instance(server_config) self.instance_cleanup.append(instance) instance.wait_until_running() instance.reload() - for device in instance.block_device_mappings: - if device['DeviceName'] == device_name: - volume_id = device['Ebs']['VolumeId'] - break - - print(volume_id) + volume_id = aws_utils.get_volume_from_bdm(device_name, instance) - waiter = boto3.client('ec2').get_waiter('volume_in_use') + self.assertTrue(volume_id, 'Should have the new volume attached') + waiter = aws_utils.get_ec2_waiter('volume_in_use') waiter.wait(VolumeIds=[volume_id]) volume = self.ec2.Volume(volume_id) - # Need to add a customer waiting to look for tag creation. - time.sleep(120) - volume.reload() - print(volume.tags) + aws_utils.wait_for_volume_tag(volume) - for tag in volume.tags: - if tag['Key'] == 'sebs' or tag['Value'] == device_name: - tag_name = tag['Key'] - tag_value = tag['Value'] - break + tag_name, tag_value = aws_utils.get_control_tag( + control_tag, volume.tags) - self.assertEqual(tag_name, 'sebs', + self.assertEqual(tag_name, control_tag, 'Volume should be tagged with our control tag.') self.assertEqual(tag_value, device_name, "Tag value should be it's device name.") diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index e5de96b..02cca60 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -1,9 +1,11 @@ import json import boto3 +import time +import subprocess def create_iam_resources(): - + print('Creating IAM resources for testing.') iam = boto3.resource('iam') assume_role_policy_doc = {'Version': '2012-10-17'} @@ -48,15 +50,109 @@ def create_iam_resources(): instance_profile.add_role( RoleName=iam_role.name ) - + print('Waiting on instance profile creation.') waiter = boto3.client('iam').get_waiter('instance_profile_exists') waiter.wait( InstanceProfileName=instance_profile.name ) + print('IAM resources created.') + return { 'role': iam_role, 'policy': iam_role_policy, 'profile': instance_profile } + + +def create_default_userdata(): + git_ref = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"] + ).strip().decode('ASCII') + + return ( + "#!/bin/bash\n" + "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" + "yum install python3 git -y\n" + f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs-test --upgrade \n" + ) + + +def get_latest_ami(): + ec2 = boto3.resource('ec2') + images = ec2.images.filter(Owners=['amazon'], + Filters=[ + {'Name': 'name', + 'Values': ['amzn2*'] + }]) + + for image in images: + ami = image + break + + return ami.id + + +def create_block_device(device_name): + return { + 'DeviceName': device_name, + 'Ebs': { + 'DeleteOnTermination': True, + 'VolumeSize': 10, + 'VolumeType': 'gp2', + 'Encrypted': False + } + } + + +def create_instance(instance_config): + ec2 = boto3.resource('ec2') + server_list = ec2.create_instances(**instance_config) + + instance = server_list[0] + + return instance + + +def get_volume_from_bdm(device_name, instance): + volume_id = '' + for device in instance.block_device_mappings: + if device['DeviceName'] == device_name: + volume_id = device['Ebs']['VolumeId'] + break + + return volume_id + + +def get_ec2_waiter(name): + waiter = boto3.client('ec2').get_waiter(name) + return waiter + + +def get_control_tag(control_tag, tags): + tag_name = '' + tag_value = '' + for tag in tags: + if tag['Key'] == control_tag: + tag_name = tag['Key'] + tag_value = tag['Value'] + break + + return tag_name, tag_value + + +def wait_for_volume_tag(volume): + i = 0 + while True: + i += 1 + + volume.reload() + + if volume.tags: + break + + time.sleep(10) + + if i > 14: + break From 4f6d69b73c96837fc582809c4d33d0b36915ccf0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 16 May 2020 12:46:01 -0400 Subject: [PATCH 076/134] fix tag name --- tests/functional/test_sebs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index dfa2c62..4d5f2f3 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -90,7 +90,7 @@ def test_new_volume(self): server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device_name)) - control_tag = 'test-new-sebs' + control_tag = 'new-volume-sebs' server_config['UserData'] += ( f'/usr/local/bin/sebs -b {device_name} -n {control_tag}\n' f'while [ ! -e {device_name} ] ; do sleep 1 ; done\n' From af2782365bd283fa2232cef54cf8cb5a7d1c2d8f Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 17 May 2020 06:38:10 -0400 Subject: [PATCH 077/134] change logger so boto doesn't log --- bin/sebs | 3 +-- sebs/app.py | 2 +- sebs/ec2.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/sebs b/bin/sebs index be34af8..5a261bf 100644 --- a/bin/sebs +++ b/bin/sebs @@ -4,8 +4,7 @@ import sys import logging from sebs import app, cli -module = sys.modules['__main__'].__file__ -log = logging.getLogger(module) +log = logging.getLogger('sebs') if __name__ == "__main__": """Main program. Sets up logging and do some work.""" diff --git a/sebs/app.py b/sebs/app.py index 92da718..a171745 100644 --- a/sebs/app.py +++ b/sebs/app.py @@ -11,7 +11,7 @@ import logging from sebs.ec2 import Instance -log = logging.getLogger(__name__) +log = logging.getLogger('sebs') def main(args): diff --git a/sebs/ec2.py b/sebs/ec2.py index 5a6f5f2..085cf2e 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -4,7 +4,7 @@ import requests from ec2_metadata import ec2_metadata -log = logging.getLogger(__name__) +log = logging.getLogger('sebs') class Instance: From 0920e79bc29c9efed89d25f4f8b0039f2893b327 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 17 May 2020 06:38:44 -0400 Subject: [PATCH 078/134] add helper functions for exisiting volumes --- tests/utils/aws_utils.py | 65 ++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 02cca60..8db2065 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -130,16 +130,14 @@ def get_ec2_waiter(name): return waiter -def get_control_tag(control_tag, tags): - tag_name = '' - tag_value = '' - for tag in tags: - if tag['Key'] == control_tag: - tag_name = tag['Key'] - tag_value = tag['Value'] - break +def has_control_tag(control_tag, device_name, volume): + + wait_for_volume_tag(volume) + + tags = [tag for tag in volume.tags if tag['Key'] + == control_tag and tag['Value'] == device_name] - return tag_name, tag_value + return bool(tags) def wait_for_volume_tag(volume): @@ -156,3 +154,52 @@ def wait_for_volume_tag(volume): if i > 14: break + + +def get_default_vpc(): + ec2 = boto3.resource('ec2') + + vpcs = ec2.vpcs.all() + + default_vpc = next(vpc for vpc in vpcs if vpc.is_default) + + return default_vpc + + +def get_avaliable_az(): + + vpc = get_default_vpc() + + az = [subnet.availability_zone for subnet in vpc.subnets.all()] + + print(az) + + return az + + +def create_existing_volume(control_tag, device_name, az): + ec2 = boto3.resource('ec2') + + volume = ec2.create_volume( + AvailabilityZone=az, + Encrypted=False, + Size=10, + VolumeType='gp2', + TagSpecifications=[ + { + 'ResourceType': 'volume', + 'Tags': [ + { + 'Key': control_tag, + 'Value': device_name + } + ] + } + ] + ) + + waiter = get_ec2_waiter('volume_available') + + waiter.wait(VolumeIds=[volume.id]) + + return volume From 93718ddae1009a2dc2c193b3b509adc7f19541e6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 17 May 2020 07:07:30 -0400 Subject: [PATCH 079/134] turn off other loggers --- sebs/ec2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sebs/ec2.py b/sebs/ec2.py index 085cf2e..559da02 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -6,6 +6,10 @@ log = logging.getLogger('sebs') +for name in logging.Logger.manager.loggerDict.keys(): + if ('boto' in name) or ('urllib3' in name): + logging.getLogger(name).setLevel(logging.WARNING) + class Instance: def __init__(self, volume_tag): From 199c284bb66924e8a4c5e995936300afe21b6ea5 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 17 May 2020 09:12:43 -0400 Subject: [PATCH 080/134] add functional tests --- tests/functional/test_sebs.py | 227 ++++++++++++++++++++++++++++++++-- 1 file changed, 216 insertions(+), 11 deletions(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index 4d5f2f3..e877aad 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -1,10 +1,9 @@ import json import boto3 -import time import unittest import warnings - from tests.utils import aws_utils +from botocore.exceptions import ClientError class TestSebs(unittest.TestCase): @@ -61,9 +60,17 @@ def tearDown(self): print(f'Deleting {instance.id}') instance.terminate() + waiter = aws_utils.get_ec2_waiter('instance_terminated') + + waiter.wait(InstanceIds=[instance.id]) + for volume in self.volume_cleanup: print(f'Deleting {volume.id}') - volume.delete() + try: + volume.delete() + except ClientError as e: + if e.response['Error']['Code'] != 'InvalidVolume.NotFound': + raise e if self.instance_profile: print('Removing role from instance profile.') @@ -85,12 +92,13 @@ def tearDown(self): def test_new_volume(self): + control_tag = 'new-volume-sebs' device_name = '/dev/xvdh' + server_config = self.default_instance.copy() server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device_name)) - control_tag = 'new-volume-sebs' server_config['UserData'] += ( f'/usr/local/bin/sebs -b {device_name} -n {control_tag}\n' f'while [ ! -e {device_name} ] ; do sleep 1 ; done\n' @@ -112,12 +120,209 @@ def test_new_volume(self): volume = self.ec2.Volume(volume_id) - aws_utils.wait_for_volume_tag(volume) + volume_tagged = aws_utils.has_control_tag( + control_tag, device_name, volume) + + self.assertTrue(volume_tagged, 'Volume should have our control tag.') + + def test_new_volumes(self): + + control_tag = 'new-volumes-sebs' + device1_name = '/dev/xvdh' + device2_name = '/dev/xvdm' + + server_config = self.default_instance.copy() + server_config['BlockDeviceMappings'].append( + aws_utils.create_block_device(device1_name)) + server_config['BlockDeviceMappings'].append( + aws_utils.create_block_device(device2_name)) + + server_config['UserData'] += ( + f'/usr/local/bin/sebs -b {device1_name} -b {device2_name} -n {control_tag}\n' + f'while [ ! -e {device1_name} ] ; do sleep 1 ; done\n' + f'while [ ! -e {device2_name} ] ; do sleep 1 ; done\n' + ) + + instance = aws_utils.create_instance(server_config) + + self.instance_cleanup.append(instance) + + instance.wait_until_running() + instance.reload() + + volume1_id = aws_utils.get_volume_from_bdm(device1_name, instance) + volume2_id = aws_utils.get_volume_from_bdm(device2_name, instance) + + self.assertTrue( + volume1_id, 'Should have the new volume attached') + self.assertTrue( + volume2_id, 'Should have the new volume attached') + + waiter = aws_utils.get_ec2_waiter('volume_in_use') + waiter.wait(VolumeIds=[volume1_id]) + waiter.wait(VolumeIds=[volume2_id]) + + volume1 = self.ec2.Volume(volume1_id) + volume2 = self.ec2.Volume(volume2_id) + + volume1_tagged = aws_utils.has_control_tag( + control_tag, device1_name, volume1) + volume2_tagged = aws_utils.has_control_tag( + control_tag, device2_name, volume2) + + self.assertTrue(volume1_tagged, 'Volume should have our control tag.') + self.assertTrue(volume2_tagged, 'Volume should have our control tag.') + + def test_existing_volume(self): + + control_tag = 'existing-volume-sebs' + device_name = '/dev/xvdh' + + az = aws_utils.get_avaliable_az() + + existing_vol = aws_utils.create_existing_volume( + control_tag, device_name, az[0]) + + print(f'Created Existing Volume {existing_vol.id}') + + server_config = self.default_instance.copy() + server_config['BlockDeviceMappings'].append( + aws_utils.create_block_device(device_name)) + server_config['Placement'] = { + 'AvailabilityZone': az[1] + } + + server_config['UserData'] += ( + f'/usr/local/bin/sebs -b {device_name} -n {control_tag}\n' + f'while [ ! -e {device_name} ] ; do sleep 1 ; done\n' + ) + + instance = aws_utils.create_instance(server_config) + + self.instance_cleanup.append(instance) + + instance.wait_until_running() + instance.reload() + + volume_id = aws_utils.get_volume_from_bdm(device_name, instance) + + default_vol = self.ec2.Volume(volume_id) + + print(f'Default Volume {default_vol.id}') + + self.assertTrue(volume_id, 'Should have the default volume attached') + + waiter = aws_utils.get_ec2_waiter('volume_deleted') + waiter.wait(VolumeIds=[volume_id]) + + instance.reload() + + new_vol_id = aws_utils.get_volume_from_bdm(device_name, instance) + + new_vol = self.ec2.Volume(new_vol_id) + self.volume_cleanup.append(new_vol) + + print(f'Final Volume {new_vol.id}') + + self.assertNotEqual(new_vol.id, default_vol.id, + 'Volume attached now should not be what we started with.') + + volume_tagged = aws_utils.has_control_tag( + control_tag, device_name, new_vol) + + waiter = aws_utils.get_ec2_waiter('volume_deleted') + + waiter.wait(VolumeIds=[existing_vol.id, default_vol.id]) + + self.assertTrue(volume_tagged, 'Volume should have our control tag.') + + def test_existing_volumes(self): + + control_tag = 'existing-volumes-sebs' + device1_name = '/dev/xvdh' + device2_name = '/dev/xvdm' + + az = aws_utils.get_avaliable_az() + + existing_vol1 = aws_utils.create_existing_volume( + control_tag, device1_name, az[0]) + + existing_vol2 = aws_utils.create_existing_volume( + control_tag, device2_name, az[0]) + + self.volume_cleanup.extend([existing_vol1, existing_vol2]) + + server_config = self.default_instance.copy() + server_config['BlockDeviceMappings'].append( + aws_utils.create_block_device(device1_name)) + server_config['BlockDeviceMappings'].append( + aws_utils.create_block_device(device2_name)) + server_config['Placement'] = { + 'AvailabilityZone': az[1] + } + + server_config['UserData'] += ( + f'/usr/local/bin/sebs -b {device1_name} -b {device2_name} -n {control_tag}\n' + f'while [ ! -e {device1_name} ] ; do sleep 1 ; done\n' + f'while [ ! -e {device2_name} ] ; do sleep 1 ; done\n' + ) + + instance = aws_utils.create_instance(server_config) + + self.instance_cleanup.append(instance) + + instance.wait_until_running() + instance.reload() + + default1_vol_id = aws_utils.get_volume_from_bdm(device1_name, instance) + default2_vol_id = aws_utils.get_volume_from_bdm(device2_name, instance) + + default_vol1 = self.ec2.Volume(default1_vol_id) + default_vol2 = self.ec2.Volume(default2_vol_id) + + self.volume_cleanup.extend([default_vol1, default_vol2]) + + self.assertTrue(default1_vol_id, + 'Should have the default volume attached') + self.assertTrue(default2_vol_id, + 'Should have the default volume attached') + + waiter = aws_utils.get_ec2_waiter('volume_deleted') + waiter.wait(VolumeIds=[default1_vol_id]) + waiter.wait(VolumeIds=[default2_vol_id]) + + instance.reload() + + new1_vol_id = aws_utils.get_volume_from_bdm(device1_name, instance) + new2_vol_id = aws_utils.get_volume_from_bdm(device2_name, instance) + + new_vol1 = self.ec2.Volume(new1_vol_id) + new_vol2 = self.ec2.Volume(new2_vol_id) + + self.volume_cleanup.extend([new_vol1, new_vol2]) + + self.volume_cleanup.append(new_vol1) + self.volume_cleanup.append(new_vol2) + + self.assertNotEqual(new_vol1.id, default_vol1.id, + 'Volume attached now should not be what we started with.') + self.assertNotEqual(new_vol2.id, default_vol2.id, + 'Volume attached now should not be what we started with.') + + volume1_tagged = aws_utils.has_control_tag( + control_tag, device1_name, new_vol1) + + volume2_tagged = aws_utils.has_control_tag( + control_tag, device2_name, new_vol2) + + waiter = aws_utils.get_ec2_waiter('volume_deleted') - tag_name, tag_value = aws_utils.get_control_tag( - control_tag, volume.tags) + waiter.wait(VolumeIds=[ + existing_vol1.id, + default_vol1.id, + existing_vol2.id, + default_vol2.id + ]) - self.assertEqual(tag_name, control_tag, - 'Volume should be tagged with our control tag.') - self.assertEqual(tag_value, device_name, - "Tag value should be it's device name.") + self.assertTrue(volume1_tagged, 'Volume should have our control tag.') + self.assertTrue(volume2_tagged, 'Volume should have our control tag.') From aaca8da7ef34b4526d6aa7f01bf9d25eeeec02b8 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sun, 17 May 2020 09:23:13 -0400 Subject: [PATCH 081/134] remove debug prints --- tests/functional/test_sebs.py | 8 ++------ tests/utils/aws_utils.py | 2 -- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index e877aad..ea91e13 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -90,6 +90,8 @@ def tearDown(self): print(f'Deleting Instance Profile: {self.instance_profile}') self.instance_profile.delete() + print('Cleanup Finished') + def test_new_volume(self): control_tag = 'new-volume-sebs' @@ -183,8 +185,6 @@ def test_existing_volume(self): existing_vol = aws_utils.create_existing_volume( control_tag, device_name, az[0]) - print(f'Created Existing Volume {existing_vol.id}') - server_config = self.default_instance.copy() server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device_name)) @@ -208,8 +208,6 @@ def test_existing_volume(self): default_vol = self.ec2.Volume(volume_id) - print(f'Default Volume {default_vol.id}') - self.assertTrue(volume_id, 'Should have the default volume attached') waiter = aws_utils.get_ec2_waiter('volume_deleted') @@ -222,8 +220,6 @@ def test_existing_volume(self): new_vol = self.ec2.Volume(new_vol_id) self.volume_cleanup.append(new_vol) - print(f'Final Volume {new_vol.id}') - self.assertNotEqual(new_vol.id, default_vol.id, 'Volume attached now should not be what we started with.') diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 8db2065..bfb3419 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -172,8 +172,6 @@ def get_avaliable_az(): az = [subnet.availability_zone for subnet in vpc.subnets.all()] - print(az) - return az From e3ebf81fde477a5b63045c3997f96708a3a5a601 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 07:21:30 -0400 Subject: [PATCH 082/134] add contributing file --- CONTRIBUTING.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..7e3ff26 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,54 @@ +## Setup + +To start contributing you will want to create a virtual environment for python3 in the root of the repo with +the name `venv`. Once your virtual environment is activated, install the requirements. + +``` +pip install -r requirements.txt +``` + +## Running Tests + +### Unit + +To run unit tests you have severals options. We recommened you install our pre-commit hook which will run +unit tests every time you commit a change. + +```BASH +bash scripts/install-hooks.bash +``` + +You can also run the tests by manually by using this script. + +```BASH +./scripts/run-tests.bash +``` + +Or by running the test command. + +```BASH +python -m unittest discover -s tests/unit/ +``` + +### Functional +Functional tests will create and destroy resources on AWS. You do not have to run these localy if you +don't want to. + +``` +python -m unittest discover -s tests/functional/ -f -c +``` + + +### Linting + +We use autopep8 to format our files. If you have installed our pre-commit hook then autopep8 will +run everytime you commit changes. + +You can also run our linting script. + +``` +./scripts/run-list.bash +``` + +Note: If files are found with issues then autopep8 will change those files for you. You will then have +to stage those files again with git. From c3a1062694db5bc86e3c255635ceb3872250889f Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 07:24:12 -0400 Subject: [PATCH 083/134] added code coverage to the requirements file --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 8e36d23..c444018 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ cached-property==1.5.1 certifi==2020.4.5.1 chardet==3.0.4 colorama==0.4.3 +coverage==5.1 docutils==0.15.2 ec2-metadata==2.2.0 idna==2.9 @@ -22,6 +23,7 @@ requests==2.23.0 rsa==3.4.2 s3transfer==0.3.3 six==1.14.0 +toml==0.10.0 typed-ast==1.4.1 urllib3==1.25.8 wrapt==1.11.2 From 66d8a9e77eddc156c7a4e3ac0e55b29fa2b96084 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 07:25:20 -0400 Subject: [PATCH 084/134] use policy with minimum privledge --- tests/utils/aws_utils.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index bfb3419..82825a4 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -29,13 +29,28 @@ def create_iam_resources(): role_policy_doc = {'Version': '2012-10-17'} - role_policy_doc['Statement'] = [{ - 'Action': [ - "ec2:*" - ], - 'Effect': 'Allow', - 'Resource': ["arn:aws:ec2:*:*:volume/*", "*"] - }] + role_policy_doc['Statement'] = [ + { + 'Action': [ + "ec2:DetachVolume", + "ec2:AttachVolume", + "ec2:DeleteVolume", + "ec2:DeleteSnapshot", + "ec2:CreateTags", + "ec2:CreateSnapshot", + "ec2:CreateVolume" + ], + 'Effect': 'Allow', + 'Resource': ["arn:aws:ec2:*:*:instance/*", + "arn:aws:ec2:*::snapshot/*", + "arn:aws:ec2:*:*:volume/*"] + }, + { + 'Action': ["ec2:DescribeInstances", "ec2:DescribeVolumes", "ec2:DescribeSnapshots"], + 'Effect': 'Allow', + 'Resource': ["*"] + } + ] iam_role_policy = iam_role.Policy('Create-Volumes') From 32ad814bb0799c23a3fc79985560862e231d3ae2 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 10:50:00 -0400 Subject: [PATCH 085/134] fix argument passing, and change test for overide-name --- tests/unit/test_cli.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index c43a87e..ca0211a 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -20,26 +20,27 @@ def test_version(self): 'Should display the proper version.') def test_single_backup(self): - args = parse_args(['-b test'], '') + args = parse_args(['-b', 'test'], '') self.assertEqual(len(args.backup), 1) def test_multiple_backup(self): - args = parse_args(['-b test1', '-b test2'], '') + args = parse_args(['-b', 'test1', '-b', 'test2'], '') self.assertEqual(len(args.backup), 2) def test_default_name(self): - args = parse_args(['-b test1', '-b test2'], '') + args = parse_args(['-b', 'test1', '-b', 'test2'], '') self.assertEqual(args.name, 'sebs') def test_override_name(self): - args = parse_args(['-b test1', '-b test2', '-n not-default'], '') + args = parse_args( + ['-b', 'test1', '-b', 'test2', '-n', 'not-default'], '') - self.assertEqual(args.name, ' not-default') + self.assertEqual(args.name, 'not-default-sebs') def test_verbose_level(self): - args = parse_args(['-b test1', '-b test2', '-vvv'], '') + args = parse_args(['-b', 'test1', '-b', 'test2', '-vvv'], '') self.assertEqual(args.verbose, 4) From 5a401f293da1a3c582f00c02b66348c25ba995d0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 10:50:22 -0400 Subject: [PATCH 086/134] change help menu for name --- sebs/cli.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sebs/cli.py b/sebs/cli.py index f70e5c3..37cc812 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -11,7 +11,7 @@ def parse_args(args, ver): # Optional argument which requires a parameter (eg. -d test) parser.add_argument("-n", "--name", default='sebs', - help=' specify a unique name.') + help=' specify a your app name.') # Optional verbosity counter (eg. -v, -vv, -vvv, etc.) parser.add_argument( @@ -31,4 +31,12 @@ def parse_args(args, ver): parser.print_help(sys.stderr) sys.exit(1) - return parser.parse_args(args) + parsed_args = parser.parse_args(args) + + parsed_args.name = parsed_args.name if 'sebs' in parsed_args.name else f'{parsed_args.name}-sebs' + + print(f'x{parsed_args.name}x') + + print(parsed_args) + + return parsed_args From 2b4863046f8d77771d5161db91458a3ea35abe8d Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 10:54:14 -0400 Subject: [PATCH 087/134] changed egg name --- tests/utils/aws_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 82825a4..d59b447 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -90,7 +90,7 @@ def create_default_userdata(): "#!/bin/bash\n" "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" "yum install python3 git -y\n" - f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs-test --upgrade \n" + f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs --upgrade \n" ) From c0ffd7ae7fae2238a5ccdd08483535fef5ca982c Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 11:26:31 -0400 Subject: [PATCH 088/134] create init test workflow --- .github/workflows/unit_test.yml | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/unit_test.yml diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml new file mode 100644 index 0000000..b7510e7 --- /dev/null +++ b/.github/workflows/unit_test.yml @@ -0,0 +1,62 @@ +name: Test + +on: + pull_request: + branches: + - develop + - master + paths: + - '**.py' # Only run this workflow when python files change + +jobs: + unit: + name: Unit + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest + + - name: Run Unit Tests + run: python -m unittest discover -s test/unit + + functional: + name: Functional + runs-on: ubuntu-latest + if: github.base_ref == 'master' + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Run Functional Tests + run: python -m unittest discover -s test/functional \ No newline at end of file From a56e2cceb4f51567eca4a196df0fa48e6ff31d6d Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 11:30:09 -0400 Subject: [PATCH 089/134] remove python 2.6 from tests --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index b7510e7..1c5a236 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8] steps: - name: Checkout Code uses: actions/checkout@v2 From 5ddc0ba8076f3e89038fc1656b777c02e949108f Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 11:32:33 -0400 Subject: [PATCH 090/134] remove pytest --- .github/workflows/unit_test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 1c5a236..8995a6e 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -35,9 +35,6 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest - name: Run Unit Tests run: python -m unittest discover -s test/unit From fdcb7018f0f1e4d333db7eedce9a77128998436f Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 11:35:24 -0400 Subject: [PATCH 091/134] fix directory name --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 8995a6e..36ee443 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -37,7 +37,7 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Run Unit Tests - run: python -m unittest discover -s test/unit + run: python -m unittest discover -s tests/unit functional: name: Functional From e3e242d031fdc26ada2fa7895bf884ea9afe7a76 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 11:38:33 -0400 Subject: [PATCH 092/134] remove debug prints --- sebs/cli.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sebs/cli.py b/sebs/cli.py index 37cc812..430d70a 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -35,8 +35,4 @@ def parse_args(args, ver): parsed_args.name = parsed_args.name if 'sebs' in parsed_args.name else f'{parsed_args.name}-sebs' - print(f'x{parsed_args.name}x') - - print(parsed_args) - return parsed_args From 37b008b386e457a671b900d83dc227a0a79b35d0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 11:42:13 -0400 Subject: [PATCH 093/134] add aws region name in setup --- tests/unit/test_stateful_volume.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_stateful_volume.py b/tests/unit/test_stateful_volume.py index 0106e66..4c0014f 100644 --- a/tests/unit/test_stateful_volume.py +++ b/tests/unit/test_stateful_volume.py @@ -23,12 +23,12 @@ def setUp(self): self.device_name = '/dev/xdf' # Setup our ec2 client stubb - ec2 = botocore.session.get_session().create_client('ec2') + ec2 = botocore.session.get_session().create_client('ec2', region_name='us-west-2') self.ec2_client = ec2 self.stub_client = Stubber(ec2) # Setup our ec2 resource stub - ec2_resource = boto3.resource('ec2') + ec2_resource = boto3.resource('ec2', region_name='us-west-2') self.stub_resource = Stubber(ec2_resource.meta.client) # Use mocks to pass out client stubb to our code From b9d75198267f08105702b1a89ed4d3c413eeda15 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 11:46:11 -0400 Subject: [PATCH 094/134] remove python 3.5 because of incompatibility with f strings --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 36ee443..2b198b8 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8] steps: - name: Checkout Code uses: actions/checkout@v2 From 7b564f986187be201ace81c6cb57388d275629a3 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 12:46:49 -0400 Subject: [PATCH 095/134] add aws credential setup --- .github/workflows/unit_test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 2b198b8..737fd98 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -52,6 +52,13 @@ jobs: with: python-version: 3.x + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Install dependencies run: pip install -r requirements.txt From 4958889fbeab53fa1897b41cda859af84555976c Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 12:52:59 -0400 Subject: [PATCH 096/134] fix test directory and add dependency --- .github/workflows/unit_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 737fd98..386325a 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -43,6 +43,7 @@ jobs: name: Functional runs-on: ubuntu-latest if: github.base_ref == 'master' + needs: unit steps: - name: Checkout Code uses: actions/checkout@v2 @@ -63,4 +64,4 @@ jobs: run: pip install -r requirements.txt - name: Run Functional Tests - run: python -m unittest discover -s test/functional \ No newline at end of file + run: python -m unittest discover -s tests/functional \ No newline at end of file From 8200bf22b45426f09984071c197109210a7c9fbb Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 13:00:27 -0400 Subject: [PATCH 097/134] change region to us-east-1 --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 386325a..1892d38 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -58,7 +58,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 + aws-region: us-east-1 - name: Install dependencies run: pip install -r requirements.txt From 381dd79479d2735858cb9ecdfd17f050a8fc1af2 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Thu, 21 May 2020 13:25:33 -0400 Subject: [PATCH 098/134] add fail fast flag for functional test --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 1892d38..1de3f27 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -64,4 +64,4 @@ jobs: run: pip install -r requirements.txt - name: Run Functional Tests - run: python -m unittest discover -s tests/functional \ No newline at end of file + run: python -m unittest discover -s tests/functional -f From 442334f446656797891899614724db61bfc4181e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 04:16:27 -0400 Subject: [PATCH 099/134] changes to setup.py to get ready for release --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b095fb4..1febe16 100644 --- a/setup.py +++ b/setup.py @@ -4,18 +4,18 @@ long_description = fh.read() setuptools.setup( - name="sebs-test", + name="sebs", version="0.0.1", author="Levi Blaney", author_email="shadycuz", - description="Create Stateful Elastic Block Device", + description="Create Stateful Elastic Block Storage on AWS.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/DontShaveTheYak/sebs", packages=setuptools.find_packages(), classifiers=[ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: GPLv3", "Operating System :: OS Independent", ], install_requires=[ From 15898ddecff8d8dce2ba2bbf120d68729b45924e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 04:41:25 -0400 Subject: [PATCH 100/134] change git_ref so it works locally and in github actions --- tests/utils/aws_utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index d59b447..7da41bf 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -1,3 +1,4 @@ +import os import json import boto3 import time @@ -82,9 +83,13 @@ def create_iam_resources(): def create_default_userdata(): - git_ref = subprocess.check_output( - ["git", "rev-parse", "--abbrev-ref", "HEAD"] - ).strip().decode('ASCII') + + git_ref = os.getenv('GITHUB_SHA') + + if not git_ref: + git_ref = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"] + ).strip().decode('ASCII') return ( "#!/bin/bash\n" From 44431bb9849a72235273a1dd5daea503a5c863f0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 04:56:08 -0400 Subject: [PATCH 101/134] fix volume not being cleaned up --- tests/functional/test_sebs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index ea91e13..2fabee2 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -185,6 +185,8 @@ def test_existing_volume(self): existing_vol = aws_utils.create_existing_volume( control_tag, device_name, az[0]) + self.volume_cleanup.append(existing_vol) + server_config = self.default_instance.copy() server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device_name)) @@ -297,9 +299,6 @@ def test_existing_volumes(self): self.volume_cleanup.extend([new_vol1, new_vol2]) - self.volume_cleanup.append(new_vol1) - self.volume_cleanup.append(new_vol2) - self.assertNotEqual(new_vol1.id, default_vol1.id, 'Volume attached now should not be what we started with.') self.assertNotEqual(new_vol2.id, default_vol2.id, From ad4d61f8bb6c29527d7be34d2ec17931ea38c275 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 04:56:28 -0400 Subject: [PATCH 102/134] change from sha to ref --- tests/utils/aws_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 7da41bf..5d9ee57 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -84,7 +84,7 @@ def create_iam_resources(): def create_default_userdata(): - git_ref = os.getenv('GITHUB_SHA') + git_ref = os.getenv('GITHUB_REF') if not git_ref: git_ref = subprocess.check_output( From ab5077f46d3f98f69217525db23fd695d8582632 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 05:17:26 -0400 Subject: [PATCH 103/134] set source branch in github action --- .github/workflows/unit_test.yml | 2 ++ tests/utils/aws_utils.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 1de3f27..bfb9f4f 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -44,6 +44,8 @@ jobs: runs-on: ubuntu-latest if: github.base_ref == 'master' needs: unit + env: + GITHUB_SOURCE_BRANCH: ${{ github.head_ref }} steps: - name: Checkout Code uses: actions/checkout@v2 diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 5d9ee57..0dee0e2 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -84,7 +84,7 @@ def create_iam_resources(): def create_default_userdata(): - git_ref = os.getenv('GITHUB_REF') + git_ref = os.getenv('GITHUB_SOURCE_BRANCH') if not git_ref: git_ref = subprocess.check_output( From eaca8145f32134dfe91019d292950ac490e6df10 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 06:30:09 -0400 Subject: [PATCH 104/134] set environment variables to increase timeout --- tests/utils/aws_utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 0dee0e2..bd76990 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -96,6 +96,8 @@ def create_default_userdata(): "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" "yum install python3 git -y\n" f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs --upgrade \n" + "export AWS_METADATA_SERVICE_NUM_ATTEMPTS=3" + "export AWS_METADATA_SERVICE_TIMEOUT=2" ) @@ -162,19 +164,24 @@ def has_control_tag(control_tag, device_name, volume): def wait_for_volume_tag(volume): i = 0 + tagged = False while True: i += 1 volume.reload() if volume.tags: + tagged = True break - time.sleep(10) + time.sleep(30) - if i > 14: + if i > 10: break + if not tagged: + raise Exception(f'{volume.id} was never tagged.') + def get_default_vpc(): ec2 = boto3.resource('ec2') From f7168ec212e24cdf5f0dd41fbbb75a49047fded8 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 06:45:38 -0400 Subject: [PATCH 105/134] add missing white space --- tests/utils/aws_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index bd76990..476a76d 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -96,8 +96,8 @@ def create_default_userdata(): "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" "yum install python3 git -y\n" f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs --upgrade \n" - "export AWS_METADATA_SERVICE_NUM_ATTEMPTS=3" - "export AWS_METADATA_SERVICE_TIMEOUT=2" + "export AWS_METADATA_SERVICE_NUM_ATTEMPTS=3 \n" + "export AWS_METADATA_SERVICE_TIMEOUT=2 \n" ) From 9428d2fc1ae1af8149391482c14796aad31a782d Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 07:25:27 -0400 Subject: [PATCH 106/134] change how version is handled for upcoming pypi release --- bin/sebs | 2 +- sebs/app.py | 1 - sebs/cli.py | 5 +++-- setup.py | 13 ++++++++++++- tests/unit/test_cli.py | 18 ++++++++++-------- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/bin/sebs b/bin/sebs index 5a261bf..8c0c5c7 100644 --- a/bin/sebs +++ b/bin/sebs @@ -11,7 +11,7 @@ if __name__ == "__main__": logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(name)s (%(levelname)s): %(message)s') try: - args = cli.parse_args(sys.argv[1:], app.__version__) + args = cli.parse_args(sys.argv[1:]) log.setLevel(max(3 - args.verbose, 0) * 10) sys.exit(app.main(args)) except KeyboardInterrupt: diff --git a/sebs/app.py b/sebs/app.py index a171745..1dde532 100644 --- a/sebs/app.py +++ b/sebs/app.py @@ -4,7 +4,6 @@ """ __authors__ = ["Levi Blaney", "Neha Singh"] -__version__ = "0.1.0" __license__ = "GPLv3" import sys diff --git a/sebs/cli.py b/sebs/cli.py index 430d70a..f21b8bf 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -1,8 +1,9 @@ import sys import argparse +from importlib import metadata -def parse_args(args, ver): +def parse_args(args): parser = argparse.ArgumentParser() @@ -25,7 +26,7 @@ def parse_args(args, ver): parser.add_argument( "--version", action="version", - version="%(prog)s (version {version})".format(version=ver)) + version="%(prog)s (version {version})".format(version=metadata.version('sebs'))) if len(args) == 0: parser.print_help(sys.stderr) diff --git a/setup.py b/setup.py index 1febe16..4910ce1 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,22 @@ +import os +import subprocess import setuptools + +release_version = os.getenv('RELEASE_VERSION') + +if not release_version: + release_version = subprocess.check_output( + ["git", "describe"] + ).strip().decode('ASCII') + + with open("README.md", "r") as fh: long_description = fh.read() setuptools.setup( name="sebs", - version="0.0.1", + version=release_version, author="Levi Blaney", author_email="shadycuz", description="Create Stateful Elastic Block Storage on AWS.", diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index ca0211a..3bdadf2 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,5 +1,6 @@ import unittest from io import StringIO +from importlib import metadata from unittest.mock import patch from sebs.cli import parse_args @@ -8,39 +9,40 @@ class TestArugmentParsing(unittest.TestCase): def test_version(self): - version = "2.1.0" + expected_version = metadata.version('sebs') with patch('sys.stdout', new=StringIO()) as fakeOutput: try: - parse_args(['--version'], version) + parse_args(['--version']) except: pass output = fakeOutput.getvalue().strip() - self.assertTrue(version in output, + + self.assertTrue(expected_version in output, 'Should display the proper version.') def test_single_backup(self): - args = parse_args(['-b', 'test'], '') + args = parse_args(['-b', 'test']) self.assertEqual(len(args.backup), 1) def test_multiple_backup(self): - args = parse_args(['-b', 'test1', '-b', 'test2'], '') + args = parse_args(['-b', 'test1', '-b', 'test2']) self.assertEqual(len(args.backup), 2) def test_default_name(self): - args = parse_args(['-b', 'test1', '-b', 'test2'], '') + args = parse_args(['-b', 'test1', '-b', 'test2']) self.assertEqual(args.name, 'sebs') def test_override_name(self): args = parse_args( - ['-b', 'test1', '-b', 'test2', '-n', 'not-default'], '') + ['-b', 'test1', '-b', 'test2', '-n', 'not-default']) self.assertEqual(args.name, 'not-default-sebs') def test_verbose_level(self): - args = parse_args(['-b', 'test1', '-b', 'test2', '-vvv'], '') + args = parse_args(['-b', 'test1', '-b', 'test2', '-vvv']) self.assertEqual(args.verbose, 4) From 62048d4526ec8a852f44b09f7c8f3860d0732fe3 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 07:32:22 -0400 Subject: [PATCH 107/134] run functional tests with verbose flag --- tests/functional/test_sebs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index 2fabee2..0eb0df0 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -102,7 +102,7 @@ def test_new_volume(self): aws_utils.create_block_device(device_name)) server_config['UserData'] += ( - f'/usr/local/bin/sebs -b {device_name} -n {control_tag}\n' + f'/usr/local/bin/sebs -b {device_name} -n {control_tag} -vvv\n' f'while [ ! -e {device_name} ] ; do sleep 1 ; done\n' ) @@ -140,7 +140,7 @@ def test_new_volumes(self): aws_utils.create_block_device(device2_name)) server_config['UserData'] += ( - f'/usr/local/bin/sebs -b {device1_name} -b {device2_name} -n {control_tag}\n' + f'/usr/local/bin/sebs -b {device1_name} -b {device2_name} -n {control_tag} -vvv\n' f'while [ ! -e {device1_name} ] ; do sleep 1 ; done\n' f'while [ ! -e {device2_name} ] ; do sleep 1 ; done\n' ) @@ -195,7 +195,7 @@ def test_existing_volume(self): } server_config['UserData'] += ( - f'/usr/local/bin/sebs -b {device_name} -n {control_tag}\n' + f'/usr/local/bin/sebs -b {device_name} -n {control_tag} -vvv\n' f'while [ ! -e {device_name} ] ; do sleep 1 ; done\n' ) @@ -260,7 +260,7 @@ def test_existing_volumes(self): } server_config['UserData'] += ( - f'/usr/local/bin/sebs -b {device1_name} -b {device2_name} -n {control_tag}\n' + f'/usr/local/bin/sebs -b {device1_name} -b {device2_name} -n {control_tag} -vvv\n' f'while [ ! -e {device1_name} ] ; do sleep 1 ; done\n' f'while [ ! -e {device2_name} ] ; do sleep 1 ; done\n' ) From 02e5f875dbf92e8fbf8760d5af656d0148d8470e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 07:37:43 -0400 Subject: [PATCH 108/134] have ci/cd install package --- .github/workflows/unit_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index bfb9f4f..a241c21 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -29,6 +29,7 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -e . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names From 42d58d6b8483c2d6b92f8e44a0b029257dddd4be Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 07:43:21 -0400 Subject: [PATCH 109/134] fetch all tags in workflow --- .github/workflows/unit_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index a241c21..de6ffb6 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -18,6 +18,9 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v2 + + - name: Fetch all Tags + run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From 1814f8e28986e0335b4ce400c851629448e9e41e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 07:46:41 -0400 Subject: [PATCH 110/134] remove depth option --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index de6ffb6..08650f6 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v2 - name: Fetch all Tags - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + run: git fetch origin +refs/tags/*:refs/tags/* - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From ece3ed22be28e8f777f2353a744752e70abd1841 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 07:50:31 -0400 Subject: [PATCH 111/134] fetch all history --- .github/workflows/unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 08650f6..4952316 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -19,8 +19,8 @@ jobs: - name: Checkout Code uses: actions/checkout@v2 - - name: Fetch all Tags - run: git fetch origin +refs/tags/*:refs/tags/* + - name: Fetch all history + run: git fetch --prune --unshallow --tags - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From 34532feb683bccaef096f20ef031f8697a69b4cc Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 07:55:57 -0400 Subject: [PATCH 112/134] make metadata work for python < 3.8 --- sebs/cli.py | 6 +++++- setup.py | 3 ++- tests/unit/test_cli.py | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/sebs/cli.py b/sebs/cli.py index f21b8bf..33444f6 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -1,6 +1,10 @@ import sys import argparse -from importlib import metadata +try: + from importlib import metadata +except ImportError: + # Running on pre-3.8 Python; use importlib-metadata package + import importlib_metadata as metadata def parse_args(args): diff --git a/setup.py b/setup.py index 4910ce1..eea371b 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,8 @@ ], install_requires=[ 'boto3', - 'ec2-metadata' + 'ec2-metadata', + 'importlib-metadata ~= 1.0 ; python_version < "3.8"' ], scripts=['bin/sebs'], python_requires='>=3.6', diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 3bdadf2..739fd59 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,9 +1,14 @@ import unittest from io import StringIO -from importlib import metadata from unittest.mock import patch from sebs.cli import parse_args +try: + from importlib import metadata +except ImportError: + # Running on pre-3.8 Python; use importlib-metadata package + import importlib_metadata as metadata + class TestArugmentParsing(unittest.TestCase): From 99d11f75e5da72c8bdc566c80349a8f166be4ccf Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 10:16:27 -0400 Subject: [PATCH 113/134] refactor sessions to avoid creating new sessions --- sebs/ec2.py | 15 ++++-- tests/unit/test_stateful_volume.py | 76 +++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 559da02..7890d67 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -13,6 +13,8 @@ class Instance: def __init__(self, volume_tag): + # Create a session so we don't have to keep getting creds. + self.session = None self.instance = self.get_instance() self.volume_tag = volume_tag self.backup = [] @@ -32,7 +34,9 @@ def get_instance(self): sys.exit(1) try: - ec2 = boto3.resource('ec2', region_name=ec2_metadata.region) + self.session = boto3.session.Session( + region_name=ec2_metadata.region) + ec2 = self.session.resource('ec2') instance = ec2.Instance(instance_id) # We have to call load to see if we are really connected instance.load() @@ -66,16 +70,17 @@ def attach_stateful_volumes(self): class StatefulVolume: - def __init__(self, instance_id, device_name, tag_name): + def __init__(self, session, instance_id, device_name, tag_name): + # Use the existing session so we dont have to keep fetching creds + self.session = session self.instance_id = instance_id self.device_name = device_name self.ready = False self.status = 'Unknown' self.volume = None self.tag_name = tag_name - self.ec2_client = boto3.client('ec2', region_name=ec2_metadata.region) - self.ec2_resource = boto3.resource( - 'ec2', region_name=ec2_metadata.region) + self.ec2_client = self.session.client('ec2') + self.ec2_resource = self.session.resource('ec2') def get_status(self): log.info(f'Checking for previous volume of {self.device_name}') diff --git a/tests/unit/test_stateful_volume.py b/tests/unit/test_stateful_volume.py index 4c0014f..1180bda 100644 --- a/tests/unit/test_stateful_volume.py +++ b/tests/unit/test_stateful_volume.py @@ -60,6 +60,14 @@ def setUp(self): self.boto3.resource = MagicMock( name='mock_resource_contructor', return_value=self.mock_resource) + self.mock_session = MagicMock(name='mock_session') + + self.mock_session.client.return_value = self.ec2_client + + self.mock_session.resource.return_value = self.mock_resource + + self.boto3.session.Session = self.mock_session + modules = { 'boto3': self.boto3, 'ec2_metadata': MagicMock() @@ -111,8 +119,11 @@ def tearDown(self): self.stub_client.deactivate() def test_class_properties(self): - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) self.stub_client.assert_no_pending_responses() @@ -152,8 +163,10 @@ def test_status_new(self): } ]}) - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) sv.get_status() @@ -186,8 +199,10 @@ def test_status_missing(self): } ]}) - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) sv.get_status() @@ -207,8 +222,10 @@ def test_status_duplicate(self): self.stub_client.add_response( 'describe_volumes', response, self.default_params) - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) sv.get_status() @@ -224,8 +241,10 @@ def test_status_not_attached(self): self.stub_client.add_response( 'describe_volumes', self.default_response, self.default_params) - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) sv.get_status() @@ -262,8 +281,10 @@ def test_volume_tagging(self): } ]}) - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) sv.get_status() @@ -273,8 +294,10 @@ def test_volume_tagging(self): Tags=[{'Key': self.tag_name, 'Value': self.device_name}]) def test_copy_new(self): - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) sv.status = 'New' response = sv.copy('fakeAZ') @@ -283,8 +306,10 @@ def test_copy_new(self): "Should do nothing if status in not 'Not Attached'.") def test_copy_same_az(self): - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) self.first_volume.availability_zone = 'fakeAZ' sv.status = 'Not Attached' @@ -312,8 +337,10 @@ def test_copy_different_az(self): self.stub_client.add_response('describe_volumes', {'Volumes': [ {'VolumeId': 'vol-2222', 'State': 'available'}]}, {'VolumeIds': ['vol-2222']}) - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) self.first_volume.availability_zone = 'fakeAZ' sv.status = 'Not Attached' @@ -331,8 +358,10 @@ def test_copy_different_az(self): self.mock_snapshot.delete.assert_called_once() def test_attach_new(self): - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) sv.status = 'New' response = sv.attach() @@ -364,8 +393,11 @@ def test_attach(self): self.stub_client.add_response('describe_volumes', {'Volumes': [ {'VolumeId': 'vol-2222', 'State': 'in-use'}]}, {'VolumeIds': ['vol-2222']}) - sv = self.StatefulVolume( - self.instance_id, self.device_name, self.tag_name) + sv = self.StatefulVolume(self.mock_session, + self.instance_id, + self.device_name, + self.tag_name) + sv.status = 'Not Attached' sv.volume = self.second_volume From 7cb4df2b16281f985012820d6702af5e32a15f40 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 10:40:33 -0400 Subject: [PATCH 114/134] pass session to volume :blush: --- sebs/ec2.py | 3 ++- tests/unit/test_instance.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sebs/ec2.py b/sebs/ec2.py index 7890d67..2102a49 100644 --- a/sebs/ec2.py +++ b/sebs/ec2.py @@ -49,7 +49,8 @@ def get_instance(self): def add_stateful_device(self, device_name): log.info(f'Handling {device_name}') - sv = StatefulVolume(self.instance.id, device_name, self.volume_tag) + sv = StatefulVolume(self.session, self.instance.id, + device_name, self.volume_tag) sv.get_status() diff --git a/tests/unit/test_instance.py b/tests/unit/test_instance.py index 3654cea..cf89c77 100644 --- a/tests/unit/test_instance.py +++ b/tests/unit/test_instance.py @@ -40,8 +40,14 @@ def test_add_device(self, mock_method, mock_volume_class): mock_method.assert_called_once() mock_volume_class.assert_called_once() - mock_volume_class.assert_called_once_with( - self.mock_instance.id, self.device_name, self.default_tag) + + # Instead of None it should be a session... + # but I can't figure out how to make a mock method (get_instance) + # set a class field self.session as a side effect. + mock_volume_class.assert_called_once_with(None, + self.mock_instance.id, + self.device_name, + self.default_tag) self.assertIn(mock_volume, server.backup, 'Should put our device in the backup list.') mock_volume.get_status.assert_called_once() @@ -62,9 +68,10 @@ def test_add_multiple_devices(self, mock_method, mock_volume_class): self.assertEqual(mock_volume_class.call_count, 2, 'Should create two volumes.') + # Same as above test, need to replace None with session mock_volume_class.assert_has_calls( - [call(self.mock_instance.id, self.device_name, self.default_tag), - call(self.mock_instance.id, '/dev/2', self.default_tag)]) + [call(None, self.mock_instance.id, self.device_name, self.default_tag), + call(None, self.mock_instance.id, '/dev/2', self.default_tag)]) self.assertEqual(len(server.backup), 2, 'Should have two volumes.') self.assertEqual(mock_volume.get_status.call_count, 2, From 5dd738fd681af49c94d8c6ebc0e697d2a851c607 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 13:15:57 -0400 Subject: [PATCH 115/134] increase time out for for volume waiter --- tests/utils/aws_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 476a76d..9941590 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -174,9 +174,9 @@ def wait_for_volume_tag(volume): tagged = True break - time.sleep(30) + time.sleep(40) - if i > 10: + if i > 14: break if not tagged: From 2716356642eaae6dadc83dee300f06b620d974ad Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 14:04:54 -0400 Subject: [PATCH 116/134] add sleep to prevent iam profile no creds error --- tests/functional/test_sebs.py | 3 +++ tests/utils/aws_utils.py | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index 0eb0df0..50e8215 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -1,3 +1,4 @@ +import time import json import boto3 import unittest @@ -92,6 +93,8 @@ def tearDown(self): print('Cleanup Finished') + time.sleep(60) + def test_new_volume(self): control_tag = 'new-volume-sebs' diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 9941590..528b541 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -94,6 +94,7 @@ def create_default_userdata(): return ( "#!/bin/bash\n" "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" + "sleep 1m\n" "yum install python3 git -y\n" f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs --upgrade \n" "export AWS_METADATA_SERVICE_NUM_ATTEMPTS=3 \n" From 97b39ae47e1b4c2c7ae6d99279ae697adf37a4b0 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Fri, 22 May 2020 14:59:43 -0400 Subject: [PATCH 117/134] increase sleep time --- tests/functional/test_sebs.py | 2 +- tests/utils/aws_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index 50e8215..51e5117 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -93,7 +93,7 @@ def tearDown(self): print('Cleanup Finished') - time.sleep(60) + time.sleep(120) def test_new_volume(self): diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 528b541..eec8d92 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -94,7 +94,7 @@ def create_default_userdata(): return ( "#!/bin/bash\n" "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" - "sleep 1m\n" + "sleep 2m\n" "yum install python3 git -y\n" f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs --upgrade \n" "export AWS_METADATA_SERVICE_NUM_ATTEMPTS=3 \n" From fd2e5ab7095fdb597895759c8e2203c4ff33d71a Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 06:18:11 -0400 Subject: [PATCH 118/134] add debug print messages --- tests/functional/test_sebs.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index 51e5117..e62357f 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -54,6 +54,8 @@ def setUp(self): }], ) + print('Finished Setup.') + def tearDown(self): print('Running Cleanup...') @@ -97,6 +99,7 @@ def tearDown(self): def test_new_volume(self): + print('Starting test_new_volume') control_tag = 'new-volume-sebs' device_name = '/dev/xvdh' @@ -130,8 +133,12 @@ def test_new_volume(self): self.assertTrue(volume_tagged, 'Volume should have our control tag.') + print('Finshed test_new_volume') + def test_new_volumes(self): + print('Starting test_new_volumes') + control_tag = 'new-volumes-sebs' device1_name = '/dev/xvdh' device2_name = '/dev/xvdm' @@ -178,8 +185,12 @@ def test_new_volumes(self): self.assertTrue(volume1_tagged, 'Volume should have our control tag.') self.assertTrue(volume2_tagged, 'Volume should have our control tag.') + print('Finished test_new_volumes') + def test_existing_volume(self): + print('Starting test_existing_volume') + control_tag = 'existing-volume-sebs' device_name = '/dev/xvdh' @@ -237,8 +248,12 @@ def test_existing_volume(self): self.assertTrue(volume_tagged, 'Volume should have our control tag.') + print('Finished test_existing_volume') + def test_existing_volumes(self): + print('Starting test_existing_volumes') + control_tag = 'existing-volumes-sebs' device1_name = '/dev/xvdh' device2_name = '/dev/xvdm' @@ -324,3 +339,5 @@ def test_existing_volumes(self): self.assertTrue(volume1_tagged, 'Volume should have our control tag.') self.assertTrue(volume2_tagged, 'Volume should have our control tag.') + + print('Finished test_existing_volumes') From 6ded3e1d375c42a346786c6703e86db654cc9fb1 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 07:34:58 -0400 Subject: [PATCH 119/134] remove sleep --- tests/utils/aws_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index eec8d92..9941590 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -94,7 +94,6 @@ def create_default_userdata(): return ( "#!/bin/bash\n" "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n" - "sleep 2m\n" "yum install python3 git -y\n" f"python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git@{git_ref}#egg=sebs --upgrade \n" "export AWS_METADATA_SERVICE_NUM_ATTEMPTS=3 \n" From 61023438d406a9de81e4e27fb9bf7333d982f360 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 07:35:48 -0400 Subject: [PATCH 120/134] adjust sleep time on volume waiter --- tests/utils/aws_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 9941590..1132c8b 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -1,7 +1,6 @@ import os import json import boto3 -import time import subprocess @@ -174,7 +173,7 @@ def wait_for_volume_tag(volume): tagged = True break - time.sleep(40) + time.sleep(30) if i > 14: break From ae9ee9324ff79ae5cd23f832cd8b5f25721bfc95 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 07:36:15 -0400 Subject: [PATCH 121/134] refactor tests to create IAM role once because of ci/cd issue --- tests/functional/test_sebs.py | 109 +++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 49 deletions(-) diff --git a/tests/functional/test_sebs.py b/tests/functional/test_sebs.py index e62357f..ae9f29b 100644 --- a/tests/functional/test_sebs.py +++ b/tests/functional/test_sebs.py @@ -9,39 +9,34 @@ class TestSebs(unittest.TestCase): - def setUp(self): - print('Setting up test environment...') + @classmethod + def setUpClass(cls): + print('Setting up environment...') + warnings.filterwarnings( "ignore", category=ResourceWarning, message="unclosed.*") - self.volume_cleanup = [] - self.instance_cleanup = [] - - self.ec2 = boto3.resource('ec2') - - self.iam = boto3.resource('iam') - iam_resources = aws_utils.create_iam_resources() - self.iam_role = iam_resources['role'] + cls.iam_role = iam_resources['role'] - self.iam_role_policy = iam_resources['policy'] + cls.iam_role_policy = iam_resources['policy'] - self.instance_profile = iam_resources['profile'] + cls.instance_profile = iam_resources['profile'] - self.default_user_data = aws_utils.create_default_userdata() + cls.default_user_data = aws_utils.create_default_userdata() ami_id = aws_utils.get_latest_ami() - self.default_instance = dict(BlockDeviceMappings=[], - ImageId=ami_id, - InstanceType='t1.micro', - MaxCount=1, - MinCount=1, - Monitoring={'Enabled': False}, - UserData=self.default_user_data, - IamInstanceProfile={ - 'Arn': self.instance_profile.arn}, + cls.default_instance = dict(BlockDeviceMappings=[], + ImageId=ami_id, + InstanceType='t1.micro', + MaxCount=1, + MinCount=1, + Monitoring={'Enabled': False}, + UserData=cls.default_user_data, + IamInstanceProfile={ + 'Arn': cls.instance_profile.arn}, TagSpecifications=[ { 'ResourceType': 'instance', @@ -54,11 +49,47 @@ def setUp(self): }], ) - print('Finished Setup.') + print('Environment setup.') + + @classmethod + def tearDownClass(cls): + print('Cleaning up environment...') + + if cls.instance_profile: + print('Removing role from instance profile.') + cls.instance_profile.remove_role( + RoleName=cls.iam_role.name + ) + + if cls.iam_role_policy: + print(f'Deleting policy: {cls.iam_role_policy}') + cls.iam_role_policy.delete() + + if cls.iam_role: + print(f'Deleting Role: {cls.iam_role}') + cls.iam_role.delete() + + if cls.instance_profile: + print(f'Deleting Instance Profile: {cls.instance_profile}') + cls.instance_profile.delete() + + print('Environment Cleanup Finished') + + def setUp(self): + + print('Setting up test.') + self.volume_cleanup = [] + self.instance_cleanup = [] + + self.ec2 = boto3.resource('ec2') + + self.iam = boto3.resource('iam') + + print('Finished test setup.') def tearDown(self): - print('Running Cleanup...') + print('Running Test Cleanup...') for instance in self.instance_cleanup: print(f'Deleting {instance.id}') instance.terminate() @@ -75,27 +106,7 @@ def tearDown(self): if e.response['Error']['Code'] != 'InvalidVolume.NotFound': raise e - if self.instance_profile: - print('Removing role from instance profile.') - self.instance_profile.remove_role( - RoleName=self.iam_role.name - ) - - if self.iam_role_policy: - print(f'Deleting policy: {self.iam_role_policy}') - self.iam_role_policy.delete() - - if self.iam_role: - print(f'Deleting Role: {self.iam_role}') - self.iam_role.delete() - - if self.instance_profile: - print(f'Deleting Instance Profile: {self.instance_profile}') - self.instance_profile.delete() - - print('Cleanup Finished') - - time.sleep(120) + print('Test Cleanup Finished') def test_new_volume(self): @@ -103,7 +114,7 @@ def test_new_volume(self): control_tag = 'new-volume-sebs' device_name = '/dev/xvdh' - server_config = self.default_instance.copy() + server_config = self.__class__.default_instance.copy() server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device_name)) @@ -143,7 +154,7 @@ def test_new_volumes(self): device1_name = '/dev/xvdh' device2_name = '/dev/xvdm' - server_config = self.default_instance.copy() + server_config = self.__class__.default_instance.copy() server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device1_name)) server_config['BlockDeviceMappings'].append( @@ -201,7 +212,7 @@ def test_existing_volume(self): self.volume_cleanup.append(existing_vol) - server_config = self.default_instance.copy() + server_config = self.__class__.default_instance.copy() server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device_name)) server_config['Placement'] = { @@ -268,7 +279,7 @@ def test_existing_volumes(self): self.volume_cleanup.extend([existing_vol1, existing_vol2]) - server_config = self.default_instance.copy() + server_config = self.__class__.default_instance.copy() server_config['BlockDeviceMappings'].append( aws_utils.create_block_device(device1_name)) server_config['BlockDeviceMappings'].append( From f92d81c7ad75613d148b7ef54e33b92025847aa6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 07:39:25 -0400 Subject: [PATCH 122/134] add back time import --- tests/utils/aws_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils/aws_utils.py b/tests/utils/aws_utils.py index 1132c8b..34baca3 100644 --- a/tests/utils/aws_utils.py +++ b/tests/utils/aws_utils.py @@ -1,4 +1,5 @@ import os +import time import json import boto3 import subprocess From 5a35a668230ebf9622249e8c3e4be9ea23ce4230 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 09:31:16 -0400 Subject: [PATCH 123/134] modify workflow to release on pypi test --- .../workflows/{unit_test.yml => build.yml} | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) rename .github/workflows/{unit_test.yml => build.yml} (72%) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/build.yml similarity index 72% rename from .github/workflows/unit_test.yml rename to .github/workflows/build.yml index 4952316..dbb9db9 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Test +name: Build on: pull_request: @@ -10,7 +10,7 @@ on: jobs: unit: - name: Unit + name: Unit Tests runs-on: ubuntu-latest strategy: matrix: @@ -44,9 +44,9 @@ jobs: run: python -m unittest discover -s tests/unit functional: - name: Functional + name: Functional Tests runs-on: ubuntu-latest - if: github.base_ref == 'master' + if: github.base_ref == 'sdfjsdlfksjd' needs: unit env: GITHUB_SOURCE_BRANCH: ${{ github.head_ref }} @@ -71,3 +71,30 @@ jobs: - name: Run Functional Tests run: python -m unittest discover -s tests/functional -f + + pypi: + name: Test Pypi + runs-on: ubuntu-latest + if: github.base_ref == 'master' + needs: unit + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From b9f739c249ec75bbcc82edf1589429138ecd50ba Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 09:42:49 -0400 Subject: [PATCH 124/134] add version calculation --- .github/workflows/build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dbb9db9..9ba002f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,11 +90,22 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine + + - name: Calculate Next Version + id: next_version + uses: K-Phoen/semver-release-action@master + with: + release_branch: master + release_strategy: none + env: + GITHUB_TOKEN: ${{ github.token }} - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} + RELEASE_VERSION: ${{ ${{ steps.next_version.outputs.tag }} }} run: | + echo "Releasing $RELEASE_VERSION" python setup.py sdist bdist_wheel twine upload dist/* From d2e03ba0e48731b84d06a720d6c1f4aef2dbabf6 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 09:48:07 -0400 Subject: [PATCH 125/134] remove typo --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ba002f..83de407 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -104,7 +104,7 @@ jobs: env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} - RELEASE_VERSION: ${{ ${{ steps.next_version.outputs.tag }} }} + RELEASE_VERSION: ${{ steps.next_version.outputs.tag }} run: | echo "Releasing $RELEASE_VERSION" python setup.py sdist bdist_wheel From 22a40ef62bf1fe97d970de7ab9d59e2f81bdd32f Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 09:55:35 -0400 Subject: [PATCH 126/134] comment out version bump --- .github/workflows/build.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83de407..5398062 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,6 +80,9 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v2 + + - name: Fetch all history + run: git fetch --prune --unshallow --tags - name: Set up Python uses: actions/setup-python@v2 @@ -91,21 +94,19 @@ jobs: python -m pip install --upgrade pip pip install setuptools wheel twine - - name: Calculate Next Version - id: next_version - uses: K-Phoen/semver-release-action@master - with: - release_branch: master - release_strategy: none - env: - GITHUB_TOKEN: ${{ github.token }} + # - name: Calculate Next Version + # id: next_version + # uses: K-Phoen/semver-release-action@master + # with: + # release_branch: master + # release_strategy: none + # env: + # GITHUB_TOKEN: ${{ github.token }} - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} - RELEASE_VERSION: ${{ steps.next_version.outputs.tag }} run: | - echo "Releasing $RELEASE_VERSION" python setup.py sdist bdist_wheel twine upload dist/* From e6b1bbaed6c74a77ce9bf77f1983fdae5540db65 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 10:04:53 -0400 Subject: [PATCH 127/134] update twine upload command for test pypi --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5398062..fa5b3cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,4 +109,4 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }} run: | python setup.py sdist bdist_wheel - twine upload dist/* + twine upload --repository testpypi dist/* From 86f6c38b9425eb6fd8d66fa041efc7e42826db02 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 10:14:08 -0400 Subject: [PATCH 128/134] change setup to submit a tag that works with pypi --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eea371b..0cd1dd5 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ if not release_version: release_version = subprocess.check_output( - ["git", "describe"] + ["git", "describe", "--abbrev=0"] ).strip().decode('ASCII') From 92838d7d5116c691c3b50a10b34fcc2c5c13030c Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 10:18:48 -0400 Subject: [PATCH 129/134] fix email for pypi --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0cd1dd5..31675ab 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ name="sebs", version=release_version, author="Levi Blaney", - author_email="shadycuz", + author_email="shadycuz@gmail.com", description="Create Stateful Elastic Block Storage on AWS.", long_description=long_description, long_description_content_type="text/markdown", From f94ccaae081ba345204d81ff713fa9b61420fea8 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 10:28:16 -0400 Subject: [PATCH 130/134] update classifiers --- setup.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 31675ab..8970625 100644 --- a/setup.py +++ b/setup.py @@ -25,9 +25,15 @@ url="https://github.com/DontShaveTheYak/sebs", packages=setuptools.find_packages(), classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GPLv3", - "Operating System :: OS Independent", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: POSIX :: Linux", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Natural Language :: English", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], install_requires=[ 'boto3', From 95fd48c2bda4bc4e413b2affdab4b6e85a6051a8 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 10:38:54 -0400 Subject: [PATCH 131/134] updating markdown files for first release --- CHANGELOG.md | 23 +++++++++ CONTRIBUTING.md | 2 + README.md | 128 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ba46a16 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Categories: +- **Added** for new features. +- **Changed** for changes in existing functionality. +- **Deprecated** for soon-to-be removed features. +- **Removed** for now removed features. +- **Fixed** for any bug fixes. +- **Security** in case of vulnerabilities. + +## [Unreleased] + +## [0.5.0] - 05/22/2020 +### Added +- Initial release with working functionality. + + +[Unreleased]: https://github.com/DontShaveTheYak/sebs/compare/v0.5.0...develop +[0.5.0]: https://github.com/DontShaveTheYak/sebs/releases/tag/v0.5.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e3ff26..53f0144 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,8 @@ don't want to. python -m unittest discover -s tests/functional/ -f -c ``` +Note: You need to push your changes upstream before you run the functional test as pip will clone down your current branch. + ### Linting diff --git a/README.md b/README.md index 95891e7..60842e7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,126 @@ -# sebs -Create (S)tateful (E)lastic (B)lock (S)torage on AWS. +# Stateful Elastic Block Storage (sebs) + +Sebs was created for the situation where you need to stick a stateful application in an AutoScaling +group with a max size of 1. Sebs will make sure that if the instance is recreated that the previous +volume is reattached back to the instance regardless of which AZ the instance is recreated in. + +## Why + +A single instance ASG is good protection in the event that the instance is [retired](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-retirement.html). +It can also help enable rolling upgrades and DR in the event that the AZ your instance is in goes down. + +### Prerequisites + +* python3 +* EC2 Instance Profile with access to create/delete snapshots and volumes. + +Example IAM Policy + +```JSON +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DetachVolume", + "ec2:AttachVolume", + "ec2:DeleteVolume", + "ec2:DeleteSnapshot", + "ec2:CreateTags", + "ec2:CreateSnapshot", + "ec2:CreateVolume" + ], + "Resource": [ + "arn:aws:ec2:*:*:instance/*", + "arn:aws:ec2:*::snapshot/*", + "arn:aws:ec2:*:*:volume/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeVolumes", + "ec2:DescribeSnapshots" + ], + "Resource": "*" + } + ] +} +``` + +### Installing + +Sebs should be run from an EC2 instance and can be run from userdata or any CaC tool. +Sebs can be installed from pip or GitHub. + +From pip + +``` +pip install sebs +``` + +From GitHub + +``` +python3 -m pip install git+https://github.com/DontShaveTheYak/sebs.git#egg=sebs +``` + +## Usage + +``` +sebs +usage: sebs [-h] -b BACKUP [-n NAME] [-v] [--version] + +optional arguments: + -h, --help show this help message and exit + -b BACKUP, --backup BACKUP + List of Devices to Backup + -n NAME, --name NAME specify a your app name. + -v, --verbose Verbosity (-v, -vv, etc) + --version show program's version number and exit +``` + +Note: If you are going to have more than one instance in a single region use sebs then you need to pass in a name. + +``` +sebs -b /dev/xvdz -n ${MY_APP_NAME} +``` + +Here is an example userdata script + +```BASH +#!/bin/bash +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + +echo "Running Sebs" +yum install python3 -y +python3 -m pip install sebs + +# On RHEL and Amazon Linux2 /usr/local/bin is not in the path for root user. +/usr/local/bin/sebs -b /dev/xvdz -n example-app + +echo 'Waiting on device /dev/xvdz to be available.' +while [ ! -e /dev/xvdz ] ; do sleep 1 ; done +echo 'Device is ready.' +``` + +On first run sebs will mark the volume mounted at `/dev/xvdz` with a control tag. +If the instance is re-created sebs will look for a volume with the control tag and +if found it will then mount that volume to the instance as the same device as before. + +## Contributing + +Please read [CONTRIBUTING.md](./CONTRIBUTING.md). + +## Versioning + +We use [SemVer](http://semver.org/) for versioning. For the versions available, +see the [tags on this repository](https://github.com/DontShaveTheYak/sebs/tags). + +To see what has changed see the [CHANGELOG](./CHANGELOG.md). + +## License + +This project is licensed under the GPLv3 License - see the [LICENSE](LICENSE) file for details From 1c3ce92e617edecf933f865a2662ee17acaccd7e Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 10:49:49 -0400 Subject: [PATCH 132/134] add date time stamp to tag for test release --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8970625..7ab654c 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import os +import datetime import subprocess import setuptools @@ -6,10 +7,14 @@ release_version = os.getenv('RELEASE_VERSION') if not release_version: - release_version = subprocess.check_output( + tag = subprocess.check_output( ["git", "describe", "--abbrev=0"] ).strip().decode('ASCII') + now = datetime.datetime.now() + + release_version = f'{tag}.dev{now.hour}{now.minute}' + with open("README.md", "r") as fh: long_description = fh.read() From a000e6aca695189b3a99f91766e39ad908f5a6fb Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 12:02:24 -0400 Subject: [PATCH 133/134] add tag and release workflows --- .github/workflows/build.yml | 2 +- .github/workflows/create_tag.yml | 22 ++++++++++++++++ .github/workflows/release.yml | 44 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/create_tag.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa5b3cd..d689866 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,7 +101,7 @@ jobs: # release_branch: master # release_strategy: none # env: - # GITHUB_TOKEN: ${{ github.token }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build and publish env: diff --git a/.github/workflows/create_tag.yml b/.github/workflows/create_tag.yml new file mode 100644 index 0000000..732308b --- /dev/null +++ b/.github/workflows/create_tag.yml @@ -0,0 +1,22 @@ +name: Tag + +on: + pull_request: + types: closed + +jobs: + tag: + name: Create Tag + runs-on: ubuntu-latest + if: github.event.pull_request.merged && github.base_ref == 'master' + steps: + - name: Checkout Code + uses: actions/checkout@master + + - name: Create Tag + uses: K-Phoen/semver-release-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + release_branch: master + release_strategy: tag \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c720790 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release + +on: create + +jobs: + release: + name: Pypi + runs-on: ubuntu-latest + if: github.event.ref_type == 'tag' + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Build and Publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + RELEASE_VERSION: ${{ github.event.ref }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* + + - name: Create Github Release + uses: Roang-zero1/github-create-release-action@master + with: + created_tag: ${{ github.event.ref }} + + - name: Update Release with Artifacts + uses: ncipollo/release-action@v1.6.1 + with: + allowUpdates: true + tag: ${{ github.event.ref }} + artifacts: "dist/*" + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 40de808c795a562029ef0fc9ad0a8ab0416b8c07 Mon Sep 17 00:00:00 2001 From: Levi Blaney Date: Sat, 23 May 2020 12:03:20 -0400 Subject: [PATCH 134/134] add back functional tests --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d689866..a509ab1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: functional: name: Functional Tests runs-on: ubuntu-latest - if: github.base_ref == 'sdfjsdlfksjd' + if: github.base_ref == 'master' needs: unit env: GITHUB_SOURCE_BRANCH: ${{ github.head_ref }}