Merge pull request #1134 from seriva/feature/integrated-spec

- Moved spec tests to data folder
- Added epicli test command for running spec tests
- Added spec test output to build folder
- Added spec test dependencies to epicli production container
- Restructured Python unit tests and fix some minor issues

core/src/epicli/.gitignore

@@ -121,8 +121,7 @@ dmypy.json
 
 # epicli specific
 external/
-tests/serverspec-cli/results/
-tests/cli/results/
+tests/results/
 .terraform
 clusters
 .vscode

core/src/epicli/.vscode/launch.json

@@ -17,28 +17,18 @@
             // "args": ["apply", "-f", "${workspaceFolder}/clusters/YOUR_DATA_YAML.yml", "--offline-requirements", "${workspaceFolder}/downloads/epirepo"]
             // "args": ["delete", "-b", "${workspaceFolder}/clusters/build/dir"]
             // "args": ["init", "-p", "PROVIDER", "-n", "NAME"]
+            // "args": ["test", "-b", "${workspaceFolder}/clusters/build/dir"]
         },
         {
-            "name": "python unit tests",
+            "name": "unit tests",
             "type": "python",
             "request": "launch",
-            "program": "${workspaceFolder}/run-tests.py",
+            "module": "pytest",
             "cwd": "${workspaceFolder}",
             "pythonPath": "${config:python.pythonPath}",
             "env": { "PYTHONPATH": "${workspaceFolder}" },
             "console": "integratedTerminal",
-            "args": ["python"]
-        },
-        {
-            "name": "server spec tests",
-            "type": "python",
-            "request": "launch",
-            "program": "${workspaceFolder}/run-tests.py",
-            "cwd": "${workspaceFolder}",
-            "pythonPath": "${config:python.pythonPath}",
-            "env": { "PYTHONPATH": "${workspaceFolder}" },
-            "console": "integratedTerminal",
-            "args": ["spec", "-i", "${workspaceFolder}/PATH_TO_CLUSTER_INVENTORY", "-u", "ADMIN_USER", "-k", "${workspaceFolder}/PATH_TO_SSH_KEY"]
+            "args": ["${workspaceFolder}/tests/", "--junitxml=${workspaceFolder}/tests/results/unit_test_results.xml"]
         }
     ]	
-}
+}

core/src/epicli/Dockerfile

@@ -12,7 +12,9 @@ COPY /dist/ /epicli
 WORKDIR /epicli
 
 RUN apt-get update \
-    && apt-get -y install gcc make musl-dev libffi-dev tar unzip openssh-client vim  
+    && apt-get -y install gcc make musl-dev libffi-dev tar unzip openssh-client vim \
+    && apt-get -y install ruby-full \
+    && gem install serverspec rake rspec_junit_formatter    
 
 RUN pip install epicli-${EPICLI_VERSION}-py3-none-any.whl
 

core/src/epicli/cli/engine/TestEngine.py

@@ -0,0 +1,51 @@
+import os
+import shutil
+
+from cli.helpers.Step import Step
+from cli.helpers.Config import Config
+from cli.helpers.build_saver import SPEC_OUTPUT_DIR, MANIFEST_FILE_NAME, ANSIBLE_OUTPUT_DIR, INVENTORY_FILE_NAME
+from cli.helpers.data_loader import load_yamls_file
+from cli.helpers.doc_list_helpers import select_single
+from cli.engine.spec.SpecCommand import SpecCommand
+
+class TestEngine(Step):
+    def __init__(self, input_data):
+        super().__init__(__name__)
+        self.build_directory = input_data.build_directory
+        self.group = input_data.group
+
+    def __enter__(self):
+        super().__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        super().__exit__(exc_type, exc_value, traceback)
+
+    def test(self):
+        # get manifest
+        path_to_manifest = os.path.join(self.build_directory, MANIFEST_FILE_NAME)
+        if not os.path.isfile(path_to_manifest):
+            raise Exception(f'No "{MANIFEST_FILE_NAME}" inside the build directory: "{self.build_directory}"')
+
+        # get inventory
+        path_to_inventory = os.path.join(self.build_directory, INVENTORY_FILE_NAME)
+        if not os.path.isfile(path_to_inventory):
+            raise Exception(f'No "{INVENTORY_FILE_NAME}" inside the build directory: "{self.build_directory}"')        
+
+        # get admin user
+        docs = load_yamls_file(path_to_manifest)
+        cluster_model = select_single(docs, lambda x: x.kind == 'epiphany-cluster')
+        admin_user = cluster_model.specification.admin_user
+        if not os.path.isfile(admin_user.key_path):
+            raise Exception(f'No SSH key file in directory: "{admin_user.key_path}"')
+
+        # get and create the spec output dir if it does not exist
+        spec_output = os.path.join(self.build_directory, SPEC_OUTPUT_DIR)
+        if not os.path.exists(spec_output):
+            os.makedirs(spec_output)        
+
+        # run the spec tests
+        spec_command = SpecCommand()
+        spec_command.run(spec_output, path_to_inventory, admin_user.name, admin_user.key_path, self.group)   
+
+        return 0

core/src/epicli/cli/engine/spec/SpecCommand.py

@@ -0,0 +1,61 @@
+import os
+import subprocess
+import shutil
+from subprocess import Popen, PIPE
+
+from cli.helpers.Log import LogPipe, Log
+from cli.helpers.Config import Config
+from cli.helpers.data_loader import DATA_FOLDER_PATH
+
+SPEC_TEST_PATH = DATA_FOLDER_PATH + '/common/tests'
+
+class SpecCommand:
+    def __init__(self):
+        self.logger = Log(__name__)
+
+
+    def __init__(self):
+        self.logger = Log(__name__)
+
+
+    def check_dependencies(self):
+        required_gems = ['serverspec', 'rake', 'rspec_junit_formatter'] 
+
+        error_str = f'''Missing Ruby or one of the following Ruby gems: {', '.join(required_gems)}
+These need to be installed to run the cluster spec tests from epicli'''
+
+        if  shutil.which('ruby') == None or shutil.which('gem') == None:
+            raise Exception(error_str)
+
+        p = subprocess.Popen(['gem', 'query', '--local'], stdout=PIPE)
+        out, err = p.communicate()
+        if all(n in out.decode('utf-8') for n in required_gems) == False: 
+            raise Exception(error_str)
+
+
+    def run(self, spec_output, inventory, user, key, group):
+        self.check_dependencies()
+
+        env = os.environ.copy()
+        env['spec_output'] = spec_output
+        env['inventory'] = inventory
+        env['user'] = user
+        env['keypath'] = key
+
+        cmd = f'rake inventory={inventory} user={user} keypath={key} spec_output={spec_output} spec:{group}'
+
+        self.logger.info(f'Running: "{cmd}"')
+
+        logpipe = LogPipe(__name__)
+        with Popen(cmd.split(' '), cwd=SPEC_TEST_PATH, env=env, stdout=logpipe, stderr=logpipe) as sp:
+            logpipe.close()
+
+        if sp.returncode != 0:
+            raise Exception(f'Error running: "{cmd}"')
+        else:
+            self.logger.info(f'Done running: "{cmd}"')       
+
+
+    @staticmethod
+    def get_spec_groups():
+        return os.listdir(SPEC_TEST_PATH + '/spec')

core/src/epicli/cli/epicli.py

@@ -16,13 +16,15 @@
 from cli.engine.InitEngine import InitEngine
 from cli.engine.PrepareEngine import PrepareEngine
 from cli.engine.UpgradeEngine import UpgradeEngine
+from cli.engine.TestEngine import TestEngine
 from cli.helpers.Log import Log
 from cli.helpers.Config import Config
 from cli.version import VERSION
 from cli.licenses import LICENSES
 from cli.helpers.query_yes_no import query_yes_no
 from cli.helpers.input_query import prompt_for_password
 from cli.helpers.build_saver import save_to_file, get_output_path
+from cli.engine.spec.SpecCommand import SpecCommand
 
 
 def main():
@@ -88,6 +90,7 @@ def debug_level(x):
     apply_parser(subparsers)
     upgrade_parser(subparsers)
     delete_parser(subparsers)
+    test_parser(subparsers)
 
     '''
     validate_parser(subparsers)
@@ -219,6 +222,22 @@ def run_upgrade(args):
     sub_parser.set_defaults(func=run_upgrade)
 
 
+def test_parser(subparsers):
+    sub_parser = subparsers.add_parser('test', description='Test a cluster from build artifacts.')
+    sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
+                            help='Absolute path to directory with build artifacts.')
+    sub_parser.add_argument('-g', '--group', choices=['all'] + SpecCommand.get_spec_groups(), default='all', action='store', dest='group',  required=False,
+                            help='Group of tests to be run, e.g. kafka.')
+
+    def run_test(args):
+        experimental_query()
+        adjust_paths_from_build(args)
+        with TestEngine(args) as engine:
+            return engine.test()
+
+    sub_parser.set_defaults(func=run_test)
+
+
 '''
 def validate_parser(subparsers):
     sub_parser = subparsers.add_parser('verify', description='Validates the configuration from file by executing a dry '
@@ -369,6 +388,9 @@ def dump_external_debug_info(title, args):
         dump_external_debug_info('ANSIBLE-VAULT VERSION', ['ansible-vault', '--version'])
         dump_external_debug_info('TERRAFORM VERSION', ['terraform', '--version'])
         dump_external_debug_info('SKOPEO VERSION', ['skopeo', '--version'])
+        dump_external_debug_info('RUBY VERSION', ['ruby', '--version'])
+        dump_external_debug_info('RUBY GEM VERSION', ['gem', '--version'])
+        dump_external_debug_info('RUBY INSTALLED GEMS', ['gem', 'query', '--local'])
 
         dump_file.write('\n\n*****LOG******\n')
         log_path = os.path.join(get_output_path(), config.log_file)

core/src/epicli/cli/helpers/build_saver.py

@@ -14,6 +14,7 @@
 INVENTORY_FILE_NAME = 'inventory'
 ANSIBLE_OUTPUT_DIR = 'ansible/'
 ANSIBLE_VAULT_OUTPUT_DIR = 'vault/'
+SPEC_OUTPUT_DIR = 'spec_tests/'
 
 BUILD_EPICLI = 'BUILD_EPICLI'
 BUILD_LEGACY = 'BUILD_LEGACY_02X'
@@ -141,4 +142,3 @@ def copy_files_recursively(src, dst):
 
 def copy_file(src, dst):
     shutil.copy2(src, dst)
-

core/src/epicli/tests/serverspec-cli/Rakefile → core/src/epicli/data/common/tests/Rakefile

@@ -3,23 +3,27 @@ require 'rspec/core/rake_task'
 
 unless ENV['inventory']
   print "ERROR: Inventory file must be specified by 'inventory' environment variable\n"
-  print "  e.g.) bundle exec rake inventory=./hosts spec:all\n"
+  print "  e.g.) rake inventory=./hosts user=operations keypath=./id_rsa spec_output=./spec_output/ spec:all\n"
   exit
 end
 
 unless ENV['user']
   print "ERROR: Service user must be specified by 'user' environment variable\n"
-  print "  e.g.) bundle exec rake inventory=./hosts user=operations spec:all\n"
+  print "  e.g.) rake inventory=./hosts user=operations keypath=./id_rsa spec_output=./spec_output/ spec:all\n"
   exit
 end
 
 unless ENV['keypath']
   print "ERROR: Private key path must be specified by 'keypath' environment variable\n"
-  print "  e.g.) bundle exec rake inventory=./hosts user=operations keypath=./id_rsa spec:all\n"
+  print "  e.g.) rake inventory=./hosts user=operations keypath=./id_rsa spec_output=./spec_output/ spec:all\n"
   exit
 end
 
-
+unless ENV['spec_output']
+  print "ERROR: Output path must be specified by 'spec_output' environment variable\n"
+  print "  e.g.) rake inventory=./hosts user=operations keypath=./id_rsa spec_output=./spec_output/ spec:all\n"
+  exit
+end
 
 groups = {}
 hosts  = []
@@ -73,7 +77,7 @@ namespace :spec do
         t.pattern = "spec/#{group}/*_spec.rb"
         t.fail_on_error = false
         t.rspec_opts = "--format documentation --format RspecJunitFormatter  \
-        --out results/" + Time.now.strftime("%Y-%m-%d_%H-%M-%S") + "_#{group}_#{groups[group][host]}.xml"
+        --out " + ENV['spec_output'] + Time.now.strftime("%Y-%m-%d_%H-%M-%S") + "_#{group}_#{groups[group][host]}.xml"
        end
     end
   end

core/src/epicli/tests/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+junit_family = xunit1
+; disable warnings since this will only complain about external Python modules we have no control over.
+addopts = -p no:warnings