Provide kubeconfig file for spec tests (#3206)

* Comply with Rubocop

* Print selected groups as yaml

* Provide kubeconfig file for spec tests

* Fix Pylint import-error issues in VSCode

* Self code review

* Add new option to launch configurations

* Fix after tests

* Update rubocop_linter_threshold

* Apply suggestions from code review

* Print selected test groups before preparing env

.devcontainer/python.env

@@ -2,4 +2,4 @@
 
 # PYTHONPATH can contain multiple locations separated by os.pathsep: semicolon (;) on Windows and colon (:) on Linux/macOS.
 # Invalid paths are ignored. To verify use "python.analysis.logLevel": "Trace".
-PYTHONPATH="ansible/playbooks/roles/repository/files/download-requirements:${PYTHONPATH}"
+PYTHONPATH=ansible/playbooks/roles/repository/files/download-requirements

.pylintrc

@@ -6,7 +6,8 @@
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+# The following line fixes 'import-error' issues in VS Code, like "Unable to import 'cli.src.ansible.AnsibleCommand'"
+init-hook='from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))'
 
 # Files or directories to be skipped. They should be base names, not
 # paths.

.rubocop.yml

@@ -11,3 +11,17 @@
 AllCops:
   Include:
     - tests/spec/**/*.rb
+
+Layout/ExtraSpacing:
+  AllowBeforeTrailingComments: true
+
+Metrics/BlockLength:
+  Exclude:
+    - tests/spec/Rakefile
+
+Naming/FileName:
+  Exclude:
+    - tests/spec/Rakefile
+
+Style/FrozenStringLiteralComment:
+  Enabled: false

.vscode/launch.json

@@ -21,8 +21,8 @@
             // "args": ["init", "-p", "<PROVIDER>", "-n", "<NAME>"]
             // "args": ["init", "-p", "<PROVIDER>", "-n", "<NAME>", "--full"]
             // "args": ["prepare", "--os", "<OS>", "--arch", "<ARCH>"]
-            // "args": ["--auto-approve", "test", "-b", "${workspaceFolder}/clusters/build/<DIR>"]
-            // "args": ["--auto-approve", "test", "-b", "${workspaceFolder}/clusters/build/<DIR>", "-i", "<GROUPS>"]
+            // "args": ["--auto-approve", "test", "-b", "${workspaceFolder}/clusters/build/<DIR>", "-k", "/etc/kubernetes/admin.conf"]
+            // "args": ["--auto-approve", "test", "-b", "${workspaceFolder}/clusters/build/<DIR>", "-i", "<GROUPS>", "-k", "/etc/kubernetes/admin.conf"]
             // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/<DIR>"]
             // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/<DIR>", "--upgrade-components", "kafka"]
         }

.vscode/settings.json

@@ -20,5 +20,6 @@
     "ruby.lint": {
         "rubocop": true
     },
+    "solargraph.diagnostics": false,
     "solargraph.formatting": true,
 }

ansible/playbooks/kubernetes_master.yml

@@ -53,9 +53,7 @@
   become: true
   become_method: sudo
   post_tasks:
-    - name: Run copy-kubeconfig from kubernetes_master role
+    - name: Run generate-kubeconfig from kubernetes_master role
       import_role:
         name: kubernetes_master
-        tasks_from: copy-kubeconfig
-  environment:
-    KUBECONFIG: "{{ kubeconfig.remote }}"
+        tasks_from: generate-kubeconfig

ci/pipelines/linters.yaml

@@ -16,7 +16,7 @@ variables:
   ansible_lint_error_threshold: 338
   pylint_score_cli_threshold: 9.44
   pylint_score_tests_threshold: 9.78
-  rubocop_linter_threshold: 207
+  rubocop_linter_threshold: 183
 
 jobs:
   - job: Run_linters

cli/epicli.py

@@ -330,6 +330,9 @@ def test_parser(subparsers):
     optional.add_argument('-i', '--include', default='all', type=comma_separated_type(choices=include_choices),
                             dest='included_groups', required=False,
                             help='Group of tests to be run, e.g. -i kafka,kafka_exporter.')
+    optional.add_argument('-k', '--kubeconfig-remote-path', type=os.path.abspath,
+                            dest='kubeconfig_remote_path', required=False,
+                            help='Absolute path to kubeconfig file on K8s master host, e.g. /etc/kubernetes/admin.conf.')
     sub_parser._action_groups.append(optional)
 
     def run_test(args):

cli/src/ansible/AnsibleCommand.py

@@ -41,7 +41,7 @@ def run_task(self, hosts, inventory, module, args=None):
         else:
             self.logger.info('Done running "' + ' '.join(cmd) + '"')
 
-    def run_task_with_retries(self, inventory, module, hosts, retries, timeout=10, args=None):
+    def run_task_with_retries(self, inventory, module, hosts, retries, delay=10, args=None):
         for i in range(retries):
             try:
                 self.run_task(hosts=hosts, inventory=inventory, module=module,
@@ -50,11 +50,14 @@ def run_task_with_retries(self, inventory, module, hosts, retries, timeout=10, a
             except Exception as e:
                 self.logger.error(e)
                 self.logger.warning('Retry running task: ' + str(i + 1) + '/' + str(retries))
-                time.sleep(timeout)
+                time.sleep(delay)
         else:
             raise Exception(f'Failed running task after {str(retries)} retries')
 
-    def run_playbook(self, inventory, playbook_path, vault_file=None):
+    def run_playbook(self, inventory, playbook_path, vault_file=None, extra_vars:list[str]=None):
+        """
+        :param extra_vars: playbook's variables passed via `--extra-vars` option
+        """
         cmd = ['ansible-playbook']
 
         if inventory is not None and len(inventory) > 0:
@@ -63,6 +66,10 @@ def run_playbook(self, inventory, playbook_path, vault_file=None):
         if vault_file is not None:
             cmd.extend(["--vault-password-file", vault_file])
 
+        if extra_vars:
+            for var in extra_vars:
+                cmd.extend(['--extra-vars', var])
+
         cmd.append(playbook_path)
 
         if Config().debug > 0:
@@ -79,15 +86,16 @@ def run_playbook(self, inventory, playbook_path, vault_file=None):
         else:
             self.logger.info('Done running "' + ' '.join(cmd) + '"')
 
-    def run_playbook_with_retries(self, inventory, playbook_path, retries, timeout=10):
+    def run_playbook_with_retries(self, inventory, playbook_path, retries, delay=10, extra_vars:list[str]=None):
         for i in range(retries):
             try:
                 self.run_playbook(inventory=inventory,
-                                  playbook_path=playbook_path)
+                                  playbook_path=playbook_path,
+                                  extra_vars=extra_vars)
                 break
             except Exception as e:
                 self.logger.error(e)
                 self.logger.warning('Retry running playbook: ' + str(i + 1) + '/' + str(retries))
-                time.sleep(timeout)
+                time.sleep(delay)
         else:
             raise Exception(f'Failed running playbook after {str(retries)} retries')

cli/src/commands/Test.py

@@ -1,20 +1,24 @@
 import os
 
-from cli.src.helpers.build_io import (ANSIBLE_INVENTORY_FILE, SPEC_OUTPUT_DIR,
-                                      load_manifest)
-from cli.src.helpers.build_io import load_inventory
+from cli.src.ansible.AnsibleCommand import AnsibleCommand
+from cli.src.helpers.build_io import (SPEC_OUTPUT_DIR,
+                                      get_inventory_path_for_build, load_inventory, load_manifest)
 from cli.src.helpers.doc_list_helpers import select_single
-from cli.src.spec.SpecCommand import SpecCommand
+from cli.src.spec.SpecCommand import SPEC_TESTS_PATH, SpecCommand
 from cli.src.Step import Step
 
 
 class Test(Step):
     def __init__(self, input_data, test_groups):
         super().__init__(__name__)
         self.build_directory = input_data.build_directory
+        self.inventory_path = get_inventory_path_for_build(self.build_directory)
         self.excluded_groups = input_data.excluded_groups
         self.included_groups = input_data.included_groups
-        self.test_groups = test_groups
+        self.kubeconfig_remote_path = input_data.kubeconfig_remote_path
+        self.all_groups = test_groups
+        self.available_groups = self.__get_available_test_groups()
+        self.selected_groups = self.__get_selected_test_groups()
 
     def __enter__(self):
         super().__enter__()
@@ -23,16 +27,75 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         pass
 
+    def __get_inventory_groups(self, append_implicit: bool) -> list[str]:
+        """
+        Get list of groups from Ansible inventory
+
+        :param append_implicit: if True, `common` group, which is not present in inventory, is appended
+        """
+        inventory_groups = load_inventory(self.inventory_path).list_groups()
+        if append_implicit:
+            inventory_groups.append('common')
+        return inventory_groups
+
+    def __get_available_test_groups(self) -> list[str]:
+        """
+        Get list of all test groups that can be run
+        """
+        inventory_groups = self.__get_inventory_groups(True)
+        return [group for group in self.all_groups if group in inventory_groups]
+
+    def __get_selected_test_groups(self) -> list[str]:
+        """
+        Get list of test groups selected to be run
+        """
+        selected_groups = ['all'] if 'all' in self.included_groups else self.included_groups
+
+        # exclude test groups
+        if self.excluded_groups:
+            included_groups = self.available_groups if 'all' in self.included_groups else self.included_groups
+            selected_groups = [group for group in included_groups if group not in self.excluded_groups]
+
+        return selected_groups
+
+    def __is_env_preparation_needed(self) -> bool:
+        """
+        Check whether additional actions are needed in order to run selected test groups
+        """
+        if self.kubeconfig_remote_path:
+            kubectl_groups = ['applications', 'kubernetes_master']
+            if any(group in kubectl_groups for group in self.selected_groups):
+                return True
+            if 'all' in self.selected_groups and any(group in kubectl_groups for group in self.available_groups):
+                return True
+
+        return False
+
+    def __prepare_env(self):
+        if self.__is_env_preparation_needed():
+            playbook_path = str(SPEC_TESTS_PATH) + '/pre_run/ansible/kubernetes_master/copy-kubeconfig.yml'
+            ansible_command = AnsibleCommand()
+            ansible_command.run_playbook(inventory=self.inventory_path,
+                                         playbook_path=playbook_path,
+                                         extra_vars=[f'kubeconfig_remote_path={self.kubeconfig_remote_path}'])
+
+    def __clean_up_env(self):
+        if self.__is_env_preparation_needed():
+            playbook_path = str(SPEC_TESTS_PATH) + '/post_run/ansible/kubernetes_master/undo-copy-kubeconfig.yml'
+            ansible_command = AnsibleCommand()
+            ansible_command.run_playbook(inventory=self.inventory_path, playbook_path=playbook_path)
+
     def test(self):
+        """
+        Run spec tests for selected groups
+        """
+        if not self.selected_groups:
+            raise Exception('No test group specified to run')
+
         # get manifest documents
         docs = load_manifest(self.build_directory)
         cluster_model = select_single(docs, lambda x: x.kind == 'epiphany-cluster')
 
-        # get inventory
-        path_to_inventory = os.path.join(self.build_directory, ANSIBLE_INVENTORY_FILE)
-        if not os.path.isfile(path_to_inventory):
-            raise Exception(f'No "{ANSIBLE_INVENTORY_FILE}" inside the build directory: "{self.build_directory}"')
-
         # get admin user
         admin_user = cluster_model.specification.admin_user
         if not os.path.isfile(admin_user.key_path):
@@ -43,29 +106,15 @@ def test(self):
         if not os.path.exists(spec_output):
             os.makedirs(spec_output)
 
-        selected_test_groups = self.included_groups
+        if 'all' not in self.selected_groups:
+            self.logger.info(f'Selected test groups: {", ".join(self.selected_groups)}')
 
-        # exclude test groups
-        if self.excluded_groups:
-            included_groups = self.included_groups
-            if 'all' in included_groups:
-                # get available test groups
-                inventory_groups = load_inventory(path_to_inventory).list_groups()
-                effective_inventory_groups = inventory_groups + ['common']
-                included_groups = [group for group in self.test_groups if group in effective_inventory_groups]
-
-            selected_test_groups = [group for group in included_groups if group not in self.excluded_groups]
-
-        # run the spec tests
-        if selected_test_groups:
-            spec_command = SpecCommand()
-            if 'all' in selected_test_groups:
-                selected_test_groups = ['all']
-            else:
-                self.logger.info(f'Selected test groups: {", ".join(selected_test_groups)}')
-
-            spec_command.run(spec_output, path_to_inventory, admin_user.name, admin_user.key_path, selected_test_groups)
-        else:
-            raise Exception('No test group specified to run')
+        self.__prepare_env()
+
+        # run tests
+        spec_command = SpecCommand()
+        spec_command.run(spec_output, self.inventory_path, admin_user.name, admin_user.key_path, self.selected_groups)
+
+        self.__clean_up_env()
 
         return 0

cli/src/spec/SpecCommand.py

@@ -1,12 +1,13 @@
 import os
 import shutil
 import subprocess
+from pathlib import Path
 from subprocess import PIPE, Popen
 
 from cli.src.helpers.data_loader import BASE_DIR
 from cli.src.Log import Log, LogPipe
 
-SPEC_TEST_PATH = BASE_DIR + '/tests/spec'
+SPEC_TESTS_PATH = Path(BASE_DIR).resolve() / 'tests' / 'spec'
 
 class SpecCommand:
     def __init__(self):
@@ -42,7 +43,7 @@ def run(self, spec_output, inventory, user, key, groups):
         self.logger.info(f'Running: "{cmd}"')
 
         logpipe = LogPipe(__name__)
-        with Popen(cmd.split(' '), cwd=SPEC_TEST_PATH, env=env, stdout=logpipe, stderr=logpipe) as sp:
+        with Popen(cmd.split(' '), cwd=SPEC_TESTS_PATH, env=env, stdout=logpipe, stderr=logpipe) as sp:
             logpipe.close()
 
         if sp.returncode != 0:
@@ -54,10 +55,7 @@ def run(self, spec_output, inventory, user, key, groups):
     @staticmethod
     def get_spec_groups() -> list[str]:
         """Get test groups based on directories."""
+        groups_path = SPEC_TESTS_PATH / 'spec'
+        groups = [str(item.name) for item in groups_path.iterdir() if item.is_dir()]
 
-        listdir = os.listdir(f'{SPEC_TEST_PATH}/spec')
-        groups = []
-        for entry in listdir:
-            if os.path.isdir(f'{SPEC_TEST_PATH}/spec/{entry}'):
-                groups = groups + [entry]
-        return sorted(groups, key=str.lower)
+        return sorted(groups)