diff --git a/docs/guides/join_nodes.md b/docs/guides/join_nodes.md index 6c3a9b8..f323c42 100644 --- a/docs/guides/join_nodes.md +++ b/docs/guides/join_nodes.md @@ -27,44 +27,30 @@ node-3 If you don't have provision a load-balancer and require the local haproxy to be deployed: ``` -ansible-playbook -i inventory enix.kubeadm.00_apiserver_proxy -l kube_control_plane:nodes-3 +ansible-playbook -i inventory enix.kubeadm.00_apiserver_proxy -e limit=nodes-3 ``` -You can skip the `-l` argument, if you're cluster doesn't have pending change you want to preserve on other nodes. -Don't forget to put all control_plane or it will fail to provision the apiserver proxy +You need to specify the `limit` variable via "extra-vars", because `-l` cannot really work in the context of ansible-kubeadm +(you need to connect to all the masters to get the IP needed to configure the loadbalancer) +### Joining nodes -### Create bootstrap-token - -Then create a bootstrap token by adding using the `bootstrap_token` tag. -Don't use a limit that skip control plane nodes. +You can join a node and skip other changes on other nodes by specify the limit variable. ``` -ansible-playbook -i inventory.cfg enix.kubeadm.01_site -t bootstrap_token +ansible-play -i inventory.cfg enix.kubeadm.01_site -e limit=nodes-3 ``` -No need to retrieve it by yourself, it will be discovered when joining the node -The token has a validity of 1H, so you don't need to repeat this step each time you try to join nodes - -### Joining nodes - -You can join a node and skip other changes to the cluster by using the `join` tag. -With the tag, you can limit to hosts you want to join. -``` -ansible-play -i inventory.cfg enix.kubeadm.01_site -t join -l nodes-3 -``` -## Alternative method +### Create bootstrap-token -You can merge the creation of the boostrap token with the joining of the action of join: +Then create a bootstrap token by adding using the `bootstrap_token` tag. +Don't use a limit that skip control plane nodes. ``` -ansible-playbook -i inventory.cfg enix.kubeadm.01_site -t bootstap_token,join -l kube_control_plane:node-3 +ansible-playbook -i inventory.cfg enix.kubeadm.01_site -t bootstrap_token ``` -Please note that you need to include a least one control plane node in the limit host pattern, -You can also skip the limit host pattern to apply to all nodes as those step are indempotent on their own: it will not mess with the current nodes. - -# To join control-plane nodes +No need to retrieve it by yourself, it will be discovered when joining the node +The token has a validity of 1H, so you don't need to repeat this step each time you try to join nodes -There is no tag for this operation, you need to apply the entire playbook for this diff --git a/playbooks/00_apiserver_proxy.yml b/playbooks/00_apiserver_proxy.yml index 779c558..be59d83 100644 --- a/playbooks/00_apiserver_proxy.yml +++ b/playbooks/00_apiserver_proxy.yml @@ -10,15 +10,10 @@ any_errors_fatal: '{{ any_errors_fatal|default(true) }}' vars: _control_plane: true - pre_tasks: - - name: 'Fail if not all master in the specified limit' - fail: - msg: 'Not all control_plane provided, ajust --limit to provid all control_plane' - when: groups[kube_cp_group|default("kube_control_plane")]|difference(ansible_play_hosts)|length > 0 roles: - role: find_ip -- hosts: '{{ kube_cp_group|default("kube_control_plane") }}:{{ kube_worker_group|default("kube_workers") }}' +- hosts: '{{ kube_cp_group|default("kube_control_plane") }}:{{ kube_worker_group|default("kube_workers") }}{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' pre_tasks: - include_role: @@ -36,7 +31,7 @@ vars: kubeadm_hook_list: ['post_apiserver_proxy'] -- hosts: 'haproxy_upgrade_group:&{{ kube_cp_group|default("kube_control_plane") }}' +- hosts: 'haproxy_upgrade_group:&{{ kube_cp_group|default("kube_control_plane") }}{{ ":" ~ limit if limit is defined else "" }}' serial: '{{ upgrade_cp_serial|default(1) }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' pre_tasks: @@ -52,7 +47,7 @@ vars: kubeadm_hook_list: ['post_proxy_upgrade_haproxy'] -- hosts: 'haproxy_upgrade_group:&{{ kube_worker_group|default("kube_workers") }}' +- hosts: 'haproxy_upgrade_group:&{{ kube_worker_group|default("kube_workers") }}{{ ":" ~ limit if limit is defined else "" }}' serial: '{{ upgrade_worker_serial|default(1) }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' pre_tasks: diff --git a/playbooks/01_site.yml b/playbooks/01_site.yml index 23c73e3..7fa6acb 100644 --- a/playbooks/01_site.yml +++ b/playbooks/01_site.yml @@ -25,7 +25,7 @@ vars: kubeadm_hook_list: ['post_preflight_cp'] -- hosts: '{{ kube_cp_group|default("kube_control_plane") }}:{{ kube_worker_group|default("kube_workers") }}' +- hosts: '{{ kube_cp_group|default("kube_control_plane") }}:{{ kube_worker_group|default("kube_workers") }}{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' roles: - role: find_ip @@ -42,7 +42,7 @@ roles: - role: process_reasons -- hosts: '{{ kube_cp_group|default("kube_control_plane") }}' +- hosts: '{{ kube_cp_group|default("kube_control_plane") }}{{ ":" ~ limit if limit is defined else "" }}{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' gather_facts: false roles: @@ -92,7 +92,7 @@ kubeadm_hook_list: ['post_config_update'] # This has to be overly cautious on package upgade -- hosts: cp_upgrade +- hosts: 'cp_upgrade{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' gather_facts: false pre_tasks: @@ -118,7 +118,7 @@ # Upgrade conrol-plane nodes - name: 'Upgrade to control plane nodes' - hosts: '{{ kube_cp_group|default("kube_control_plane") }}:&nodes_upgrade' + hosts: '{{ kube_cp_group|default("kube_control_plane") }}:&nodes_upgrade{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' serial: '{{ upgrade_cp_serial|default(1) }}' gather_facts: false @@ -147,7 +147,7 @@ # Upgrade worker nodes - name: 'Upgrade to workers nodes' - hosts: '{{ kube_worker_group|default("kube_workers") }}:&nodes_upgrade' + hosts: '{{ kube_worker_group|default("kube_workers") }}:&nodes_upgrade{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' serial: '{{ upgrade_worker_serial|default(1) }}' gather_facts: false @@ -174,7 +174,7 @@ # Join control-plane nodes - name: 'Join new control plane nodes' - hosts: '{{ kube_cp_group|default("kube_control_plane") }}' + hosts: '{{ kube_cp_group|default("kube_control_plane") }}{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' gather_facts: false vars: @@ -198,7 +198,7 @@ # Join worker nodes - name: 'Join new workers nodes' - hosts: '{{ kube_worker_group|default("kube_workers") }}' + hosts: '{{ kube_worker_group|default("kube_workers") }}{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' gather_facts: false tags: ['join'] @@ -218,7 +218,7 @@ kubeadm_hook_list: ['post_workers_join', 'post_nodes_join'] - name: 'Finally executing post_run hook on all hosts' - hosts: '{{ kube_cp_group|default("kube_control_plane") }}:{{ kube_worker_group|default("kube_workers") }}' + hosts: '{{ kube_cp_group|default("kube_control_plane") }}:{{ kube_worker_group|default("kube_workers") }}{{ ":" ~ limit if limit is defined else "" }}' any_errors_fatal: '{{ any_errors_fatal|default(true) }}' gather_facts: false tasks: diff --git a/roles/bootstrap_token/tasks/main.yaml b/roles/bootstrap_token/tasks/main.yaml index e708521..7bd2ce3 100644 --- a/roles/bootstrap_token/tasks/main.yaml +++ b/roles/bootstrap_token/tasks/main.yaml @@ -4,6 +4,7 @@ nodes_to_join: >- {{ q('inventory_hostnames', kube_cp_group ~ ':' ~ kube_worker_group) |map('extract', hostvars) + |selectattr('_kubelet_config_stat', 'defined') |rejectattr('_kubelet_config_stat.stat.exists') |map(attribute='inventory_hostname')|list }} run_once: true diff --git a/roles/common_vars/defaults/main.yml b/roles/common_vars/defaults/main.yml index 8a0019f..e2ba02c 100644 --- a/roles/common_vars/defaults/main.yml +++ b/roles/common_vars/defaults/main.yml @@ -6,3 +6,6 @@ kube_cp_group: kube_control_plane kube_worker_group: kube_workers cp_node: '{{ (groups.cp_running|default(groups[kube_cp_group]))|first }}' + +_target_kube_version: '{{ hostvars[cp_node]._target_kube_version }}' +_target_kubeadm_version: '{{ hostvars[cp_node]._target_kubeadm_version }}' diff --git a/roles/packages/meta/main.yml b/roles/packages/meta/main.yml index 94c1f15..6e32029 100644 --- a/roles/packages/meta/main.yml +++ b/roles/packages/meta/main.yml @@ -1,19 +1,4 @@ --- dependencies: - role: packages_common -galaxy_info: - author: Julien Girardin - description: Install kubernetes related packages - company: Enix - license: Apache - min_ansible_version: 2.7 - platforms: - - name: Ubuntu - versions: - - 18.04 - - 20.04 - galaxy_tags: - - kubernetes - - kubeadm - - kubelet - - kubectl + - role: common_vars diff --git a/roles/upload_certs/tasks/main.yml b/roles/upload_certs/tasks/main.yml index 5ee9768..ba2e6f3 100644 --- a/roles/upload_certs/tasks/main.yml +++ b/roles/upload_certs/tasks/main.yml @@ -15,7 +15,7 @@ kubeadm init phase upload-certs --upload-certs --certificate-key {{ cert_encryption_key }} - no_log: '{{ sensitive_debug|bool }}' + no_log: '{{ not sensitive_debug|bool }}' run_once: true delegate_to: '{{ cp_node }}' when: _cp_to_join|length > 0 diff --git a/tests/conftest.py b/tests/conftest.py index d604c60..1275190 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,8 +95,8 @@ def vagrant(tmpdir): return LocalVagrant(inventory_dir_copy=tmpdir) -@then("Set cluster {variable} = {value}") -@given("The cluster {variable) = {value}") +@then(parsers.parse("Set cluster {variable} = {value}")) +@given(parsers.parse("The cluster {variable} = {value}")) def cluster_set_param(provider, variable, value): provider.vars[variable] = value # Refresh infrastructure @@ -160,7 +160,7 @@ def ansible_extra_args(request): @when( parsers.re( - r"I (?Pdry-)?run the playbooks?:?\s+(?P.+?)(?P\s+with error:?\s+)?(?(with_err)(?P.+)|\Z)", + r"I (?Pdry-)?run the playbooks?:?\s+(?P.+?)(?P\s+with error:?\s+)?(?(with_err)(?P.+)|\Z)", re.DOTALL, ) ) @@ -171,7 +171,7 @@ def ansible_playbook( galaxy_deps, ansible_extra_args, results, - playbooks, + arguments, dry_run, error, ): @@ -179,18 +179,12 @@ def ansible_playbook( dry_run = True else: dry_run = False - playbook_list = re.findall(r"[\w./]+", playbooks) - if not all(os.path.exists(p) for p in playbook_list): - playbook_list_subdir = [os.path.join("playbooks", p) for p in playbook_list] - if all(os.path.exists(p) for p in playbook_list_subdir): - playbook_list = playbook_list_subdir - else: - raise ValueError("All playbooks could not be found") + argument_list = re.findall(r"[^\s]+", arguments) result = run_ansible_playbook( virtualenv, - playbook_list, - ansible_extra_args=ansible_extra_args, inventory=inventory, + arguments=argument_list, + ansible_extra_args=ansible_extra_args, dry_run=dry_run, ) if error: @@ -206,9 +200,9 @@ def ansible_playbook( def ansible_kubeadm(inventory, virtualenv, galaxy_deps, ansible_extra_args, results): result = run_ansible_playbook( virtualenv, + inventory, ["tests/playbooks/verify.yml"], ansible_extra_args=ansible_extra_args, - inventory=inventory, ) assert_ansible_error(result) diff --git a/tests/features/haproxy.feature b/tests/features/haproxy.feature index f00e300..7a42657 100644 --- a/tests/features/haproxy.feature +++ b/tests/features/haproxy.feature @@ -18,8 +18,8 @@ Feature: Haproxy apiserver_proxy_use_docker: true kube_version: 1.23 When I run the playbook tests/playbooks/prepare.yml - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml When I run the playbook tests/playbooks/cni.yml Then I should have a working cluster @@ -27,14 +27,14 @@ Feature: Haproxy When With those group_vars on group all: apiserver_proxy_use_docker: When I reset tasks counters - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml with error: As docker has been deprecated When With those group_vars on group all: apiserver_proxy_use_docker: false When I reset tasks counters - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml Then I should have a working cluster diff --git a/tests/features/install.feature b/tests/features/install.feature index 99a499a..01990c6 100644 --- a/tests/features/install.feature +++ b/tests/features/install.feature @@ -17,16 +17,16 @@ Feature: Install cgroupDriver: "systemd" kube_version: When I run the playbook tests/playbooks/prepare.yml - When I dry-run the playbooks 00_apiserver_proxy.yml - 01_site.yml - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + When I dry-run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml When I run the playbook tests/playbooks/cni.yml Then I should have a working cluster When I reset tasks counters - And I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + And I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml Then I should see no orange/yellow changed tasks Examples: diff --git a/tests/features/join_nodes.feature b/tests/features/join_nodes.feature index 8c5348a..d0acd16 100644 --- a/tests/features/join_nodes.feature +++ b/tests/features/join_nodes.feature @@ -1,10 +1,10 @@ -Feature: Upgrade - A test to upgrade a kubeadm cluster +Feature: Join Nodes + A test to join nodes to a kubeadm cluster - Scenario: Upgrade via ansible-kubeadm + Scenario: Join nodes via ansible-kubeadm Given I want ansible 3 Given The cluster control_plane_count = 1 - Given The cluster worker_count = 1 + Given The cluster worker_count = 0 Given Some running VMs When With those group_vars on group all: @@ -18,14 +18,19 @@ Feature: Upgrade cgroupDriver: "systemd" kube_version: 1.23 When I run the playbook tests/playbooks/prepare.yml - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml When I run the playbook tests/playbooks/cni.yml + Then I should have a working cluster - Then Set cluster worker_count = 2 + Then Set cluster worker_count = 1 + When I run the playbook tests/playbooks/prepare.yml + When I run the playbooks playbooks/01_site.yml -e "limit=*-node-1" + Then I should have a working cluster - When With those group_vars on group all: kube_version: 1.24 - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + Then Set cluster control_plane_count = 2 + When I run the playbook tests/playbooks/prepare.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml Then I should have a working cluster diff --git a/tests/features/upgrade.feature b/tests/features/upgrade.feature index 111cd01..4420435 100644 --- a/tests/features/upgrade.feature +++ b/tests/features/upgrade.feature @@ -18,13 +18,13 @@ Feature: Upgrade kube_version: action_reasons_review_skip: true When I run the playbook tests/playbooks/prepare.yml - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml When I run the playbook tests/playbooks/cni.yml When With those group_vars on group all: kube_version: - When I run the playbooks 00_apiserver_proxy.yml - 01_site.yml + When I run the playbooks playbooks/00_apiserver_proxy.yml + playbooks/01_site.yml Then I should have a working cluster diff --git a/tests/helpers/ansible.py b/tests/helpers/ansible.py index 9f809ee..01595e3 100644 --- a/tests/helpers/ansible.py +++ b/tests/helpers/ansible.py @@ -2,6 +2,7 @@ import os import ansible_runner +import ansible_runner.interface def install_ansible(virtualenv, version=None): @@ -22,20 +23,15 @@ def install_galaxy_deps(virtualenv): "install", "-r", os.path.join(test_dir, "ansible.requirements.yml"), - "-p", - os.path.join(test_dir, "playbooks/roles"), ] ) def run_ansible_playbook( - virtualenv, playbooks, dry_run=False, ansible_extra_args=None, **kwargs + virtualenv, inventory, arguments, dry_run=False, ansible_extra_args=None, **kwargs ): - if isinstance(playbooks, str): - playbooks = [playbooks] - playbooks = [ - os.path.join(os.path.dirname(__file__), "../..", pbk) for pbk in playbooks - ] + if isinstance(arguments, str): + arguments = [arguments] # ansible_runner has several "bugs": # - Don't accept multiple playbooks on the parameter "playbook" (which is supposed to accept list) # - If you pass custom binary it cannot say if ansible or ansible-playbook so doesn't inject playbook anymore @@ -43,15 +39,21 @@ def run_ansible_playbook( envvars = dict(os.environ) envvars.setdefault("ANSIBLE_HOST_KEY_CHECKING", "false") envvars.setdefault("ANSIBLE_FORCE_COLOR", "true") - cmdline = " ".join(itertools.chain(ansible_extra_args or [], playbooks)) + cmdline_args = [ + "-i", + inventory, + *itertools.chain(arguments, ansible_extra_args or []), + ] if dry_run: - cmdline += " -C" - return ansible_runner.run( - binary=os.path.join(virtualenv.virtualenv, "bin/ansible-playbook"), - cmdline=cmdline, + cmdline_args += ["-C"] + runner = ansible_runner.interface.init_command_config( + executable_cmd=os.path.join(virtualenv.virtualenv, "bin/ansible-playbook"), + cmdline_args=cmdline_args, envvars=envvars, **kwargs ) + runner.run() + return runner def assert_ansible_error(run): diff --git a/tests/test_basic.py b/tests/test_basic.py index 1f18d33..64e8d5b 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -17,3 +17,8 @@ def test_upgrade(operating_system): @scenario("features/haproxy.feature", "Test upgrade to haproxy pkg") def test_haproxy(operating_system): pass + + +@scenario("features/join_nodes.feature", "Join nodes via ansible-kubeadm") +def test_join_nodes(operating_system): + pass