From d649a115d8bb902c7991691e23ca76cedf50a4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A9ri=20Le=20Bouder?= Date: Wed, 30 Jun 2021 12:24:42 -0400 Subject: [PATCH] aws: introduce ansible-test-splitter job This commit changes the way we split up the AWS jobs. We've got now a first job called `ansible-test-splitter` that is in charge of: - identify what target needs to be run (like the `ansible-test` role) - split up the workload on different jobs This logic is handled by a new role called `ansible-test-splitter` and these two tasks are done through two Python scripts. Why Python? Well, the two tasks were getting overcomplicated and the use of Python simplify a lot the troubleshoot and the code complexity. In addition, we can later create action plugin from these scripts. Finally, the commit also introduces a semaphore for the AWS jobs. It's a way to control the number of paralel jobs. --- playbooks/ansible-test-splitter/run.yaml | 11 +++ .../ansible-test-splitter/defaults/main.yaml | 3 + .../files/split_targets.py | 57 +++++++++++++++ .../files/test_changed.py | 45 ++++++++++++ .../tasks/ansible_test_changed.yaml | 13 ++++ roles/ansible-test-splitter/tasks/main.yaml | 7 ++ .../tasks/split_targets.yaml | 18 +++++ zuul.d/ansible-cloud-jobs.yaml | 69 +++++++++++++++---- zuul.d/project-templates.yaml | 69 ++++++++++++------- 9 files changed, 254 insertions(+), 38 deletions(-) create mode 100644 playbooks/ansible-test-splitter/run.yaml create mode 100644 roles/ansible-test-splitter/defaults/main.yaml create mode 100644 roles/ansible-test-splitter/files/split_targets.py create mode 100644 roles/ansible-test-splitter/files/test_changed.py create mode 100644 roles/ansible-test-splitter/tasks/ansible_test_changed.yaml create mode 100644 roles/ansible-test-splitter/tasks/main.yaml create mode 100644 roles/ansible-test-splitter/tasks/split_targets.yaml diff --git a/playbooks/ansible-test-splitter/run.yaml b/playbooks/ansible-test-splitter/run.yaml new file mode 100644 index 000000000..31c28bbcb --- /dev/null +++ b/playbooks/ansible-test-splitter/run.yaml @@ -0,0 +1,11 @@ +--- +- hosts: controller + tasks: + - name: Run ansible-test-splitter + import_role: + name: ansible-test-splitter + vars: + ansible_test_test_command: "{{ ansible_test_command }}" + ansible_test_location: "{{ ansible_user_dir }}/{{ zuul.projects[ansible_collections_repo].src_dir }}" + ansible_test_git_branch: "{{ zuul.projects['github.com/ansible/ansible'].checkout }}" + ansible_test_ansible_path: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/ansible/ansible'].src_dir }}" diff --git a/roles/ansible-test-splitter/defaults/main.yaml b/roles/ansible-test-splitter/defaults/main.yaml new file mode 100644 index 000000000..cf90de94c --- /dev/null +++ b/roles/ansible-test-splitter/defaults/main.yaml @@ -0,0 +1,3 @@ +--- +ansible_test_splitter__test_changed: false +ansible_test_splitter__children_prefix: please_adjust_this diff --git a/roles/ansible-test-splitter/files/split_targets.py b/roles/ansible-test-splitter/files/split_targets.py new file mode 100644 index 000000000..447e33135 --- /dev/null +++ b/roles/ansible-test-splitter/files/split_targets.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +from pathlib import PosixPath +import random +import sys +import json + +job_prefix = sys.argv[1] +if len(sys.argv) == 3: + targets_from_cli = sys.argv[2].split(" ") +else: + targets_from_cli = [] +jobs = [f"{job_prefix}{i}" for i in range(10)] +targets_per_job = 20 +slow_targets = [] +regular_targets = [] + +batches = [] + +targets = PosixPath("tests/integration/targets/") +for target in targets.glob("*"): + aliases = target / "aliases" + if not target.is_dir(): + continue + if not aliases.is_file(): + continue + if targets_from_cli and target.name not in targets_from_cli: + continue + lines = aliases.read_text().split("\n") + if "disabled" in lines: + continue + if "unstable" in lines: + continue + if "slow" in lines or "# reason: slow" in lines: + batches.append([target.name]) + else: + regular_targets.append(target.name) + +random.shuffle(regular_targets) + +splitted_targets = len(regular_targets) % targets_per_job + +splitted_targets = [] +while regular_targets: + batches.append( + [regular_targets.pop() for i in range(targets_per_job) if regular_targets] + ) + + +result = { + "data": { + "zuul": {"child_jobs": jobs[0:len(batches)]}, + "child": {"targets_to_test": batches}, + } +} + +print(json.dumps(result)) diff --git a/roles/ansible-test-splitter/files/test_changed.py b/roles/ansible-test-splitter/files/test_changed.py new file mode 100644 index 000000000..8d6caf250 --- /dev/null +++ b/roles/ansible-test-splitter/files/test_changed.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +from pathlib import PosixPath +import sys +import subprocess + +targets_to_test = [] +targets_dir = PosixPath("tests/integration/targets") +zuul_branch = sys.argv[1] +diff = subprocess.check_output( + ["git", "diff", f"origin/{zuul_branch}", "--name-only"] +).decode() +module_files = [PosixPath(d) for d in diff.split("\n") if d.startswith("plugins/")] +for i in module_files: + if not i.is_file(): + continue + target_name = i.stem + + for t in targets_dir.iterdir(): + aliases = t / "aliases" + if not aliases.is_file(): + continue + # There is a target with the module name, let's take that + if t.name == target_name: + targets_to_test.append(target_name) + break + alias_content = aliases.read_text().split("\n") + # The target name is in the aliases file + if target_name in alias_content: + targets_to_test.append(target_name) + break + +target_files = [ + PosixPath(d) for d in diff.split("\n") if d.startswith("tests/integration/targets/") +] +for i in target_files: + splitted = str(i).split("/") + if len(splitted) < 5: + continue + target_name = splitted[3] + aliases = targets_dir / target_name / "aliases" + if aliases.is_file(): + targets_to_test.append(target_name) + +print(" ".join(list(set(targets_to_test)))) diff --git a/roles/ansible-test-splitter/tasks/ansible_test_changed.yaml b/roles/ansible-test-splitter/tasks/ansible_test_changed.yaml new file mode 100644 index 000000000..0a8ed1aa6 --- /dev/null +++ b/roles/ansible-test-splitter/tasks/ansible_test_changed.yaml @@ -0,0 +1,13 @@ +--- +- copy: + src: test_changed.py + dest: /tmp/test_changed.py + mode: '0700' + +- name: Identify the changed targets + command: python3 /tmp/test_changed.py "{{ zuul.branch }}" + args: + chdir: "{{ ansible_test_location }}" + register: _result +- set_fact: + ansible_test_splitter__changed_targets: "{{ _result.stdout }}" diff --git a/roles/ansible-test-splitter/tasks/main.yaml b/roles/ansible-test-splitter/tasks/main.yaml new file mode 100644 index 000000000..c9c2653b5 --- /dev/null +++ b/roles/ansible-test-splitter/tasks/main.yaml @@ -0,0 +1,7 @@ +--- +- name: Identiy the targets associated with the changed files + import_tasks: ansible_test_changed.yaml + when: ansible_test_splitter__test_changed|bool + +- name: Split targets + import_tasks: split_targets.yaml diff --git a/roles/ansible-test-splitter/tasks/split_targets.yaml b/roles/ansible-test-splitter/tasks/split_targets.yaml new file mode 100644 index 000000000..2336dbd1a --- /dev/null +++ b/roles/ansible-test-splitter/tasks/split_targets.yaml @@ -0,0 +1,18 @@ +--- +- copy: + src: split_targets.py + dest: /tmp/split_targets.py + mode: '0700' + +- name: Split the workload + command: python3 /tmp/split_targets.py "{{ ansible_test_splitter__children_prefix }}" "{{ ansible_test_splitter__changed_targets|default('') }}" + args: + chdir: "{{ ansible_test_location }}" + register: _result +- debug: var=_result +- set_fact: + for_zuul_return: '{{ _result.stdout | from_json }}' +- debug: var=for_zuul_return +- name: Register the result + zuul_return: + data: "{{ for_zuul_return.data }}" diff --git a/zuul.d/ansible-cloud-jobs.yaml b/zuul.d/ansible-cloud-jobs.yaml index e5da79701..ce6cb2a47 100644 --- a/zuul.d/ansible-cloud-jobs.yaml +++ b/zuul.d/ansible-cloud-jobs.yaml @@ -298,12 +298,30 @@ ### AWS +- job: + name: ansible-test-splitter + run: playbooks/ansible-test-splitter/run.yaml + nodeset: controller-python36 + required-projects: + - name: github.com/ansible/ansible + timeout: 1000 + vars: + ansible_collections_repo: "{{ zuul.project.canonical_name }}" + ansible_test_location: "{{ ansible_user_dir }}/{{ zuul.projects[zuul.project.canonical_name].src_dir }}" + ansible_test_splitter__test_changed: true + ansible_test_splitter__children_prefix: ansible-test-cloud-integration-aws-py36_ + +- semaphore: + name: ansible-test-cloud-integration-aws + max: 6 + - job: name: ansible-test-cloud-integration-aws-py36 parent: ansible-core-ci-aws-session nodeset: controller-python36 dependencies: - name: build-ansible-collection + - name: ansible-test-splitter pre-run: - playbooks/ansible-test-base/pre.yaml - playbooks/ansible-cloud/aws/pre.yaml @@ -319,45 +337,68 @@ ansible_test_command: integration ansible_test_enable_ara: false ansible_test_python: 3.6 - ansible_test_changed: true - ansible_test_split_in: 6 + ansible_test_retry_on_error: true + semaphore: ansible-test-cloud-integration-aws - job: - name: ansible-test-cloud-integration-aws-py36_1_of_6 + name: ansible-test-cloud-integration-aws-py36_0 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_do_number: 1 + ansible_test_integration_targets: "{{ child.targets_to_test[0]|join(' ') }}" - job: - name: ansible-test-cloud-integration-aws-py36_2_of_6 + name: ansible-test-cloud-integration-aws-py36_1 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_do_number: 2 + ansible_test_integration_targets: "{{ child.targets_to_test[1]|join(' ') }}" - job: - name: ansible-test-cloud-integration-aws-py36_3_of_6 + name: ansible-test-cloud-integration-aws-py36_2 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_do_number: 3 + ansible_test_integration_targets: "{{ child.targets_to_test[2]|join(' ') }}" + +- job: + name: ansible-test-cloud-integration-aws-py36_3 + parent: ansible-test-cloud-integration-aws-py36 + vars: + ansible_test_integration_targets: "{{ child.targets_to_test[3]|join(' ') }}" + +- job: + name: ansible-test-cloud-integration-aws-py36_4 + parent: ansible-test-cloud-integration-aws-py36 + vars: + ansible_test_integration_targets: "{{ child.targets_to_test[4]|join(' ') }}" - job: - name: ansible-test-cloud-integration-aws-py36_4_of_6 + name: ansible-test-cloud-integration-aws-py36_5 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_do_number: 4 + ansible_test_integration_targets: "{{ child.targets_to_test[5]|join(' ') }}" - job: - name: ansible-test-cloud-integration-aws-py36_5_of_6 + name: ansible-test-cloud-integration-aws-py36_6 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_do_number: 5 + ansible_test_integration_targets: "{{ child.targets_to_test[6]|join(' ') }}" - job: - name: ansible-test-cloud-integration-aws-py36_6_of_6 + name: ansible-test-cloud-integration-aws-py36_7 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_do_number: 6 + ansible_test_integration_targets: "{{ child.targets_to_test[7]|join(' ') }}" +- job: + name: ansible-test-cloud-integration-aws-py36_8 + parent: ansible-test-cloud-integration-aws-py36 + vars: + ansible_test_integration_targets: "{{ child.targets_to_test[8]|join(' ') }}" + +- job: + name: ansible-test-cloud-integration-aws-py36_9 + parent: ansible-test-cloud-integration-aws-py36 + vars: + ansible_test_integration_targets: "{{ child.targets_to_test[9]|join(' ') }}" #### units - job: name: ansible-test-units-community-aws-python38 diff --git a/zuul.d/project-templates.yaml b/zuul.d/project-templates.yaml index 40e47341c..6fc20306a 100644 --- a/zuul.d/project-templates.yaml +++ b/zuul.d/project-templates.yaml @@ -76,12 +76,18 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/community.aws - name: github.com/ansible-collections/community.general - - ansible-test-cloud-integration-aws-py36_1_of_6 - - ansible-test-cloud-integration-aws-py36_2_of_6 - - ansible-test-cloud-integration-aws-py36_3_of_6 - - ansible-test-cloud-integration-aws-py36_4_of_6 - - ansible-test-cloud-integration-aws-py36_5_of_6 - - ansible-test-cloud-integration-aws-py36_6_of_6 + - ansible-test-splitter + - ansible-test-cloud-integration-aws-py36_0 + - ansible-test-cloud-integration-aws-py36_1 + - ansible-test-cloud-integration-aws-py36_2 + - ansible-test-cloud-integration-aws-py36_3 + - ansible-test-cloud-integration-aws-py36_4 + - ansible-test-cloud-integration-aws-py36_5 + - ansible-test-cloud-integration-aws-py36_6 + - ansible-test-cloud-integration-aws-py36_7 + - ansible-test-cloud-integration-aws-py36_8 + - ansible-test-cloud-integration-aws-py36_9 + gate: jobs: - ansible-test-sanity-aws-ansible-2.9-python36 @@ -97,12 +103,17 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/community.aws - name: github.com/ansible-collections/community.general - - ansible-test-cloud-integration-aws-py36_1_of_6 - - ansible-test-cloud-integration-aws-py36_2_of_6 - - ansible-test-cloud-integration-aws-py36_3_of_6 - - ansible-test-cloud-integration-aws-py36_4_of_6 - - ansible-test-cloud-integration-aws-py36_5_of_6 - - ansible-test-cloud-integration-aws-py36_6_of_6 + - ansible-test-splitter + - ansible-test-cloud-integration-aws-py36_0 + - ansible-test-cloud-integration-aws-py36_1 + - ansible-test-cloud-integration-aws-py36_2 + - ansible-test-cloud-integration-aws-py36_3 + - ansible-test-cloud-integration-aws-py36_4 + - ansible-test-cloud-integration-aws-py36_5 + - ansible-test-cloud-integration-aws-py36_6 + - ansible-test-cloud-integration-aws-py36_7 + - ansible-test-cloud-integration-aws-py36_8 + - ansible-test-cloud-integration-aws-py36_9 - project-template: name: ansible-collections-community-aws @@ -119,12 +130,17 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/ansible.utils - name: github.com/ansible-collections/community.general - - ansible-test-cloud-integration-aws-py36_1_of_6 - - ansible-test-cloud-integration-aws-py36_2_of_6 - - ansible-test-cloud-integration-aws-py36_3_of_6 - - ansible-test-cloud-integration-aws-py36_4_of_6 - - ansible-test-cloud-integration-aws-py36_5_of_6 - - ansible-test-cloud-integration-aws-py36_6_of_6 + - ansible-test-splitter + - ansible-test-cloud-integration-aws-py36_0 + - ansible-test-cloud-integration-aws-py36_1 + - ansible-test-cloud-integration-aws-py36_2 + - ansible-test-cloud-integration-aws-py36_3 + - ansible-test-cloud-integration-aws-py36_4 + - ansible-test-cloud-integration-aws-py36_5 + - ansible-test-cloud-integration-aws-py36_6 + - ansible-test-cloud-integration-aws-py36_7 + - ansible-test-cloud-integration-aws-py36_8 + - ansible-test-cloud-integration-aws-py36_9 - ansible-galaxy-importer: voting: false gate: @@ -140,12 +156,17 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/ansible.utils - name: github.com/ansible-collections/community.general - - ansible-test-cloud-integration-aws-py36_1_of_6 - - ansible-test-cloud-integration-aws-py36_2_of_6 - - ansible-test-cloud-integration-aws-py36_3_of_6 - - ansible-test-cloud-integration-aws-py36_4_of_6 - - ansible-test-cloud-integration-aws-py36_5_of_6 - - ansible-test-cloud-integration-aws-py36_6_of_6 + - ansible-test-splitter + - ansible-test-cloud-integration-aws-py36_0 + - ansible-test-cloud-integration-aws-py36_1 + - ansible-test-cloud-integration-aws-py36_2 + - ansible-test-cloud-integration-aws-py36_3 + - ansible-test-cloud-integration-aws-py36_4 + - ansible-test-cloud-integration-aws-py36_5 + - ansible-test-cloud-integration-aws-py36_6 + - ansible-test-cloud-integration-aws-py36_7 + - ansible-test-cloud-integration-aws-py36_8 + - ansible-test-cloud-integration-aws-py36_9 - ansible-galaxy-importer: voting: false