introduce ansible-test-splitter job

playbooks/ansible-test-splitter/run.yaml

@@ -0,0 +1,11 @@
+---
+- hosts: controller
+  tasks:
+    - name: Run ansible-test-splitter
+      import_role:
+        name: ansible-test-splitter
+      vars:
+        ansible_test_test_command: "{{ ansible_test_command }}"
+        ansible_test_location: "{{ ansible_user_dir }}/{{ zuul.projects[ansible_collections_repo].src_dir }}"
+        ansible_test_git_branch: "{{ zuul.projects['github.com/ansible/ansible'].checkout }}"
+        ansible_test_ansible_path: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/ansible/ansible'].src_dir }}"

roles/ansible-test-splitter/defaults/main.yaml

@@ -0,0 +1,3 @@
+---
+ansible_test_splitter__test_changed: false
+ansible_test_splitter__children_prefix: please_adjust_this

roles/ansible-test-splitter/files/split_targets.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+from pathlib import PosixPath
+import random
+import sys
+import json
+
+job_prefix = sys.argv[1]
+if len(sys.argv) == 3:
+    targets_from_cli = sys.argv[2].split(" ")
+else:
+    targets_from_cli = []
+jobs = [f"{job_prefix}{i}" for i in range(10)]
+targets_per_job = 20
+slow_targets = []
+regular_targets = []
+
+batches = []
+
+targets = PosixPath("tests/integration/targets/")
+for target in targets.glob("*"):
+    aliases = target / "aliases"
+    if not target.is_dir():
+        continue
+    if not aliases.is_file():
+        continue
+    if targets_from_cli and target.name not in targets_from_cli:
+        continue
+    lines = aliases.read_text().split("\n")
+    if "disabled" in lines:
+        continue
+    if "unstable" in lines:
+        continue
+    if "slow" in lines or "# reason: slow" in lines:
+        batches.append([target.name])
+    else:
+        regular_targets.append(target.name)
+
+random.shuffle(regular_targets)
+
+splitted_targets = len(regular_targets) % targets_per_job
+
+splitted_targets = []
+while regular_targets:
+    batches.append(
+        [regular_targets.pop() for i in range(targets_per_job) if regular_targets]
+    )
+
+
+result = {
+    "data": {
+        "zuul": {"child_jobs": jobs[0:len(batches)]},
+        "child": {"targets_to_test": batches},
+    }
+}
+
+print(json.dumps(result))

roles/ansible-test-splitter/files/test_changed.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+from pathlib import PosixPath
+import sys
+import subprocess
+
+targets_to_test = []
+targets_dir = PosixPath("tests/integration/targets")
+zuul_branch = sys.argv[1]
+diff = subprocess.check_output(
+    ["git", "diff", f"origin/{zuul_branch}", "--name-only"]
+).decode()
+module_files = [PosixPath(d) for d in diff.split("\n") if d.startswith("plugins/")]
+for i in module_files:
+    if not i.is_file():
+        continue
+    target_name = i.stem
+
+    for t in targets_dir.iterdir():
+        aliases = t / "aliases"
+        if not aliases.is_file():
+            continue
+        # There is a target with the module name, let's take that
+        if t.name == target_name:
+            targets_to_test.append(target_name)
+            break
+        alias_content = aliases.read_text().split("\n")
+        # The target name is in the aliases file
+        if target_name in alias_content:
+            targets_to_test.append(target_name)
+            break
+
+target_files = [
+    PosixPath(d) for d in diff.split("\n") if d.startswith("tests/integration/targets/")
+]
+for i in target_files:
+    splitted = str(i).split("/")
+    if len(splitted) < 5:
+        continue
+    target_name = splitted[3]
+    aliases = targets_dir / target_name / "aliases"
+    if aliases.is_file():
+        targets_to_test.append(target_name)
+
+print(" ".join(list(set(targets_to_test))))

roles/ansible-test-splitter/tasks/ansible_test_changed.yaml

@@ -0,0 +1,13 @@
+---
+- copy:
+    src: test_changed.py
+    dest: /tmp/test_changed.py
+    mode: '0700'
+
+- name: Identify the changed targets
+  command: python3 /tmp/test_changed.py "{{ zuul.branch }}"
+  args:
+    chdir: "{{ ansible_test_location }}"
+  register: _result
+- set_fact:
+    ansible_test_splitter__changed_targets: "{{ _result.stdout }}"

roles/ansible-test-splitter/tasks/main.yaml

@@ -0,0 +1,7 @@
+---
+- name: Identiy the targets associated with the changed files
+  import_tasks: ansible_test_changed.yaml
+  when: ansible_test_splitter__test_changed|bool
+
+- name: Split targets
+  import_tasks: split_targets.yaml

roles/ansible-test-splitter/tasks/split_targets.yaml

@@ -0,0 +1,18 @@
+---
+- copy:
+    src: split_targets.py
+    dest: /tmp/split_targets.py
+    mode: '0700'
+
+- name: Split the workload
+  command: python3 /tmp/split_targets.py "{{ ansible_test_splitter__children_prefix }}" "{{ ansible_test_splitter__changed_targets|default('') }}"
+  args:
+    chdir: "{{ ansible_test_location }}"
+  register: _result
+- debug: var=_result
+- set_fact:
+    for_zuul_return: '{{ _result.stdout | from_json }}'
+- debug: var=for_zuul_return
+- name: Register the result
+  zuul_return:
+    data: "{{ for_zuul_return.data }}"

zuul.d/ansible-cloud-jobs.yaml

@@ -298,12 +298,30 @@
 
 
 ### AWS
+- job:
+    name: ansible-test-splitter
+    run: playbooks/ansible-test-splitter/run.yaml
+    nodeset: controller-python36
+    required-projects:
+      - name: github.com/ansible/ansible
+    timeout: 1000
+    vars:
+      ansible_collections_repo: "{{ zuul.project.canonical_name }}"
+      ansible_test_location: "{{ ansible_user_dir }}/{{ zuul.projects[zuul.project.canonical_name].src_dir }}"
+      ansible_test_splitter__test_changed: true
+      ansible_test_splitter__children_prefix: ansible-test-cloud-integration-aws-py36_
+
+- semaphore:
+    name: ansible-test-cloud-integration-aws
+    max: 6
+
 - job:
     name: ansible-test-cloud-integration-aws-py36
     parent: ansible-core-ci-aws-session
     nodeset: controller-python36
     dependencies:
       - name: build-ansible-collection
+      - name: ansible-test-splitter
     pre-run:
       - playbooks/ansible-test-base/pre.yaml
       - playbooks/ansible-cloud/aws/pre.yaml
@@ -319,45 +337,68 @@
       ansible_test_command: integration
       ansible_test_enable_ara: false
       ansible_test_python: 3.6
-      ansible_test_changed: true
-      ansible_test_split_in: 6
+      ansible_test_retry_on_error: true
+    semaphore: ansible-test-cloud-integration-aws
 
 - job:
-    name: ansible-test-cloud-integration-aws-py36_1_of_6
+    name: ansible-test-cloud-integration-aws-py36_0
     parent: ansible-test-cloud-integration-aws-py36
     vars:
-      ansible_test_do_number: 1
+      ansible_test_integration_targets: "{{ child.targets_to_test[0]|join(' ') }}"
 
 - job:
-    name: ansible-test-cloud-integration-aws-py36_2_of_6
+    name: ansible-test-cloud-integration-aws-py36_1
     parent: ansible-test-cloud-integration-aws-py36
     vars:
-      ansible_test_do_number: 2
+      ansible_test_integration_targets: "{{ child.targets_to_test[1]|join(' ') }}"
 
 - job:
-    name: ansible-test-cloud-integration-aws-py36_3_of_6
+    name: ansible-test-cloud-integration-aws-py36_2
     parent: ansible-test-cloud-integration-aws-py36
     vars:
-      ansible_test_do_number: 3
+      ansible_test_integration_targets: "{{ child.targets_to_test[2]|join(' ') }}"
+
+- job:
+    name: ansible-test-cloud-integration-aws-py36_3
+    parent: ansible-test-cloud-integration-aws-py36
+    vars:
+      ansible_test_integration_targets: "{{ child.targets_to_test[3]|join(' ') }}"
+
+- job:
+    name: ansible-test-cloud-integration-aws-py36_4
+    parent: ansible-test-cloud-integration-aws-py36
+    vars:
+      ansible_test_integration_targets: "{{ child.targets_to_test[4]|join(' ') }}"
 
 - job:
-    name: ansible-test-cloud-integration-aws-py36_4_of_6
+    name: ansible-test-cloud-integration-aws-py36_5
     parent: ansible-test-cloud-integration-aws-py36
     vars:
-      ansible_test_do_number: 4
+      ansible_test_integration_targets: "{{ child.targets_to_test[5]|join(' ') }}"
 
 - job:
-    name: ansible-test-cloud-integration-aws-py36_5_of_6
+    name: ansible-test-cloud-integration-aws-py36_6
     parent: ansible-test-cloud-integration-aws-py36
     vars:
-      ansible_test_do_number: 5
+      ansible_test_integration_targets: "{{ child.targets_to_test[6]|join(' ') }}"
 
 - job:
-    name: ansible-test-cloud-integration-aws-py36_6_of_6
+    name: ansible-test-cloud-integration-aws-py36_7
     parent: ansible-test-cloud-integration-aws-py36
     vars:
-      ansible_test_do_number: 6
+      ansible_test_integration_targets: "{{ child.targets_to_test[7]|join(' ') }}"
 
+- job:
+    name: ansible-test-cloud-integration-aws-py36_8
+    parent: ansible-test-cloud-integration-aws-py36
+    vars:
+      ansible_test_integration_targets: "{{ child.targets_to_test[8]|join(' ') }}"
+
+- job:
+    name: ansible-test-cloud-integration-aws-py36_9
+    parent: ansible-test-cloud-integration-aws-py36
+    vars:
+      ansible_test_integration_targets: "{{ child.targets_to_test[9]|join(' ') }}"
 #### units
 - job:
     name: ansible-test-units-community-aws-python38

zuul.d/project-templates.yaml

@@ -76,12 +76,18 @@
               - name: github.com/ansible-collections/ansible.netcommon
               - name: github.com/ansible-collections/community.aws
               - name: github.com/ansible-collections/community.general
-        - ansible-test-cloud-integration-aws-py36_1_of_6
-        - ansible-test-cloud-integration-aws-py36_2_of_6
-        - ansible-test-cloud-integration-aws-py36_3_of_6
-        - ansible-test-cloud-integration-aws-py36_4_of_6
-        - ansible-test-cloud-integration-aws-py36_5_of_6
-        - ansible-test-cloud-integration-aws-py36_6_of_6
+        - ansible-test-splitter
+        - ansible-test-cloud-integration-aws-py36_0
+        - ansible-test-cloud-integration-aws-py36_1
+        - ansible-test-cloud-integration-aws-py36_2
+        - ansible-test-cloud-integration-aws-py36_3
+        - ansible-test-cloud-integration-aws-py36_4
+        - ansible-test-cloud-integration-aws-py36_5
+        - ansible-test-cloud-integration-aws-py36_6
+        - ansible-test-cloud-integration-aws-py36_7
+        - ansible-test-cloud-integration-aws-py36_8
+        - ansible-test-cloud-integration-aws-py36_9
+
     gate:
       jobs:
         - ansible-test-sanity-aws-ansible-2.9-python36
@@ -97,12 +103,17 @@
               - name: github.com/ansible-collections/ansible.netcommon
               - name: github.com/ansible-collections/community.aws
               - name: github.com/ansible-collections/community.general
-        - ansible-test-cloud-integration-aws-py36_1_of_6
-        - ansible-test-cloud-integration-aws-py36_2_of_6
-        - ansible-test-cloud-integration-aws-py36_3_of_6
-        - ansible-test-cloud-integration-aws-py36_4_of_6
-        - ansible-test-cloud-integration-aws-py36_5_of_6
-        - ansible-test-cloud-integration-aws-py36_6_of_6
+        - ansible-test-splitter
+        - ansible-test-cloud-integration-aws-py36_0
+        - ansible-test-cloud-integration-aws-py36_1
+        - ansible-test-cloud-integration-aws-py36_2
+        - ansible-test-cloud-integration-aws-py36_3
+        - ansible-test-cloud-integration-aws-py36_4
+        - ansible-test-cloud-integration-aws-py36_5
+        - ansible-test-cloud-integration-aws-py36_6
+        - ansible-test-cloud-integration-aws-py36_7
+        - ansible-test-cloud-integration-aws-py36_8
+        - ansible-test-cloud-integration-aws-py36_9
 
 - project-template:
     name: ansible-collections-community-aws
@@ -119,12 +130,17 @@
               - name: github.com/ansible-collections/ansible.netcommon
               - name: github.com/ansible-collections/ansible.utils
               - name: github.com/ansible-collections/community.general
-        - ansible-test-cloud-integration-aws-py36_1_of_6
-        - ansible-test-cloud-integration-aws-py36_2_of_6
-        - ansible-test-cloud-integration-aws-py36_3_of_6
-        - ansible-test-cloud-integration-aws-py36_4_of_6
-        - ansible-test-cloud-integration-aws-py36_5_of_6
-        - ansible-test-cloud-integration-aws-py36_6_of_6
+        - ansible-test-splitter
+        - ansible-test-cloud-integration-aws-py36_0
+        - ansible-test-cloud-integration-aws-py36_1
+        - ansible-test-cloud-integration-aws-py36_2
+        - ansible-test-cloud-integration-aws-py36_3
+        - ansible-test-cloud-integration-aws-py36_4
+        - ansible-test-cloud-integration-aws-py36_5
+        - ansible-test-cloud-integration-aws-py36_6
+        - ansible-test-cloud-integration-aws-py36_7
+        - ansible-test-cloud-integration-aws-py36_8
+        - ansible-test-cloud-integration-aws-py36_9
         - ansible-galaxy-importer:
             voting: false
     gate:
@@ -140,12 +156,17 @@
               - name: github.com/ansible-collections/ansible.netcommon
               - name: github.com/ansible-collections/ansible.utils
               - name: github.com/ansible-collections/community.general
-        - ansible-test-cloud-integration-aws-py36_1_of_6
-        - ansible-test-cloud-integration-aws-py36_2_of_6
-        - ansible-test-cloud-integration-aws-py36_3_of_6
-        - ansible-test-cloud-integration-aws-py36_4_of_6
-        - ansible-test-cloud-integration-aws-py36_5_of_6
-        - ansible-test-cloud-integration-aws-py36_6_of_6
+        - ansible-test-splitter
+        - ansible-test-cloud-integration-aws-py36_0
+        - ansible-test-cloud-integration-aws-py36_1
+        - ansible-test-cloud-integration-aws-py36_2
+        - ansible-test-cloud-integration-aws-py36_3
+        - ansible-test-cloud-integration-aws-py36_4
+        - ansible-test-cloud-integration-aws-py36_5
+        - ansible-test-cloud-integration-aws-py36_6
+        - ansible-test-cloud-integration-aws-py36_7
+        - ansible-test-cloud-integration-aws-py36_8
+        - ansible-test-cloud-integration-aws-py36_9
         - ansible-galaxy-importer:
             voting: false