From 384bc5ca74c22f2cf1dbc628987e59aaefc9aea8 Mon Sep 17 00:00:00 2001 From: Jill Rouleau Date: Fri, 30 Jul 2021 10:34:52 -0700 Subject: [PATCH] Revert "aws: introduce ansible-test-splitter job (#978)" This reverts commit c3d89eb95df6863de4bf5be846d31d04644f97fc. --- playbooks/ansible-test-splitter/run.yaml | 11 --- .../ansible-test-splitter/defaults/main.yaml | 3 - .../files/split_targets.py | 57 --------------- .../files/test_changed.py | 45 ------------ .../tasks/ansible_test_changed.yaml | 13 ---- roles/ansible-test-splitter/tasks/main.yaml | 7 -- .../tasks/split_targets.yaml | 18 ----- zuul.d/ansible-cloud-jobs.yaml | 69 ++++--------------- zuul.d/project-templates.yaml | 69 +++++++------------ 9 files changed, 38 insertions(+), 254 deletions(-) delete mode 100644 playbooks/ansible-test-splitter/run.yaml delete mode 100644 roles/ansible-test-splitter/defaults/main.yaml delete mode 100644 roles/ansible-test-splitter/files/split_targets.py delete mode 100644 roles/ansible-test-splitter/files/test_changed.py delete mode 100644 roles/ansible-test-splitter/tasks/ansible_test_changed.yaml delete mode 100644 roles/ansible-test-splitter/tasks/main.yaml delete mode 100644 roles/ansible-test-splitter/tasks/split_targets.yaml diff --git a/playbooks/ansible-test-splitter/run.yaml b/playbooks/ansible-test-splitter/run.yaml deleted file mode 100644 index 31c28bbcb..000000000 --- a/playbooks/ansible-test-splitter/run.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: controller - tasks: - - name: Run ansible-test-splitter - import_role: - name: ansible-test-splitter - vars: - ansible_test_test_command: "{{ ansible_test_command }}" - ansible_test_location: "{{ ansible_user_dir }}/{{ zuul.projects[ansible_collections_repo].src_dir }}" - ansible_test_git_branch: "{{ zuul.projects['github.com/ansible/ansible'].checkout }}" - ansible_test_ansible_path: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/ansible/ansible'].src_dir }}" diff --git a/roles/ansible-test-splitter/defaults/main.yaml b/roles/ansible-test-splitter/defaults/main.yaml deleted file mode 100644 index cf90de94c..000000000 --- a/roles/ansible-test-splitter/defaults/main.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -ansible_test_splitter__test_changed: false -ansible_test_splitter__children_prefix: please_adjust_this diff --git a/roles/ansible-test-splitter/files/split_targets.py b/roles/ansible-test-splitter/files/split_targets.py deleted file mode 100644 index 447e33135..000000000 --- a/roles/ansible-test-splitter/files/split_targets.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import PosixPath -import random -import sys -import json - -job_prefix = sys.argv[1] -if len(sys.argv) == 3: - targets_from_cli = sys.argv[2].split(" ") -else: - targets_from_cli = [] -jobs = [f"{job_prefix}{i}" for i in range(10)] -targets_per_job = 20 -slow_targets = [] -regular_targets = [] - -batches = [] - -targets = PosixPath("tests/integration/targets/") -for target in targets.glob("*"): - aliases = target / "aliases" - if not target.is_dir(): - continue - if not aliases.is_file(): - continue - if targets_from_cli and target.name not in targets_from_cli: - continue - lines = aliases.read_text().split("\n") - if "disabled" in lines: - continue - if "unstable" in lines: - continue - if "slow" in lines or "# reason: slow" in lines: - batches.append([target.name]) - else: - regular_targets.append(target.name) - -random.shuffle(regular_targets) - -splitted_targets = len(regular_targets) % targets_per_job - -splitted_targets = [] -while regular_targets: - batches.append( - [regular_targets.pop() for i in range(targets_per_job) if regular_targets] - ) - - -result = { - "data": { - "zuul": {"child_jobs": jobs[0:len(batches)]}, - "child": {"targets_to_test": batches}, - } -} - -print(json.dumps(result)) diff --git a/roles/ansible-test-splitter/files/test_changed.py b/roles/ansible-test-splitter/files/test_changed.py deleted file mode 100644 index 8d6caf250..000000000 --- a/roles/ansible-test-splitter/files/test_changed.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import PosixPath -import sys -import subprocess - -targets_to_test = [] -targets_dir = PosixPath("tests/integration/targets") -zuul_branch = sys.argv[1] -diff = subprocess.check_output( - ["git", "diff", f"origin/{zuul_branch}", "--name-only"] -).decode() -module_files = [PosixPath(d) for d in diff.split("\n") if d.startswith("plugins/")] -for i in module_files: - if not i.is_file(): - continue - target_name = i.stem - - for t in targets_dir.iterdir(): - aliases = t / "aliases" - if not aliases.is_file(): - continue - # There is a target with the module name, let's take that - if t.name == target_name: - targets_to_test.append(target_name) - break - alias_content = aliases.read_text().split("\n") - # The target name is in the aliases file - if target_name in alias_content: - targets_to_test.append(target_name) - break - -target_files = [ - PosixPath(d) for d in diff.split("\n") if d.startswith("tests/integration/targets/") -] -for i in target_files: - splitted = str(i).split("/") - if len(splitted) < 5: - continue - target_name = splitted[3] - aliases = targets_dir / target_name / "aliases" - if aliases.is_file(): - targets_to_test.append(target_name) - -print(" ".join(list(set(targets_to_test)))) diff --git a/roles/ansible-test-splitter/tasks/ansible_test_changed.yaml b/roles/ansible-test-splitter/tasks/ansible_test_changed.yaml deleted file mode 100644 index 0a8ed1aa6..000000000 --- a/roles/ansible-test-splitter/tasks/ansible_test_changed.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- copy: - src: test_changed.py - dest: /tmp/test_changed.py - mode: '0700' - -- name: Identify the changed targets - command: python3 /tmp/test_changed.py "{{ zuul.branch }}" - args: - chdir: "{{ ansible_test_location }}" - register: _result -- set_fact: - ansible_test_splitter__changed_targets: "{{ _result.stdout }}" diff --git a/roles/ansible-test-splitter/tasks/main.yaml b/roles/ansible-test-splitter/tasks/main.yaml deleted file mode 100644 index c9c2653b5..000000000 --- a/roles/ansible-test-splitter/tasks/main.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Identiy the targets associated with the changed files - import_tasks: ansible_test_changed.yaml - when: ansible_test_splitter__test_changed|bool - -- name: Split targets - import_tasks: split_targets.yaml diff --git a/roles/ansible-test-splitter/tasks/split_targets.yaml b/roles/ansible-test-splitter/tasks/split_targets.yaml deleted file mode 100644 index 2336dbd1a..000000000 --- a/roles/ansible-test-splitter/tasks/split_targets.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- copy: - src: split_targets.py - dest: /tmp/split_targets.py - mode: '0700' - -- name: Split the workload - command: python3 /tmp/split_targets.py "{{ ansible_test_splitter__children_prefix }}" "{{ ansible_test_splitter__changed_targets|default('') }}" - args: - chdir: "{{ ansible_test_location }}" - register: _result -- debug: var=_result -- set_fact: - for_zuul_return: '{{ _result.stdout | from_json }}' -- debug: var=for_zuul_return -- name: Register the result - zuul_return: - data: "{{ for_zuul_return.data }}" diff --git a/zuul.d/ansible-cloud-jobs.yaml b/zuul.d/ansible-cloud-jobs.yaml index 0b597cce8..d03a9da4d 100644 --- a/zuul.d/ansible-cloud-jobs.yaml +++ b/zuul.d/ansible-cloud-jobs.yaml @@ -259,30 +259,12 @@ ### AWS -- job: - name: ansible-test-splitter - run: playbooks/ansible-test-splitter/run.yaml - nodeset: controller-python36-f34 - required-projects: - - name: github.com/ansible/ansible - timeout: 1000 - vars: - ansible_collections_repo: "{{ zuul.project.canonical_name }}" - ansible_test_location: "{{ ansible_user_dir }}/{{ zuul.projects[zuul.project.canonical_name].src_dir }}" - ansible_test_splitter__test_changed: true - ansible_test_splitter__children_prefix: ansible-test-cloud-integration-aws-py36_ - -- semaphore: - name: ansible-test-cloud-integration-aws - max: 6 - - job: name: ansible-test-cloud-integration-aws-py36 parent: ansible-core-ci-aws-session nodeset: controller-python36-f34 dependencies: - name: build-ansible-collection - - name: ansible-test-splitter pre-run: - playbooks/ansible-test-base/pre.yaml - playbooks/ansible-cloud/aws/pre.yaml @@ -298,68 +280,45 @@ ansible_test_command: integration ansible_test_enable_ara: false ansible_test_python: 3.6 - ansible_test_retry_on_error: true - semaphore: ansible-test-cloud-integration-aws - -- job: - name: ansible-test-cloud-integration-aws-py36_0 - parent: ansible-test-cloud-integration-aws-py36 - vars: - ansible_test_integration_targets: "{{ child.targets_to_test[0]|join(' ') }}" + ansible_test_changed: true + ansible_test_split_in: 6 - job: - name: ansible-test-cloud-integration-aws-py36_1 + name: ansible-test-cloud-integration-aws-py36_1_of_6 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_integration_targets: "{{ child.targets_to_test[1]|join(' ') }}" - -- job: - name: ansible-test-cloud-integration-aws-py36_2 - parent: ansible-test-cloud-integration-aws-py36 - vars: - ansible_test_integration_targets: "{{ child.targets_to_test[2]|join(' ') }}" - -- job: - name: ansible-test-cloud-integration-aws-py36_3 - parent: ansible-test-cloud-integration-aws-py36 - vars: - ansible_test_integration_targets: "{{ child.targets_to_test[3]|join(' ') }}" + ansible_test_do_number: 1 - job: - name: ansible-test-cloud-integration-aws-py36_4 + name: ansible-test-cloud-integration-aws-py36_2_of_6 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_integration_targets: "{{ child.targets_to_test[4]|join(' ') }}" + ansible_test_do_number: 2 - job: - name: ansible-test-cloud-integration-aws-py36_5 + name: ansible-test-cloud-integration-aws-py36_3_of_6 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_integration_targets: "{{ child.targets_to_test[5]|join(' ') }}" + ansible_test_do_number: 3 - job: - name: ansible-test-cloud-integration-aws-py36_6 + name: ansible-test-cloud-integration-aws-py36_4_of_6 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_integration_targets: "{{ child.targets_to_test[6]|join(' ') }}" + ansible_test_do_number: 4 - job: - name: ansible-test-cloud-integration-aws-py36_7 + name: ansible-test-cloud-integration-aws-py36_5_of_6 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_integration_targets: "{{ child.targets_to_test[7]|join(' ') }}" + ansible_test_do_number: 5 - job: - name: ansible-test-cloud-integration-aws-py36_8 + name: ansible-test-cloud-integration-aws-py36_6_of_6 parent: ansible-test-cloud-integration-aws-py36 vars: - ansible_test_integration_targets: "{{ child.targets_to_test[8]|join(' ') }}" + ansible_test_do_number: 6 -- job: - name: ansible-test-cloud-integration-aws-py36_9 - parent: ansible-test-cloud-integration-aws-py36 - vars: - ansible_test_integration_targets: "{{ child.targets_to_test[9]|join(' ') }}" #### units - job: name: ansible-test-units-community-aws-python38 diff --git a/zuul.d/project-templates.yaml b/zuul.d/project-templates.yaml index ce86f6033..fa944522a 100644 --- a/zuul.d/project-templates.yaml +++ b/zuul.d/project-templates.yaml @@ -76,18 +76,12 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/community.aws - name: github.com/ansible-collections/community.general - - ansible-test-splitter - - ansible-test-cloud-integration-aws-py36_0 - - ansible-test-cloud-integration-aws-py36_1 - - ansible-test-cloud-integration-aws-py36_2 - - ansible-test-cloud-integration-aws-py36_3 - - ansible-test-cloud-integration-aws-py36_4 - - ansible-test-cloud-integration-aws-py36_5 - - ansible-test-cloud-integration-aws-py36_6 - - ansible-test-cloud-integration-aws-py36_7 - - ansible-test-cloud-integration-aws-py36_8 - - ansible-test-cloud-integration-aws-py36_9 - + - ansible-test-cloud-integration-aws-py36_1_of_6 + - ansible-test-cloud-integration-aws-py36_2_of_6 + - ansible-test-cloud-integration-aws-py36_3_of_6 + - ansible-test-cloud-integration-aws-py36_4_of_6 + - ansible-test-cloud-integration-aws-py36_5_of_6 + - ansible-test-cloud-integration-aws-py36_6_of_6 gate: jobs: - ansible-test-sanity-aws-ansible-2.9-python36 @@ -103,17 +97,12 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/community.aws - name: github.com/ansible-collections/community.general - - ansible-test-splitter - - ansible-test-cloud-integration-aws-py36_0 - - ansible-test-cloud-integration-aws-py36_1 - - ansible-test-cloud-integration-aws-py36_2 - - ansible-test-cloud-integration-aws-py36_3 - - ansible-test-cloud-integration-aws-py36_4 - - ansible-test-cloud-integration-aws-py36_5 - - ansible-test-cloud-integration-aws-py36_6 - - ansible-test-cloud-integration-aws-py36_7 - - ansible-test-cloud-integration-aws-py36_8 - - ansible-test-cloud-integration-aws-py36_9 + - ansible-test-cloud-integration-aws-py36_1_of_6 + - ansible-test-cloud-integration-aws-py36_2_of_6 + - ansible-test-cloud-integration-aws-py36_3_of_6 + - ansible-test-cloud-integration-aws-py36_4_of_6 + - ansible-test-cloud-integration-aws-py36_5_of_6 + - ansible-test-cloud-integration-aws-py36_6_of_6 - project-template: name: ansible-collections-community-aws @@ -130,17 +119,12 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/ansible.utils - name: github.com/ansible-collections/community.general - - ansible-test-splitter - - ansible-test-cloud-integration-aws-py36_0 - - ansible-test-cloud-integration-aws-py36_1 - - ansible-test-cloud-integration-aws-py36_2 - - ansible-test-cloud-integration-aws-py36_3 - - ansible-test-cloud-integration-aws-py36_4 - - ansible-test-cloud-integration-aws-py36_5 - - ansible-test-cloud-integration-aws-py36_6 - - ansible-test-cloud-integration-aws-py36_7 - - ansible-test-cloud-integration-aws-py36_8 - - ansible-test-cloud-integration-aws-py36_9 + - ansible-test-cloud-integration-aws-py36_1_of_6 + - ansible-test-cloud-integration-aws-py36_2_of_6 + - ansible-test-cloud-integration-aws-py36_3_of_6 + - ansible-test-cloud-integration-aws-py36_4_of_6 + - ansible-test-cloud-integration-aws-py36_5_of_6 + - ansible-test-cloud-integration-aws-py36_6_of_6 - ansible-galaxy-importer: voting: false gate: @@ -156,17 +140,12 @@ - name: github.com/ansible-collections/ansible.netcommon - name: github.com/ansible-collections/ansible.utils - name: github.com/ansible-collections/community.general - - ansible-test-splitter - - ansible-test-cloud-integration-aws-py36_0 - - ansible-test-cloud-integration-aws-py36_1 - - ansible-test-cloud-integration-aws-py36_2 - - ansible-test-cloud-integration-aws-py36_3 - - ansible-test-cloud-integration-aws-py36_4 - - ansible-test-cloud-integration-aws-py36_5 - - ansible-test-cloud-integration-aws-py36_6 - - ansible-test-cloud-integration-aws-py36_7 - - ansible-test-cloud-integration-aws-py36_8 - - ansible-test-cloud-integration-aws-py36_9 + - ansible-test-cloud-integration-aws-py36_1_of_6 + - ansible-test-cloud-integration-aws-py36_2_of_6 + - ansible-test-cloud-integration-aws-py36_3_of_6 + - ansible-test-cloud-integration-aws-py36_4_of_6 + - ansible-test-cloud-integration-aws-py36_5_of_6 + - ansible-test-cloud-integration-aws-py36_6_of_6 - ansible-galaxy-importer: voting: false