Update eco job output

test/eco/bootstrap.result

@@ -1 +1,9 @@
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 0
+
+STDERR:
 WARNING  Loading custom .yamllint config file, this extends our internal yamllint config.
+
+
+STDOUT:

test/eco/colsystem.result

@@ -1,10 +1,22 @@
-WARNING  Listing 2 violation(s) that are fatal
-.ansible-lint:1: load-failure [Errno 2] No such file or directory: '~/.cache/ansible-lint-eco/colsystem/tests/ansible-lint.yml' (filenotfounderror)
-playbooks/molecule/sudo/molecule.yml:17: yaml line too long (576 > 160 characters) (line-length)
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 2
+
+STDERR:
+WARNING  Listing 5 violation(s) that are fatal
 You can skip specific rules or tags by adding them to your configuration file:
-# .ansible-lint
+# .config/ansible-lint.yml
 warn_list:  # or 'skip_list' to silence them completely
-  - load-failure  # Failed to load or parse file
-  - yaml  # Violations reported by yamllint
+  - experimental  # all rules tagged as experimental
+  - load-failure  # Failed to load or parse file.
+  - yaml  # Violations reported by yamllint.
 
-Finished with 2 failure(s), 0 warning(s) on 291 files.
+Finished with 2 failure(s), 3 warning(s) on 279 files.
+
+
+STDOUT:
+.ansible-lint:1: load-failure [Errno 2] No such file or directory: '~/.cache/ansible-lint-eco/colsystem/tests/ansible-lint.yml' (filenotfounderror)
+playbooks/molecule/sudo/molecule.yml:17: yaml line too long (576 > 160 characters) (line-length)
+roles/dotfiles/tasks/03_vim.yml:25: risky-file-permissions File permissions unset or incorrect.
+roles/mythtv/tasks/main.yml:10: risky-file-permissions File permissions unset or incorrect.
+roles/packages_workstation/tasks/Linux/06_blu_ray.yml:9: risky-file-permissions File permissions unset or incorrect.

test/eco/debops.result

@@ -1 +1,15 @@
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 2
+
+STDERR:
 WARNING  Loading custom .yamllint config file, this extends our internal yamllint config.
+WARNING  Listing 3 violation(s) that are fatal
+
+Finished with 3 failure(s), 0 warning(s) on 4129 files.
+
+
+STDOUT:
+../../../.ansible/collections/ansible_collections/debops/debops/roles/btrfs/tasks/main.yml:21:3: syntax-check couldn't resolve module/action 'btrfs_subvolume'. This often indicates a misspelling, missing collection, or incorrect module path.
+ansible/playbooks/site.yml:1: syntax-check Ansible syntax check failed.
+ansible/playbooks/srv.yml:1: syntax-check Ansible syntax check failed.

test/eco/docker-rootless.result

@@ -1 +1,18 @@
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 0
+
+STDERR:
 WARNING  Loading custom .yamllint.yml config file, this extends our internal yamllint config.
+WARNING  Listing 2 violation(s) that are fatal
+You can skip specific rules or tags by adding them to your configuration file:
+# .config/ansible-lint.yml
+warn_list:  # or 'skip_list' to silence them completely
+  - experimental  # all rules tagged as experimental
+
+Finished with 0 failure(s), 2 warning(s) on 29 files.
+
+
+STDOUT:
+tasks/docker_install_rootless.yml:22: risky-file-permissions File permissions unset or incorrect.
+tasks/docker_install_rootless.yml:32: risky-file-permissions File permissions unset or incorrect.

test/eco/hardening.result

@@ -1,8 +1,16 @@
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 0
+
+STDERR:
 WARNING  Loading custom .yamllint.yml config file, this extends our internal yamllint config.
 WARNING  Listing 4 violation(s) that are fatal
+
+Finished with 0 failure(s), 4 warning(s) on 121 files.
+
+
+STDOUT:
 defaults/main/sshd.yml:20: yaml line too long (143 > 120 characters) (line-length)
 tasks/auditd.yml:21: yaml line too long (122 > 120 characters) (line-length)
 tasks/packagemgmt.yml:164: yaml line too long (129 > 120 characters) (line-length)
 tasks/packagemgmt.yml:192: yaml line too long (162 > 120 characters) (line-length)
-
-Finished with 0 failure(s), 4 warning(s) on 121 files.

test/eco/mysql.result

@@ -1,10 +1,17 @@
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 2
+
+STDERR:
 WARNING  Loading custom .yamllint config file, this extends our internal yamllint config.
-WARNING  Listing 2 violation(s) that are fatal
-../../ansible-lint/314d98/roles/geerlingguy.mysql/tasks/replication.yml:37: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors
-tasks/replication.yml:37: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors
+WARNING  Listing 1 violation(s) that are fatal
 You can skip specific rules or tags by adding them to your configuration file:
-# .ansible-lint
+# .config/ansible-lint.yml
 warn_list:  # or 'skip_list' to silence them completely
-  - experimental  # all rules tagged as experimental
+  - internal-error  # Unexpected internal error.
+
+Finished with 1 failure(s), 0 warning(s) on 28 files.
+
 
-Finished with 0 failure(s), 2 warning(s) on 46 files.
+STDOUT:
+molecule/default/converge.yml:7:7: internal-error the role 'geerlingguy.mysql' was not found in ~/.cache/ansible-lint-eco/mysql/molecule/default/roles:~/.cache/ansible-compat/2b326e/roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.cache/ansible-lint-eco/mysql/molecule/default

test/eco/tripleo-ansible.result

@@ -1,7 +1,15 @@
-INFO     Running ansible-galaxy role install --roles-path ~/.cache/ansible-lint/3378c2/roles -vr requirements.yml
-INFO     Running ansible-galaxy collection install -p ~/.cache/ansible-lint/3378c2/collections -vr requirements.yml
-INFO     Added ANSIBLE_LIBRARY=plugins/modules:~/.cache/ansible-lint/314d98/modules:plugins/modules:~/.cache/ansible-lint/314d98/modules:~/.cache/ansible-lint/3378c2/modules
-INFO     Added ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/3378c2/collections
-INFO     Added ANSIBLE_ROLES_PATH=~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.cache/ansible-lint/314d98/roles:~/.cache/ansible-lint/314d98/roles:roles:~/.cache/ansible-lint/3378c2/roles
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 0
+
+STDERR:
+INFO     Running ansible-galaxy role install -vr requirements.yml --roles-path ~/.cache/ansible-compat/3378c2/roles
+INFO     Running ansible-galaxy collection install -vr requirements.yml -p ~/.cache/ansible-compat/3378c2/collections
+INFO     Set ANSIBLE_LIBRARY=~/.cache/ansible-compat/3378c2/modules:~/.ansible/plugins/modules:/usr/share/ansible/plugins/modules
+INFO     Set ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-compat/3378c2/collections:~/.ansible/collections:/usr/share/ansible/collections
+INFO     Set ANSIBLE_ROLES_PATH=~/.cache/ansible-compat/3378c2/roles:roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles
 INFO     Discovered files to lint using: git ls-files --cached --others --exclude-standard -z
 INFO     Excluded removed files using: git ls-files --deleted -z
+
+
+STDOUT:

test/eco/zuul-jobs.result

@@ -1,25 +1,40 @@
-INFO     Added ANSIBLE_LIBRARY=plugins/modules:~/.cache/ansible-lint/314d98/modules:plugins/modules:~/.cache/ansible-lint/314d98/modules:~/.cache/ansible-lint/6f378d/modules
-INFO     Added ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections
-INFO     Added ANSIBLE_ROLES_PATH=~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.cache/ansible-lint/314d98/roles:~/.cache/ansible-lint/314d98/roles:roles
+CMD: ansible-lint -f pep8 -x fqcn-builtins
+
+RC: 2
+
+STDERR:
+INFO     Set ANSIBLE_LIBRARY=~/.cache/ansible-compat/6f378d/modules:~/.ansible/plugins/modules:/usr/share/ansible/plugins/modules
+INFO     Set ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-compat/6f378d/collections:~/.ansible/collections:/usr/share/ansible/collections
+INFO     Set ANSIBLE_ROLES_PATH=~/.cache/ansible-compat/6f378d/roles:roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles
 WARNING  Loading custom .yamllint config file, this extends our internal yamllint config.
 INFO     Discovered files to lint using: git ls-files --cached --others --exclude-standard -z
 INFO     Excluded removed files using: git ls-files --deleted -z
 INFO     Discovered files to lint using: git ls-files --cached --others --exclude-standard -z
 INFO     Excluded removed files using: git ls-files --deleted -z
-WARNING  Listing 9 violation(s) that are fatal
-roles/emit-job-header/tasks/main.yaml:26: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors
-roles/ensure-pip/tasks/Debian.yaml:17: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors
-test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect
-test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:6: risky-file-permissions File permissions unset or incorrect
-test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: no-loop-var-prefix Role loop_var should use configured prefix.: zj_
-test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect
-test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:17: risky-file-permissions File permissions unset or incorrect
-test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:25: risky-file-permissions File permissions unset or incorrect
-test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:30: risky-file-permissions File permissions unset or incorrect
+WARNING  Listing 16 violation(s) that are fatal
 You can skip specific rules or tags by adding them to your configuration file:
-# .ansible-lint
+# .config/ansible-lint.yml
 warn_list:  # or 'skip_list' to silence them completely
   - experimental  # all rules tagged as experimental
-  - no-loop-var-prefix  # Role loop_var should use configured prefix.: zj_
+  - no-loop-var-prefix  # Role loop_var should use configured prefix.
+
+Finished with 1 failure(s), 15 warning(s) on 1242 files.
+
 
-Finished with 1 failure(s), 8 warning(s) on 1242 files.
+STDOUT:
+roles/emit-job-header/tasks/main.yaml:26: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors.
+roles/ensure-chart-testing/tasks/main.yaml:27: risky-file-permissions File permissions unset or incorrect.
+roles/ensure-dhall/tasks/main.yaml:15: risky-file-permissions File permissions unset or incorrect.
+roles/ensure-go/tasks/install-go.yaml:12: risky-file-permissions File permissions unset or incorrect.
+roles/ensure-packer/tasks/install-packer.yaml:21: risky-file-permissions File permissions unset or incorrect.
+roles/ensure-pip/tasks/Debian.yaml:17: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors.
+roles/ensure-pip/tasks/source.yaml:7: risky-file-permissions File permissions unset or incorrect.
+roles/ensure-terraform/tasks/install-terraform.yaml:32: risky-file-permissions File permissions unset or incorrect.
+roles/phoronix-combine-results/tasks/fetch-result.yaml:8: risky-file-permissions File permissions unset or incorrect.
+test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect.
+test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:6: risky-file-permissions File permissions unset or incorrect.
+test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: no-loop-var-prefix Role loop_var should use configured prefix.
+test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect.
+test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:17: risky-file-permissions File permissions unset or incorrect.
+test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:25: risky-file-permissions File permissions unset or incorrect.
+test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:30: risky-file-permissions File permissions unset or incorrect.

test/test_eco.py

@@ -1,10 +1,13 @@
 """Test a set of 3rd party Ansible repositories for possible regressions."""
 import os
 import pathlib
+import shlex
 import subprocess
 
 import pytest
 
+from ansiblelint.testing import run_ansible_lint
+
 eco_repos = {
     "bootstrap": [
         "https://github.com/robertdebock/ansible-role-bootstrap",
@@ -47,12 +50,28 @@ def test_eco(repo: str) -> None:
         subprocess.run(f"git clone {url} {cache_dir}/{repo}", shell=True, check=True)
     # run ansible lint and paths from user home in order to produce
     # consistent results regardless on its location.
-    subprocess.run(
-        # we exclude `fqcn-builtins` until repository owners fix it.
-        f'ansible-lint -f pep8 -x fqcn-builtins 2>&1 | sed "s:${{HOME}}:~:g" > {my_dir}/{repo}.result',
-        shell=True,
-        check=False,
+
+    # we exclude `fqcn-builtins` until repository owners fix it.
+    args = ["-f", "pep8", "-x", "fqcn-builtins"]
+    result = run_ansible_lint(
+        *args,
+        executable="ansible-lint",
         cwd=f"{cache_dir}/{repo}",
     )
+
+    def rel_home(text: str) -> str:
+        """Replace full path to home directory with ~."""
+        return text.replace(os.path.expanduser("~"), "~")
+
+    result_txt = f"CMD: {shlex.join(result.args)}\n\nRC: {result.returncode}\n\nSTDERR:\n{result.stderr}\n\nSTDOUT:\n{result.stdout}"
+
+    with open(f"{my_dir}/{repo}.result", "w", encoding="utf-8") as f:
+        f.write(rel_home(result_txt))
     # fail if result is different than our expected one:
-    subprocess.run(f"git diff HEAD test/eco/{repo}.result", shell=True, check=True)
+    result = subprocess.run(
+        f"git diff --exit-code test/eco/{repo}.result",
+        shell=True,
+        check=False,
+        capture_output=True,
+    )
+    assert result.returncode == 0, result_txt

tox.ini

@@ -11,6 +11,8 @@ isolated_build = true
 requires =
   setuptools >= 41.4.0
   pip >= 19.3.0
+  # tox-extra ensure non-zero exit code if git reports dirty after test run
+  tox-extra >= 0.3.0
 skip_missing_interpreters = True
 
 [testenv]
@@ -153,9 +155,10 @@ changedir = {toxinidir}/docs
 [testenv:eco]
 description = Perform ecosystem impact (downstream testing) https://github.com/ansible-community/ansible-lint/discussions/1403
 skip_install = true
-deps = tox>=4.0.0b1
+deps =
+  {[testenv]deps}
 commands =
-  python -m tox -e py -- -n auto -m eco
+  pytest -n auto -m eco
 
 [testenv:packaging]
 basepython = python3