diff --git a/test/eco/bootstrap.result b/test/eco/bootstrap.result index d2bf41a090..986daad5cc 100644 --- a/test/eco/bootstrap.result +++ b/test/eco/bootstrap.result @@ -1 +1,9 @@ +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 0 + +STDERR: WARNING Loading custom .yamllint config file, this extends our internal yamllint config. + + +STDOUT: diff --git a/test/eco/colsystem.result b/test/eco/colsystem.result index 813cfdad3e..1a2e6ee408 100644 --- a/test/eco/colsystem.result +++ b/test/eco/colsystem.result @@ -1,10 +1,22 @@ -WARNING Listing 2 violation(s) that are fatal -.ansible-lint:1: load-failure [Errno 2] No such file or directory: '~/.cache/ansible-lint-eco/colsystem/tests/ansible-lint.yml' (filenotfounderror) -playbooks/molecule/sudo/molecule.yml:17: yaml line too long (576 > 160 characters) (line-length) +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 2 + +STDERR: +WARNING Listing 5 violation(s) that are fatal You can skip specific rules or tags by adding them to your configuration file: -# .ansible-lint +# .config/ansible-lint.yml warn_list: # or 'skip_list' to silence them completely - - load-failure # Failed to load or parse file - - yaml # Violations reported by yamllint + - experimental # all rules tagged as experimental + - load-failure # Failed to load or parse file. + - yaml # Violations reported by yamllint. -Finished with 2 failure(s), 0 warning(s) on 291 files. +Finished with 2 failure(s), 3 warning(s) on 279 files. + + +STDOUT: +.ansible-lint:1: load-failure [Errno 2] No such file or directory: '~/.cache/ansible-lint-eco/colsystem/tests/ansible-lint.yml' (filenotfounderror) +playbooks/molecule/sudo/molecule.yml:17: yaml line too long (576 > 160 characters) (line-length) +roles/dotfiles/tasks/03_vim.yml:25: risky-file-permissions File permissions unset or incorrect. +roles/mythtv/tasks/main.yml:10: risky-file-permissions File permissions unset or incorrect. +roles/packages_workstation/tasks/Linux/06_blu_ray.yml:9: risky-file-permissions File permissions unset or incorrect. diff --git a/test/eco/debops.result b/test/eco/debops.result index d2bf41a090..3bd8b568b1 100644 --- a/test/eco/debops.result +++ b/test/eco/debops.result @@ -1 +1,15 @@ +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 2 + +STDERR: WARNING Loading custom .yamllint config file, this extends our internal yamllint config. +WARNING Listing 3 violation(s) that are fatal + +Finished with 3 failure(s), 0 warning(s) on 4129 files. + + +STDOUT: +../../../.ansible/collections/ansible_collections/debops/debops/roles/btrfs/tasks/main.yml:21:3: syntax-check couldn't resolve module/action 'btrfs_subvolume'. This often indicates a misspelling, missing collection, or incorrect module path. +ansible/playbooks/site.yml:1: syntax-check Ansible syntax check failed. +ansible/playbooks/srv.yml:1: syntax-check Ansible syntax check failed. diff --git a/test/eco/docker-rootless.result b/test/eco/docker-rootless.result index 044e47dd4c..20dfdc066b 100644 --- a/test/eco/docker-rootless.result +++ b/test/eco/docker-rootless.result @@ -1 +1,18 @@ +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 0 + +STDERR: WARNING Loading custom .yamllint.yml config file, this extends our internal yamllint config. +WARNING Listing 2 violation(s) that are fatal +You can skip specific rules or tags by adding them to your configuration file: +# .config/ansible-lint.yml +warn_list: # or 'skip_list' to silence them completely + - experimental # all rules tagged as experimental + +Finished with 0 failure(s), 2 warning(s) on 29 files. + + +STDOUT: +tasks/docker_install_rootless.yml:22: risky-file-permissions File permissions unset or incorrect. +tasks/docker_install_rootless.yml:32: risky-file-permissions File permissions unset or incorrect. diff --git a/test/eco/hardening.result b/test/eco/hardening.result index 7defc6e43c..a7807374c7 100644 --- a/test/eco/hardening.result +++ b/test/eco/hardening.result @@ -1,8 +1,16 @@ +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 0 + +STDERR: WARNING Loading custom .yamllint.yml config file, this extends our internal yamllint config. WARNING Listing 4 violation(s) that are fatal + +Finished with 0 failure(s), 4 warning(s) on 121 files. + + +STDOUT: defaults/main/sshd.yml:20: yaml line too long (143 > 120 characters) (line-length) tasks/auditd.yml:21: yaml line too long (122 > 120 characters) (line-length) tasks/packagemgmt.yml:164: yaml line too long (129 > 120 characters) (line-length) tasks/packagemgmt.yml:192: yaml line too long (162 > 120 characters) (line-length) - -Finished with 0 failure(s), 4 warning(s) on 121 files. diff --git a/test/eco/mysql.result b/test/eco/mysql.result index 5af8036efe..f8f4c0a229 100644 --- a/test/eco/mysql.result +++ b/test/eco/mysql.result @@ -1,10 +1,17 @@ +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 2 + +STDERR: WARNING Loading custom .yamllint config file, this extends our internal yamllint config. -WARNING Listing 2 violation(s) that are fatal -../../ansible-lint/314d98/roles/geerlingguy.mysql/tasks/replication.yml:37: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors -tasks/replication.yml:37: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors +WARNING Listing 1 violation(s) that are fatal You can skip specific rules or tags by adding them to your configuration file: -# .ansible-lint +# .config/ansible-lint.yml warn_list: # or 'skip_list' to silence them completely - - experimental # all rules tagged as experimental + - internal-error # Unexpected internal error. + +Finished with 1 failure(s), 0 warning(s) on 28 files. + -Finished with 0 failure(s), 2 warning(s) on 46 files. +STDOUT: +molecule/default/converge.yml:7:7: internal-error the role 'geerlingguy.mysql' was not found in ~/.cache/ansible-lint-eco/mysql/molecule/default/roles:~/.cache/ansible-compat/2b326e/roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.cache/ansible-lint-eco/mysql/molecule/default diff --git a/test/eco/tripleo-ansible.result b/test/eco/tripleo-ansible.result index 3fab39bbaf..9450d4b92f 100644 --- a/test/eco/tripleo-ansible.result +++ b/test/eco/tripleo-ansible.result @@ -1,7 +1,15 @@ -INFO Running ansible-galaxy role install --roles-path ~/.cache/ansible-lint/3378c2/roles -vr requirements.yml -INFO Running ansible-galaxy collection install -p ~/.cache/ansible-lint/3378c2/collections -vr requirements.yml -INFO Added ANSIBLE_LIBRARY=plugins/modules:~/.cache/ansible-lint/314d98/modules:plugins/modules:~/.cache/ansible-lint/314d98/modules:~/.cache/ansible-lint/3378c2/modules -INFO Added ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/3378c2/collections -INFO Added ANSIBLE_ROLES_PATH=~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.cache/ansible-lint/314d98/roles:~/.cache/ansible-lint/314d98/roles:roles:~/.cache/ansible-lint/3378c2/roles +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 0 + +STDERR: +INFO Running ansible-galaxy role install -vr requirements.yml --roles-path ~/.cache/ansible-compat/3378c2/roles +INFO Running ansible-galaxy collection install -vr requirements.yml -p ~/.cache/ansible-compat/3378c2/collections +INFO Set ANSIBLE_LIBRARY=~/.cache/ansible-compat/3378c2/modules:~/.ansible/plugins/modules:/usr/share/ansible/plugins/modules +INFO Set ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-compat/3378c2/collections:~/.ansible/collections:/usr/share/ansible/collections +INFO Set ANSIBLE_ROLES_PATH=~/.cache/ansible-compat/3378c2/roles:roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles INFO Discovered files to lint using: git ls-files --cached --others --exclude-standard -z INFO Excluded removed files using: git ls-files --deleted -z + + +STDOUT: diff --git a/test/eco/zuul-jobs.result b/test/eco/zuul-jobs.result index cf3019cc15..c9c97050ed 100644 --- a/test/eco/zuul-jobs.result +++ b/test/eco/zuul-jobs.result @@ -1,25 +1,40 @@ -INFO Added ANSIBLE_LIBRARY=plugins/modules:~/.cache/ansible-lint/314d98/modules:plugins/modules:~/.cache/ansible-lint/314d98/modules:~/.cache/ansible-lint/6f378d/modules -INFO Added ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections:~/.cache/ansible-lint/314d98/collections -INFO Added ANSIBLE_ROLES_PATH=~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.cache/ansible-lint/314d98/roles:~/.cache/ansible-lint/314d98/roles:roles +CMD: ansible-lint -f pep8 -x fqcn-builtins + +RC: 2 + +STDERR: +INFO Set ANSIBLE_LIBRARY=~/.cache/ansible-compat/6f378d/modules:~/.ansible/plugins/modules:/usr/share/ansible/plugins/modules +INFO Set ANSIBLE_COLLECTIONS_PATH=~/.cache/ansible-compat/6f378d/collections:~/.ansible/collections:/usr/share/ansible/collections +INFO Set ANSIBLE_ROLES_PATH=~/.cache/ansible-compat/6f378d/roles:roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles WARNING Loading custom .yamllint config file, this extends our internal yamllint config. INFO Discovered files to lint using: git ls-files --cached --others --exclude-standard -z INFO Excluded removed files using: git ls-files --deleted -z INFO Discovered files to lint using: git ls-files --cached --others --exclude-standard -z INFO Excluded removed files using: git ls-files --deleted -z -WARNING Listing 9 violation(s) that are fatal -roles/emit-job-header/tasks/main.yaml:26: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors -roles/ensure-pip/tasks/Debian.yaml:17: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors -test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect -test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:6: risky-file-permissions File permissions unset or incorrect -test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: no-loop-var-prefix Role loop_var should use configured prefix.: zj_ -test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect -test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:17: risky-file-permissions File permissions unset or incorrect -test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:25: risky-file-permissions File permissions unset or incorrect -test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:30: risky-file-permissions File permissions unset or incorrect +WARNING Listing 16 violation(s) that are fatal You can skip specific rules or tags by adding them to your configuration file: -# .ansible-lint +# .config/ansible-lint.yml warn_list: # or 'skip_list' to silence them completely - experimental # all rules tagged as experimental - - no-loop-var-prefix # Role loop_var should use configured prefix.: zj_ + - no-loop-var-prefix # Role loop_var should use configured prefix. + +Finished with 1 failure(s), 15 warning(s) on 1242 files. + -Finished with 1 failure(s), 8 warning(s) on 1242 files. +STDOUT: +roles/emit-job-header/tasks/main.yaml:26: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors. +roles/ensure-chart-testing/tasks/main.yaml:27: risky-file-permissions File permissions unset or incorrect. +roles/ensure-dhall/tasks/main.yaml:15: risky-file-permissions File permissions unset or incorrect. +roles/ensure-go/tasks/install-go.yaml:12: risky-file-permissions File permissions unset or incorrect. +roles/ensure-packer/tasks/install-packer.yaml:21: risky-file-permissions File permissions unset or incorrect. +roles/ensure-pip/tasks/Debian.yaml:17: ignore-errors Use failed_when and specify error conditions instead of using ignore_errors. +roles/ensure-pip/tasks/source.yaml:7: risky-file-permissions File permissions unset or incorrect. +roles/ensure-terraform/tasks/install-terraform.yaml:32: risky-file-permissions File permissions unset or incorrect. +roles/phoronix-combine-results/tasks/fetch-result.yaml:8: risky-file-permissions File permissions unset or incorrect. +test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect. +test-playbooks/registry/roles/ensure-registry-cert/tasks/main.yaml:6: risky-file-permissions File permissions unset or incorrect. +test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: no-loop-var-prefix Role loop_var should use configured prefix. +test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:1: risky-file-permissions File permissions unset or incorrect. +test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:17: risky-file-permissions File permissions unset or incorrect. +test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:25: risky-file-permissions File permissions unset or incorrect. +test-playbooks/registry/roles/run-test-intermediate-registry/tasks/main.yaml:30: risky-file-permissions File permissions unset or incorrect. diff --git a/test/test_eco.py b/test/test_eco.py index e226e5b0bd..1e53cacd07 100644 --- a/test/test_eco.py +++ b/test/test_eco.py @@ -1,10 +1,13 @@ """Test a set of 3rd party Ansible repositories for possible regressions.""" import os import pathlib +import shlex import subprocess import pytest +from ansiblelint.testing import run_ansible_lint + eco_repos = { "bootstrap": [ "https://github.com/robertdebock/ansible-role-bootstrap", @@ -47,12 +50,28 @@ def test_eco(repo: str) -> None: subprocess.run(f"git clone {url} {cache_dir}/{repo}", shell=True, check=True) # run ansible lint and paths from user home in order to produce # consistent results regardless on its location. - subprocess.run( - # we exclude `fqcn-builtins` until repository owners fix it. - f'ansible-lint -f pep8 -x fqcn-builtins 2>&1 | sed "s:${{HOME}}:~:g" > {my_dir}/{repo}.result', - shell=True, - check=False, + + # we exclude `fqcn-builtins` until repository owners fix it. + args = ["-f", "pep8", "-x", "fqcn-builtins"] + result = run_ansible_lint( + *args, + executable="ansible-lint", cwd=f"{cache_dir}/{repo}", ) + + def rel_home(text: str) -> str: + """Replace full path to home directory with ~.""" + return text.replace(os.path.expanduser("~"), "~") + + result_txt = f"CMD: {shlex.join(result.args)}\n\nRC: {result.returncode}\n\nSTDERR:\n{result.stderr}\n\nSTDOUT:\n{result.stdout}" + + with open(f"{my_dir}/{repo}.result", "w", encoding="utf-8") as f: + f.write(rel_home(result_txt)) # fail if result is different than our expected one: - subprocess.run(f"git diff HEAD test/eco/{repo}.result", shell=True, check=True) + result = subprocess.run( + f"git diff --exit-code test/eco/{repo}.result", + shell=True, + check=False, + capture_output=True, + ) + assert result.returncode == 0, result.stdout diff --git a/tox.ini b/tox.ini index 08c5f7aa54..049faf64ff 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,8 @@ isolated_build = true requires = setuptools >= 41.4.0 pip >= 19.3.0 + # tox-extra ensure non-zero exit code if git reports dirty after test run + tox-extra >= 0.3.0 skip_missing_interpreters = True [testenv] @@ -153,9 +155,10 @@ changedir = {toxinidir}/docs [testenv:eco] description = Perform ecosystem impact (downstream testing) https://github.com/ansible-community/ansible-lint/discussions/1403 skip_install = true -deps = tox>=4.0.0b1 +deps = + {[testenv]deps} commands = - python -m tox -e py -- -n auto -m eco + pytest -n auto -m eco [testenv:packaging] basepython = python3