Update Ansible to 9.9.0 (hitachienergy#3469)

* Update AlmaLinux image to 8.7.20221110 * Increase timeout to 30 * Increase `retries-count` from 3 to 10 for `download-requirements.py` * Fix `epicli - list index out of range` * Fix pytest tests in VS Code --------- Co-authored-by: przemyslavic <[email protected]>
przemyslavic · Nov 18, 2024 · d41fdc1 · d41fdc1
1 parent 717d9c9
commit d41fdc1
Show file tree

Hide file tree

Showing 43 changed files with 251 additions and 213 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -64,7 +64,7 @@ RUN : INSTALL PIP REQUIREMENTS \
  && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
     --requirement /requirements.txt \
  && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
-    poetry pylint==2.* pylint_junit ansible-lint==6.5.0 ansible-lint-to-junit-xml yamllint pytest pytest_mock setuptools twine wheel
+    poetry pylint==2.* pylint_junit ansible-lint==24.7.0 ansible-lint-junit==0.17.8 yamllint pytest pytest_mock setuptools twine wheel
 
 RUN : SETUP USER AND OTHERS \
  && groupadd --gid $USER_GID $USERNAME \

diff --git a/.devcontainer/gen-dependency-info.py b/.devcontainer/gen-dependency-info.py
@@ -91,7 +91,7 @@ def _main() -> None:
     # Write licenses 'cli/licenses.py'
     licenses_content = """\
     # This is a generated file so don`t change this manually.
-    # To re-generate run 'python gen-dependency-info.py <GitHub_PAT>' from the project root.
+    # To re-generate run 'python3 gen-dependency-info.py <GitHub_PAT>' from the project root.
 
     LICENSES = """
 

diff --git a/.devcontainer/poetry.lock b/.devcontainer/poetry.lock
diff --git a/.devcontainer/pyproject.toml b/.devcontainer/pyproject.toml
@@ -13,7 +13,7 @@ jsonschema = "*"
 python-json-logger = "*"
 "ruamel.yaml" = "*"
 click = "*"
-ansible = "6.7.0"
+ansible = "~9"
 azure-cli = "2.63.0"
 
 [build-system]

diff --git a/.devcontainer/requirements.md b/.devcontainer/requirements.md
@@ -46,7 +46,7 @@ The main reasons for this 2-way approach:
 2. Run the following to update the `license.py` and generate the `DEPENDENCIES.md` file:
 
     ```shell
-    python gen-dependency-info.py YOUR-GITHUB-PAT
+    python3 gen-dependency-info.py YOUR-GITHUB-PAT
     ```
 
     For obtaining a GitHub Personal Access Token, check [here](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).

diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt
@@ -1,6 +1,6 @@
 adal==1.2.7 ; python_full_version == "3.10.14"
-ansible-core==2.13.13 ; python_full_version == "3.10.14"
-ansible==6.7.0 ; python_full_version == "3.10.14"
+ansible-core==2.16.10 ; python_full_version == "3.10.14"
+ansible==9.9.0 ; python_full_version == "3.10.14"
 antlr4-python3-runtime==4.13.2 ; python_full_version == "3.10.14"
 applicationinsights==0.11.10 ; python_full_version == "3.10.14"
 argcomplete==3.3.0 ; python_full_version == "3.10.14"
@@ -90,15 +90,15 @@ azure-synapse-artifacts==0.19.0 ; python_full_version == "3.10.14"
 azure-synapse-managedprivateendpoints==0.4.0 ; python_full_version == "3.10.14"
 azure-synapse-spark==0.2.0 ; python_full_version == "3.10.14"
 bcrypt==4.2.0 ; python_full_version == "3.10.14"
-boto3==1.35.6 ; python_full_version == "3.10.14"
-botocore==1.35.6 ; python_full_version == "3.10.14"
-certifi==2024.7.4 ; python_full_version == "3.10.14"
-cffi==1.17.0 ; python_full_version == "3.10.14"
+boto3==1.35.12 ; python_full_version == "3.10.14"
+botocore==1.35.12 ; python_full_version == "3.10.14"
+certifi==2024.8.30 ; python_full_version == "3.10.14"
+cffi==1.17.1 ; python_full_version == "3.10.14"
 chardet==5.2.0 ; python_full_version == "3.10.14"
 charset-normalizer==3.3.2 ; python_full_version == "3.10.14"
 click==8.1.7 ; python_full_version == "3.10.14"
 colorama==0.4.6 ; python_full_version == "3.10.14"
-cryptography==43.0.0 ; python_full_version == "3.10.14"
+cryptography==43.0.1 ; python_full_version == "3.10.14"
 decorator==5.1.1 ; python_full_version == "3.10.14"
 deprecated==1.2.14 ; python_full_version == "3.10.14"
 distro==1.9.0 ; sys_platform == "linux" and python_full_version == "3.10.14"
@@ -145,7 +145,7 @@ referencing==0.35.1 ; python_full_version == "3.10.14"
 requests-oauthlib==2.0.0 ; python_full_version == "3.10.14"
 requests==2.32.3 ; python_full_version == "3.10.14"
 requests[socks]==2.32.3 ; python_full_version == "3.10.14"
-resolvelib==0.8.1 ; python_full_version == "3.10.14"
+resolvelib==1.0.1 ; python_full_version == "3.10.14"
 rpds-py==0.20.0 ; python_full_version == "3.10.14"
 ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_full_version == "3.10.14"
 ruamel-yaml==0.18.6 ; python_full_version == "3.10.14"

diff --git a/.gitignore b/.gitignore
@@ -127,3 +127,4 @@ clusters
 #.vscode
 #.devcontainer
 *.dump
+sandbox/
diff --git a/.pylintrc b/.pylintrc
@@ -349,8 +349,8 @@ int-import-graph=
 [EXCEPTIONS]
 
 # Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
+# ["builtins.BaseException", "builtins.Exception"]
+overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"]
 
 
 [TYPING]

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,12 +1,13 @@
 {
+    // Ansible
+    "ansible.python.interpreterPath": "/usr/local/bin/python3",
     // Common
     "trailing-spaces.highlightCurrentLine": false, // highlighting currently edited line can be annoying
     "trailing-spaces.syntaxIgnore": [
         "markdown" // "two spaces" syntax is for line breaks
     ],
 
     // Python
-    "pylint.enabled": true,
     "pylint.args": [
         "--rcfile=${workspaceFolder}/.pylintrc"
     ],

diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -6,7 +6,7 @@
   "tasks": [
     {
       "label": "Ensure test_results directory",
-      "command": "mkdir -p ${workspaceFolder}/test_results",
+      "command": "mkdir -p test_results",
       "presentation": {
         "echo": false,
         "reveal": "silent",
@@ -15,28 +15,29 @@
     },
     {
       "label": "Pytest",
-      "command": "pytest --junitxml=${workspaceFolder}/test_results/pytest_results.xml",
+      "command": "pytest --junitxml=test_results/pytest_results.xml",
       "dependsOn": ["Ensure test_results directory"],
       "group": "test",
     },
     {
       "label": "Pylint epicli [junit]",
-      "command": "pylint --rcfile .pylintrc ./cli ./tests --output ${workspaceFolder}/test_results/pylint_results.xml",
+      "command": "pylint --rcfile .pylintrc ./cli ./tests/unit --output test_results/pylint_results.xml --recursive y",
       "dependsOn": ["Ensure test_results directory"],
       "group": "test",
     },
     {
       "label": "Pylint epicli [terminal]",
-      "command": "pylint --rcfile .pylintrc ./cli ./tests --output-format text",
+      "command": "pylint --rcfile .pylintrc ./cli ./tests/unit --output-format text --recursive y",
       "group": "test",
     },
     {
-      "label": "Pylint repository modules",
+      "label": "Pylint repository role plugins",
       "command": "pylint",
       "args": [
         "--rcfile", ".pylintrc",
         "--output-format", "text",
-        "./ansible/playbooks/roles/repository/library/tests",
+        "--recursive", "y",
+        "ansible/playbooks/roles/repository/library",
       ],
       "group": "test",
     },
@@ -54,13 +55,13 @@
     },
     {
       "label": "Ansible-lint [junit]",
-      "command": "ansible-lint --nocolor -q -c .ansible-lint -p ${workspaceFolder}/ansible > ${workspaceFolder}/test_results/ansiblelint_results.txt ; ansible-lint-to-junit-xml ${workspaceFolder}/test_results/ansiblelint_results.txt > ${workspaceFolder}/test_results/ansiblelint_results.xml",
+      "command": "ansible-lint --nocolor -q -c .ansible-lint -p ansible > test_results/ansible-lint_results.txt ; ansible-lint-junit test_results/ansible-lint_results.txt -o test_results/ansible-lint_results.xml",
       "dependsOn": ["Ensure test_results directory"],
       "group": "test",
     },
     {
       "label": "Ansible-lint [terminal]",
-      "command": "ansible-lint -q -c .ansible-lint -p ${workspaceFolder}/ansible",
+      "command": "ansible-lint -q -c .ansible-lint -p ansible",
       "group": "test",
     },
     {

diff --git a/ansible/playbooks/roles/applications/tasks/applications/rabbitmq/set-erlang-cookie.yml b/ansible/playbooks/roles/applications/tasks/applications/rabbitmq/set-erlang-cookie.yml
@@ -9,13 +9,12 @@
   when: data.rabbitmq.cluster.cookie is undefined
   block:
     - name: Get existing erlang cookie
-      command: kubectl -n {{ namespace_name }} get secret {{ _cookie_secret_name }} --template={{ _template }}
+      command: kubectl -n {{ namespace_name }} get secret {{ _cookie_secret_name }} --template='{{ _template }}'
       register: get_secret_result
       vars:
         _cookie_secret_name: >-
           {{ rabbitmq_service_name }}-cookie
-        _template: >-
-          {{ '{{ .data.cookie }}' }}
+        _template: '{% raw %}{{ .data.cookie }}{% endraw %}'
 
     - name: Use existing erlang cookie
       set_fact:

diff --git a/ansible/playbooks/roles/keycloak/tasks/create-database.yml b/ansible/playbooks/roles/keycloak/tasks/create-database.yml
@@ -15,7 +15,7 @@
     postgres_primary: "{{ item.item }}"
   loop: "{{ in_recovery_state.results }}"
   loop_control:
-    label: {in_recovery: "{{ item.in_recovery }}"}
+    label: "in_recovery: {{ item.in_recovery }}"
   when: not item.in_recovery
 
 - name: Create DB objects

diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml
@@ -66,8 +66,8 @@
       admin: "{{ certificates.files.admin.cert.subject }}"
       node: "{{ certificates.files.node.cert.subject }}"
     _epiphany_dn_attributes:
-      admin: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.admin.keys()) }}"
-      node: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.node.keys()) }}"
+      admin: "{{ certificates.dn_attributes_order | select('in', _epiphany_subjects.admin.keys()) }}"
+      node: "{{ certificates.dn_attributes_order | select('in', _epiphany_subjects.node.keys()) }}"
     _epiphany_dns:
       admin: >-
         {{ _epiphany_dn_attributes.admin | zip(_epiphany_dn_attributes.admin | map('extract', _epiphany_subjects.admin))

diff --git a/ansible/playbooks/roles/preflight/tasks/common/check-user.yml b/ansible/playbooks/roles/preflight/tasks/common/check-user.yml
@@ -1,14 +1,12 @@
 ---
+  # Ignore warning: "Consider using 'become', 'become_method', and 'become_user' rather than running sudo".
+  # sudo used by purpose, for 'become', 'failed_when' is ignored when sudo requires password and task fails
+  # with short message: "Missing sudo password".
 - name: Check if user has passwordless root privileges through sudo
   command: sudo -n -u root test -w /etc/sudoers
   changed_when: false
   failed_when: test_privileges.rc not in [0, 1]
   register: test_privileges
-  # Disable warning: "Consider using 'become', 'become_method', and 'become_user' rather than running sudo".
-  # sudo used by purpose, for 'become', 'failed_when' is ignored when sudo requires password and task fails
-  # with short message: "Missing sudo password".
-  args:
-    warn: false
 
 - name: Assert root privileges
   assert:

diff --git a/ansible/playbooks/roles/repository/files/download-requirements/conftest.py b/ansible/playbooks/roles/repository/files/download-requirements/conftest.py
diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/__init__.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/__init__.py
diff --git a/ansible/playbooks/roles/repository/tasks/download-requirements.yml b/ansible/playbooks/roles/repository/tasks/download-requirements.yml
@@ -13,6 +13,7 @@
         --manifest "{{ download_requirements_manifest }}" \
         --no-logfile \
         --repos-backup-file /var/tmp/enabled-system-repos.tar \
+        --retries-count 10 \
         --verbose |&
     tee >(systemd-cat --identifier=download-requirements.py)
   args:
@@ -31,6 +32,7 @@
         "{{ download_requirements_os_name }}" \
         --no-logfile \
         --repos-backup-file /var/tmp/enabled-system-repos.tar \
+        --retries-count 10 \
         --verbose |&
     tee >(systemd-cat --identifier=download-requirements.py)
   args:

diff --git a/ansible/playbooks/roles/upgrade/tasks/docker.yml b/ansible/playbooks/roles/upgrade/tasks/docker.yml
@@ -13,8 +13,6 @@
   command: >-
     {{ 'dnf check-update docker-ce' if ansible_os_family == 'RedHat' else
        'apt list --upgradable docker-ce' if ansible_os_family == 'Debian' else None }}
-  args:
-    warn: no
   register: check_docker_update
   failed_when: check_docker_update.rc not in [0, 100] # dnf check-update returns code 100 if there are packages available for update
   changed_when: false

diff --git a/ci/ansible/playbooks/os/ubuntu/upgrade-release.yml b/ci/ansible/playbooks/os/ubuntu/upgrade-release.yml
@@ -44,8 +44,6 @@
           until: result is succeeded
           register: result
           changed_when: false
-          args:
-            warn: false  # do not suggest apt module
 
         # 3) Fully update the system
 

diff --git a/ci/pipelines/linters.yaml b/ci/pipelines/linters.yaml
@@ -13,10 +13,10 @@ pool:
   name: $(agentPoolName)
 
 variables:
-  ansible_lint_error_threshold: 338
-  pylint_score_cli_threshold: 9.41
-  pylint_score_tests_threshold: 9.78
-  rubocop_linter_threshold: 183
+  ansible_lint_error_threshold: 1009
+  pylint_score_cli_threshold: 9.52
+  pylint_score_tests_threshold: 9.85
+  rubocop_linter_threshold: 169
 
 jobs:
   - job: Run_linters
@@ -30,11 +30,11 @@ jobs:
 
       - task: Bash@3
         displayName: Install Ansible Lint and its dependencies
-        # Installing Ansible 5.2.0 to be compatible with the epicli image.
+        # Installing Ansible 9.9.0 to be compatible with the epicli image.
         inputs:
           targetType: inline
           script: |
-            python3 -m pip install --upgrade ansible==6.2.0 ansible-compat==3.* ansible-lint==6.5.0 ansible-lint-junit==0.16 lxml pip setuptools
+            python3 -m pip install --upgrade ansible==9.9.0 ansible-lint==24.7.0 ansible-lint-junit==0.17.8 lxml pip setuptools
 
       - task: Bash@3
         displayName: Run Ansible Lint
@@ -79,6 +79,7 @@ jobs:
             python3 -m pylint ./cli \
             --rcfile .pylintrc \
             --fail-under=$(pylint_score_cli_threshold) \
+            --recursive y \
             --output cli_code_results.xml
 
       - task: PublishTestResults@2
@@ -96,8 +97,8 @@ jobs:
             python3 -m pylint ./tests \
             --rcfile .pylintrc \
             --fail-under=$(pylint_score_tests_threshold) \
-            --output test_code_results.xml \
-            --disable=F0401 # Disable import-error checking
+            --recursive y \
+            --output test_code_results.xml
 
       - task: PublishTestResults@2
         displayName: Publish Pylint test results for test code

diff --git a/cli/licenses.py b/cli/licenses.py
@@ -14,12 +14,20 @@
     },
     {
         "Name": "ansible-core",
-        "Version": "2.17.3",
+        "Version": "2.16.10",
         "Summary": "Radically simple IT automation",
         "Home-page": "https://ansible.com/",
         "Author": "Ansible, Inc.",
         "License": "GPLv3+"
     },
+    {
+        "Name": "ansible",
+        "Version": "9.9.0",
+        "Summary": "Radically simple IT automation",
+        "Home-page": "https://ansible.com/",
+        "Author": "Ansible, Inc.",
+        "License": "GPL-3.0-or-later"
+    },
     {
         "Name": "antlr4-python3-runtime",
         "Version": "4.13.2",
@@ -85,8 +93,7 @@
         "Author": "Microsoft Corporation",
         "License": "MIT License",
         "License URL": "https://api.github.com/repos/azure/azure-cli/license",
-        "License repo": "MIT License\n\nCopyright (c) 2016 Microsoft Corporation\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n",
-        "License text": "MIT License\n\nCopyright (c) [year] [fullname]\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
+        "License repo": "MIT License\n\nCopyright (c) 2016 Microsoft Corporation\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
     },
     {
         "Name": "azure-cli-telemetry",
@@ -968,7 +975,7 @@
     },
     {
         "Name": "boto3",
-        "Version": "1.35.6",
+        "Version": "1.35.12",
         "Summary": "The AWS SDK for Python",
         "Home-page": "https://github.com/boto/boto3",
         "Author": "Amazon Web Services",
@@ -979,7 +986,7 @@
     },
     {
         "Name": "botocore",
-        "Version": "1.35.6",
+        "Version": "1.35.12",
         "Summary": "Low-level, data-driven core of boto 3.",
         "Home-page": "https://github.com/boto/botocore",
         "Author": "Amazon Web Services",
@@ -990,7 +997,7 @@
     },
     {
         "Name": "certifi",
-        "Version": "2024.7.4",
+        "Version": "2024.8.30",
         "Summary": "Python package for providing Mozilla's CA Bundle.",
         "Home-page": "https://github.com/certifi/python-certifi",
         "Author": "Kenneth Reitz",
@@ -1000,7 +1007,7 @@
     },
     {
         "Name": "cffi",
-        "Version": "1.17.0",
+        "Version": "1.17.1",
         "Summary": "Foreign Function Interface for Python calling C code.",
         "Home-page": "http://cffi.readthedocs.org",
         "Author": "Armin Rigo, Maciej Fijalkowski",
@@ -1046,7 +1053,7 @@
     },
     {
         "Name": "cryptography",
-        "Version": "43.0.0",
+        "Version": "43.0.1",
         "Summary": "cryptography is a package which provides cryptographic recipes and primitives to Python developers.",
         "Home-page": "",
         "Author": "The cryptography developers <[email protected]>",
@@ -1465,7 +1472,7 @@
     },
     {
         "Name": "resolvelib",
-        "Version": "0.8.1",
+        "Version": "1.0.1",
         "Summary": "Resolve abstract dependencies into concrete ones",
         "Home-page": "https://github.com/sarugaku/resolvelib",
         "Author": "Tzu-ping Chung",

diff --git a/cli/src/ansible/AnsibleConfigFileCreator.py b/cli/src/ansible/AnsibleConfigFileCreator.py
@@ -53,6 +53,7 @@ def process_ansible_options(self):
         self.add_setting('defaults', 'interpreter_python', 'auto')
         if not Config().no_color:
             self.add_setting('defaults', 'force_color', 'true')
+        self.add_setting('defaults', 'timeout', 30)
 
     def create(self):
         self.logger.info('Creating ansible.cfg')