Update develop with the latest changes from v2.0 branch (#3475)

* Switch from 'k8s.gcr.io' to 'registry.k8s.io' * Update Ansible to 9.9.0 (#3469) * Update AlmaLinux image to 8.7.20221110 * Increase timeout to 30 * Increase `retries-count` from 3 to 10 for `download-requirements.py` * Fix `epicli - list index out of range` * Fix pytest tests in VS Code --------- Co-authored-by: przemyslavic <[email protected]> * Prepare v2.0.17 release (#3472) * Add missing dependency for postgresql-client-common package * Update AlmaLinux AWS OS image * Update version and changelog * Update pylint score threshold --------- Co-authored-by: Tomasz Baran <[email protected]>
hitachienergy · Nov 18, 2024 · d736c2a · d736c2a
1 parent fd8ed96
commit d736c2a
Show file tree

Hide file tree

Showing 53 changed files with 396 additions and 350 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -64,7 +64,7 @@ RUN : INSTALL PIP REQUIREMENTS \
  && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
     --requirement /requirements.txt \
  && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
-    poetry pylint==2.* pylint_junit ansible-lint==6.5.0 ansible-lint-to-junit-xml yamllint pytest pytest_mock setuptools twine wheel
+    poetry pylint==2.* pylint_junit ansible-lint==24.7.0 ansible-lint-junit==0.17.8 yamllint pytest pytest_mock setuptools twine wheel
 
 RUN : SETUP USER AND OTHERS \
  && groupadd --gid $USER_GID $USERNAME \

diff --git a/.devcontainer/gen-dependency-info.py b/.devcontainer/gen-dependency-info.py
@@ -91,7 +91,7 @@ def _main() -> None:
     # Write licenses 'cli/licenses.py'
     licenses_content = """\
     # This is a generated file so don`t change this manually.
-    # To re-generate run 'python gen-dependency-info.py <GitHub_PAT>' from the project root.
+    # To re-generate run 'python3 gen-dependency-info.py <GitHub_PAT>' from the project root.
 
     LICENSES = """
 

diff --git a/.devcontainer/poetry.lock b/.devcontainer/poetry.lock
diff --git a/.devcontainer/pyproject.toml b/.devcontainer/pyproject.toml
@@ -13,7 +13,7 @@ jsonschema = "*"
 python-json-logger = "*"
 "ruamel.yaml" = "*"
 click = "*"
-ansible = "6.7.0"
+ansible = "~9"
 azure-cli = "2.63.0"
 
 [build-system]

diff --git a/.devcontainer/requirements.md b/.devcontainer/requirements.md
@@ -46,7 +46,7 @@ The main reasons for this 2-way approach:
 2. Run the following to update the `license.py` and generate the `DEPENDENCIES.md` file:
 
     ```shell
-    python gen-dependency-info.py YOUR-GITHUB-PAT
+    python3 gen-dependency-info.py YOUR-GITHUB-PAT
     ```
 
     For obtaining a GitHub Personal Access Token, check [here](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).

diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt
@@ -1,6 +1,6 @@
 adal==1.2.7 ; python_full_version == "3.10.14"
-ansible-core==2.13.13 ; python_full_version == "3.10.14"
-ansible==6.7.0 ; python_full_version == "3.10.14"
+ansible-core==2.16.10 ; python_full_version == "3.10.14"
+ansible==9.9.0 ; python_full_version == "3.10.14"
 antlr4-python3-runtime==4.13.2 ; python_full_version == "3.10.14"
 applicationinsights==0.11.10 ; python_full_version == "3.10.14"
 argcomplete==3.3.0 ; python_full_version == "3.10.14"
@@ -90,15 +90,15 @@ azure-synapse-artifacts==0.19.0 ; python_full_version == "3.10.14"
 azure-synapse-managedprivateendpoints==0.4.0 ; python_full_version == "3.10.14"
 azure-synapse-spark==0.2.0 ; python_full_version == "3.10.14"
 bcrypt==4.2.0 ; python_full_version == "3.10.14"
-boto3==1.35.6 ; python_full_version == "3.10.14"
-botocore==1.35.6 ; python_full_version == "3.10.14"
-certifi==2024.7.4 ; python_full_version == "3.10.14"
-cffi==1.17.0 ; python_full_version == "3.10.14"
+boto3==1.35.12 ; python_full_version == "3.10.14"
+botocore==1.35.12 ; python_full_version == "3.10.14"
+certifi==2024.8.30 ; python_full_version == "3.10.14"
+cffi==1.17.1 ; python_full_version == "3.10.14"
 chardet==5.2.0 ; python_full_version == "3.10.14"
 charset-normalizer==3.3.2 ; python_full_version == "3.10.14"
 click==8.1.7 ; python_full_version == "3.10.14"
 colorama==0.4.6 ; python_full_version == "3.10.14"
-cryptography==43.0.0 ; python_full_version == "3.10.14"
+cryptography==43.0.1 ; python_full_version == "3.10.14"
 decorator==5.1.1 ; python_full_version == "3.10.14"
 deprecated==1.2.14 ; python_full_version == "3.10.14"
 distro==1.9.0 ; sys_platform == "linux" and python_full_version == "3.10.14"
@@ -145,7 +145,7 @@ referencing==0.35.1 ; python_full_version == "3.10.14"
 requests-oauthlib==2.0.0 ; python_full_version == "3.10.14"
 requests==2.32.3 ; python_full_version == "3.10.14"
 requests[socks]==2.32.3 ; python_full_version == "3.10.14"
-resolvelib==0.8.1 ; python_full_version == "3.10.14"
+resolvelib==1.0.1 ; python_full_version == "3.10.14"
 rpds-py==0.20.0 ; python_full_version == "3.10.14"
 ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_full_version == "3.10.14"
 ruamel-yaml==0.18.6 ; python_full_version == "3.10.14"

diff --git a/.gitignore b/.gitignore
@@ -127,3 +127,4 @@ clusters
 #.vscode
 #.devcontainer
 *.dump
+sandbox/
diff --git a/.pylintrc b/.pylintrc
@@ -349,8 +349,8 @@ int-import-graph=
 [EXCEPTIONS]
 
 # Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
+# ["builtins.BaseException", "builtins.Exception"]
+overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"]
 
 
 [TYPING]

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,12 +1,13 @@
 {
+    // Ansible
+    "ansible.python.interpreterPath": "/usr/local/bin/python3",
     // Common
     "trailing-spaces.highlightCurrentLine": false, // highlighting currently edited line can be annoying
     "trailing-spaces.syntaxIgnore": [
         "markdown" // "two spaces" syntax is for line breaks
     ],
 
     // Python
-    "pylint.enabled": true,
     "pylint.args": [
         "--rcfile=${workspaceFolder}/.pylintrc"
     ],

diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -6,7 +6,7 @@
   "tasks": [
     {
       "label": "Ensure test_results directory",
-      "command": "mkdir -p ${workspaceFolder}/test_results",
+      "command": "mkdir -p test_results",
       "presentation": {
         "echo": false,
         "reveal": "silent",
@@ -15,28 +15,29 @@
     },
     {
       "label": "Pytest",
-      "command": "pytest --junitxml=${workspaceFolder}/test_results/pytest_results.xml",
+      "command": "pytest --junitxml=test_results/pytest_results.xml",
       "dependsOn": ["Ensure test_results directory"],
       "group": "test",
     },
     {
       "label": "Pylint epicli [junit]",
-      "command": "pylint --rcfile .pylintrc ./cli ./tests --output ${workspaceFolder}/test_results/pylint_results.xml",
+      "command": "pylint --rcfile .pylintrc ./cli ./tests/unit --output test_results/pylint_results.xml --recursive y",
       "dependsOn": ["Ensure test_results directory"],
       "group": "test",
     },
     {
       "label": "Pylint epicli [terminal]",
-      "command": "pylint --rcfile .pylintrc ./cli ./tests --output-format text",
+      "command": "pylint --rcfile .pylintrc ./cli ./tests/unit --output-format text --recursive y",
       "group": "test",
     },
     {
-      "label": "Pylint repository modules",
+      "label": "Pylint repository role plugins",
       "command": "pylint",
       "args": [
         "--rcfile", ".pylintrc",
         "--output-format", "text",
-        "./ansible/playbooks/roles/repository/library/tests",
+        "--recursive", "y",
+        "ansible/playbooks/roles/repository/library",
       ],
       "group": "test",
     },
@@ -54,13 +55,13 @@
     },
     {
       "label": "Ansible-lint [junit]",
-      "command": "ansible-lint --nocolor -q -c .ansible-lint -p ${workspaceFolder}/ansible > ${workspaceFolder}/test_results/ansiblelint_results.txt ; ansible-lint-to-junit-xml ${workspaceFolder}/test_results/ansiblelint_results.txt > ${workspaceFolder}/test_results/ansiblelint_results.xml",
+      "command": "ansible-lint --nocolor -q -c .ansible-lint -p ansible > test_results/ansible-lint_results.txt ; ansible-lint-junit test_results/ansible-lint_results.txt -o test_results/ansible-lint_results.xml",
       "dependsOn": ["Ensure test_results directory"],
       "group": "test",
     },
     {
       "label": "Ansible-lint [terminal]",
-      "command": "ansible-lint -q -c .ansible-lint -p ${workspaceFolder}/ansible",
+      "command": "ansible-lint -q -c .ansible-lint -p ansible",
       "group": "test",
     },
     {

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.0.16dev
+2.0.18dev
diff --git a/ansible/playbooks/roles/applications/tasks/applications/rabbitmq/set-erlang-cookie.yml b/ansible/playbooks/roles/applications/tasks/applications/rabbitmq/set-erlang-cookie.yml
@@ -9,13 +9,12 @@
   when: data.rabbitmq.cluster.cookie is undefined
   block:
     - name: Get existing erlang cookie
-      command: kubectl -n {{ namespace_name }} get secret {{ _cookie_secret_name }} --template={{ _template }}
+      command: kubectl -n {{ namespace_name }} get secret {{ _cookie_secret_name }} --template='{{ _template }}'
       register: get_secret_result
       vars:
         _cookie_secret_name: >-
           {{ rabbitmq_service_name }}-cookie
-        _template: >-
-          {{ '{{ .data.cookie }}' }}
+        _template: '{% raw %}{{ .data.cookie }}{% endraw %}'
 
     - name: Use existing erlang cookie
       set_fact:

diff --git a/ansible/playbooks/roles/containerd/templates/config.toml.j2 b/ansible/playbooks/roles/containerd/templates/config.toml.j2
@@ -55,7 +55,7 @@ oom_score = 0
     stream_idle_timeout = "4h0m0s"
     enable_selinux = false
     selinux_category_range = 1024
-    sandbox_image = "{{ image_registry_address }}/k8s.gcr.io/pause:3.5"
+    sandbox_image = "{{ image_registry_address }}/registry.k8s.io/pause:3.5"
     stats_collect_period = 10
     systemd_cgroup = false
     enable_tls_streaming = false

diff --git a/ansible/playbooks/roles/keycloak/tasks/create-database.yml b/ansible/playbooks/roles/keycloak/tasks/create-database.yml
@@ -15,7 +15,7 @@
     postgres_primary: "{{ item.item }}"
   loop: "{{ in_recovery_state.results }}"
   loop_control:
-    label: {in_recovery: "{{ item.in_recovery }}"}
+    label: "in_recovery: {{ item.in_recovery }}"
   when: not item.in_recovery
 
 - name: Create DB objects

diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml
@@ -66,8 +66,8 @@
       admin: "{{ certificates.files.admin.cert.subject }}"
       node: "{{ certificates.files.node.cert.subject }}"
     _epiphany_dn_attributes:
-      admin: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.admin.keys()) }}"
-      node: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.node.keys()) }}"
+      admin: "{{ certificates.dn_attributes_order | select('in', _epiphany_subjects.admin.keys()) }}"
+      node: "{{ certificates.dn_attributes_order | select('in', _epiphany_subjects.node.keys()) }}"
     _epiphany_dns:
       admin: >-
         {{ _epiphany_dn_attributes.admin | zip(_epiphany_dn_attributes.admin | map('extract', _epiphany_subjects.admin))

diff --git a/ansible/playbooks/roles/preflight/tasks/common/check-user.yml b/ansible/playbooks/roles/preflight/tasks/common/check-user.yml
@@ -1,14 +1,12 @@
 ---
+  # Ignore warning: "Consider using 'become', 'become_method', and 'become_user' rather than running sudo".
+  # sudo used by purpose, for 'become', 'failed_when' is ignored when sudo requires password and task fails
+  # with short message: "Missing sudo password".
 - name: Check if user has passwordless root privileges through sudo
   command: sudo -n -u root test -w /etc/sudoers
   changed_when: false
   failed_when: test_privileges.rc not in [0, 1]
   register: test_privileges
-  # Disable warning: "Consider using 'become', 'become_method', and 'become_user' rather than running sudo".
-  # sudo used by purpose, for 'become', 'failed_when' is ignored when sudo requires password and task fails
-  # with short message: "Missing sudo password".
-  args:
-    warn: false
 
 - name: Assert root privileges
   assert:

diff --git a/ansible/playbooks/roles/repository/files/download-requirements/conftest.py b/ansible/playbooks/roles/repository/files/download-requirements/conftest.py
diff --git a/...le/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml b/...le/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml
@@ -33,26 +33,26 @@ images:
       sha1: 996f919bfc7db09e30f759d5987395bce287e63d
     # K8s
     # v1.22.4
-    'k8s.gcr.io/kube-apiserver:v1.22.4':
-      sha1: eea0329e33a54d62b0b1cfd6e54169a0f70201a9
+    'registry.k8s.io/kube-apiserver:v1.22.4':
+      sha1: 2fad255ce92b7ee12f206be9d9d098194a8bc151
 
-    'k8s.gcr.io/kube-controller-manager:v1.22.4':
-      sha1: 8fb47001234a741bcd9b1f76d863b8a818728145
+    'registry.k8s.io/kube-controller-manager:v1.22.4':
+      sha1: 51f7911deeadd69d90486a52a4f28e46acf47ef8
 
-    'k8s.gcr.io/kube-scheduler:v1.22.4':
-      sha1: 687d3836b2dfc047a3f146254efa0b1f6c692f69
+    'registry.k8s.io/kube-scheduler:v1.22.4':
+      sha1: eb715ec0defec88b946b4bb37a41b012968b0ac8
 
-    'k8s.gcr.io/kube-proxy:v1.22.4':
-      sha1: b3ce65eedfbf046f619d6f65f6d4cfe442f181a0
+    'registry.k8s.io/kube-proxy:v1.22.4':
+      sha1: 182c5c09e343f3d4e2072cd9e1f9bd93ca254892
 
-    'k8s.gcr.io/coredns/coredns:v1.8.4':
-      sha1: e89c23e998fa93d17a09d1b7cef6aae5cb24b049
+    'registry.k8s.io/coredns/coredns:v1.8.4':
+      sha1: a1cce4643a1e5e8455c96e3adaee141cd8732768
 
-    'k8s.gcr.io/etcd:3.5.0-0':
-      sha1: 3aa7fb15e643e0d199e7b7f7f165533bc4a67f39
+    'registry.k8s.io/etcd:3.5.0-0':
+      sha1: b20cc5eb05db5efaa0dbd17a4ce952bf1032958a
 
-    'k8s.gcr.io/pause:3.5':
-      sha1: 8be7eaa14c9425a6286635d0440f6611ca115d67
+    'registry.k8s.io/pause:3.5':
+      sha1: 95ce209a3229d539db6ec24722e0a86f67e8a157
 
     'quay.io/coreos/flannel:v0.14.0':
       sha1: 13b299bcaa8e0c78d9fe1f5cd9b2155651976d92

diff --git a/...pository/files/download-requirements/requirements/x86_64/debian/ubuntu-20.04/packages.yml b/...pository/files/download-requirements/requirements/x86_64/debian/ubuntu-20.04/packages.yml
@@ -92,6 +92,7 @@ packages:
 
     # postgres related packages
     # if version is not specified, it's not related to postgres version and the latest is used
+    - 'libipc-run-perl' # for postgresql-client-common
     - 'pgdg-keyring'
     - 'postgresql-13-pgaudit=1.5.0*'
     - 'postgresql-10-repmgr=5.2.1*'

diff --git a/...ble/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml b/...ble/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml
@@ -40,26 +40,26 @@ images:
 
     # K8s
     # v1.22.4
-    'k8s.gcr.io/kube-apiserver:v1.22.4':
-      sha1: 3c83ee79ae018a583f2d4642b8829570fd4f1d9f
+    'registry.k8s.io/kube-apiserver:v1.22.4':
+      sha1: 753ff29805d9ac873f409f0cd8dec0818dba4b2d
 
-    'k8s.gcr.io/kube-controller-manager:v1.22.4':
-      sha1: 4e17c60205a0ea8304dbb57aadf2befd31173fc0
+    'registry.k8s.io/kube-controller-manager:v1.22.4':
+      sha1: 3e6936ed6984505bcd194da11b8fcd3e77b52ac4
 
-    'k8s.gcr.io/kube-scheduler:v1.22.4':
-      sha1: b157699b85ac5f9c5175e8c469d1ba1ecdd3c4ff
+    'registry.k8s.io/kube-scheduler:v1.22.4':
+      sha1: d1fa695dbae08408cf22c8e14b6a70058d5873e1
 
-    'k8s.gcr.io/kube-proxy:v1.22.4':
-      sha1: 6bab7b8f18941f120c14068d80814621b64f098e
+    'registry.k8s.io/kube-proxy:v1.22.4':
+      sha1: b0e3b50e84060be942e48ea67a5c4a95b7355973
 
-    'k8s.gcr.io/coredns/coredns:v1.8.4':
-      sha1: 282c7b3836bc5d3673027871ffe2140e980df097
+    'registry.k8s.io/coredns/coredns:v1.8.4':
+      sha1: 447ce8fb5c1ef69298658d976e8aa3a55da28675
 
-    'k8s.gcr.io/etcd:3.5.0-0':
-      sha1: c6354fbd7424494ceb6deb24128574e59e258fa6
+    'registry.k8s.io/etcd:3.5.0-0':
+      sha1: 78f42080da842ff9454ba9420ad48d90e84de045
 
-    'k8s.gcr.io/pause:3.5':
-      sha1: d0359d4eec7d50783612e9cd59a54a5e517c774e
+    'registry.k8s.io/pause:3.5':
+      sha1: 98f71abc4bdab5a3d5eb779b18483caed2544c5d
 
     'quay.io/coreos/flannel:v0.14.0':
       sha1: 092a48b4eebb7a2328124a89fb7f2b9b4bcb3991
@@ -77,20 +77,20 @@ images:
       sha1: c096f70cf3b486b3f0d084c85d33deab6b9423ef
 
   rook:
-    'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0':
-      sha1: 8567876a11c527e9d406d3f3efa09e3cee437985
+    'registry.k8s.io/sig-storage/csi-attacher:v3.4.0':
+      sha1: a65c9b964b98b0a908a577350a03a5166ab467b3
 
-    'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0':
-      sha1: 07104d23bbb224f81ef6b8fd379b01a7cbba0946
+    'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.0':
+      sha1: 49304df65bfbb463bb4be7db688a95c2b1c1a1a5
 
-    'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0':
-      sha1: 55c9d55eb0f2cbe9e37e5464e578b2d7fa45f8f8
+    'registry.k8s.io/sig-storage/csi-provisioner:v3.1.0':
+      sha1: 0d909f211b06853aaaea6e2533f91e7b7ecfd210
 
-    'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0':
-      sha1: 289bafda08f126423dcf7ad78f08a5d8ed57a82e
+    'registry.k8s.io/sig-storage/csi-resizer:v1.4.0':
+      sha1: 8acc84db171b9b8efc17290e17ea7e12e9a09219
 
-    'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1':
-      sha1: 8deb564fde820c7f795d8b425f867d4194edd088
+    'registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1':
+      sha1: d5d1b7d2b3eee884dda7d5cd9a6b553d4d997d50
 
     'quay.io/ceph/ceph:v16.2.7-20220510':
       sha1: 7ffb69fe844f59486704ea80d6802d7d3416b455

diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/__init__.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/__init__.py