[v2.0] Update v2.0 branch with the latest changes from develop

.devcontainer/Dockerfile

@@ -7,6 +7,7 @@ ARG USER_GID=$USER_UID
 ARG AWS_CLI_VERSION=2.0.30
 ARG HELM_VERSION=3.3.1
 ARG KUBECTL_VERSION=1.22.4
+ARG KUBELOGIN_VERSION=0.0.33
 ARG TERRAFORM_VERSION=1.1.3
 
 RUN : INSTALL APT REQUIREMENTS \
@@ -35,6 +36,11 @@ RUN : INSTALL HELM BINARY \
  && chmod +x ./kubectl \
  && mv ./kubectl /usr/local/bin/kubectl \
  && kubectl version --client \
+ && : INSTALL KUBELOGIN BINARY \
+ && curl -fsSLO https://github.com/Azure/kubelogin/releases/download/v${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip \
+ && unzip -j kubelogin-linux-amd64.zip -d /usr/local/bin \
+ && rm kubelogin-linux-amd64.zip \
+ && kubelogin --version \
  && : INSTALL TERRAFORM BINARY \
  && curl -fsSLO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
  && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \
@@ -48,6 +54,7 @@ RUN : INSTALL HELM BINARY \
  && aws --version
 
 RUN : INSTALL GEM REQUIREMENTS \
+ && gem install net-ssh -v 6.1.0 \
  && gem install \
     bcrypt_pbkdf ed25519 rake rspec_junit_formatter rubocop rubocop-junit_formatter serverspec solargraph
 

Dockerfile

@@ -7,6 +7,7 @@ ARG USER_GID=$USER_UID
 ARG AWS_CLI_VERSION=2.0.30
 ARG HELM_VERSION=3.3.1
 ARG KUBECTL_VERSION=1.22.4
+ARG KUBELOGIN_VERSION=0.0.33
 ARG TERRAFORM_VERSION=1.1.3
 
 ENV EPICLI_DOCKER_SHARED_DIR=/shared
@@ -28,6 +29,11 @@ RUN : INSTALL APT REQUIREMENTS \
     && chmod +x ./kubectl \
     && mv ./kubectl /usr/local/bin/kubectl \
     && kubectl version --client \
+    && : INSTALL KUBELOGIN BINARY \
+    && curl -fsSLO https://github.com/Azure/kubelogin/releases/download/v${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip \
+    && unzip -j kubelogin-linux-amd64.zip -d /usr/local/bin \
+    && rm kubelogin-linux-amd64.zip \
+    && kubelogin --version \
     && : INSTALL TERRAFORM BINARY \
     && curl -fsSLO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
     && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \
@@ -42,6 +48,7 @@ RUN : INSTALL APT REQUIREMENTS \
     && aws --version \
 \
     && : INSTALL GEM REQUIREMENTS \
+    && gem install net-ssh -v 6.1.0 \
     && gem install \
         bcrypt_pbkdf ed25519 rake rspec_junit_formatter serverspec \
 \

ansible/playbooks/roles/applications/templates/rabbitmq/03-rabbitmq.yml.j2

@@ -47,7 +47,8 @@ data:
       ## Set to "hostname" to use pod hostnames.
       ## When this value is changed, so should the variable used to set the RABBITMQ_NODENAME
       ## environment variable.
-      cluster_formation.k8s.address_type = ip
+      cluster_formation.k8s.address_type = hostname
+      cluster_formation.k8s.hostname_suffix = .{{ rabbitmq_service_name }}.{{ namespace_name }}.svc.cluster.local
       ## How often should node cleanup checks run?
       cluster_formation.node_cleanup.interval = 30
       ## Set to false if automatic removal of unknown/absent nodes
@@ -120,9 +121,6 @@ spec:
                 fieldPath: status.podIP
           - name: RABBITMQ_USE_LONGNAME
             value: "true"
-          # See a note on cluster_formation.k8s.address_type in the config file section
-          - name: RABBITMQ_NODENAME
-            value: "rabbit@$(MY_POD_IP)"
           - name: K8S_SERVICE_NAME
             value: "{{ rabbitmq_service_name }}"
           - name: RABBITMQ_ERLANG_COOKIE

ansible/playbooks/roles/common/tasks/main.yml

@@ -91,6 +91,30 @@
     owner: root
     group: root
 
+- name: Customize CoreDNS for AKS
+  when: k8s_as_cloud_service
+  run_once: true
+  delegate_to: localhost
+  become: false
+  environment:
+    KUBECONFIG: "{{ kubeconfig.local }}"
+  block:
+
+  - name: Render Epiphany hosts template
+    template:
+      src: coredns-epi.yml.j2
+      dest: "/tmp/coredns-epi.yml"
+      mode: u=rw,go=r
+
+  - name: Apply custom CoreDNS configmap
+    become: false
+    command: |
+      kubectl apply -f /tmp/coredns-epi.yml
+
+  - name: Restart CoreDNS
+    command: |
+      kubectl -n kube-system rollout restart deployment coredns
+
 - include_tasks: epiuser.yml
   tags:
     - epiuser

ansible/playbooks/roles/common/templates/coredns-epi.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom # this is the name of the configmap you can overwrite with your changes
+  namespace: kube-system
+data:
+    epi.override: | # you may select any name here, but it must end with the .override file extension
+          hosts {
+              {% for host in play_hosts %}
+              {% set name = host.split('.') %}
+              {{ hostvars[host]['ansible_default_ipv4']['address'] }} {{ host }} {{ name[0] }}
+              {% endfor %}
+              fallthrough
+          }

ansible/playbooks/roles/elasticsearch_curator/tasks/install-es-curator-Debian.yml

@@ -5,3 +5,7 @@
   apt:
     name: "elasticsearch-curator={{ curator_version }}"
     state: present
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"

ansible/playbooks/roles/firewall/tasks/Debian/install-firewall.yml

@@ -14,6 +14,10 @@
     name: ufw
     state: absent
     purge: true
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"
 
 - name: Install firewalld package
   block:
@@ -22,6 +26,9 @@
         name: firewalld
         state: present
       register: install_firewalld
+      until: install_firewalld is succeeded
+      retries: "{{ retries }}"
+      delay: "{{ delay }}"
 
     - name: Stop and mask service for consistency with RHEL
       block:

ansible/playbooks/roles/grafana/tasks/install-packages-Debian.yml

@@ -3,3 +3,7 @@
   apt:
     name: "{{ _package_name }}"
     state: present
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"

ansible/playbooks/roles/haproxy_runc/tasks/install-packages-Debian.yml

@@ -6,3 +6,7 @@
       - containerd.io={{ containerd_defaults.containerd_version }}-*  # provides "runc"
     state: present
     allow_downgrade: true
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"

ansible/playbooks/roles/keycloak/templates/dynamic-chart-values.yml.j2

@@ -7,7 +7,7 @@ Keep data structure in sync with Helm chart's values.
 ref: https://github.com/codecentric/helm-charts/blob/keycloakx-1.6.1/charts/keycloakx/values.yaml
 -#}
 
-{% if specification.image_registry.use_local %}
+{% if specification.image_registry.use_local and not k8s_as_cloud_service %}
 image:
   repository: {{ image_registry_address }}/{{ _chart_values.image.repository }}
 dbchecker:

ansible/playbooks/roles/postgres_exporter/tasks/main.yml

@@ -8,6 +8,10 @@
     state: present
     install_recommends: no
   when: ansible_os_family == 'Debian'
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"
 
 - name: Create postgres_exporter system group
   group:

ansible/playbooks/roles/postgresql/templates/postgresql.conf.j2

@@ -772,6 +772,9 @@ default_text_search_config = 'pg_catalog.english'
 #include_if_exists = '...'		# include file only if it exists
 #include = '...'			# include file
 include = 'postgresql-epiphany.conf'	# Epiphany managed configuration that overrides settings above
+{% if specification.custom_postgresql_config %}
+include_if_exists = '{{ specification.custom_postgresql_config }}'
+{% endif %}
 
 
 #------------------------------------------------------------------------------

ansible/playbooks/roles/preflight/tasks/common/check-routing.yml

@@ -37,7 +37,7 @@
 - name: Validate if ansible_default_ipv4.address matches address from inventory
   when:
     - common_vars.provider == "any"
-    - common_vars.specification.cloud is undefined
+    - not k8s_as_cloud_service
   assert:
     that: ansible_default_ipv4.address == ansible_host
     fail_msg: >-

ansible/playbooks/roles/prometheus/files/secret.yml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: prometheus
+  namespace: kube-system
+  annotations:
+    kubernetes.io/service-account.name: prometheus
+type: kubernetes.io/service-account-token

ansible/playbooks/roles/prometheus/tasks/configure-k8s-apps-monitoring.yml

@@ -30,6 +30,14 @@
   run_once: true
   delegate_to: localhost
 
+# Starting from K8s v1.24, secrets are not automatically generated when service accounts are created
+- name: Create secret
+  when: k8s_as_cloud_service is defined and k8s_as_cloud_service
+  become: false
+  command: "kubectl apply -f {{ role_path }}/files/secret.yml"
+  run_once: true
+  delegate_to: localhost
+
 - name: Get kubernetes bearer token for prometheus
   become: false
   shell: |-

ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2

@@ -104,7 +104,51 @@ scrape_configs:
 
 # Scrape config for nodes (kubelet).
 
-  - job_name: 'kubernetes-pods'
+  - job_name: 'kubernetes-pods-http'
+    kubernetes_sd_configs:
+    - role: pod
+      api_server: "{{ api_server_address }}"
+      tls_config:
+        insecure_skip_verify: true
+      bearer_token: "{{ bearer_token }}"
+    tls_config:
+      insecure_skip_verify: true
+    bearer_token: "{{ bearer_token }}"
+    scheme: http
+    relabel_configs:
+      - action: keep
+        regex: true
+        source_labels:
+        - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+      - action: replace
+        regex: (.+)
+        source_labels:
+        - __meta_kubernetes_pod_annotation_prometheus_io_path
+        target_label: __metrics_path__
+      - action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        source_labels:
+        - __address__
+        - __meta_kubernetes_pod_annotation_prometheus_io_port
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_namespace
+        target_label: kubernetes_namespace
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_name
+        target_label: kubernetes_pod_name
+      - action: drop
+        source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+        regex: https
+
+# To scrape metrics over HTTPS, you need to annotate application pods that expose an HTTPS endpoint
+# with the following key/value pair: prometheus.io/scheme="https".
+  - job_name: 'kubernetes-pods-https'
     kubernetes_sd_configs:
     - role: pod
       api_server: "{{ api_server_address }}"
@@ -142,6 +186,9 @@ scrape_configs:
         source_labels:
         - __meta_kubernetes_pod_name
         target_label: kubernetes_pod_name
+      - action: keep
+        source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+        regex: https
 
 # Scrape config for Kubelet cAdvisor.
 

ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml

@@ -24,3 +24,7 @@
       - rabbitmq-server={{ versions.debian.rabbitmq }}
     update_cache: true
     state: present
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"

ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/redhat/packages.yml

@@ -43,6 +43,8 @@ packages:
     - 'fping'
     - 'fuse-overlayfs'  # for docker-ce-rootless-extras
     - 'fuse3'  # for docker-ce-rootless-extras
+    - 'glibc-common'  # for samba packages
+    - 'glibc-langpack-en'  # for samba packages
     - 'gnutls'  # for cifs-utils
     - 'gssproxy'  # for nfs-utils
     - 'htop'

ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml

@@ -45,6 +45,8 @@ packages:
     - 'fping'
     - 'fuse-overlayfs'  # for docker-ce-rootless-extras
     - 'fuse3'  # for docker-ce-rootless-extras
+    - 'glibc-common'  # for samba packages
+    - 'glibc-langpack-en'  # for samba packages
     - 'gnutls'  # for cifs-utils
     - 'gssproxy'  # for nfs-utils
     - 'htop'

ansible/playbooks/roles/repository/tasks/Debian/setup.yml

@@ -7,6 +7,10 @@
           - apache2
           - libdpkg-perl  # required by dpkg-scanpackages script
         cache_valid_time: 3600  # 1 h
+      register: result
+      until: result is succeeded
+      retries: "{{ retries }}"
+      delay: "{{ delay }}"
       when: not offline_mode
 
     - name: Create epirepo repository

ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-kubeadm.yml

@@ -36,6 +36,10 @@
       vars:
         packages_to_install_with_version: >-
           {{ packages_to_install.items() | map('join', '=') | list }}
+      register: result
+      until: result is succeeded
+      retries: "{{ retries }}"
+      delay: "{{ delay }}"
 
     - name: K8s/install | Include hold packages task
       include_tasks: hold-packages.yml

ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages.yml

@@ -7,6 +7,10 @@
 - name: Install K8s packages
   when: packages_to_install_or_upgrade | length
      or packages_to_downgrade | length
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"
   vars:
     target_packages:
       kubeadm: "{{ version }}-00" # kubeadm may be removed as dependency and then has to be reinstalled
@@ -52,6 +56,10 @@
       apt:
         name: "{{ packages_to_downgrade }}"
         state: absent
+      register: result
+      until: result is succeeded
+      retries: "{{ retries }}"
+      delay: "{{ delay }}"
       when: packages_to_downgrade | length
 
     - name: "K8s/install | Install packages: {{ packages_to_install_with_version | join( ', ' ) }}"

ansible/playbooks/roles/upgrade/tasks/legacy-haproxy/Debian/uninstall-haproxy.yml

@@ -4,3 +4,7 @@
     name: haproxy
     update_cache: true
     state: absent
+  register: result
+  until: result is succeeded
+  retries: "{{ retries }}"
+  delay: "{{ delay }}"

ci/ansible/playbooks/os/ubuntu/upgrade-release.yml

@@ -58,6 +58,10 @@
         - name: Install all updates
           apt:
             upgrade: safe  # equivalent of 'apt upgrade'
+          register: result
+          until: result is succeeded
+          retries: "{{ retries }}"
+          delay: "{{ delay }}"
 
         # 4) Reboot the system (to use the latest kernel)
 
@@ -85,6 +89,10 @@
           apt:
             autoremove: true
             purge: true
+          register: result
+          until: result is succeeded
+          retries: "{{ retries }}"
+          delay: "{{ delay }}"
 
         # 6) Upgrade OS release