[develop] AKS and Epiphany integration (#3409)

.devcontainer/Dockerfile

@@ -7,6 +7,7 @@ ARG USER_GID=$USER_UID
 ARG AWS_CLI_VERSION=2.0.30
 ARG HELM_VERSION=3.3.1
 ARG KUBECTL_VERSION=1.22.4
+ARG KUBELOGIN_VERSION=0.0.33
 ARG TERRAFORM_VERSION=1.1.3
 
 RUN : INSTALL APT REQUIREMENTS \
@@ -35,6 +36,11 @@ RUN : INSTALL HELM BINARY \
  && chmod +x ./kubectl \
  && mv ./kubectl /usr/local/bin/kubectl \
  && kubectl version --client \
+ && : INSTALL KUBELOGIN BINARY \
+ && curl -fsSLO https://github.com/Azure/kubelogin/releases/download/v${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip \
+ && unzip -j kubelogin-linux-amd64.zip -d /usr/local/bin \
+ && rm kubelogin-linux-amd64.zip \
+ && kubelogin --version \
  && : INSTALL TERRAFORM BINARY \
  && curl -fsSLO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
  && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \

Dockerfile

@@ -7,6 +7,7 @@ ARG USER_GID=$USER_UID
 ARG AWS_CLI_VERSION=2.0.30
 ARG HELM_VERSION=3.3.1
 ARG KUBECTL_VERSION=1.22.4
+ARG KUBELOGIN_VERSION=0.0.33
 ARG TERRAFORM_VERSION=1.1.3
 
 ENV EPICLI_DOCKER_SHARED_DIR=/shared
@@ -28,6 +29,11 @@ RUN : INSTALL APT REQUIREMENTS \
     && chmod +x ./kubectl \
     && mv ./kubectl /usr/local/bin/kubectl \
     && kubectl version --client \
+    && : INSTALL KUBELOGIN BINARY \
+    && curl -fsSLO https://github.com/Azure/kubelogin/releases/download/v${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip \
+    && unzip -j kubelogin-linux-amd64.zip -d /usr/local/bin \
+    && rm kubelogin-linux-amd64.zip \
+    && kubelogin --version \
     && : INSTALL TERRAFORM BINARY \
     && curl -fsSLO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
     && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \

ansible/playbooks/roles/common/tasks/main.yml

@@ -91,6 +91,30 @@
     owner: root
     group: root
 
+- name: Customize CoreDNS for AKS
+  when: k8s_as_cloud_service
+  run_once: true
+  delegate_to: localhost
+  become: false
+  environment:
+    KUBECONFIG: "{{ kubeconfig.local }}"
+  block:
+
+  - name: Render Epiphany hosts template
+    template:
+      src: coredns-epi.yml.j2
+      dest: "/tmp/coredns-epi.yml"
+      mode: u=rw,go=r
+
+  - name: Apply custom CoreDNS configmap
+    become: false
+    command: |
+      kubectl apply -f /tmp/coredns-epi.yml
+
+  - name: Restart CoreDNS
+    command: |
+      kubectl -n kube-system rollout restart deployment coredns
+
 - include_tasks: epiuser.yml
   tags:
     - epiuser

ansible/playbooks/roles/common/templates/coredns-epi.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom # this is the name of the configmap you can overwrite with your changes
+  namespace: kube-system
+data:
+    epi.override: | # you may select any name here, but it must end with the .override file extension
+          hosts {
+              {% for host in play_hosts %}
+              {% set name = host.split('.') %}
+              {{ hostvars[host]['ansible_default_ipv4']['address'] }} {{ host }} {{ name[0] }}
+              {% endfor %}
+              fallthrough
+          }

ansible/playbooks/roles/keycloak/templates/dynamic-chart-values.yml.j2

@@ -7,7 +7,7 @@ Keep data structure in sync with Helm chart's values.
 ref: https://github.com/codecentric/helm-charts/blob/keycloakx-1.6.1/charts/keycloakx/values.yaml
 -#}
 
-{% if specification.image_registry.use_local %}
+{% if specification.image_registry.use_local and not k8s_as_cloud_service %}
 image:
   repository: {{ image_registry_address }}/{{ _chart_values.image.repository }}
 dbchecker:

ansible/playbooks/roles/preflight/tasks/common/check-routing.yml

@@ -37,7 +37,7 @@
 - name: Validate if ansible_default_ipv4.address matches address from inventory
   when:
     - common_vars.provider == "any"
-    - common_vars.specification.cloud is undefined
+    - not k8s_as_cloud_service
   assert:
     that: ansible_default_ipv4.address == ansible_host
     fail_msg: >-

ansible/playbooks/roles/prometheus/files/secret.yml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: prometheus
+  namespace: kube-system
+  annotations:
+    kubernetes.io/service-account.name: prometheus
+type: kubernetes.io/service-account-token

ansible/playbooks/roles/prometheus/tasks/configure-k8s-apps-monitoring.yml

@@ -30,6 +30,14 @@
   run_once: true
   delegate_to: localhost
 
+# Starting from K8s v1.24, secrets are not automatically generated when service accounts are created
+- name: Create secret
+  when: k8s_as_cloud_service is defined and k8s_as_cloud_service
+  become: false
+  command: "kubectl apply -f {{ role_path }}/files/secret.yml"
+  run_once: true
+  delegate_to: localhost
+
 - name: Get kubernetes bearer token for prometheus
   become: false
   shell: |-

ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2

@@ -104,7 +104,51 @@ scrape_configs:
 
 # Scrape config for nodes (kubelet).
 
-  - job_name: 'kubernetes-pods'
+  - job_name: 'kubernetes-pods-http'
+    kubernetes_sd_configs:
+    - role: pod
+      api_server: "{{ api_server_address }}"
+      tls_config:
+        insecure_skip_verify: true
+      bearer_token: "{{ bearer_token }}"
+    tls_config:
+      insecure_skip_verify: true
+    bearer_token: "{{ bearer_token }}"
+    scheme: http
+    relabel_configs:
+      - action: keep
+        regex: true
+        source_labels:
+        - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+      - action: replace
+        regex: (.+)
+        source_labels:
+        - __meta_kubernetes_pod_annotation_prometheus_io_path
+        target_label: __metrics_path__
+      - action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        source_labels:
+        - __address__
+        - __meta_kubernetes_pod_annotation_prometheus_io_port
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_namespace
+        target_label: kubernetes_namespace
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_name
+        target_label: kubernetes_pod_name
+      - action: drop
+        source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+        regex: https
+
+# To scrape metrics over HTTPS, you need to annotate application pods that expose an HTTPS endpoint
+# with the following key/value pair: prometheus.io/scheme="https".
+  - job_name: 'kubernetes-pods-https'
     kubernetes_sd_configs:
     - role: pod
       api_server: "{{ api_server_address }}"
@@ -142,6 +186,9 @@ scrape_configs:
         source_labels:
         - __meta_kubernetes_pod_name
         target_label: kubernetes_pod_name
+      - action: keep
+        source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+        regex: https
 
 # Scrape config for Kubelet cAdvisor.
 

cli/src/ansible/AnsibleVarsGenerator.py

@@ -148,10 +148,7 @@ def populate_group_vars(self, ansible_dir):
         main_vars['full_download'] = Config().full_download
 
         # Consider to move this to the provider level.
-        if self.cluster_model.provider != 'any':
-            main_vars['k8s_as_cloud_service'] = self.cluster_model.specification.cloud.k8s_as_cloud_service
-        else:
-            main_vars['k8s_as_cloud_service'] = False
+        main_vars['k8s_as_cloud_service'] = self.cluster_model.specification.cloud.k8s_as_cloud_service
 
         if self.is_upgrade_run:
             shared_config_doc = self.get_shared_config_from_manifest()

docs/changelogs/CHANGELOG-2.0.md

@@ -1,11 +1,12 @@
 <!-- markdownlint-disable-file no-duplicate-header -->
 # Changelog 2.0
 
-## [2.0.10] 2023-11-03
+## [2.0.10] YYYY-MM-DD
 
 ### Added
 
 - [#3408](https://github.com/hitachienergy/epiphany/issues/3408) - Apply changes made by the Lumada team
+- [#3410](https://github.com/hitachienergy/epiphany/issues/3410) - AKS and Epiphany integration
 
 ## [2.0.9] 2023-10-19
 

schema/any/defaults/epiphany-cluster.yml

@@ -9,6 +9,8 @@ specification:
   admin_user:
     name: operations # YOUR-ADMIN-USERNAME
     key_path: /root/.ssh/epiphany-operations/id_rsa # YOUR-SSH-KEY-PATH
+  cloud:
+    k8s_as_cloud_service: False
   components:
     kubernetes_master:
       count: 1