From f34f59da87aec531c4e708ea30fac45eb7ed7a00 Mon Sep 17 00:00:00 2001 From: przemyslavic <43173646+przemyslavic@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:01:20 +0100 Subject: [PATCH] [develop] AKS and Epiphany integration (#3409) --- .devcontainer/Dockerfile | 6 +++ Dockerfile | 6 +++ ansible/playbooks/roles/common/tasks/main.yml | 24 +++++++++ .../roles/common/templates/coredns-epi.yml.j2 | 14 ++++++ .../templates/dynamic-chart-values.yml.j2 | 2 +- .../preflight/tasks/common/check-routing.yml | 2 +- .../roles/prometheus/files/secret.yml | 8 +++ .../tasks/configure-k8s-apps-monitoring.yml | 8 +++ .../prometheus/templates/prometheus.yml.j2 | 49 ++++++++++++++++++- cli/src/ansible/AnsibleVarsGenerator.py | 5 +- docs/changelogs/CHANGELOG-2.0.md | 3 +- schema/any/defaults/epiphany-cluster.yml | 2 + 12 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 ansible/playbooks/roles/common/templates/coredns-epi.yml.j2 create mode 100644 ansible/playbooks/roles/prometheus/files/secret.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5333577bd8..cfbbd1a2ee 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,6 +7,7 @@ ARG USER_GID=$USER_UID ARG AWS_CLI_VERSION=2.0.30 ARG HELM_VERSION=3.3.1 ARG KUBECTL_VERSION=1.22.4 +ARG KUBELOGIN_VERSION=0.0.33 ARG TERRAFORM_VERSION=1.1.3 RUN : INSTALL APT REQUIREMENTS \ @@ -35,6 +36,11 @@ RUN : INSTALL HELM BINARY \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ && kubectl version --client \ + && : INSTALL KUBELOGIN BINARY \ + && curl -fsSLO https://github.com/Azure/kubelogin/releases/download/v${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip \ + && unzip -j kubelogin-linux-amd64.zip -d /usr/local/bin \ + && rm kubelogin-linux-amd64.zip \ + && kubelogin --version \ && : INSTALL TERRAFORM BINARY \ && curl -fsSLO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \ && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \ diff --git a/Dockerfile b/Dockerfile index 842c89611c..e99092d84c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ ARG USER_GID=$USER_UID ARG AWS_CLI_VERSION=2.0.30 ARG HELM_VERSION=3.3.1 ARG KUBECTL_VERSION=1.22.4 +ARG KUBELOGIN_VERSION=0.0.33 ARG TERRAFORM_VERSION=1.1.3 ENV EPICLI_DOCKER_SHARED_DIR=/shared @@ -28,6 +29,11 @@ RUN : INSTALL APT REQUIREMENTS \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ && kubectl version --client \ + && : INSTALL KUBELOGIN BINARY \ + && curl -fsSLO https://github.com/Azure/kubelogin/releases/download/v${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip \ + && unzip -j kubelogin-linux-amd64.zip -d /usr/local/bin \ + && rm kubelogin-linux-amd64.zip \ + && kubelogin --version \ && : INSTALL TERRAFORM BINARY \ && curl -fsSLO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \ && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \ diff --git a/ansible/playbooks/roles/common/tasks/main.yml b/ansible/playbooks/roles/common/tasks/main.yml index 14071887ac..08b0a57c6b 100644 --- a/ansible/playbooks/roles/common/tasks/main.yml +++ b/ansible/playbooks/roles/common/tasks/main.yml @@ -91,6 +91,30 @@ owner: root group: root +- name: Customize CoreDNS for AKS + when: k8s_as_cloud_service + run_once: true + delegate_to: localhost + become: false + environment: + KUBECONFIG: "{{ kubeconfig.local }}" + block: + + - name: Render Epiphany hosts template + template: + src: coredns-epi.yml.j2 + dest: "/tmp/coredns-epi.yml" + mode: u=rw,go=r + + - name: Apply custom CoreDNS configmap + become: false + command: | + kubectl apply -f /tmp/coredns-epi.yml + + - name: Restart CoreDNS + command: | + kubectl -n kube-system rollout restart deployment coredns + - include_tasks: epiuser.yml tags: - epiuser diff --git a/ansible/playbooks/roles/common/templates/coredns-epi.yml.j2 b/ansible/playbooks/roles/common/templates/coredns-epi.yml.j2 new file mode 100644 index 0000000000..8801fda4e7 --- /dev/null +++ b/ansible/playbooks/roles/common/templates/coredns-epi.yml.j2 @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom # this is the name of the configmap you can overwrite with your changes + namespace: kube-system +data: + epi.override: | # you may select any name here, but it must end with the .override file extension + hosts { + {% for host in play_hosts %} + {% set name = host.split('.') %} + {{ hostvars[host]['ansible_default_ipv4']['address'] }} {{ host }} {{ name[0] }} + {% endfor %} + fallthrough + } diff --git a/ansible/playbooks/roles/keycloak/templates/dynamic-chart-values.yml.j2 b/ansible/playbooks/roles/keycloak/templates/dynamic-chart-values.yml.j2 index affd81810c..24b76ccca4 100644 --- a/ansible/playbooks/roles/keycloak/templates/dynamic-chart-values.yml.j2 +++ b/ansible/playbooks/roles/keycloak/templates/dynamic-chart-values.yml.j2 @@ -7,7 +7,7 @@ Keep data structure in sync with Helm chart's values. ref: https://github.com/codecentric/helm-charts/blob/keycloakx-1.6.1/charts/keycloakx/values.yaml -#} -{% if specification.image_registry.use_local %} +{% if specification.image_registry.use_local and not k8s_as_cloud_service %} image: repository: {{ image_registry_address }}/{{ _chart_values.image.repository }} dbchecker: diff --git a/ansible/playbooks/roles/preflight/tasks/common/check-routing.yml b/ansible/playbooks/roles/preflight/tasks/common/check-routing.yml index 17d4bcb103..d0eba56439 100644 --- a/ansible/playbooks/roles/preflight/tasks/common/check-routing.yml +++ b/ansible/playbooks/roles/preflight/tasks/common/check-routing.yml @@ -37,7 +37,7 @@ - name: Validate if ansible_default_ipv4.address matches address from inventory when: - common_vars.provider == "any" - - common_vars.specification.cloud is undefined + - not k8s_as_cloud_service assert: that: ansible_default_ipv4.address == ansible_host fail_msg: >- diff --git a/ansible/playbooks/roles/prometheus/files/secret.yml b/ansible/playbooks/roles/prometheus/files/secret.yml new file mode 100644 index 0000000000..60a1fa97c4 --- /dev/null +++ b/ansible/playbooks/roles/prometheus/files/secret.yml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: prometheus + namespace: kube-system + annotations: + kubernetes.io/service-account.name: prometheus +type: kubernetes.io/service-account-token diff --git a/ansible/playbooks/roles/prometheus/tasks/configure-k8s-apps-monitoring.yml b/ansible/playbooks/roles/prometheus/tasks/configure-k8s-apps-monitoring.yml index 77fd4e37b6..b43319f9b3 100644 --- a/ansible/playbooks/roles/prometheus/tasks/configure-k8s-apps-monitoring.yml +++ b/ansible/playbooks/roles/prometheus/tasks/configure-k8s-apps-monitoring.yml @@ -30,6 +30,14 @@ run_once: true delegate_to: localhost +# Starting from K8s v1.24, secrets are not automatically generated when service accounts are created +- name: Create secret + when: k8s_as_cloud_service is defined and k8s_as_cloud_service + become: false + command: "kubectl apply -f {{ role_path }}/files/secret.yml" + run_once: true + delegate_to: localhost + - name: Get kubernetes bearer token for prometheus become: false shell: |- diff --git a/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 b/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 index 247f785ffe..5e7a641125 100644 --- a/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 @@ -104,7 +104,51 @@ scrape_configs: # Scrape config for nodes (kubelet). - - job_name: 'kubernetes-pods' + - job_name: 'kubernetes-pods-http' + kubernetes_sd_configs: + - role: pod + api_server: "{{ api_server_address }}" + tls_config: + insecure_skip_verify: true + bearer_token: "{{ bearer_token }}" + tls_config: + insecure_skip_verify: true + bearer_token: "{{ bearer_token }}" + scheme: http + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name + - action: drop + source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + regex: https + +# To scrape metrics over HTTPS, you need to annotate application pods that expose an HTTPS endpoint +# with the following key/value pair: prometheus.io/scheme="https". + - job_name: 'kubernetes-pods-https' kubernetes_sd_configs: - role: pod api_server: "{{ api_server_address }}" @@ -142,6 +186,9 @@ scrape_configs: source_labels: - __meta_kubernetes_pod_name target_label: kubernetes_pod_name + - action: keep + source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + regex: https # Scrape config for Kubelet cAdvisor. diff --git a/cli/src/ansible/AnsibleVarsGenerator.py b/cli/src/ansible/AnsibleVarsGenerator.py index 484fb6df5d..98d936f52f 100644 --- a/cli/src/ansible/AnsibleVarsGenerator.py +++ b/cli/src/ansible/AnsibleVarsGenerator.py @@ -148,10 +148,7 @@ def populate_group_vars(self, ansible_dir): main_vars['full_download'] = Config().full_download # Consider to move this to the provider level. - if self.cluster_model.provider != 'any': - main_vars['k8s_as_cloud_service'] = self.cluster_model.specification.cloud.k8s_as_cloud_service - else: - main_vars['k8s_as_cloud_service'] = False + main_vars['k8s_as_cloud_service'] = self.cluster_model.specification.cloud.k8s_as_cloud_service if self.is_upgrade_run: shared_config_doc = self.get_shared_config_from_manifest() diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index 715224b562..1c1b581430 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -1,11 +1,12 @@ # Changelog 2.0 -## [2.0.10] 2023-11-03 +## [2.0.10] YYYY-MM-DD ### Added - [#3408](https://github.com/hitachienergy/epiphany/issues/3408) - Apply changes made by the Lumada team +- [#3410](https://github.com/hitachienergy/epiphany/issues/3410) - AKS and Epiphany integration ## [2.0.9] 2023-10-19 diff --git a/schema/any/defaults/epiphany-cluster.yml b/schema/any/defaults/epiphany-cluster.yml index f3dc9a0a08..e9899ee060 100644 --- a/schema/any/defaults/epiphany-cluster.yml +++ b/schema/any/defaults/epiphany-cluster.yml @@ -9,6 +9,8 @@ specification: admin_user: name: operations # YOUR-ADMIN-USERNAME key_path: /root/.ssh/epiphany-operations/id_rsa # YOUR-SSH-KEY-PATH + cloud: + k8s_as_cloud_service: False components: kubernetes_master: count: 1