deploy_kube.yaml

---
- hosts: all
  vars:
    container_runtime: containerd
    kube_dns_domain: kube.mccormicom.com
    cni_plugin: calico
    pod_network_cidr: 10.244.0.0/16
    install_basic_tools: no
    kubeadm_reset: yes
    disable_ipv6: False
    kubernetes_dashboard: True
    cert_manager: True
    cert_manager_manifest: https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml
    metrics_server: True
    metrics_server_manifest: https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
    metrics_server_disable_tls_verify: True
    flannel_manifest: https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
    calico_manifest: https://raw.githubusercontent.com/projectcalico/calico/refs/heads/master/manifests/calico.yaml
    weave_manifest: https://cloud.weave.works/k8s/v1.16/net.yaml
    control_plane_endpoint: kube-cp.mccormicom.com
  become: yes
  tasks:
    - name: Check assertions.
      assert:
        that:
          - (ansible_distribution == "Debian" and ansible_distribution_major_version|int > 11) or
            (ansible_distribution == "CentOS" and ansible_distribution_major_version|int > 6) or
            ansible_distribution == "Archlinux"
          - container_runtime == "docker" or container_runtime == "containerd" or container_runtime == "cri-o"
          - cni_plugin == "flannel" or cni_plugin == "calico" or cni_plugin == "weave"

    - name: Still cant get containerd and debian to get along.
      fail:
      when: container_runtime == "containerd" and ansible_distribution == "Debian"

# Prework
    - name: Let iptables see bridged traffic
      copy:
        dest: /etc/modules-load.d/k8s.conf
        content: "br_netfilter"

#    - name: Load overlay module for containerd.

    - name: Manually load br_netfilter once.
      shell:
        cmd: lsmod | grep br_netfilter || modprobe br_netfilter

    - name: Configure sysctl.
      sysctl:
        name: net.ipv4.ip_forward
        value: '1'
        sysctl_set: yes
        state: present
        reload: yes

    - name: Grab current default grub config.
      shell:
        cmd: grep GRUB_CMDLINE_LINUX /etc/default/grub | cut -d= -f2- | tr -d '"'
      register: current_default_grub_config
      when: disable_ipv6

    - name: Disable IPv6 in grub config.
      lineinfile:
        path: /etc/default/grub
        regexp: ^GRUB_CMDLINE_LINUX
        line: "GRUB_CMDLINE_LINUX='{{ current_default_grub_config.stdout }} ipv6.disable=0'"
      when:
        - disable_ipv6
        - "'ipv6.disable=0' not in current_default_grub_config.stdout"
      register: grub_cfg_update

    - name: Disable IPv6 immediately.
      command: sysctl net.ipv6.conf.all.disable_ipv6=1
      when: disable_ipv6

    - name: Rebuild grub config.
      shell:
        cmd: grub2-mkconfig -o /boot/grub2/grub.cfg || grub-mkconfig -o /boot/grub/grub.cfg
      when:
        - disable_ipv6
        - grub_cfg_update.changed

    - name: Disable swap permanently.
      mount:
        fstype: swap
        path: none
        state: absent

    - name: Disable swap now.
      command: swapoff -a

    - name: Install basic tools.
      package:
        name: "{{ item }}"
      loop:
        - vim
        - sudo
        - curl
      when: install_basic_tools

    - name: Install required packages.
      package:
        name: "{{ item }}"
      loop:
        - ca-certificates
        - git
        - jq

    - name: Install required Redhat/CentOS packages.
      package:
        name: yum-utils
      when: ansible_distribution == "CentOS"

    - name: Install required Debian packages.
      package:
        name: "{{ item }}"
      when: ansible_distribution == "Debian"
      loop:
        - curl
        - gpg
#        - gnupg1

# Repos
## Repos Debian
    - name: Configure Kubernetes apt source.
      copy:
        dest: /etc/apt/sources.list.d/kubernetes.list
        content: "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /"
      when:
        - ansible_distribution == "Debian"

    - name: Deploy keys for Kubernetes apt source.
      get_url:
        url: https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key
        dest: /etc/apt/keyrings/kubernetes-apt-keyring.gpg.armor
      when:
        - ansible_distribution == "Debian"

    - name: Dearmor key for Kubernetes apt source.
      shell:
        cmd: gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg /etc/apt/keyrings/kubernetes-apt-keyring.gpg.armor && rm /etc/apt/keyrings/kubernetes-apt-keyring.gpg.armor
        creates: /etc/apt/keyrings/kubernetes-apt-keyring.gpg
      when:
        - ansible_distribution == "Debian"

#    - name: Configure repo for docker.io package. Might be needed for Debian 10 but not Debian 11. 

## Repos Redhat
    - name: Configure docker/containerd repo for Redhat family.
      command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
      when:
        - container_runtime is defined
        - container_runtime|lower == "docker" or container_runtime|lower == "containerd"
        - ansible_distribution == "CentOS" and ansible_distribution_major_version|int > 6

    - name: Configure cri-o repo for Redhat family.
      command: "{{item}}" 
      when:
        - container_runtime is defined
        - container_runtime == "cri-o"
        - ansible_distribution == "CentOS"
      loop:
        - yum-config-manager --add-repo https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/CentOS_8/devel:kubic:libcontainers:stable.repo
        - yum-config-manager --add-repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.21:/1.21.2/CentOS_8/devel:kubic:libcontainers:stable:cri-o:1.21:1.21.2.repo 

    - name: Configure kubernetes repo for Redhat family.
      copy:
        dest: /etc/yum.repos.d/kubernetes.repo
        content: |
                 [kubernetes]
                 name=Kubernetes
                 baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
                 enabled=1
                 gpgcheck=1
                 repo_gpgcheck=1
                 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
      when:
        - ansible_distribution == "CentOS"

## Repos Debian
    - name: Configure apt repos for cri-o.
      copy:
        dest: /etc/apt/sources.list.d/cri-o.list
        content: "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.29/deb/ /"
      when:
        - container_runtime is defined
        - container_runtime == "cri-o"
        - ansible_distribution == "Debian"

    - name: Deploy keys for cri-o apt source.
      get_url:
        url: https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.29/deb/Release.key
        dest: /etc/apt/keyrings/cri-o-apt-keyring.gpg.armor
      when:
        - container_runtime is defined
        - container_runtime == "cri-o"
        - ansible_distribution == "Debian"

    - name: Dearmor key for cri-o apt source.
      shell:
        cmd: gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg /etc/apt/keyrings/cri-o-apt-keyring.gpg.armor && rm /etc/apt/keyrings/cri-o-apt-keyring.gpg.armor
        creates: /etc/apt/keyrings/cri-o-apt-keyring.gpg
      when:
        - container_runtime is defined
        - container_runtime == "cri-o"
        - ansible_distribution == "Debian"

# Packages
## Packages Debian
    - name: Update apt cache.
      apt:
        update_cache: yes
      when:
        - ansible_distribution == "Debian" or ansible_distribution == "Ubuntu" 

## Packages Arch
    - name: Gather the package facts.
      command: pacman -Q
      register: package_facts
      changed_when: False
      when: ansible_distribution == "Archlinux"

    - name: Force Arch to replace iptables and install the kubernetes packages.
      shell:
        cmd: "(echo y ; echo Y) | pacman -S kube-apiserver kube-controller-manager kube-proxy kube-scheduler kubelet" 
      when:
        - ansible_distribution == "Archlinux"
        - "'kubelet' not in package_facts.stdout"

## Packages All Distros
    - name: Install kubernetes packages.
      package:
        name: "{{ item }}"
      loop:
        - kubelet
        - kubeadm
        - kubectl

    - name: Pick a docker package name.
      set_fact:
        docker_package_name: "{{item.package}}"
      when: container_runtime == "docker" and ansible_distribution == item.os
      loop:
        - { os: 'Debian', package: 'docker.io' }
        - { os: 'CentOS', package: 'docker-ce' }
        - { os: 'Archlinux', package: 'docker' }

    - name: Install docker package.
      package:
        name: "{{docker_package_name}}" 
      when:
        - container_runtime is defined
        - container_runtime|lower == "docker" or container_runtime|lower == "docker.io" 

    - name: Install containerd package if selected.
      package:
        name: containerd
      when:
        - container_runtime is defined
        - container_runtime|lower == "containerd" or container_runtime|lower == "containerd.io"

    - name: Install cri-o packages if selected.
      package:
        name: "{{item}}" 
      when:
        - container_runtime is defined
        - container_runtime == "cri-o"
      loop:
        - cri-o

# Configs
## Configs All Distros
    - name: Configure containerd if selected.
      copy:
        src: containerd_config.toml
        dest: /etc/containerd/config.toml
      when:
        - container_runtime is defined
        - container_runtime|lower == "containerd"
        - ansible_distribution != "Archlinux"

## Configs Arch
    - name: Configure some default container repos for Arch.
      copy:
        dest: /etc/containers/registries.conf
        content: |
                 [registries.search]
                 registries = ['registry.access.redhat.com', 'registry.redhat.io', 'docker.io', 'quay.io']
      when:
        - ansible_distribution == "Archlinux"
        - container_runtime == "cri-o"

## Configs Redhat
    - name: Configure docker to use systemd cgroup driver.
      copy:
        dest: /etc/docker/daemon.json
        content: |
                 {
                   "exec-opts": ["native.cgroupdriver=systemd"]
                 }
      when:
        - container_runtime is defined
        - container_runtime|lower == "docker" or container_runtime|lower == "docker.io"
        - ansible_distribution == "CentOS" and ansible_distribution_major_version|int > 6

    - name: KeyPaths is invalid. KeyPath is valid. (CentOS Stream 9 Bug?)
      lineinfile:
        path: /etc/containers/policy.json
        regexp: '"keyPaths":'
        line: '                    "keyPath": "/etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release"'
      when:
        - ansible_distribution == "CentOS" and ansible_distribution_major_version|int > 6
      loop:
        - 1
        - 2

# Services
    - name: Enable and start the appropriate service.
      service:
        name: "{{ item }}"
        state: started
        enabled: yes
      when: container_runtime|regex_replace('-', '') == item or item == "kubelet"
      loop:
        - containerd
        - docker
        - crio
        - kubelet

# Notes on trying to get containerd working.

#    - name: Create symlink for containerd.
#      file:
#        src: /opt/cni/bin
#        dest: /usr/lib/cni
#        state: link
#      when:
#        - container_runtime is defined
#        - container_runtime|lower == "containerd"

#    - name: Configure containerd.
#      lineinfile:
#        path: /etc/containerd/config.toml
#        regex: '^      bin_dir'
#        line: '      bin_dir = "/opt/cni/bin/"'
#      when:
#        - container_runtime is defined
#        - container_runtime|lower == "containerd"

#    - name: Configure flannel for containerd.
#      copy:
#        dest: /run/flannel/subnet.env
#        content: |
#                 FLANNEL_NETWORK=10.44.0.0/16
#                 FLANNEL_SUBNET=10.44.0.1/24
#                 FLANNEL_MTU=1450
#                 FLANNEL_IPMASQ=true
#      when:
#        - container_runtime is defined
#        - container_runtime|lower == "containerd"
#        - cni_plugin is defined
#        - cni_plugin|lower == "flannel"

#root@kube07:~# cat /run/flannel/subnet.env
#FLANNEL_NETWORK=10.44.0.0/16
#FLANNEL_SUBNET=10.44.0.1/24
#FLANNEL_MTU=1450
#FLANNEL_IPMASQ=true

# Actions
    - name: Refresh facts to ensure they are available in future plays.
      set_fact:
        container_runtime: "{{container_runtime}}"
        kube_dns_domain: "{{kube_dns_domain}}"
        cni_plugin: "{{cni_plugin}}"
        pod_network_cidr: "{{pod_network_cidr}}"
        kubeadm_reset: "{{kubeadm_reset}}"
        flannel_manifest: "{{flannel_manifest}}"
        calico_manifest: "{{calico_manifest}}"
        weave_manifest: "{{weave_manifest}}"
        control_plane_endpoint: "{{control_plane_endpoint}}"
        kubernetes_dashboard: "{{kubernetes_dashboard}}"
        cert_manager: "{{cert_manager}}"
        cert_manager_manifest: "{{cert_manager_manifest}}"
        metrics_server: "{{metrics_server}}"
        metrics_server_manifest: "{{metrics_server_manifest}}"
        metrics_server_disable_tls_verify: "{{metrics_server_disable_tls_verify}}"

    - name: Reset kubeadm for a master.
      command: kubeadm reset -f
      when: kubeadm_reset
      failed_when: false
      run_once: True

    - name: Reset kubeadm if requested.
      command: kubeadm reset -f
      when: kubeadm_reset
      failed_when: false

    - name: Stop services if reset was requested.
      service:
        name: "{{item}}"
        state: stopped 
      when: kubeadm_reset
      failed_when: false
      loop:
        - containerd
        - docker
        - crio
        - kubelet

    - name: Start only the requested runtime.
      when: kubeadm_reset
      service:
        name: "{{container_runtime | regex_replace('-', '') }}"
        state: started

## Actions Redhat
    - name: Kill the firewall. (Redhat family)
      service:
        name: firewalld
        state: stopped
      when:
        - ansible_distribution == "CentOS" and ansible_distribution_major_version|int > 6 

- hosts: masters
  become: yes
  run_once: yes
  tasks:
    - name: Pre-pull Kubernetes images.
      command: kubeadm config images pull

    - name: Initialize Kubernetes stack.
      command: >
               kubeadm init
               --service-dns-domain {{ kube_dns_domain }}
               --pod-network-cidr {{ pod_network_cidr }}
               --control-plane-endpoint {{control_plane_endpoint}}
               --upload-certs
      register: kubeadm_init

    - name: Extract tokens from init command. 
      set_fact:
        kubeadm_cert_key: "{{ (kubeadm_init.stdout | regex_search('[-][-]certificate-key[ ][a-z0-9]+')).split(' ')[1] }}"
        kubeadm_token: "{{(kubeadm_init.stdout | regex_findall('[-][-]token[ ][a-z0-9.]+') | last).split(' ')[1]}}"
        kubeadm_token_hash: "{{kubeadm_init.stdout | regex_findall('sha256:[a-z0-9]+') | last}}"

#    - name: debug extracted tokens 
#      debug:
#        msg:
#          - "{{kubeadm_token}}"
#          - "{{kubeadm_token_hash}}"
#          - "{{kubeadm_cert_key}}"

    - name: Wait for kubectl to start responding.
      wait_for:
        host: localhost 
        port: 6443

    - name: Pick a CNI Plugin manifest.
      set_fact:
        cni_plugin_manifest: "{{ item }}"
      when: cni_plugin|string in item
      loop:
        - "{{ flannel_manifest }}"
        - "{{ calico_manifest }}"
        - "{{ weave_manifest }}"

    - name: Wait for kubectl to start responding.
      wait_for:
        host: localhost
        port: 6443

    - name: Initialize CNI Plugin.
      shell:
        cmd: "kubectl apply -f {{cni_plugin_manifest}}"
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf

    - name: Store variables for next play.
      add_host:
        hostname: dummy
        kubeadm_token: "{{kubeadm_token}}"
        kubeadm_token_hash: "{{kubeadm_token_hash}}"
        master_ip: "{{ansible_default_ipv4.address}}"

- hosts: masters
  become: yes
  tasks:
    - name: Join the cluster. (Secondary Masters)
      command: "kubeadm join {{hostvars['dummy']['master_ip']}}:6443 --token {{hostvars['dummy']['kubeadm_token']}} --discovery-token-ca-cert-hash {{hostvars['dummy']['kubeadm_token_hash']}} --control-plane --certificate-key {{kubeadm_cert_key}}"
      when:
        - ansible_default_ipv4.address != hostvars['dummy']['master_ip']

    - name: Generate a list of secondary masters.
      run_once: yes
      shell:
        cmd: "kubectl get node | grep '<none>' | awk '{print $1}' | sort"
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      register: kube_secondary_masters
      when:
        - ansible_default_ipv4.address == hostvars['dummy']['master_ip']

    - name: Store variables for next play.
      add_host:
        hostname: dummy
        kube_secondary_masters: "{{kube_secondary_masters}}"

- hosts: masters
  become: yes
  run_once: yes
  tasks:
    - name: Assign control-plane role to secondary master servers.
      command: "kubectl label node {{item}} node-role.kubernetes.io/control-plane="
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      with_items:
        - "{{hostvars['dummy']['kube_secondary_masters']['stdout_lines']}}"

    - name: Assign master role to secondary master servers.
      command: "kubectl label node {{item}} node-role.kubernetes.io/master="
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      with_items:
        - "{{hostvars['dummy']['kube_secondary_masters']['stdout_lines']}}"

#    - name: Assign etcd role to secondary master servers.
#      command: "kubectl label node {{item}} node-role.kubernetes.io/etcd="
#      environment:
#        KUBECONFIG: /etc/kubernetes/admin.conf
#      with_items:
#        - "{{hostvars['dummy']['kube_secondary_masters']['stdout_lines']}}"

- hosts: workers
  become: yes
  tasks:
    - name: Check assertions.
      assert:
        that:
          - hostvars['dummy']['kubeadm_token'] is defined
          - hostvars['dummy']['kubeadm_token_hash'] is defined
          - hostvars['dummy']['master_ip'] is defined

    - name: Join the cluster. (Worker)
      command: "kubeadm join {{hostvars['dummy']['master_ip']}}:6443 --token {{hostvars['dummy']['kubeadm_token']}} --discovery-token-ca-cert-hash {{hostvars['dummy']['kubeadm_token_hash']}}"

- hosts: masters
  become: yes
  run_once: yes
  tasks:
    - name: Generate a list of worker nodes.
      shell:
        cmd: "kubectl get node | grep '<none>' | awk '{print $1}' | sort"
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      register: kube_workers

    - name: Assign worker role to all workers.
      command: "kubectl label node {{item}} node-role.kubernetes.io/worker="
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      loop: "{{ kube_workers.stdout_lines }}"

    - name: Start up a test workload to ensure everything is working as expected.
      command: "kubectl run --image nginx nginx"
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf

    - name: Clone Ingress Controller repo.
      git:
        repo: https://github.com/nginxinc/kubernetes-ingress.git
        dest: /tmp/ingress-controller/
        version: v2.4.2

    - name: Deploy an Ingress Controller.
      shell:
        cmd: |
             kubectl apply -f /tmp/ingress-controller/deployments/common/ns-and-sa.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/rbac/rbac.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/default-server-secret.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/nginx-config.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/ingress-class.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/crds/k8s.nginx.org_virtualservers.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/crds/k8s.nginx.org_transportservers.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/crds/k8s.nginx.org_policies.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/common/crds/k8s.nginx.org_globalconfigurations.yaml
             kubectl apply -f /tmp/ingress-controller/deployments/daemon-set/nginx-ingress.yaml
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf

    - name: Deploy Cert-Manager.
      command: "kubectl apply -f {{cert_manager_manifest}}"
      when: cert_manager
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf

    - name: Deploy Metrics-Server.
      command: "kubectl apply -f {{metrics_server_manifest}}"
      when: metrics_server
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf

    - name: Pull existing Metrics Server config.
      shell:
        cmd: "kubectl get deploy/metrics-server -n kube-system -o json | jq .spec.template.spec.containers[].args | grep -v '[[]' | grep -v '[]]'"
      when: metrics_server_disable_tls_verify
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      register: metrics_server_args

    - name: Add the insecure-tls flag to the metric server args.
      set_fact:
        new_metrics_args: "{{ metrics_server_args.stdout | regex_replace('\"', '') | regex_replace(',\n', '\n') | replace(' ', '') | split('\n') + ['--kubelet-insecure-tls'] }}"
      when: metrics_server_disable_tls_verify

    - name: Ensure we match the expected json format.
      set_fact:
        new_metrics_args: "{{ new_metrics_args | to_json }}"
      when: metrics_server_disable_tls_verify

    - name: Disable TLS verification for Metrics Server.
      command: |
               kubectl patch deployment metrics-server -n kube-system -p '{"spec": {"template": {"spec": {"containers": [{"name": "metrics-server", "args": {{ new_metrics_args }} }]}}}}'
      when: metrics_server_disable_tls_verify
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf

    - name: Wait a moment for cert-manager to be available.
      pause:
        seconds: 15
      when: cert_manager

    - name: Deploy the Kubernetes Dashboard if requested.
      command: "{{ item }}"
      when: kubernetes_dashboard
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      loop:
        - kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml
#       - kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v3.0.0-alpha0/charts/kubernetes-dashboard.yaml
        - kubectl delete clusterrole kubernetes-dashboard
        - kubectl create clusterrole kubernetes-dashboard --verb=* --resource=*.*
        - kubectl create token kubernetes-dashboard -n kubernetes-dashboard
      register: kubectl_kubernetes_dashboard

    - name: Grab the Kubernetes Dashboard service IP.
      environment:
        KUBECONFIG: /etc/kubernetes/admin.conf
      shell:
        cmd: "kubectl get service kubernetes-dashboard -n kubernetes-dashboard | awk '{print $3}' | tail -1"
      when: kubernetes_dashboard
      register: kubernetes_dashboard_service

    - name: Use this token to finish your Kubernetes Dashboard config.
      debug:
        msg:
          - "URL: https://{{ kubernetes_dashboard_service.stdout }}/#/workloads?namespace=_all"
          - "Token: {{ kubectl_kubernetes_dashboard.results[-1].stdout_lines | last }}"
      when: kubernetes_dashboard