full-cluster.yaml

node_defaults:
  keyfile: "/home/username/.ssh/id_rsa"
  password: '{{ env.PASS }}'     #Either keyfile or password can be used.
  username: "centos"

vrrp_ips:
- hosts:
  - name: k8s-lb-1
    priority: 254
  - name: k8s-lb-2
    priority: 253
  id: d8efc729e4
  interface: eth0
  ip: 192.168.0.250
  password: 11a1aabe
  router_id: '250'

nodes:
  - name: "k8s-lb-1"
    address: "10.101.0.1"
    internal_address: "192.168.0.1"
    roles: ["balancer"]
  - name: "k8s-lb-2"
    address: "10.101.0.2"
    internal_address: "192.168.0.2"
    roles: ["balancer"]
  - name: "k8s-control-plane-1"
    address: "10.101.0.3"
    internal_address: "192.168.0.3"
    roles: ["control-plane"]
  - name: "k8s-control-plane-2"
    address: "10.101.0.4"
    internal_address: "192.168.0.4"
    roles: ["control-plane"]
  - name: "k8s-control-plane-3"
    address: "10.101.0.5"
    internal_address: "192.168.0.5"
    roles: ["control-plane"]
  - name: "k8s-worker-1"
    address: "10.101.0.6"
    internal_address: "192.168.0.6"
    roles: ["worker"]
    taints:
      - "node-role.kubernetes.io/remove-example:NoSchedule-"
      - "node-role.kubernetes.io/add-example=add-example:NoSchedule"
  - name: "k8s-worker-2"
    address: "10.101.0.7"
    internal_address: "192.168.0.7"
    roles: ["worker"]
    labels:
      netcracker-infra: infra
      region: europe
  - name: "k8s-worker-3"
    address: "10.101.0.8"
    internal_address: "192.168.0.8"
    roles: ["worker"]
    labels:
      netcracker-infra: infra
      region: asia

cluster_name: "k8s.example.com"
public_cluster_ip: "10.101.0.1"

services:

  kubeadm:
    kubernetesVersion: v1.30.1
    controlPlaneEndpoint: 'k8s.example.com:6443'
    imageRepository: artifactory.example.com:5443/registry.k8s.io
    networking:
      podSubnet: 10.30.0.0/24
      serviceSubnet: 172.30.0.0/24
    apiServer:
      certSANs:
        - 10.101.0.1
        - 10.101.0.2
        - 10.101.0.3
        - 10.101.0.4
        - 10.101.0.5
        - 10.101.0.6
        - 10.101.0.7
        - 10.101.0.8
        - 192.168.0.1
        - 192.168.0.2
        - 192.168.0.3
        - 192.168.0.4
        - 192.168.0.5
        - 192.168.0.6
        - 192.168.0.7
        - 192.168.0.8
        - k8s-lb-1
        - k8s-lb-2
        - k8s-control-plane-1
        - k8s-control-plane-2
        - k8s-control-plane-3
        - k8s-worker-1
        - k8s-worker-2
        - k8s-worker-3
      extraArgs:
        enable-admission-plugins: NodeRestriction,PodNodeSelector

  cri:
    containerRuntime: containerd

    containerdConfig:
      version: 2
      plugins."io.containerd.grpc.v1.cri":
        sandbox_image: registry.k8s.io/pause:3.9
      plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc:
        runtime_type: "io.containerd.runc.v2"
      plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options:
        SystemdCgroup: true
      plugins."io.containerd.grpc.v1.cri".registry:
        config_path: "/etc/containerd/certs.d"
      plugins."io.containerd.grpc.v1.cri".registry.configs."artifactory.example.com:5443".auth:
        auth: "bmMtdXNlcjperfr="

    containerdRegistriesConfig:
      artifactory.example.com:5443:
        host."https://artifactory.example.com:5443":
          capabilities: ["pull", "resolve"]
          skip_verify: true

  resolv.conf:
    search: default
    nameservers:
      - 1.1.1.1
      - 1.0.0.1
      - 2606:4700:4700::1111
      - 2606:4700:4700::1001

  etc_hosts:
    1.1.1.1:
      - example.com

  ntp:
    # For RHEL OS use chrony:
    chrony:
      servers:
        - 0.de.pool.ntp.org iburst
        - 1.de.pool.ntp.org iburst
      makestep: 5 10
      rtcsync: True

    # For Debian OS use timesyncd. Uncomment and customize the following section:
    # timesyncd:
    #   Time:
    #     NTP:
    #       - 0.de.pool.ntp.org
    #       - 1.de.pool.ntp.org
    #     FallbackNTP:
    #       - 2.de.pool.ntp.org
    #       - 3.de.pool.ntp.org
    #     RootDistanceMaxSec: 10
    #     PollIntervalMinSec: 64
    #     PollIntervalMaxSec: 1024

  thirdparties:
    /usr/bin/kubeadm:
      source: 'https://example.com/kubernetes/v1.30.1/bin/linux/amd64/kubeadm'
      sha1: e5cdfcda337a5c8d59035da9db0c2b02913271d1
      groups:
        - control-plane
        - worker
    /opt/cni/cni-plugins-linux.tgz:
      source: 'https://example.com/cni-plugins/v0.8.3/cni-plugins-linux-amd64-v0.8.3.tgz'
      sha1: f662ec5b648e114802276f8f353ad48a3376da47
      unpack: '/opt/cni/bin'
      group: control-plane
    /usr/bin/calicoctl:
      source: 'https://example.com/calico/v3.27.3/calicoctl-linux-amd64'

  kernel_security:
    # For RHEL OS use selinux:
    selinux:
      state: enforcing
      policy: targeted
      permissive:
        - http_port_t
        - http_cache_port_t
        - commplex_main_port_t
    # For Debian OS use apparmor:
    apparmor:
      complain:
        - man_filter
      disable:
        - man_groff

  modprobe:
    # For RHEL OS:
    rhel:
    - br_netfilter
    - ip_vs
    - ip_vs_rr
    - ip_vs_wrr
    - ip_vs_sh
    - ip6table_filter
    - nf_conntrack_ipv6
    - nf_nat_masquerade_ipv6
    - nf_reject_ipv6
    - nf_defrag_ipv6
    # For Debian OS:
    debian:
    - br_netfilter
    - ip_vs
    - ip_vs_rr
    - ip_vs_wrr
    - ip_vs_sh
    - ip6table_filter
    - nf_conntrack
    - nf_nat
    - nf_reject_ipv6
    - nf_defrag_ipv6

  sysctl:
    net.bridge.bridge-nf-call-iptables: 1
    net.ipv4.ip_forward: 1
    net.ipv4.ip_nonlocal_bind: 1
    net.bridge.bridge-nf-call-ip6tables: 1
    net.ipv6.conf.all.forwarding: 1
    net.ipv6.ip_nonlocal_bind: 1

  loadbalancer:
    haproxy:
      defaults:
        timeout_connect: '10s'
        timeout_client: '1m'
        timeout_server: '1m'
        timeout_tunnel: '60m'
        timeout_client_fin: '1m'
        maxconn: 10000
      keep_configs_updated: True

  packages:
    package_manager:
      replace-repositories: true
      # For RHEL OS use yum-like format:
      repositories:
        centos-sclo-haproxy18:
          name: "CentOS SCLO rh-haproxy18"
          enabled: 1
          gpgcheck: 0
          baseurl: "http://mirror.centos.org/centos/7/sclo/x86_64/rh/rh-haproxy18/"
      # For Debian OS use apt-like format:
      # repositories:
      #   - "deb [arch=amd64 trusted=yes] http://example.com/deb/ubuntu/ focal main restricted"
    associations:
      haproxy:
        executable_name: '/bin/haproxy'
        package_name: 'haproxy'
        service_name: 'haproxy'
        config_location: '/etc/haproxy/haproxy.cfg'
      containerd:
        package_name:
          - 'containerd.io-1.6*'
    install:
      include:
      - ethtool
      - ebtables
      - socat
      - curl
      - openssl
      - unzip
      - policycoreutils-python

rbac:
  account_defaults:
    namespace: kube-system
    configs:
      - apiVersion: v1
        kind: ServiceAccount
        metadata: {}
        secrets: []
      - apiVersion: rbac.authorization.k8s.io/v1
        kind: ClusterRoleBinding
        metadata: {}
        roleRef:
          apiGroup: rbac.authorization.k8s.io
          kind: ClusterRole
        subjects:
          - kind: ServiceAccount
      - apiVersion: v1
        kind: Secret
        metadata:
          annotations: {}
        type: kubernetes.io/service-account-token

  accounts:
    - name: superadmin
      role: cluster-admin
    - name: superadmin2
      role: cluster-admin
      namespace: kube-system

  admission: pss
  pss:
    pod-security: enabled
    defaults:
      enforce: restricted
      enforce-version: latest
      audit: restricted
      audit-version: latest
      warn: restricted
      warn-version: latest
    exemptions:
      usernames: ["example-user1", "example-user2"]
      runtimeClasses: ["example-class-1", "example-class-2"]
      namespaces: ["kube-system", "example-namespace-1", "example-namespace-2"]

plugin_defaults:
  installation:
    registry: artifactory.example.com:5443

plugins:

  calico:
    install: true
    version: v3.27.3
    installation:
      priority: 0
    mode: ipip
    crossSubnet: true
    natOutgoing: true
    mtu: 1440
    typha:
      enabled: false
      replicas: 2
      image: calico/typha:v3.27.3
      nodeSelector:
        kubernetes.io/os: linux
    cni:
      image: calico/cni:v3.27.3
    node:
      image: calico/node:v3.27.3
    kube-controllers:
      image: calico/kube-controllers:v3.27.3
      nodeSelector:
        kubernetes.io/os: linux
    apiserver:
      image: calico/apiserver:v3.27.3
      enabled: true
      nodeSelector:
        kubernetes.io/os: linux
      expect:
        apiservice:
          retries: 60

  nginx-ingress-controller:
    install: true
    installation:
      priority: 1
    controller:
      image: registry.k8s.io/ingress-nginx/controller:v1.10.1
      ssl:
        enableSslPassthrough: false
        default-certificate:
          paths:
            cert: /path/to/cert
            key: /path/to/key
      nodeSelector:
        kubernetes.io/os: linux

  kubernetes-dashboard:
    install: true
    installation:
      priority: 2
    hostname: 'dashboard.k8s.example.com'
    dashboard:
      image: kubernetesui/dashboard:v2.7.0
      nodeSelector:
        kubernetes.io/os: linux
    metrics-scraper:
      image: kubernetesui/metrics-scraper:v1.0.8
      nodeSelector:
        kubernetes.io/os: linux
    ingress:
      metadata:
        name: kubernetes-dashboard
        namespace: kubernetes-dashboard
        annotations:
          nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
          nginx.ingress.kubernetes.io/ssl-redirect: "true"
          nginx.ingress.kubernetes.io/rewrite-target: /
          nginx.ingress.kubernetes.io/secure-backends: "true"
          nginx.ingress.kubernetes.io/ssl-passthrough: "true"
      spec:
        tls:
          - hosts:
            - '{{ plugins["kubernetes-dashboard"].hostname }}'
        rules:
          - host: '{{ plugins["kubernetes-dashboard"].hostname }}'
            http:
              paths:
                - path: /
                  pathType: Prefix
                  backend:
                    service:
                      name: kubernetes-dashboard
                      port:
                        number: 443