Merge pull request #22 from flatcar/tormath1/brigthbox

brightbox: kubernetes bootstrap example

brightbox/README.md

@@ -0,0 +1,69 @@
+# Flatcar Provisioning Automation for Brightbox
+
+This repository provides tools to automate Kubernetes provisioning on [Brightbox][brightbox] using [Terraform][terraform] and Flatcar via the Systemd sysext approach: https://www.flatcar.org/docs/latest/container-runtimes/getting-started-with-kubernetes/#deploy-a-kubernetes-cluster-with-flatcar
+
+:warning: This is really for demo purposes but it can serve as a foundation (for example do not pass the admin configuration through HTTP for workers to join) :warning:
+
+## Features
+
+- Minimal configuration required (demo deployment works with default settings w/o any customisation, just run `terraform apply`!).
+- Deploy one or multiple workers.
+
+## Prerequisites
+
+1. Brightbox credentials: `api_client`, `api_secret`.
+2. A public SSH key to install on the control plane
+
+## HowTo
+
+This will create a server in 'gb1-a' using a medium instance size for the control plane and small instance sizes for the three workers.
+See "Customisation" below for advanced settings.
+
+1. Clone the repo.
+2. Add credentials and a SSH key in a `terraform.tfvars` file, expected credentials name can be found in `provider.tf`
+3. Run
+   ```shell
+   terraform init
+   ```
+4. Plan and apply.
+   Invoke Terraform:
+   ```shell
+   terraform plan
+   terraform apply
+   ```
+
+Terraform will print the control plane information (ipv4) after deployment concluded. You can now easily fetch the kubernetes `admin` configuration via a secure channel:
+
+```
+$ scp core@<IP from the output>:/home/core/.kube/config ~/.kube/config
+$ kubectl get nodes
+NAME                          STATUS     ROLES           AGE   VERSION
+srv-cruzw.gb1.brightbox.com   NotReady   <none>          55s   v1.29.2
+srv-fltor.gb1.brightbox.com   NotReady   control-plane   72s   v1.29.2
+srv-gvzhx.gb1.brightbox.com   NotReady   <none>          59s   v1.29.2
+srv-mipnf.gb1.brightbox.com   NotReady   <none>          60s   v1.29.2
+```
+
+From now, you can operate the Kubernetes cluster as usual (deploy CNI, deploy workloads, etc.)
+
+_NOTE_:
+* Server IP address can be found at any moment after deployment by running `terraform output`
+* If you update server configuration(s) in `server-configs` and re-run `terraform apply`, the instance will be **replaced**.
+Consider adding [`create_before_destroy`](https://www.terraform.io/docs/configuration/meta-arguments/lifecycle.html#syntax-and-arguments) to the `brightbox_server` resource in [`compute.tf`](compute.tf) to avoid services becoming unavailable during reprovisioning.
+
+### Customisation
+
+The provisioning automation can be customised via settings in `terraform.tfvars`:
+  - `ssh_keys`: SSH public keys to add to core user's `authorized_keys` (needed for fetching the Kubernetes configuration)
+  - `release_channel`: Select one of "lts", "stable", "beta", or "alpha".
+    Read more about channels [here](https://www.flatcar.org/releases).
+  - `flatcar_version`: Select the desired Flatcar version for the given channel (default to "current", which is the latest).
+  - `zone`: Where to deploy servers
+  - `control_plane_type`: Which instance type used for deploying the controle plane
+  - `worker_type`: Which instance type used for deploying the workers
+  - `kubernetes_version`: The Kubernetes version to deploy (NOTE: It has to be released on the Flatcar sysext bakery: https://github.com/flatcar/sysext-bakery/releases/tag/latest)
+  - `workers`: How many workers to deploy
+
+[butane]: https://www.flatcar.org/docs/latest/provisioning/config-transpiler/configuration/
+[brightbox]: https://www.brightbox.com/
+[terraform]: https://www.terraform.io/

brightbox/compute.tf

@@ -0,0 +1,55 @@
+data "brightbox_image" "flatcar" {
+  name        = "^flatcar-${var.release_channel}.*server$"
+  arch        = "x86_64"
+  official    = true
+  most_recent = true
+}
+
+resource "brightbox_server" "control-plane" {
+  image         = data.brightbox_image.flatcar.id
+  name          = "control-plane"
+  zone          = var.zone
+  type          = var.control_plane_type
+  user_data     = data.ct_config.config-control-plane.rendered
+  server_groups = [brightbox_server_group.kubernetes.id]
+  depends_on    = [brightbox_firewall_policy.kubernetes]
+}
+
+resource "brightbox_server" "worker" {
+  count     = var.workers
+  image     = data.brightbox_image.flatcar.id
+  name      = "worker-${count.index}"
+  zone      = var.zone
+  type      = var.worker_type
+  user_data = data.ct_config.config-worker.rendered
+}
+
+data "ct_config" "config-control-plane" {
+  strict = true
+  content = templatefile("${path.module}/server-configs/control-plane.yaml.tmpl", {
+    kubernetes_version = var.kubernetes_version
+  })
+  snippets = [
+    data.template_file.core_user.rendered
+  ]
+}
+
+data "ct_config" "config-worker" {
+  strict = true
+  content = templatefile("${path.module}/server-configs/worker.yaml.tmpl", {
+    kubernetes_version = var.kubernetes_version
+    control_plane_ip   = brightbox_cloudip.control-plane.public_ipv4
+  })
+}
+
+data "template_file" "core_user" {
+  template = file("${path.module}/core-user.yaml.tmpl")
+  vars = {
+    ssh_keys = jsonencode(var.ssh_keys)
+  }
+}
+
+resource "brightbox_cloudip" "control-plane" {
+  target = brightbox_server.control-plane.interface
+  name   = "control-plane public address"
+}

brightbox/core-user.yaml.tmpl

@@ -0,0 +1,7 @@
+variant: flatcar
+version: 1.0.0
+
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys: ${ssh_keys}

brightbox/network.tf

@@ -0,0 +1,40 @@
+resource "brightbox_server_group" "kubernetes" {
+  name = "Kubernetes server group"
+}
+
+resource "brightbox_firewall_policy" "kubernetes" {
+  name         = "Kubernetes firewall policy"
+  server_group = brightbox_server_group.kubernetes.id
+}
+
+resource "brightbox_firewall_rule" "kubernetes_api" {
+  destination_port = 6443
+  protocol         = "tcp"
+  source           = "any"
+  description      = "Kubernetes API access from anywhere"
+  firewall_policy  = brightbox_firewall_policy.kubernetes.id
+}
+
+resource "brightbox_firewall_rule" "ssh" {
+  destination_port = 22
+  protocol         = "tcp"
+  source           = "any"
+  description      = "SSH access from anywhere"
+  firewall_policy  = brightbox_firewall_policy.kubernetes.id
+}
+
+resource "brightbox_firewall_rule" "workers" {
+  protocol        = "tcp"
+  source          = data.brightbox_server_group.default.id
+  firewall_policy = brightbox_firewall_policy.kubernetes.id
+}
+
+resource "brightbox_firewall_rule" "internet" {
+  protocol        = "tcp"
+  destination     = "any"
+  firewall_policy = brightbox_firewall_policy.kubernetes.id
+}
+
+data "brightbox_server_group" "default" {
+  name = "^default$"
+}

brightbox/outputs.tf

@@ -0,0 +1,3 @@
+output "ipv4-control-plane" {
+  value = brightbox_cloudip.control-plane.public_ipv4
+}

brightbox/provider.tf

@@ -0,0 +1,23 @@
+terraform {
+  required_version = ">= 0.14.0"
+  required_providers {
+    brightbox = {
+      source  = "brightbox/brightbox"
+      version = "3.4.3"
+    }
+    ct = {
+      source  = "poseidon/ct"
+      version = "0.11.0"
+    }
+    template = {
+      source  = "hashicorp/template"
+      version = "~> 2.2.0"
+    }
+  }
+}
+
+provider "brightbox" {
+  apiclient = var.api_client
+  apisecret = var.api_secret
+}
+

brightbox/server-configs/control-plane.yaml.tmpl

@@ -0,0 +1,66 @@
+---
+version: 1.0.0
+variant: flatcar
+storage:
+  links:
+    - target: /opt/extensions/kubernetes/kubernetes-${kubernetes_version}-x86-64.raw
+      path: /etc/extensions/kubernetes.raw
+      hard: false
+  files:
+    - path: /etc/sysupdate.kubernetes.d/kubernetes.conf
+      contents:
+        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes.conf
+    - path: /etc/sysupdate.d/noop.conf
+      contents:
+        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
+    - path: /opt/extensions/kubernetes/kubernetes-${kubernetes_version}-x86-64.raw
+      contents:
+        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${kubernetes_version}-x86-64.raw
+systemd:
+  units:
+    - name: systemd-sysupdate.timer
+      enabled: true
+    - name: systemd-sysupdate.service
+      dropins:
+        - name: kubernetes.conf
+          contents: |
+            [Service]
+            ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
+            ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
+            ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
+            ExecStartPost=/usr/bin/sh -c "[[ $(cat /tmp/kubernetes) != $(cat /tmp/kubernetes-new) ]] && touch /run/reboot-required"
+    - name: kubeadm.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubeadm service
+        Requires=containerd.service
+        After=containerd.service coreos-metadata.service
+        Requires=coreos-metadata.service
+        ConditionPathExists=!/etc/kubernetes/kubelet.conf
+        [Service]
+        EnvironmentFile=/run/metadata/flatcar
+        ExecStartPre=/usr/bin/kubeadm init --control-plane-endpoint "$${COREOS_OPENSTACK_IPV4_PUBLIC}:6443"
+        ExecStartPre=/usr/bin/mkdir -p /home/core/.kube /var/www
+        ExecStartPre=/usr/bin/cp /etc/kubernetes/admin.conf /home/core/.kube/config
+        ExecStartPre=/usr/bin/cp /etc/kubernetes/admin.conf /var/www
+        ExecStartPre=/usr/bin/chmod a+r /var/www/admin.conf
+        ExecStart=/usr/bin/chown -R core:core /home/core/.kube
+        [Install]
+        WantedBy=multi-user.target
+    - name: nginx.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=NGINX
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker rm --force nginx1
+        ExecStart=/usr/bin/docker run --name nginx1 -p 8080:80 --volume "/var/www:/usr/share/nginx/html:ro" --pull always --log-driver=journald docker.io/nginx:1
+        ExecStop=/usr/bin/docker stop nginx1
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target

brightbox/server-configs/worker.yaml.tmpl

@@ -0,0 +1,47 @@
+---
+version: 1.0.0
+variant: flatcar
+storage:
+  links:
+    - target: /opt/extensions/kubernetes/kubernetes-${kubernetes_version}-x86-64.raw
+      path: /etc/extensions/kubernetes.raw
+      hard: false
+  files:
+    - path: /etc/sysupdate.kubernetes.d/kubernetes.conf
+      contents:
+        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes.conf
+    - path: /etc/sysupdate.d/noop.conf
+      contents:
+        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
+    - path: /opt/extensions/kubernetes/kubernetes-${kubernetes_version}-x86-64.raw
+      contents:
+        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${kubernetes_version}-x86-64.raw
+systemd:
+  units:
+    - name: systemd-sysupdate.timer
+      enabled: true
+    - name: systemd-sysupdate.service
+      dropins:
+        - name: kubernetes.conf
+          contents: |
+            [Service]
+            ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
+            ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
+            ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
+            ExecStartPost=/usr/bin/sh -c "[[ $(cat /tmp/kubernetes) != $(cat /tmp/kubernetes-new) ]] && touch /run/reboot-required"
+    - name: kubeadm.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubeadm service
+        Requires=containerd.service
+        After=containerd.service
+        ConditionPathExists=!/etc/kubernetes/kubelet.conf
+        [Service]
+        Restart=on-failure
+        StartLimitInterval=0
+        RestartSec=10
+        ExecStartPre=/usr/bin/curl -fsSL http://${control_plane_ip}:8080/admin.conf -o /tmp/admin.conf
+        ExecStart=/usr/bin/kubeadm join --discovery-file /tmp/admin.conf
+        [Install]
+        WantedBy=multi-user.target

brightbox/variables.tf

@@ -0,0 +1,56 @@
+variable "ssh_keys" {
+  type        = list(string)
+  default     = []
+  description = "Additional SSH public keys for user 'core'."
+}
+
+variable "release_channel" {
+  type        = string
+  description = "Release channel"
+  default     = "stable"
+
+  validation {
+    condition     = contains(["lts", "stable", "beta", "alpha"], var.release_channel)
+    error_message = "release_channel must be lts, stable, beta, or alpha."
+  }
+}
+
+variable "api_client" {
+  type        = string
+  description = "Brightbox API client"
+}
+
+variable "api_secret" {
+  type        = string
+  description = "Brightbox API secret"
+}
+
+variable "zone" {
+  type        = string
+  description = "Brightbox zone"
+  default     = "gb1-a"
+}
+
+variable "control_plane_type" {
+  type        = string
+  description = "Brightbox control plane instance type"
+  default     = "4gb.ssd"
+}
+
+variable "worker_type" {
+  type        = string
+  description = "Brightbox worker instance type"
+  default     = "1gb.ssd"
+}
+
+variable "kubernetes_version" {
+  type        = string
+  description = "Kubernetes version"
+  default     = "v1.29.2"
+}
+
+variable "workers" {
+  type        = number
+  description = "Number of workers"
+  default     = "3"
+}