Merge pull request #92 from PyratLabs/feature-systemd_documentation

systemd unit ordering + documentation
PyratLabs · Feb 16, 2021 · 05242ba · 05242ba
2 parents c80898d + c2348df
commit 05242ba
Show file tree

Hide file tree

Showing 57 changed files with 383 additions and 140 deletions.
diff --git a/.ansible-lint b/.ansible-lint
@@ -1,4 +1,4 @@
 ---
 
 skip_list:
-    - '106'
+  - role-name
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,19 @@
 ---
 -->
 
+## 2021-02-16, v2.6.0
+
+### Notable changes
+
+  - Tidy up of `when` params and `assert` tasks to be more readable.
+  - Added feature to tweak K3S service dependencies.
+  - Updated documentation:
+    - Node labels and component arguments
+    - systemd config
+    - Use alternate CNI (Calico example)
+
+---
+
 ## 2021-01-31, v2.5.3
 
 ### Notable changes

diff --git a/README.md b/README.md
@@ -71,13 +71,29 @@ consistency. These are generally cluster-level configuration.
 | `k3s_github_url`                 | Set the GitHub URL to install k3s from.                                         | https://github.com/k3s-io/k3s  |
 | `k3s_install_dir`                | Installation directory for k3s.                                                 | `/usr/local/bin`               |
 | `k3s_install_hard_links`         | Install using hard links rather than symbolic links.                            | `false`                        |
-| `k3s_start_on_boot`              | Start k3s on boot.                                                              | `true`                         |
 | `k3s_server_manifests_templates` | A list of Auto-Deploying Manifests Templates.                                   | []                             |
 | `k3s_use_experimental`           | Allow the use of experimental features in k3s.                                  | `false`                        |
 | `k3s_use_unsupported_config`     | Allow the use of unsupported configurations in k3s.                             | `false`                        |
 | `k3s_etcd_datastore`             | Enable etcd embedded datastore (read notes below).                              | `false`                        |
 | `k3s_debug`                      | Enable debug logging on the k3s service.                                        | `false`                        |
 
+### K3S Service Configuration
+
+The below variables change how and when the systemd service unit file for K3S
+is run. Use this with caution, please refer to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd.unit.html#%5BUnit%5D%20Section%20Options)
+for more information.
+
+| Variable               | Description                                                    | Default Value |
+|------------------------|----------------------------------------------------------------|---------------|
+| `k3s_start_on_boot`    | Start k3s on boot.                                             | `true`        |
+| `k3s_service_requires` | List of required systemd units to k3s service unit.            | []            |
+| `k3s_service_wants`    | List of "wanted" systemd unit to k3s (weaker than "requires"). | []\*          |
+| `k3s_service_before`   | Start k3s before a defined list of systemd units.              | []            |
+| `k3s_service_after`    | Start k3s after a defined list of systemd units.               | []\*          |
+
+\* The systemd unit template **always** specifies `network-online.target` for
+`wants` and `after`.
+
 ### Group/Host Variables
 
 Below are variables that are set against individual or groups of play hosts.

diff --git a/defaults/main.yml b/defaults/main.yml
@@ -35,9 +35,6 @@ k3s_install_dir: /usr/local/bin
 # Install using hard links rather than symbolic links
 k3s_install_hard_links: false
 
-# Start k3s on system boot
-k3s_start_on_boot: true
-
 # A list of templates used for preconfigure the cluster.
 k3s_server_manifests_templates: []
 
@@ -50,6 +47,24 @@ k3s_use_unsupported_config: false
 # Enable etcd embedded datastore
 k3s_etcd_datastore: false
 
+##
+# Systemd config
+##
+
+# Start k3s on system boot
+k3s_start_on_boot: true
+
+# List of required systemd units to k3s service unit.
+k3s_service_requires: []
+
+# List of "wanted" systemd unit to k3s (weaker than "requires").
+k3s_service_wants: []
+
+# Start k3s before a defined list of systemd units.
+k3s_service_before: []
+
+# Start k3s after a defined list of systemd units.
+k3s_service_after: []
 
 ##
 # Server Configuration

diff --git a/documentation/README.md b/documentation/README.md
@@ -31,6 +31,9 @@ minimum configuration.
 
   - [Setting up 2-node HA control plane with external datastore](configuration/2-node-ha-ext-datastore.md)
   - [Provision multiple standalone k3s nodes](configuration/multiple-standalone-k3s-nodes.md)
+  - [Set node labels and component arguments](configuration/node-labels-and-component-args.md)
+  - [Use an alternate CNI](configuration/use-an-alternate-cni.md)
+  - [Start K3S after another service](configuration/systemd-config.md)
 
 ### Operations
 

diff --git a/documentation/configuration/2-node-ha-ext-datastore.md b/documentation/configuration/2-node-ha-ext-datastore.md
@@ -64,6 +64,8 @@ https://rancher.com/docs/k3s/latest/en/installation/datastore/#datastore-endpoin
 
 k3s_server:
   datastore-endpoint: postgres://postgres:verybadpass@database:5432/postgres?sslmode=disable
+  node-taint:
+    - "k3s-controlplane=true:NoExecute"
 ```
 
 Your worker nodes need to know how to connect to the control plane, this is

diff --git a/documentation/configuration/node-labels-and-component-args.md b/documentation/configuration/node-labels-and-component-args.md
@@ -0,0 +1,39 @@
+# Configure node labels and component arguments
+
+The following command line arguments can be specified multiple times with
+`key=value` pairs:
+
+  - `--kube-kubelet-arg`
+  - `--kube-proxy-arg`
+  - `--kube-apiserver-arg`
+  - `--kube-scheduler-arg`
+  - `--kube-controller-manager-arg`
+  - `--kube-cloud-controller-manager-arg`
+  - `--node-label`
+  - `--node-taint`
+
+In the config file, this is done by defining a list of values for each
+command like argument, for example:
+
+```yaml
+---
+
+k3s_server:
+  # Set the plugins registry directory
+  kubelet-arg:
+    - "volume-plugin-dir=/var/lib/rancher/k3s/agent/kubelet/plugins_registry"
+  # Set the pod eviction timeout and node monitor grace period
+  kube-controller-manager-arg:
+    - "pod-eviction-timeout=2m"
+    - "node-monitor-grace-period=30s"
+  # Set API server feature gate
+  kube-apiserver-arg:
+    - "feature-gates=RemoveSelfLink=false"
+  # Laels to apply to a node
+  node-label:
+    - "NodeTier=development"
+    - "NodeLocation=eu-west-2a"
+  # Stop k3s control plane having workloads scheduled on them
+  node-taint:
+    - "k3s-controlplane=true:NoExecute"
+```
diff --git a/documentation/configuration/systemd-config.md b/documentation/configuration/systemd-config.md
@@ -0,0 +1,19 @@
+# systemd config
+
+Below are examples to tweak how and when K3S starts up.
+
+## Wanted service units
+
+In this example, we're going to start K3S after Wireguard. Our example server
+has a Wireguard connection `wg0`. We are using "wants" rather than "requires"
+as it's a weaker requirement that Wireguard must be running. We then want
+K3S to start after Wireguard has started.
+
+```yaml
+---
+
+k3s_service_wants:
+  - [email protected]
+k3s_service_after:
+  - [email protected]
+```
diff --git a/documentation/configuration/use-an-alternate-cni.md b/documentation/configuration/use-an-alternate-cni.md
@@ -0,0 +1,63 @@
+# Use an alternate CNI
+
+K3S ships with Flannel, however sometimes you want an different CNI such as
+Calico, Canal or Weave Net. To do this you will need to disable Flannel with
+`flannel-backend: "none"`, specify a `cluster-cidr` and add your CNI manifests
+to the `k3s_server_manifests_templates`.
+
+## Calico example
+
+The below is based on the
+[Calico quickstart documentation](https://docs.projectcalico.org/getting-started/kubernetes/quickstart).
+
+Steps:
+
+  1. Download `tigera-operator.yaml` to the manifests directory.
+  1. Download `custom-resources.yaml` to the manifests directory.
+  1. Choose a `cluster-cidr` (we are using 192.168.0.0/16)
+  1. Set `k3s_server` and `k3s_server_manifest_templates` as per the below,
+     ensure the paths to manifests are correct for your project repo.
+
+```yaml
+---
+
+# K3S Server config, don't deploy flannel and set cluster pod CIDR.
+k3s_server:
+  cluster-cidr: 192.168.0.0/16
+  flannel-backend: "none"
+
+# Deploy the following k3s server templates.
+k3s_server_manifests_templates:
+  - "manifests/calico/tigera-operator.yaml"
+  - "manifests/calico/custom-resources.yaml"
+```
+
+All nodes should come up as "Ready", below is a 3-node cluster:
+
+```text
+ $ kubectl get nodes -o wide -w
+NAME     STATUS   ROLES                       AGE    VERSION        INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION     CONTAINER-RUNTIME
+kube-0   Ready    control-plane,etcd,master   114s   v1.20.2+k3s1   10.10.9.2     10.10.9.2     Ubuntu 20.04.1 LTS   5.4.0-56-generic   containerd://1.4.3-k3s1
+kube-1   Ready    control-plane,etcd,master   80s    v1.20.2+k3s1   10.10.9.3     10.10.9.3     Ubuntu 20.04.1 LTS   5.4.0-56-generic   containerd://1.4.3-k3s1
+kube-2   Ready    control-plane,etcd,master   73s    v1.20.2+k3s1   10.10.9.4     10.10.9.4     Ubuntu 20.04.1 LTS   5.4.0-56-generic   containerd://1.4.3-k3s1
+```
+
+Pods should be deployed with deployed within the CIDR specified in our config
+file.
+
+```text
+$ kubectl get pods -o wide -A
+NAMESPACE         NAME                                      READY   STATUS      RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
+calico-system     calico-kube-controllers-cfb4ff54b-8rp8r   1/1     Running     0          5m4s    192.168.145.65   kube-0   <none>           <none>
+calico-system     calico-node-2cm2m                         1/1     Running     0          5m4s    10.10.9.2        kube-0   <none>           <none>
+calico-system     calico-node-2s6lx                         1/1     Running     0          4m42s   10.10.9.4        kube-2   <none>           <none>
+calico-system     calico-node-zwqjz                         1/1     Running     0          4m49s   10.10.9.3        kube-1   <none>           <none>
+calico-system     calico-typha-7b6747d665-78swq             1/1     Running     0          3m5s    10.10.9.4        kube-2   <none>           <none>
+calico-system     calico-typha-7b6747d665-8ff66             1/1     Running     0          3m5s    10.10.9.3        kube-1   <none>           <none>
+calico-system     calico-typha-7b6747d665-hgplx             1/1     Running     0          5m5s    10.10.9.2        kube-0   <none>           <none>
+kube-system       coredns-854c77959c-6qhgt                  1/1     Running     0          5m20s   192.168.145.66   kube-0   <none>           <none>
+kube-system       helm-install-traefik-4czr9                0/1     Completed   0          5m20s   192.168.145.67   kube-0   <none>           <none>
+kube-system       metrics-server-86cbb8457f-qcxf5           1/1     Running     0          5m20s   192.168.145.68   kube-0   <none>           <none>
+kube-system       traefik-6f9cbd9bd4-7h4rl                  1/1     Running     0          2m50s   192.168.126.65   kube-1   <none>           <none>
+tigera-operator   tigera-operator-b6c4bfdd9-29hhr           1/1     Running     0          5m20s   10.10.9.2        kube-0   <none>           <none>
+```
diff --git a/documentation/quickstart-cluster.md b/documentation/quickstart-cluster.md
@@ -86,7 +86,7 @@ Here is our playbook for the k3s cluster (`cluster.yml`):
   vars:
     k3s_become_for_all: true
   roles:
-    - xanmanning.k3s
+    - role: xanmanning.k3s
 ```
 
 ## Execution

diff --git a/documentation/quickstart-ha-cluster.md b/documentation/quickstart-ha-cluster.md
@@ -94,7 +94,7 @@ Here is our playbook for the k3s cluster (`ha_cluster.yml`):
     k3s_etcd_datastore: true
     k3s_use_experimental: true  # Note this is required for k3s < v1.19.5+k3s1
   roles:
-    - xanmanning.k3s
+    - role: xanmanning.k3s
 ```
 
 ## Execution

diff --git a/documentation/quickstart-single-node.md b/documentation/quickstart-single-node.md
@@ -68,7 +68,7 @@ Here is our playbook for a single node k3s cluster (`single_node.yml`):
   vars:
     k3s_become_for_all: true
   roles:
-    - xanmanning.k3s
+    - role: xanmanning.k3s
 ```
 
 ## Execution

diff --git a/molecule/autodeploy/converge.yml b/molecule/autodeploy/converge.yml
@@ -13,4 +13,4 @@
     k3s_server_manifests_templates:
       - "molecule/autodeploy/templates/00-ns-monitoring.yml.j2"
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/autodeploy/molecule.yml b/molecule/autodeploy/molecule.yml
@@ -7,7 +7,7 @@ driver:
 lint: |
   set -e
   yamllint -s .
-  ansible-lint
+  ansible-lint --exclude molecule/
 platforms:
   - name: node1
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest"

diff --git a/molecule/debug/molecule.yml b/molecule/debug/molecule.yml
@@ -7,7 +7,7 @@ driver:
 lint: |
   set -e
   yamllint -s .
-  ansible-lint
+  ansible-lint --exclude molecule/
 platforms:
   - name: node1
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest"

diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml
@@ -6,4 +6,4 @@
     molecule_is_test: true
     k3s_install_hard_links: true
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml
@@ -7,7 +7,7 @@ driver:
 lint: |
   set -e
   yamllint -s .
-  ansible-lint
+  ansible-lint --exclude molecule/
 platforms:
   - name: node1
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest"

diff --git a/molecule/default/playbook-download.yml b/molecule/default/playbook-download.yml
@@ -6,4 +6,4 @@
     molecule_is_test: true
     k3s_state: downloaded
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/playbook-restart-cluster.yml b/molecule/default/playbook-restart-cluster.yml
@@ -6,4 +6,4 @@
     molecule_is_test: true
     k3s_state: restarted
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/playbook-rootless.yml b/molecule/default/playbook-rootless.yml
@@ -12,4 +12,4 @@
       rootless: true
     k3s_install_dir: "/home/{{ ansible_user_id }}/bin"
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/playbook-standalone.yml b/molecule/default/playbook-standalone.yml
@@ -6,4 +6,4 @@
     molecule_is_test: true
     k3s_build_cluster: false
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/playbook-start-cluster.yml b/molecule/default/playbook-start-cluster.yml
@@ -6,4 +6,4 @@
     molecule_is_test: true
     k3s_state: started
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/playbook-stop-cluster.yml b/molecule/default/playbook-stop-cluster.yml
@@ -6,4 +6,4 @@
     molecule_is_test: true
     k3s_state: stopped
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/playbook-uninstall-cluster.yml b/molecule/default/playbook-uninstall-cluster.yml
@@ -6,4 +6,4 @@
     molecule_is_test: true
     k3s_state: uninstalled
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/docker/converge.yml b/molecule/docker/converge.yml
@@ -10,4 +10,4 @@
     k3s_agent:
       docker: true
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/docker/molecule.yml b/molecule/docker/molecule.yml
@@ -7,7 +7,7 @@ driver:
 lint: |
   set -e
   yamllint -s .
-  ansible-lint
+  ansible-lint --exclude molecule/
 platforms:
   - name: node1
     image: "${MOLECULE_DISTRO:-geerlingguy/docker-ubuntu2004-ansible:latest}"

diff --git a/molecule/highavailabilitydb/converge.yml b/molecule/highavailabilitydb/converge.yml
@@ -14,4 +14,4 @@
         k3s_control_node: true
       when: inventory_hostname in ['node2', 'node3']
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/highavailabilitydb/molecule.yml b/molecule/highavailabilitydb/molecule.yml
@@ -7,7 +7,7 @@ driver:
 lint: |
   set -e
   yamllint -s .
-  ansible-lint
+  ansible-lint --exclude molecule/
 platforms:
   - name: node1
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest"

diff --git a/molecule/highavailabilityetcd/converge.yml b/molecule/highavailabilityetcd/converge.yml
@@ -14,4 +14,4 @@
       ansible.builtin.set_fact:
         k3s_control_node: true
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/highavailabilityetcd/molecule.yml b/molecule/highavailabilityetcd/molecule.yml
@@ -7,7 +7,7 @@ driver:
 lint: |
   set -e
   yamllint -s .
-  ansible-lint
+  ansible-lint --exclude molecule/
 platforms:
   - name: node1
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest"

diff --git a/molecule/nodeploy/converge.yml b/molecule/nodeploy/converge.yml
@@ -7,4 +7,4 @@
     k3s_server: "{{ lookup('file', 'k3s_server.yml') | from_yaml }}"
     k3s_agent: "{{ lookup('file', 'k3s_agent.yml') | from_yaml }}"
   roles:
-    - role: xanmanning.k3s
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"