Merge pull request #18 from alexzhc/main

Add "Job" mode and experiment DRBD v9.1.x
hwameistor · Sep 9, 2022 · c81a345 · c81a345
2 parents aaad620 + 2e2ab32
commit c81a345
Show file tree

Hide file tree

Showing 13 changed files with 344 additions and 59 deletions.
diff --git a/.github/workflows/release-drbd9-images.yml b/.github/workflows/release-drbd9-images.yml
@@ -5,10 +5,7 @@ on:
     branches: [ "main" ]
     paths:
     - 'docker-drbd9/**'
-  pull_request:
-    branches: [ "main" ]
-    paths:
-    - 'docker-drbd9/**'
+
 jobs:
 
   build:

diff --git a/.github/workflows/release-shipper-images.yaml b/.github/workflows/release-shipper-images.yaml
@@ -5,10 +5,7 @@ on:
     branches: [ "main" ]
     paths:
     - 'docker-shipper/*'
-  pull_request:
-    branches: [ "main" ]
-    paths:
-    - 'docker-shipper/*'
+
 jobs:
 
   build:

diff --git a/Makefile b/Makefile
@@ -1,6 +1,10 @@
-DRBD_VER ?= 9.0.32-1
-DRBD_UTILS_VER ?= 9.12.1
-DRBD_HEADERS_SHA ?= c757cf357edef67751b8f45a6ea894d287180087
+DRBD_VER ?= 9.0.32-1 # another tested value is: 9.1.11
+DRBD_UTILS_VER ?= 9.12.1 # another tested value is: 9.21.4
+
+# Pick a commit according to date from: https://github.com/LINBIT/drbd-headers/commits/master
+# For utils 9.21.4: fc45d779096ae5943ea3f56934a1f9b48ffb8e41
+DRBD_HEADERS_SHA ?= c757cf357edef67751b8f45a6ea894d287180087 # for utils 9.12.1
+
 KVER := $(shell uname -r)
 DIST ?= rhel7
 ENTRY ?= /pkgs/entrypoint.adapter.sh
@@ -9,7 +13,7 @@ REG ?= daocloud.io/daocloud # Test Registry
 
 drbd9:
 	 cd docker-drbd9 && \
-	 ./build.sh $(DRBD_VER)
+	 ./build.sh $(DRBD_VER) rhel7 rhel8 bionic focal jammy
 
 compiler-centos7:
 	cd docker-shipper && \

diff --git a/README.md b/README.md
@@ -7,25 +7,27 @@
 1. Adapt host OS type automatically;
 2. Drop drbd kernel modules and drbd-utils to the host;
 3. Use pre-built kernel modules for stock RHEL/CentOS hosts;
-4. Use dynamically built kernel modules for un-stock RHEL/CentOS hosts and Ubuntu hosts.
+4. Use dynamically built kernel modules for un-stock RHEL/CentOS hosts and Ubuntu hosts;
+5. Provide two deployment mode: Job (default) and DaemonSet
 
 ![flowchart](flowchart.drawio.svg)
+
 * Yellow: LINBIT's logic
 * Blue: DaoCloud's Logic
 
 ## Official DRBD Docker Images
 
 LINBIT/drbd <https://github.com/LINBIT/drbd/tree/drbd-9.1/docker>
 
+## DRBD Version
+
+* kernel module v9.1.8 with utils v9.12.1
+* (EXPERIMENTAL!) kernel module v9.1.11 with utils v9.21.4
+
 ## Arch Support
 
 * x86_64
 
-## DRBD Versions
-
-* DRBD: v9.0.32-1
-* DRBD Utils: v9.12.1
-
 ## OS Distro Support
 
 * RHEL/CentOS 7
@@ -36,28 +38,50 @@ LINBIT/drbd <https://github.com/LINBIT/drbd/tree/drbd-9.1/docker>
 
 ### Not Supported, but for test only
 
-* Ubuntu 22 Jammy (will install DRBD v9.1.10 )
+* Ubuntu 22 Jammy ( experimental: will always install DRBD v9.1.11 )
 
 ## Kubernetes Version
 
 * 18+
 
-## DRBD Version:
-* v9.1.8
-
 ## Guide
 
 ### Dependency
+
 For dynamically built kernels, the host must have kernel source installed.
-```
+
+```console
 # RHEL/CentOS
 $ yum install -y kernel-devel-$(uname -r)
 
 # Ubuntu
 $ apt-get install -y linux-headers-$(uname -r)
 ```
 
+> **Note:** 
+> 
+> For major releases of stock RHEL/CentOS 7 and 8, `kernel-devel` is not needed
+
+### OS Distros
+
+By default, OS distros are auto-detected by helm `lookup` function.
+
+However, in `DaemonSet` mode, it can be overridden by adding values to the array `distros: []` in `values.yaml`.
+
+**Distros that are not supported will be ignored.**
+
+For example:
+
+```yaml
+distros: 
+- rhel7
+- rhel8
+- bionic
+#- focal
+```
+
 ### Deploy by Helm Charts
+
 Deploy the below `DaemonSet`. It will bring up a pod on each kubernetes worker node to install DRBD modules and tools:
 
 ```console
@@ -79,22 +103,46 @@ $ helm install drbd-adapter ./drbd-adapter \
     --set registry=daocloud.io/daocloud
 ```
 
-### Set OS Distros
+#### Experiment DRBD v9.1.11
 
-By default, OS distros are auto-detected by helm `lookup` function.
+Only for Experiments!
 
-However, it can be overridden by adding values to the array `distros: []` in `values.yaml`.
+```console
+ $ helm install drbd-adapter ./drbd-adapter \
+    -n hwameistor --create-namespace \
+    --set imagePullPolicy=Always \
+    --set registry=daocloud.io/daocloud \
+    --set drbdVersion=v9.1.11
+```
 
-**Distros that are not supported will be ignored.**
+### Deployment Examples
 
-For example:
+#### Job
 
-```yaml
-distros: 
-- rhel7
-- rhel8
-- bionic
-#- focal
+Set `DeployKind: job` in `values.yaml`, which is the default:
+
+```console
+$ kubectl get po -l app=drbd-adapter -o wide
+NAME                                     READY   STATUS      RESTARTS   AGE   IP            NODE       
+drbd-adapter-k8s-worker-1-rhel7-fqpfg    0/2     Completed   0          36m   10.1.44.70    k8s-worker-1
+drbd-adapter-k8s-worker-2-rhel8-k45hp    0/2     Completed   0          36m   10.1.82.97    k8s-worker-2
+drbd-adapter-k8s-worker-3-bionic-rr7bv   0/2     Completed   0          36m   10.1.15.220   k8s-worker-3
+drbd-adapter-k8s-worker-4-focal-xcmnx    0/2     Completed   0          36m   10.1.57.106   k8s-worker-4
+drbd-adapter-k8s-worker-5-jammy-7xf4g    0/2     Completed   0          36m   10.1.42.42    k8s-worker-5
+```
+
+#### DaemonSet
+
+Set `DeployKind: daemonset` in `values.yaml`.
+
+```console
+$ kubectl -n hwameistor get po -l app=drbd-adapter -o wide
+NAME                 READY   STATUS    RESTARTS   AGE   IP            NODE        
+drbd-adapter-5w74s   1/1     Running   0          11m   10.6.254.23   k8s-worker-3
+drbd-adapter-7766x   1/1     Running   0          11m   10.6.254.21   k8s-worker-1
+drbd-adapter-cq52p   1/1     Running   0          11m   10.6.254.24   k8s-worker-4
+drbd-adapter-hlpvc   1/1     Running   0          11m   10.6.254.22   k8s-worker-2
+drbd-adapter-slm5z   1/1     Running   0          11m   10.6.254.25   k8s-worker-5
 ```
 
 ### Post-installation Check
@@ -108,5 +156,43 @@ $ modinfo drbd
 
 $ lsmod | grep drbd
 
-$ drbdadm --version
+$ drbdadm --ver
 ```
+
+### Cluster Expansion
+
+After expanding Kubernetes cluster, to install DRBD on new nodes
+
+#### DaemonSet
+
+`DaemonSet` will automatically expand to the new nodes unless `affinity` and `tolerations` forbid it.
+
+#### Job
+
+Charts need to be reapplied for the helm `lookup` function to identify the new nodes.
+
+```console
+$ helm upgrade drbd-adapter ./drbd-adapter -n hwameistor
+```
+
+### Deploy on Kubernetes master nodes
+
+By default, master nodes are avoided. To deploy on master nodes, modify `values.yaml` as below:
+
+#### DaemonSet
+
+Remove `nodeAffinity`
+
+#### Job
+
+Set `deployOnMaster: "no"`
+
+## Common Issues
+
+### 1. Pods get stuck at `ContainerCreating`
+
+Cause:
+    A possible cause is that `/etc/drbd.conf` and `/etc/centos-release` get created as directories instead of files. This is a Kubernetes problem. You may find from `kubectl describe pod` that those two paths fail to mount.
+
+Solution:
+    Delete those two directories on hosts, and then uninstall chart release and re-install it.
diff --git a/docker-drbd9/build.sh b/docker-drbd9/build.sh
@@ -8,7 +8,8 @@ sed -i "s/^ENV DRBD_VERSION.*/ENV DRBD_VERSION ${DRBD_VER}/" Dockerfile.*
 
 [ -f ./drbd.tar.gz ] || wget https://pkg.linbit.com/downloads/drbd/"$([[ $DRBD_VER =~ ^9.0 ]] && echo 9.0 || echo 9 )"/drbd-${DRBD_VER}.tar.gz -O ./drbd.tar.gz
 
-for i in rhel7 rhel8 bionic focal jammy; do
+shift
+for i in $@; do
     df="Dockerfile.${i}"
     [ -f $df ] && \
     docker build . -f "$df"\

diff --git a/docker-shipper/Dockerfile.shipper b/docker-shipper/Dockerfile.shipper
@@ -75,6 +75,6 @@ COPY --from=compiler-centos8 /pkgs/ /files/
 
 COPY . /files/
 
-RUN chmod -v +x /files/entrypoint.adapter.sh
+RUN chmod -v +x /files/entrypoint.adapter.sh /files/drbd.modules
 
 CMD mv -vf /files/* /pkgs/
diff --git a/docker-shipper/drbd.modules b/docker-shipper/drbd.modules
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for i in drbd drbd_transport_tcp; do
+    /sbin/modinfo $i && /sbin/modprobe $i
+done
diff --git a/docker-shipper/drbd.modules-load.conf b/docker-shipper/drbd.modules-load.conf
@@ -1,2 +1,2 @@
 drbd
-drbd_transport_tcp
+drbd_transport_tcp
diff --git a/docker-shipper/entrypoint.adapter.sh b/docker-shipper/entrypoint.adapter.sh
@@ -16,10 +16,11 @@ if [ -z $host_dist ] \
    host_dist=rhel8
 fi
 
-# Gracefully exit for distro mismatch, so that next initContainer may start
+# For DaemonSet: Gracefully exit for distro mismatch, so that next initContainer may start
+# For Job: Exit failure
 if [[ $host_dist != $image_dist ]]; then 
    echo "Image type does not match OS type, skip !" 
-   exit 0
+   [[ $LB_SKIP == 'yes' ]] && exit 0 || exit 1
 fi
 
 ## Unload current drbd modules from kernel if it is lower than the target version 
@@ -70,6 +71,7 @@ if [[ $LB_DROP == yes ]]; then
 
    # onboot load modules 
    cp -vf /pkgs/drbd.modules-load.conf /etc/modules-load.d/drbd.conf
+   cp -vf /pkgs/drbd.modules /etc/sysconfig/modules/
 
    # drop drbd utils and set up conf directories
    cp -vf /pkgs/utils/* /usr-local/bin/
@@ -81,3 +83,10 @@ if [[ $LB_DROP == yes ]]; then
       ln -svf /$i /usr-local/$i
    done 
 fi
+
+# Check if DRBD is loaded correctly
+if [[ $( cat /proc/drbd | awk '/^version/ {print $2}' ) == $DRBD_VERSION ]]; then
+   exit 0
+else 
+   exit 1
+fi
diff --git a/helm/drbd-adapter/Chart.yaml b/helm/drbd-adapter/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: drbd-adapter
-description: DRBD kernel loader that auto-adapt distros
+description: A DRBD kernel loader that auto-adapts OS distros
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.0
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to