From 2ad1573ffc4a3d9a3886347562846febaae49648 Mon Sep 17 00:00:00 2001 From: alexzhc Date: Fri, 9 Sep 2022 15:00:07 +0800 Subject: [PATCH 1/2] Add deploy kind: job Signed-off-by: alexzhc --- .github/workflows/release-drbd9-images.yml | 5 +- .github/workflows/release-shipper-images.yaml | 5 +- Makefile | 12 +- README.md | 128 ++++++++++--- docker-drbd9/build.sh | 3 +- docker-shipper/Dockerfile.shipper | 2 +- docker-shipper/drbd.modules | 5 + docker-shipper/drbd.modules-load.conf | 2 +- docker-shipper/entrypoint.adapter.sh | 13 +- helm/drbd-adapter/Chart.yaml | 4 +- .../{drbd-adapter.yaml => daemonset.yaml} | 28 ++- helm/drbd-adapter/templates/job.yaml | 168 ++++++++++++++++++ helm/drbd-adapter/values.yaml | 24 ++- 13 files changed, 340 insertions(+), 59 deletions(-) create mode 100644 docker-shipper/drbd.modules rename helm/drbd-adapter/templates/{drbd-adapter.yaml => daemonset.yaml} (85%) create mode 100644 helm/drbd-adapter/templates/job.yaml diff --git a/.github/workflows/release-drbd9-images.yml b/.github/workflows/release-drbd9-images.yml index f29ab27..fe3e4fc 100644 --- a/.github/workflows/release-drbd9-images.yml +++ b/.github/workflows/release-drbd9-images.yml @@ -5,10 +5,7 @@ on: branches: [ "main" ] paths: - 'docker-drbd9/**' - pull_request: - branches: [ "main" ] - paths: - - 'docker-drbd9/**' + jobs: build: diff --git a/.github/workflows/release-shipper-images.yaml b/.github/workflows/release-shipper-images.yaml index 4e6ea2f..64fb66f 100644 --- a/.github/workflows/release-shipper-images.yaml +++ b/.github/workflows/release-shipper-images.yaml @@ -5,10 +5,7 @@ on: branches: [ "main" ] paths: - 'docker-shipper/*' - pull_request: - branches: [ "main" ] - paths: - - 'docker-shipper/*' + jobs: build: diff --git a/Makefile b/Makefile index 2777e7b..5e1f459 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ -DRBD_VER ?= 9.0.32-1 -DRBD_UTILS_VER ?= 9.12.1 -DRBD_HEADERS_SHA ?= c757cf357edef67751b8f45a6ea894d287180087 +DRBD_VER ?= 9.0.32-1 # another tested value is: 9.1.11 +DRBD_UTILS_VER ?= 9.12.1 # another tested value is: 9.21.4 + +# Pick a commit according to date from: https://github.com/LINBIT/drbd-headers/commits/master +# For utils 9.21.4: fc45d779096ae5943ea3f56934a1f9b48ffb8e41 +DRBD_HEADERS_SHA ?= c757cf357edef67751b8f45a6ea894d287180087 # for utils 9.12.1 + KVER := $(shell uname -r) DIST ?= rhel7 ENTRY ?= /pkgs/entrypoint.adapter.sh @@ -9,7 +13,7 @@ REG ?= daocloud.io/daocloud # Test Registry drbd9: cd docker-drbd9 && \ - ./build.sh $(DRBD_VER) + ./build.sh $(DRBD_VER) rhel7 rhel8 bionic focal jammy compiler-centos7: cd docker-shipper && \ diff --git a/README.md b/README.md index 945e366..b632df2 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,11 @@ 1. Adapt host OS type automatically; 2. Drop drbd kernel modules and drbd-utils to the host; 3. Use pre-built kernel modules for stock RHEL/CentOS hosts; -4. Use dynamically built kernel modules for un-stock RHEL/CentOS hosts and Ubuntu hosts. +4. Use dynamically built kernel modules for un-stock RHEL/CentOS hosts and Ubuntu hosts; +5. Provide two deployment mode: Job (default) and DaemonSet ![flowchart](flowchart.drawio.svg) + * Yellow: LINBIT's logic * Blue: DaoCloud's Logic @@ -17,15 +19,15 @@ LINBIT/drbd +## DRBD Version + +* kernel module v9.1.8 with utils v9.12.1 +* (EXPERIMENTAL!) kernel module v9.1.11 with utils v9.21.4 + ## Arch Support * x86_64 -## DRBD Versions - -* DRBD: v9.0.32-1 -* DRBD Utils: v9.12.1 - ## OS Distro Support * RHEL/CentOS 7 @@ -36,20 +38,19 @@ LINBIT/drbd ### Not Supported, but for test only -* Ubuntu 22 Jammy (will install DRBD v9.1.10 ) +* Ubuntu 22 Jammy ( experimental: will always install DRBD v9.1.11 ) ## Kubernetes Version * 18+ -## DRBD Version: -* v9.1.8 - ## Guide ### Dependency + For dynamically built kernels, the host must have kernel source installed. -``` + +```console # RHEL/CentOS $ yum install -y kernel-devel-$(uname -r) @@ -57,7 +58,26 @@ $ yum install -y kernel-devel-$(uname -r) $ apt-get install -y linux-headers-$(uname -r) ``` +### OS Distros + +By default, OS distros are auto-detected by helm `lookup` function. + +However, in `DaemonSet` mode, it can be overridden by adding values to the array `distros: []` in `values.yaml`. + +**Distros that are not supported will be ignored.** + +For example: + +```yaml +distros: +- rhel7 +- rhel8 +- bionic +#- focal +``` + ### Deploy by Helm Charts + Deploy the below `DaemonSet`. It will bring up a pod on each kubernetes worker node to install DRBD modules and tools: ```console @@ -79,22 +99,46 @@ $ helm install drbd-adapter ./drbd-adapter \ --set registry=daocloud.io/daocloud ``` -### Set OS Distros +#### Experiment DRBD v9.1.11 -By default, OS distros are auto-detected by helm `lookup` function. +Only for Experiments! -However, it can be overridden by adding values to the array `distros: []` in `values.yaml`. +```console + $ helm install drbd-adapter ./drbd-adapter \ + -n hwameistor --create-namespace \ + --set imagePullPolicy=Always \ + --set registry=daocloud.io/daocloud \ + --set drbdVersion=v9.1.11 +``` -**Distros that are not supported will be ignored.** +### Deployment Examples -For example: +#### Job -```yaml -distros: -- rhel7 -- rhel8 -- bionic -#- focal +Set `DeployKind: job` in `values.yaml`, which is the default: + +```console +$ kubectl get po -l app=drbd-adapter -o wide +NAME READY STATUS RESTARTS AGE IP NODE +drbd-adapter-k8s-worker-1-rhel7-fqpfg 0/2 Completed 0 36m 10.1.44.70 k8s-worker-1 +drbd-adapter-k8s-worker-2-rhel8-k45hp 0/2 Completed 0 36m 10.1.82.97 k8s-worker-2 +drbd-adapter-k8s-worker-3-bionic-rr7bv 0/2 Completed 0 36m 10.1.15.220 k8s-worker-3 +drbd-adapter-k8s-worker-4-focal-xcmnx 0/2 Completed 0 36m 10.1.57.106 k8s-worker-4 +drbd-adapter-k8s-worker-5-jammy-7xf4g 0/2 Completed 0 36m 10.1.42.42 k8s-worker-5 +``` + +#### DaemonSet + +Set `DeployKind: daemonset` in `values.yaml`. + +```console +$ kubectl -n hwameistor get po -l app=drbd-adapter -o wide +NAME READY STATUS RESTARTS AGE IP NODE +drbd-adapter-5w74s 1/1 Running 0 11m 10.6.254.23 k8s-worker-3 +drbd-adapter-7766x 1/1 Running 0 11m 10.6.254.21 k8s-worker-1 +drbd-adapter-cq52p 1/1 Running 0 11m 10.6.254.24 k8s-worker-4 +drbd-adapter-hlpvc 1/1 Running 0 11m 10.6.254.22 k8s-worker-2 +drbd-adapter-slm5z 1/1 Running 0 11m 10.6.254.25 k8s-worker-5 ``` ### Post-installation Check @@ -108,5 +152,43 @@ $ modinfo drbd $ lsmod | grep drbd -$ drbdadm --version +$ drbdadm --ver +``` + +### Cluster Expansion + +After expanding Kubernetes cluster, to install DRBD on new nodes + +#### DaemonSet + +`DaemonSet` will automatically expand to the new nodes unless `affinity` and `tolerations` forbid it. + +#### Job + +Charts need to be reapplied for the helm `lookup` function to identify the new nodes. + +```console +$ helm upgrade drbd-adapter ./drbd-adapter -n hwameistor ``` + +### Deploy on Kubernetes master nodes + +By default, master nodes are avoided. To deploy on master nodes, modify `values.yaml` as below: + +#### DaemonSet + +Remove `nodeAffinity` + +#### Job + +Set `deployOnMaster: "no"` + +## Common Issues + +### 1. Pods get stuck at `ContainerCreating` + +Cause: + A possible cause is that `/etc/drbd.conf` and `/etc/centos-release` get created as directories instead of files. This is a Kubernetes problem. You may find from `kubectl describe pod` that those two paths fail to mount. + +Solution: + Delete those two directories on hosts, and then uninstall chart release and re-install it. diff --git a/docker-drbd9/build.sh b/docker-drbd9/build.sh index 77ed533..eeb396b 100755 --- a/docker-drbd9/build.sh +++ b/docker-drbd9/build.sh @@ -8,7 +8,8 @@ sed -i "s/^ENV DRBD_VERSION.*/ENV DRBD_VERSION ${DRBD_VER}/" Dockerfile.* [ -f ./drbd.tar.gz ] || wget https://pkg.linbit.com/downloads/drbd/"$([[ $DRBD_VER =~ ^9.0 ]] && echo 9.0 || echo 9 )"/drbd-${DRBD_VER}.tar.gz -O ./drbd.tar.gz -for i in rhel7 rhel8 bionic focal jammy; do +shift +for i in $@; do df="Dockerfile.${i}" [ -f $df ] && \ docker build . -f "$df"\ diff --git a/docker-shipper/Dockerfile.shipper b/docker-shipper/Dockerfile.shipper index 91f65db..5fbfc70 100644 --- a/docker-shipper/Dockerfile.shipper +++ b/docker-shipper/Dockerfile.shipper @@ -75,6 +75,6 @@ COPY --from=compiler-centos8 /pkgs/ /files/ COPY . /files/ -RUN chmod -v +x /files/entrypoint.adapter.sh +RUN chmod -v +x /files/entrypoint.adapter.sh /files/drbd.modules CMD mv -vf /files/* /pkgs/ diff --git a/docker-shipper/drbd.modules b/docker-shipper/drbd.modules new file mode 100644 index 0000000..e10bfdd --- /dev/null +++ b/docker-shipper/drbd.modules @@ -0,0 +1,5 @@ +#!/bin/sh + +for i in drbd drbd_transport_tcp; do + /sbin/modinfo $i && /sbin/modprobe $i +done diff --git a/docker-shipper/drbd.modules-load.conf b/docker-shipper/drbd.modules-load.conf index 069f6c2..6a43e68 100644 --- a/docker-shipper/drbd.modules-load.conf +++ b/docker-shipper/drbd.modules-load.conf @@ -1,2 +1,2 @@ drbd -drbd_transport_tcp \ No newline at end of file +drbd_transport_tcp diff --git a/docker-shipper/entrypoint.adapter.sh b/docker-shipper/entrypoint.adapter.sh index 5b63afe..fc26aa7 100755 --- a/docker-shipper/entrypoint.adapter.sh +++ b/docker-shipper/entrypoint.adapter.sh @@ -16,10 +16,11 @@ if [ -z $host_dist ] \ host_dist=rhel8 fi -# Gracefully exit for distro mismatch, so that next initContainer may start +# For DaemonSet: Gracefully exit for distro mismatch, so that next initContainer may start +# For Job: Exit failure if [[ $host_dist != $image_dist ]]; then echo "Image type does not match OS type, skip !" - exit 0 + [[ $LB_SKIP == 'yes' ]] && exit 0 || exit 1 fi ## Unload current drbd modules from kernel if it is lower than the target version @@ -70,6 +71,7 @@ if [[ $LB_DROP == yes ]]; then # onboot load modules cp -vf /pkgs/drbd.modules-load.conf /etc/modules-load.d/drbd.conf + cp -vf /pkgs/drbd.modules /etc/sysconfig/modules/ # drop drbd utils and set up conf directories cp -vf /pkgs/utils/* /usr-local/bin/ @@ -81,3 +83,10 @@ if [[ $LB_DROP == yes ]]; then ln -svf /$i /usr-local/$i done fi + +# Check if DRBD is loaded correctly +if [[ $( cat /proc/drbd | awk '/^version/ {print $2}' ) == $DRBD_VERSION ]]; then + exit 0 +else + exit 1 +fi diff --git a/helm/drbd-adapter/Chart.yaml b/helm/drbd-adapter/Chart.yaml index 19232a9..e2ebcf4 100644 --- a/helm/drbd-adapter/Chart.yaml +++ b/helm/drbd-adapter/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: drbd-adapter -description: DRBD kernel loader that auto-adapt distros +description: A DRBD kernel loader that auto-adapts OS distros # A chart can be either an 'application' or a 'library' chart. # @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/drbd-adapter/templates/drbd-adapter.yaml b/helm/drbd-adapter/templates/daemonset.yaml similarity index 85% rename from helm/drbd-adapter/templates/drbd-adapter.yaml rename to helm/drbd-adapter/templates/daemonset.yaml index 685174a..85487b8 100644 --- a/helm/drbd-adapter/templates/drbd-adapter.yaml +++ b/helm/drbd-adapter/templates/daemonset.yaml @@ -1,17 +1,20 @@ +{{ if eq $.Values.deployKind "daemonset" }} apiVersion: apps/v1 kind: DaemonSet metadata: - name: drbd-adapter + name: {{ $.Values.prefix }}drbd-adapter labels: - k8s-app: drbd-adapter + app: {{ $.Values.prefix }}drbd-adapter + drbd-version: {{ $.Values.drbdVersion }} spec: selector: matchLabels: - k8s-app: drbd-adapter + app: {{ $.Values.prefix }}drbd-adapter template: metadata: labels: - k8s-app: drbd-adapter + app: {{ $.Values.prefix }}drbd-adapter + drbd-version: {{ $.Values.drbdVersion }} spec: initContainers: - name: shipper @@ -21,8 +24,7 @@ spec: - name: pkgs mountPath: /pkgs {{- $distros := $.Values.distros }} - {{- if $distros -}} - {{ else }} + {{- if not $distros -}} {{- $distros = append $distros "flatcar" }} {{- range $index, $node := (lookup "v1" "Node" "" "").items }} {{- $osImage := lower $node.status.nodeInfo.osImage }} @@ -45,7 +47,7 @@ spec: {{- if regexMatch "^(rhel[78]|bionic|focal|jammy)$" $dist }} {{- $tag := $.Values.drbdVersion }} {{- if eq $dist "jammy" }} - {{- $tag = "v9.1.10" }} + {{- $tag = "v9.1.11" }} {{- end }} - name: {{ $dist }} image: {{ $.Values.registry }}/drbd9-{{ $dist }}:{{ $tag }} @@ -54,6 +56,8 @@ spec: securityContext: privileged: true env: + - name: LB_SKIP + value: 'yes' - name: LB_DROP value: {{ $.Values.drop | quote }} - name: LB_UPGRADE @@ -85,13 +89,15 @@ spec: readOnly: true - name: etc-modules-load mountPath: /etc/modules-load.d + - name: etc-sysconfig-modules + mountPath: /etc/sysconfig/modules {{- end }} {{- end }} containers: - name: monitor # Source registry: quay.io/piraeusdatastore/drbd-reactor image: {{ $.Values.registry}}/drbd-reactor:{{ $.Values.drbdReactorVersion}} - imagePullPolicy: {{ .Values.imagePullPolicy}} + imagePullPolicy: {{ $.Values.imagePullPolicy}} {{- if regexMatch "^9.1." $.Values.drbdVersion }} # DRBD Reactor only supports DRBD 9.1.x command: - /usr/sbin/drbd-reactor @@ -142,6 +148,12 @@ spec: - name: etc-modules-load hostPath: path: /etc/modules-load.d + type: DirectoryOrCreate + - name: etc-sysconfig-modules + hostPath: + path: /etc/sysconfig/modules + type: DirectoryOrCreate affinity: nodeAffinity: {{- toYaml $.Values.affinity.nodeAffinity | nindent 10 }} +{{- end }} \ No newline at end of file diff --git a/helm/drbd-adapter/templates/job.yaml b/helm/drbd-adapter/templates/job.yaml new file mode 100644 index 0000000..88c1279 --- /dev/null +++ b/helm/drbd-adapter/templates/job.yaml @@ -0,0 +1,168 @@ +{{ if eq $.Values.deployKind "job" }} +{{ range $index, $node := (lookup "v1" "Node" "" "").items }} +{{ $nodeName := $node.metadata.name }} +{{ $osImage := lower $node.status.nodeInfo.osImage }} +{{ $tag := $.Values.drbdVersion }} +{{ $distro := "unsupported" }} +{{ if regexMatch "(red hat enterprise|centos) .*7" $osImage }} + {{ $distro = "rhel7" }} +{{ else if regexMatch "(red hat enterprise|centos) .*8" $osImage }} + {{ $distro = "rhel8" }} +{{ else if regexMatch "ubuntu .*18" $osImage }} + {{ $distro = "bionic" }} +{{ else if regexMatch "ubuntu .*20" $osImage }} + {{ $distro = "focal" }} +{{ else if regexMatch "ubuntu .*22" $osImage }} + {{ $distro = "jammy" }} + {{ $tag = "v9.1.11" }} +{{ end }} +{{ if and (ne $.Values.deployOnMaster "yes") (or (hasKey $node.metadata.labels "node-role.kubernetes.io/master") (hasKey $node.metadata.labels "node-role.kubernetes.io/control-plane"))}} +{{ else }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: drbd-adapter-{{ $nodeName }}-{{ $distro }} + labels: + app: drbd-adapter + drbd-version: {{ $.Values.drbdVersion }} +spec: + ttlSecondsAfterFinished: 3600 + backoffLimit: 0 + template: + metadata: + labels: + app: drbd-adapter + drbd-version: {{ $.Values.drbdVersion }} + spec: + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: {{ $nodeName }} + containers: + - name: shipper + image: {{ $.Values.registry }}/drbd9-shipper:{{ $tag }} + imagePullPolicy: {{ $.Values.imagePullPolicy }} + volumeMounts: + - name: pkgs + mountPath: /pkgs + {{ if ne $distro "unsupported" }} + - name: {{ $distro }} + image: {{ $.Values.registry }}/drbd9-{{ $distro }}:{{ $tag }} + imagePullPolicy: {{ $.Values.imagePullPolicy }} + command: [ /pkgs/entrypoint.adapter.sh ] + securityContext: + privileged: true + env: + - name: LB_SKIP + value: 'no' + - name: LB_DROP + value: 'yes' + - name: LB_UPGRADE + value: {{ $.Values.upgrade | quote }} + volumeMounts: + {{ if regexMatch "^rhel[78]$" $distro }} + - name: centos-release + mountPath: /etc/centos-release + readOnly: true + {{ end }} + - name: pkgs + mountPath: /pkgs + - name: os-release + mountPath: /etc/host-release + readOnly: true + - name: usr-src + mountPath: /usr/src + readOnly: true + - name: lib-modules + mountPath: /lib/modules + - name: usr-local + mountPath: /usr-local + - name: etc-drbd-conf + mountPath: /etc/drbd.conf + - name: etc-drbd-d + mountPath: /etc/drbd.d + - name: var-lib-drbd + mountPath: /var/lib/drbd + readOnly: true + - name: etc-modules-load + mountPath: /etc/modules-load.d + - name: etc-sysconfig-modules + mountPath: /etc/sysconfig/modules + {{ end }} + volumes: + - name: pkgs + emptyDir: {} + - name: os-release + hostPath: + path: /etc/os-release + - name: centos-release + hostPath: + path: /etc/centos-release + - name: usr-src + hostPath: + path: /usr/src + - name: lib-modules + hostPath: + path: /lib/modules + - name: usr-local + hostPath: + path: /usr/local + - name: etc-drbd-conf + hostPath: + path: /etc/drbd.conf + type: FileOrCreate + - name: etc-drbd-d + hostPath: + path: /etc/drbd.d + type: DirectoryOrCreate + - name: var-lib-drbd + hostPath: + path: /var/lib/drbd + type: DirectoryOrCreate + - name: etc-modules-load + hostPath: + path: /etc/modules-load.d + volumes: + - name: pkgs + emptyDir: {} + - name: os-release + hostPath: + path: /etc/os-release + - name: centos-release + hostPath: + path: /etc/centos-release + - name: usr-src + hostPath: + path: /usr/src + - name: lib-modules + hostPath: + path: /lib/modules + - name: usr-local + hostPath: + path: /usr/local + - name: etc-drbd-conf + hostPath: + path: /etc/drbd.conf + type: FileOrCreate + - name: etc-drbd-d + hostPath: + path: /etc/drbd.d + type: DirectoryOrCreate + - name: var-lib-drbd + hostPath: + path: /var/lib/drbd + type: DirectoryOrCreate + - name: etc-modules-load + hostPath: + path: /etc/modules-load.d + type: DirectoryOrCreate + - name: etc-sysconfig-modules + hostPath: + path: /etc/sysconfig/modules + type: DirectoryOrCreate + affinity: + nodeAffinity: + {{- toYaml $.Values.affinity.nodeAffinity | nindent 10 }} +{{ end }} +{{ end }} +{{ end }} \ No newline at end of file diff --git a/helm/drbd-adapter/values.yaml b/helm/drbd-adapter/values.yaml index 2cac597..392b5f6 100644 --- a/helm/drbd-adapter/values.yaml +++ b/helm/drbd-adapter/values.yaml @@ -1,11 +1,12 @@ -registry: ghcr.io/hwameistor +deployKind: job # Deployment mode: job|daemonset +prefix: "" # prefix to the pod name, such as "hwameistor-" +registry: ghcr.io/hwameistor # Users in China may use mirror "daocloud.io/daocloud" imagePullPolicy: IfNotPresent # For testing, set 'Always' -drbdVersion: v9.0.32-1 -drbdUtilsVersion: v9.12.1 +drbdVersion: v9.0.32-1 # another supported value is: 9.1.11 drbdReactorVersion: v0.8.0 -# Here are all supported distros, remove the ones that are not in your cluster +# Here are all supported distros, remove the ones that are not in your cluster distros: [] # Must be set as "[]" (empty array) to enable AutoDetect" # - rhel7 # - rhel8 @@ -13,13 +14,18 @@ distros: [] # Must be set as "[]" (empty array) to enable AutoDetect" # - focal # - jammy -# by default drop binary files to the host -drop: "yes" - -# by default upgrade if possible +# by default upgrade if possible: (yes|no) upgrade: "no" -# Aovid master nodes +# only apply to "deployKind: job" +# by default NOT deploying on masters (yes|no) +deployOnMaster: "no" + +# only apply to "deployKind: daemonset" +# by default drop binary files to the host: (yes|no), +drop: "yes" + +# avoid master nodes, you may defind more here to avoid more nodes affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: From 2e2ab32b4919af99b783fc4aacc8d344b6600ecc Mon Sep 17 00:00:00 2001 From: alexzhc Date: Fri, 9 Sep 2022 15:06:15 +0800 Subject: [PATCH 2/2] updated readme Signed-off-by: alexzhc --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b632df2..7b70940 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,10 @@ $ yum install -y kernel-devel-$(uname -r) $ apt-get install -y linux-headers-$(uname -r) ``` +> **Note:** +> +> For major releases of stock RHEL/CentOS 7 and 8, `kernel-devel` is not needed + ### OS Distros By default, OS distros are auto-detected by helm `lookup` function.