✨ Add sentinel file to signal successful bootstrapping

kubernetes-sigs · Jan 20, 2021 · cd3f067 · cd3f067
1 parent c615219
commit cd3f067
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 11 deletions.
diff --git a/bootstrap/kubeadm/internal/cloudinit/cloudinit.go b/bootstrap/kubeadm/internal/cloudinit/cloudinit.go
@@ -26,7 +26,10 @@ import (
 )
 
 const (
-	standardJoinCommand            = "kubeadm join --config /run/kubeadm/kubeadm-join-config.yaml %s"
+	standardJoinCommand = "kubeadm join --config /run/kubeadm/kubeadm-join-config.yaml %s"
+	// sentinelFileCommand writes a file to /run/cluster-api to signal successful Kubernetes bootstrapping in a way that
+	// works both for Linux and Windows OS.
+	sentinelFileCommand            = "echo success > /run/cluster-api/bootstrap-success.complete"
 	retriableJoinScriptName        = "/usr/local/bin/kubeadm-bootstrap-script"
 	retriableJoinScriptOwner       = "root"
 	retriableJoinScriptPermissions = "0755"
@@ -50,6 +53,7 @@ type BaseUserData struct {
 	UseExperimentalRetry bool
 	KubeadmCommand       string
 	KubeadmVerbosity     string
+	SentinelFileCommand  string
 }
 
 func (input *BaseUserData) prepare() error {
@@ -64,6 +68,7 @@ func (input *BaseUserData) prepare() error {
 		}
 		input.WriteFiles = append(input.WriteFiles, *joinScriptFile)
 	}
+	input.SentinelFileCommand = sentinelFileCommand
 	return nil
 }
 

diff --git a/bootstrap/kubeadm/internal/cloudinit/controlplane_init.go b/bootstrap/kubeadm/internal/cloudinit/controlplane_init.go
@@ -31,9 +31,13 @@ const (
 {{.ClusterConfiguration | Indent 6}}
       ---
 {{.InitConfiguration | Indent 6}}
+-   path: /run/cluster-api/placeholder
+    owner: root:root
+    permissions: '0640'
+    content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)"
 runcmd:
 {{- template "commands" .PreKubeadmCommands }}
-  - 'kubeadm init --config /run/kubeadm/kubeadm.yaml {{.KubeadmVerbosity}}'
+  - 'kubeadm init --config /run/kubeadm/kubeadm.yaml {{.KubeadmVerbosity}} && {{ .SentinelFileCommand }}'
 {{- template "commands" .PostKubeadmCommands }}
 {{- template "ntp" .NTP }}
 {{- template "users" .Users }}
@@ -57,6 +61,7 @@ func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) {
 	input.Header = cloudConfigHeader
 	input.WriteFiles = input.Certificates.AsFiles()
 	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
+	input.SentinelFileCommand = sentinelFileCommand
 	userData, err := generate("InitControlplane", controlPlaneCloudInit, input)
 	if err != nil {
 		return nil, err

diff --git a/bootstrap/kubeadm/internal/cloudinit/controlplane_join.go b/bootstrap/kubeadm/internal/cloudinit/controlplane_join.go
@@ -29,9 +29,13 @@ const (
     permissions: '0640'
     content: |
 {{.JoinConfiguration | Indent 6}}
+-   path: /run/cluster-api/placeholder
+    owner: root:root
+    permissions: '0640'
+    content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)"
 runcmd:
 {{- template "commands" .PreKubeadmCommands }}
-  - {{ .KubeadmCommand }}
+  - {{ .KubeadmCommand }} && {{ .SentinelFileCommand }}
 {{- template "commands" .PostKubeadmCommands }}
 {{- template "ntp" .NTP }}
 {{- template "users" .Users }}

diff --git a/bootstrap/kubeadm/internal/cloudinit/node.go b/bootstrap/kubeadm/internal/cloudinit/node.go
@@ -25,9 +25,13 @@ const (
     content: |
       ---
 {{.JoinConfiguration | Indent 6}}
+-   path: /run/cluster-api/placeholder
+    owner: root:root
+    permissions: '0640'
+    content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)"
 runcmd:
 {{- template "commands" .PreKubeadmCommands }}
-  - {{ .KubeadmCommand }}
+  - {{ .KubeadmCommand }} && {{ .SentinelFileCommand }}
 {{- template "commands" .PostKubeadmCommands }}
 {{- template "ntp" .NTP }}
 {{- template "users" .Users }}

diff --git a/docs/book/src/developer/providers/bootstrap.md b/docs/book/src/developer/providers/bootstrap.md
@@ -59,6 +59,10 @@ The following diagram shows the typical logic for a bootstrap provider:
 1. Set `status.ready` to true
 1. Patch the resource to persist changes
 
+## Sentinel File
+
+A bootstrap provider's bootstrap data must create `/run/cluster-api/bootstrap-success.complete` (or `C:\run\cluster-api\bootstrap-success.complete` for Windows machines) upon successful bootstrapping of a Kubernetes node. This allows infrastructure providers to detect and act on bootstrap failures.
+
 ## RBAC
 
 ### Provider controller

diff --git a/test/infrastructure/docker/cloudinit/runcmd.go b/test/infrastructure/docker/cloudinit/runcmd.go
@@ -18,7 +18,6 @@ package cloudinit
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -92,15 +91,18 @@ func (a *runCmd) Commands() ([]Cmd, error) {
 func hackKubeadmIgnoreErrors(c Cmd) Cmd {
 	// case kubeadm commands are defined as a string
 	if c.Cmd == "/bin/sh" && len(c.Args) >= 2 {
-		if c.Args[0] == "-c" && (strings.Contains(c.Args[1], "kubeadm init") || strings.Contains(c.Args[1], "kubeadm join")) {
-			c.Args[1] = fmt.Sprintf("%s %s", c.Args[1], "--ignore-preflight-errors=all")
+		if c.Args[0] == "-c" {
+			c.Args[1] = strings.Replace(c.Args[1], "kubeadm init", "kubeadm init --ignore-preflight-errors=all", 1)
+			c.Args[1] = strings.Replace(c.Args[1], "kubeadm join", "kubeadm join --ignore-preflight-errors=all", 1)
 		}
 	}
 
 	// case kubeadm commands are defined as a list
 	if c.Cmd == "kubeadm" && len(c.Args) >= 1 {
 		if c.Args[0] == "init" || c.Args[0] == "join" {
-			c.Args = append(c.Args, "--ignore-preflight-errors=all")
+			c.Args = append(c.Args, "")                 // make space
+			copy(c.Args[2:], c.Args[1:])                // shift elements
+			c.Args[1] = "--ignore-preflight-errors=all" // insert the additional arg
 		}
 	}
 

diff --git a/test/infrastructure/docker/cloudinit/runcmd_test.go b/test/infrastructure/docker/cloudinit/runcmd_test.go
@@ -68,7 +68,7 @@ func TestRunCmdRun(t *testing.T) {
 				},
 			},
 			expectedCmds: []Cmd{
-				{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --config /run/kubeadm/kubeadm.yaml --ignore-preflight-errors=all"}},
+				{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config /run/kubeadm/kubeadm.yaml"}},
 			},
 		},
 	}
@@ -98,11 +98,11 @@ runcmd:
 
 	r.Cmds[0] = hackKubeadmIgnoreErrors(r.Cmds[0])
 
-	expected0 := Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --config=/run/kubeadm/kubeadm.yaml --ignore-preflight-errors=all"}}
+	expected0 := Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config=/run/kubeadm/kubeadm.yaml"}}
 	g.Expect(r.Cmds[0]).To(Equal(expected0))
 
 	r.Cmds[1] = hackKubeadmIgnoreErrors(r.Cmds[1])
 
-	expected1 := Cmd{Cmd: "kubeadm", Args: []string{"join", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml", "--ignore-preflight-errors=all"}}
+	expected1 := Cmd{Cmd: "kubeadm", Args: []string{"join", "--ignore-preflight-errors=all", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml"}}
 	g.Expect(r.Cmds[1]).To(Equal(expected1))
 }