diff --git a/bootstrap/kubeadm/internal/cloudinit/cloudinit.go b/bootstrap/kubeadm/internal/cloudinit/cloudinit.go index dae81bee6eb3..73923a1e7858 100644 --- a/bootstrap/kubeadm/internal/cloudinit/cloudinit.go +++ b/bootstrap/kubeadm/internal/cloudinit/cloudinit.go @@ -26,7 +26,10 @@ import ( ) const ( - standardJoinCommand = "kubeadm join --config /run/kubeadm/kubeadm-join-config.yaml %s" + standardJoinCommand = "kubeadm join --config /run/kubeadm/kubeadm-join-config.yaml %s" + // sentinelFileCommand writes a file to /run/cluster-api to signal successful Kubernetes bootstrapping in a way that + // works both for Linux and Windows OS. + sentinelFileCommand = "echo success > /run/cluster-api/bootstrap-success.complete" retriableJoinScriptName = "/usr/local/bin/kubeadm-bootstrap-script" retriableJoinScriptOwner = "root" retriableJoinScriptPermissions = "0755" @@ -50,6 +53,7 @@ type BaseUserData struct { UseExperimentalRetry bool KubeadmCommand string KubeadmVerbosity string + SentinelFileCommand string } func (input *BaseUserData) prepare() error { @@ -64,6 +68,7 @@ func (input *BaseUserData) prepare() error { } input.WriteFiles = append(input.WriteFiles, *joinScriptFile) } + input.SentinelFileCommand = sentinelFileCommand return nil } diff --git a/bootstrap/kubeadm/internal/cloudinit/controlplane_init.go b/bootstrap/kubeadm/internal/cloudinit/controlplane_init.go index 35d4d4a30540..c6cd12d5665e 100644 --- a/bootstrap/kubeadm/internal/cloudinit/controlplane_init.go +++ b/bootstrap/kubeadm/internal/cloudinit/controlplane_init.go @@ -31,9 +31,13 @@ const ( {{.ClusterConfiguration | Indent 6}} --- {{.InitConfiguration | Indent 6}} +- path: /run/cluster-api/placeholder + owner: root:root + permissions: '0640' + content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" runcmd: {{- template "commands" .PreKubeadmCommands }} - - 'kubeadm init --config /run/kubeadm/kubeadm.yaml {{.KubeadmVerbosity}}' + - 'kubeadm init --config /run/kubeadm/kubeadm.yaml {{.KubeadmVerbosity}} && {{ .SentinelFileCommand }}' {{- template "commands" .PostKubeadmCommands }} {{- template "ntp" .NTP }} {{- template "users" .Users }} @@ -57,6 +61,7 @@ func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) { input.Header = cloudConfigHeader input.WriteFiles = input.Certificates.AsFiles() input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) + input.SentinelFileCommand = sentinelFileCommand userData, err := generate("InitControlplane", controlPlaneCloudInit, input) if err != nil { return nil, err diff --git a/bootstrap/kubeadm/internal/cloudinit/controlplane_join.go b/bootstrap/kubeadm/internal/cloudinit/controlplane_join.go index 103ba0d903af..ddb04190c487 100644 --- a/bootstrap/kubeadm/internal/cloudinit/controlplane_join.go +++ b/bootstrap/kubeadm/internal/cloudinit/controlplane_join.go @@ -29,9 +29,13 @@ const ( permissions: '0640' content: | {{.JoinConfiguration | Indent 6}} +- path: /run/cluster-api/placeholder + owner: root:root + permissions: '0640' + content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" runcmd: {{- template "commands" .PreKubeadmCommands }} - - {{ .KubeadmCommand }} + - {{ .KubeadmCommand }} && {{ .SentinelFileCommand }} {{- template "commands" .PostKubeadmCommands }} {{- template "ntp" .NTP }} {{- template "users" .Users }} diff --git a/bootstrap/kubeadm/internal/cloudinit/node.go b/bootstrap/kubeadm/internal/cloudinit/node.go index 5da4cc8fa41a..5bb466e608dd 100644 --- a/bootstrap/kubeadm/internal/cloudinit/node.go +++ b/bootstrap/kubeadm/internal/cloudinit/node.go @@ -25,9 +25,13 @@ const ( content: | --- {{.JoinConfiguration | Indent 6}} +- path: /run/cluster-api/placeholder + owner: root:root + permissions: '0640' + content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" runcmd: {{- template "commands" .PreKubeadmCommands }} - - {{ .KubeadmCommand }} + - {{ .KubeadmCommand }} && {{ .SentinelFileCommand }} {{- template "commands" .PostKubeadmCommands }} {{- template "ntp" .NTP }} {{- template "users" .Users }} diff --git a/docs/book/src/developer/providers/bootstrap.md b/docs/book/src/developer/providers/bootstrap.md index 2a7a72f33041..79d39835fae6 100644 --- a/docs/book/src/developer/providers/bootstrap.md +++ b/docs/book/src/developer/providers/bootstrap.md @@ -59,6 +59,10 @@ The following diagram shows the typical logic for a bootstrap provider: 1. Set `status.ready` to true 1. Patch the resource to persist changes +## Sentinel File + +A bootstrap provider's bootstrap data must create `/run/cluster-api/bootstrap-success.complete` (or `C:\run\cluster-api\bootstrap-success.complete` for Windows machines) upon successful bootstrapping of a Kubernetes node. This allows infrastructure providers to detect and act on bootstrap failures. + ## RBAC ### Provider controller diff --git a/test/infrastructure/docker/cloudinit/runcmd.go b/test/infrastructure/docker/cloudinit/runcmd.go index 104e9c34f0df..e7f1a0571e2f 100644 --- a/test/infrastructure/docker/cloudinit/runcmd.go +++ b/test/infrastructure/docker/cloudinit/runcmd.go @@ -18,7 +18,6 @@ package cloudinit import ( "encoding/json" - "fmt" "strings" "github.com/pkg/errors" @@ -92,15 +91,18 @@ func (a *runCmd) Commands() ([]Cmd, error) { func hackKubeadmIgnoreErrors(c Cmd) Cmd { // case kubeadm commands are defined as a string if c.Cmd == "/bin/sh" && len(c.Args) >= 2 { - if c.Args[0] == "-c" && (strings.Contains(c.Args[1], "kubeadm init") || strings.Contains(c.Args[1], "kubeadm join")) { - c.Args[1] = fmt.Sprintf("%s %s", c.Args[1], "--ignore-preflight-errors=all") + if c.Args[0] == "-c" { + c.Args[1] = strings.Replace(c.Args[1], "kubeadm init", "kubeadm init --ignore-preflight-errors=all", 1) + c.Args[1] = strings.Replace(c.Args[1], "kubeadm join", "kubeadm join --ignore-preflight-errors=all", 1) } } // case kubeadm commands are defined as a list if c.Cmd == "kubeadm" && len(c.Args) >= 1 { if c.Args[0] == "init" || c.Args[0] == "join" { - c.Args = append(c.Args, "--ignore-preflight-errors=all") + c.Args = append(c.Args, "") // make space + copy(c.Args[2:], c.Args[1:]) // shift elements + c.Args[1] = "--ignore-preflight-errors=all" // insert the additional arg } } diff --git a/test/infrastructure/docker/cloudinit/runcmd_test.go b/test/infrastructure/docker/cloudinit/runcmd_test.go index 7d5b4b1de5e4..0de69b08141f 100644 --- a/test/infrastructure/docker/cloudinit/runcmd_test.go +++ b/test/infrastructure/docker/cloudinit/runcmd_test.go @@ -68,7 +68,7 @@ func TestRunCmdRun(t *testing.T) { }, }, expectedCmds: []Cmd{ - {Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --config /run/kubeadm/kubeadm.yaml --ignore-preflight-errors=all"}}, + {Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config /run/kubeadm/kubeadm.yaml"}}, }, }, } @@ -98,11 +98,11 @@ runcmd: r.Cmds[0] = hackKubeadmIgnoreErrors(r.Cmds[0]) - expected0 := Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --config=/run/kubeadm/kubeadm.yaml --ignore-preflight-errors=all"}} + expected0 := Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config=/run/kubeadm/kubeadm.yaml"}} g.Expect(r.Cmds[0]).To(Equal(expected0)) r.Cmds[1] = hackKubeadmIgnoreErrors(r.Cmds[1]) - expected1 := Cmd{Cmd: "kubeadm", Args: []string{"join", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml", "--ignore-preflight-errors=all"}} + expected1 := Cmd{Cmd: "kubeadm", Args: []string{"join", "--ignore-preflight-errors=all", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml"}} g.Expect(r.Cmds[1]).To(Equal(expected1)) }