From 44eaa7d7f09e45c1a70b9df49ec20bfed417823e Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 27 Apr 2021 22:13:17 +0300 Subject: [PATCH] feat: inject iPXE script into the iPXE binaries Fixes #227 See also https://github.com/talos-systems/pkgs/pull/267 The rough idea is that iPXE binaries we're building contain placeholder script which is replaced on the fly with the actual script which depends on Sidero API endpoint. Patching is a bit tricky for `undionly.kpxe`, as it is compressed, so we patch uncompressed version and compress the final asset after that. Signed-off-by: Andrey Smirnov --- Dockerfile | 21 ++-- Makefile | 2 +- .../internal/ipxe/ipxe_server.go | 30 +++++- .../internal/ipxe/patch.go | 102 ++++++++++++++++++ .../content/docs/v0.3/Guides/bootstrapping.md | 9 +- .../content/docs/v0.3/Guides/first-cluster.md | 6 ++ sfyra/pkg/bootstrap/cluster.go | 2 - sfyra/pkg/vm/set.go | 2 +- 8 files changed, 150 insertions(+), 24 deletions(-) create mode 100644 app/metal-controller-manager/internal/ipxe/patch.go diff --git a/Dockerfile b/Dockerfile index e5cd57153..84a9b2a6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,10 @@ FROM --platform=arm64 ghcr.io/talos-systems/linux-firmware:${PKGS} AS pkg-linux- FROM ghcr.io/talos-systems/musl:${PKGS} AS pkg-musl FROM --platform=amd64 ghcr.io/talos-systems/kernel:${PKGS} AS pkg-kernel-amd64 FROM --platform=arm64 ghcr.io/talos-systems/kernel:${PKGS} AS pkg-kernel-arm64 +FROM ghcr.io/talos-systems/liblzma:${PKGS} AS pkg-liblzma +FROM ghcr.io/talos-systems/ipxe:${PKGS} AS pkg-ipxe +FROM --platform=amd64 ghcr.io/talos-systems/ipxe:${PKGS} AS pkg-ipxe-amd64 +FROM --platform=arm64 ghcr.io/talos-systems/ipxe:${PKGS} AS pkg-ipxe-arm64 # The base target provides the base for running various tasks against the source # code @@ -122,11 +126,6 @@ ARG TARGETARCH RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=${TARGETARCH} go build -ldflags "-s -w" -o /manager ./app/metal-controller-manager RUN chmod +x /manager -FROM scratch AS assets -ADD http://boot.ipxe.org/undionly.kpxe /amd64/undionly.kpxe -ADD http://boot.ipxe.org/ipxe.efi /amd64/ipxe.efi -ADD http://boot.ipxe.org/arm64-efi/ipxe.efi /arm64/ipxe.efi - FROM base AS agent-build-amd64 RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o /agent ./app/metal-controller-manager/cmd/agent RUN chmod +x /agent @@ -149,21 +148,23 @@ COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2 ./lib/firmware/bnx2 COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2x ./lib/firmware/bnx2x RUN set -o pipefail && find . 2>/dev/null | cpio -H newc -o | xz -v -C crc32 -0 -e -T 0 -z >/initramfs.xz -FROM scratch AS metal-controller-manager +FROM scratch AS metal-controller-manager-image COPY --from=pkg-ca-certificates / / COPY --from=pkg-fhs / / COPY --from=pkg-musl / / COPY --from=pkg-libressl / / +COPY --from=pkg-liblzma / / COPY --from=pkg-ipmitool / / -COPY --from=assets /amd64/undionly.kpxe /var/lib/sidero/tftp/undionly.kpxe -COPY --from=assets /amd64/undionly.kpxe /var/lib/sidero/tftp/undionly.kpxe.0 -COPY --from=assets /amd64/ipxe.efi /var/lib/sidero/tftp/ipxe.efi -COPY --from=assets /arm64/ipxe.efi /var/lib/sidero/tftp/ipxe-arm64.efi +COPY --from=pkg-ipxe-amd64 /usr/libexec/ /var/lib/sidero/ipxe/amd64 +COPY --from=pkg-ipxe-arm64 /usr/libexec/ /var/lib/sidero/ipxe/arm64 +COPY --from=pkg-ipxe /usr/libexec/zbin /bin/zbin COPY --from=initramfs-archive-amd64 /initramfs.xz /var/lib/sidero/env/agent-amd64/initramfs.xz COPY --from=initramfs-archive-arm64 /initramfs.xz /var/lib/sidero/env/agent-arm64/initramfs.xz COPY --from=pkg-kernel-amd64 /boot/vmlinuz /var/lib/sidero/env/agent-amd64/vmlinuz COPY --from=pkg-kernel-arm64 /boot/vmlinuz /var/lib/sidero/env/agent-arm64/vmlinuz COPY --from=build-metal-controller-manager /manager /manager + +FROM metal-controller-manager-image AS metal-controller-manager LABEL org.opencontainers.image.source https://github.com/talos-systems/sidero ENTRYPOINT [ "/manager" ] diff --git a/Makefile b/Makefile index cb70f2ec6..53d1e1975 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ TEST_PKGS ?= ./... TALOS_RELEASE ?= v0.9.3 TOOLS ?= ghcr.io/talos-systems/tools:v0.5.0 -PKGS ?= v0.5.0 +PKGS ?= v0.5.0-8-gb0d9cd2 SFYRA_CLUSTERCTL_CONFIG ?= $(HOME)/.cluster-api/clusterctl.sfyra.yaml diff --git a/app/metal-controller-manager/internal/ipxe/ipxe_server.go b/app/metal-controller-manager/internal/ipxe/ipxe_server.go index 67da9f4a7..feffaf8e9 100644 --- a/app/metal-controller-manager/internal/ipxe/ipxe_server.go +++ b/app/metal-controller-manager/internal/ipxe/ipxe_server.go @@ -12,6 +12,7 @@ import ( "log" "net" "net/http" + "strconv" "strings" "text/template" @@ -36,16 +37,28 @@ var ( ErrBootFromDisk = errors.New("boot from disk") ) +const iPXEPort = 8081 + +// bootFile is used when iPXE is booted without embedded script via iPXE request http://endpoint:8081/boot.ipxe. const bootFile = `#!ipxe chain ipxe?uuid=${uuid}&mac=${mac:hexhyp}&domain=${domain}&hostname=${hostname}&serial=${serial}&arch=${buildarch} ` +// bootTemplate is embedded into iPXE binary when that binary is sent to the node. +// +// bootTemplate should be kept in sync with the bootFile above. +var bootTemplate = template.Must(template.New("iPXE embedded").Parse(`dhcp +chain http://{{ .Endpoint }}:{{ .Port }}/ipxe?uuid=${uuid}&mac=${mac:hexhyp}&domain=${domain}&hostname=${hostname}&serial=${serial}&arch=${buildarch} +`)) + +// ipxeTemplate is returned as response to `chain` request from the bootFile/bootTemplate to boot actual OS (or Sidero agent). var ipxeTemplate = template.Must(template.New("iPXE config").Parse(`#!ipxe kernel /env/{{ .Env.Name }}/{{ .KernelAsset }} {{range $arg := .Env.Spec.Kernel.Args}} {{$arg}}{{end}} initrd /env/{{ .Env.Name }}/{{ .InitrdAsset }} boot `)) +// ipxeBootFromDisk script is used to skip PXE booting and boot from disk. const ipxeBootFromDisk = `#!ipxe exit ` @@ -151,7 +164,7 @@ func ipxeHandler(w http.ResponseWriter, r *http.Request) { return } - if env.ObjectMeta.Name != "agent" { + if !strings.HasPrefix(env.ObjectMeta.Name, "agent") { if err = markAsPXEBooted(server); err != nil { log.Printf("error marking server as PXE booted: %s", err) } @@ -163,6 +176,19 @@ func ServeIPXE(endpoint, args string, mgrClient client.Client) error { extraAgentKernelArgs = args c = mgrClient + var embeddedScriptBuf bytes.Buffer + + if err := bootTemplate.Execute(&embeddedScriptBuf, map[string]string{ + "Endpoint": apiEndpoint, + "Port": strconv.Itoa(iPXEPort), + }); err != nil { + return err + } + + if err := PatchBinaries(embeddedScriptBuf.Bytes()); err != nil { + return err + } + mux := http.NewServeMux() mux.Handle("/boot.ipxe", logRequest(http.HandlerFunc(bootFileHandler))) @@ -172,7 +198,7 @@ func ServeIPXE(endpoint, args string, mgrClient client.Client) error { log.Println("Listening...") - return http.ListenAndServe(":8081", mux) + return http.ListenAndServe(fmt.Sprintf(":%d", iPXEPort), mux) } func logRequest(next http.Handler) http.Handler { diff --git a/app/metal-controller-manager/internal/ipxe/patch.go b/app/metal-controller-manager/internal/ipxe/patch.go new file mode 100644 index 000000000..60c06ff5f --- /dev/null +++ b/app/metal-controller-manager/internal/ipxe/patch.go @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package ipxe + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// PatchBinaries patches iPXE binaries on the fly with the new embedded script. +// +// This relies on special build in `pkgs/ipxe` where a placeholder iPXE script is embedded. +// EFI iPXE binaries are uncompressed, so these are patched directly. +// BIOS amd64 undionly.pxe is compressed, so we instead patch uncompressed version and compress it back using zbin. +// (zbin is built with iPXE). +func PatchBinaries(script []byte) error { + if err := patchScript("/var/lib/sidero/ipxe/amd64/ipxe.efi", "/var/lib/sidero/tftp/ipxe.efi", script); err != nil { + return err + } + + if err := patchScript("/var/lib/sidero/ipxe/arm64/ipxe.efi", "/var/lib/sidero/tftp/ipxe-arm64.efi", script); err != nil { + return err + } + + if err := patchScript("/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin", "/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin.patched", script); err != nil { + return err + } + + if err := compressKPXE("/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin.patched", "/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.zinfo", "/var/lib/sidero/tftp/undionly.kpxe"); err != nil { + return err + } + + if err := compressKPXE("/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin.patched", "/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.zinfo", "/var/lib/sidero/tftp/undionly.kpxe.0"); err != nil { + return err + } + + return nil +} + +var ( + placeholderStart = []byte("# *PLACEHOLDER START*") + placeholderEnd = []byte("# *PLACEHOLDER END*") +) + +func patchScript(source, destination string, script []byte) error { + contents, err := os.ReadFile(source) + if err != nil { + return err + } + + start := bytes.Index(contents, placeholderStart) + if start == -1 { + return fmt.Errorf("placeholder start not found in %q", source) + } + + end := bytes.Index(contents, placeholderEnd) + if end == -1 { + return fmt.Errorf("placeholder end not found in %q", source) + } + + if end < start { + return fmt.Errorf("placeholder end before start") + } + + end += len(placeholderEnd) + + length := end - start + + if len(script) > length { + return fmt.Errorf("script size %d is larger than placeholder space %d", len(script), length) + } + + script = append(script, bytes.Repeat([]byte{'\n'}, length-len(script))...) + + copy(contents[start:end], script) + + if err = os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return err + } + + return os.WriteFile(destination, contents, 0o644) +} + +// compressPXE is equivalent to: ./util/zbin bin/undionly.kpxe.bin bin/undionly.kpxe.zinfo > bin/undionly.kpxe.zbin. +func compressKPXE(binFile, infoFile, outFile string) error { + out, err := os.Create(outFile) + if err != nil { + return err + } + + defer out.Close() + + cmd := exec.Command("/bin/zbin", binFile, infoFile) + cmd.Stdout = out + + return cmd.Run() +} diff --git a/docs/website/content/docs/v0.3/Guides/bootstrapping.md b/docs/website/content/docs/v0.3/Guides/bootstrapping.md index f9528ddab..57b75a784 100644 --- a/docs/website/content/docs/v0.3/Guides/bootstrapping.md +++ b/docs/website/content/docs/v0.3/Guides/bootstrapping.md @@ -61,14 +61,7 @@ allow bootp; allow booting; next-server 192.168.1.150; -if exists user-class and option user-class = "iPXE" { - filename "http://192.168.1.150:8081/boot.ipxe"; -} elsif substring (option vendor-class-identifier, 0, 10) = "HTTPClient" { - option vendor-class-identifier "HTTPClient"; - filename "http://192.168.1.150:8081/tftp/ipxe.efi"; -} else { - filename "ipxe.efi"; -} +filename "ipxe.efi"; host talos-mgmt-0 { fixed-address 192.168.254.2; diff --git a/docs/website/content/docs/v0.3/Guides/first-cluster.md b/docs/website/content/docs/v0.3/Guides/first-cluster.md index cdbdd2a35..f4ba29989 100644 --- a/docs/website/content/docs/v0.3/Guides/first-cluster.md +++ b/docs/website/content/docs/v0.3/Guides/first-cluster.md @@ -81,6 +81,12 @@ host talos-mgmt-0 { } ``` +There are multiple ways to boot the via iPXE: + +* if the node has built-in iPXE, direct URL to the iPXE script can be used: `http://192.168.254.2:8081/boot.ipxe`. +* depending on the boot mode (BIOS or UEFI), either `ipxe.efi` or `undionly.kpxe` can be used (these images contain embedded iPXE scripts). +* iPXE binaries can be delivered either over TFTP or HTTP (HTTP support depends on node firmware). + ## Register the Servers At this point, any servers on the same network as Sidero should PXE boot using the Sidero PXE service. diff --git a/sfyra/pkg/bootstrap/cluster.go b/sfyra/pkg/bootstrap/cluster.go index e8040f204..263e63a96 100644 --- a/sfyra/pkg/bootstrap/cluster.go +++ b/sfyra/pkg/bootstrap/cluster.go @@ -134,8 +134,6 @@ func (cluster *Cluster) findExisting(ctx context.Context) error { return err } - config.Context = cluster.options.Name - _, cidr, err := net.ParseCIDR(cluster.options.CIDR) if err != nil { return err diff --git a/sfyra/pkg/vm/set.go b/sfyra/pkg/vm/set.go index 35fde833a..b077eb95c 100644 --- a/sfyra/pkg/vm/set.go +++ b/sfyra/pkg/vm/set.go @@ -161,7 +161,7 @@ func (set *Set) create(ctx context.Context) error { }, PXEBooted: true, TFTPServer: set.options.BootSource.String(), - IPXEBootFilename: fmt.Sprintf("http://%s:8081/boot.ipxe", set.options.BootSource), + IPXEBootFilename: "undionly.kpxe", SkipInjectingConfig: true, }) }