Skip to content

Commit

Permalink
feat: inject iPXE script into the iPXE binaries
Browse files Browse the repository at this point in the history
Fixes #227

See also siderolabs/pkgs#267

The rough idea is that iPXE binaries we're building contain placeholder
script which is replaced on the fly with the actual script which depends
on Sidero API endpoint. Patching is a bit tricky for `undionly.kpxe`, as
it is compressed, so we patch uncompressed version and compress the
final asset after that.

Signed-off-by: Andrey Smirnov <[email protected]>
  • Loading branch information
smira authored and talos-bot committed May 5, 2021
1 parent 1659b96 commit 44eaa7d
Show file tree
Hide file tree
Showing 8 changed files with 150 additions and 24 deletions.
21 changes: 11 additions & 10 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ FROM --platform=arm64 ghcr.io/talos-systems/linux-firmware:${PKGS} AS pkg-linux-
FROM ghcr.io/talos-systems/musl:${PKGS} AS pkg-musl
FROM --platform=amd64 ghcr.io/talos-systems/kernel:${PKGS} AS pkg-kernel-amd64
FROM --platform=arm64 ghcr.io/talos-systems/kernel:${PKGS} AS pkg-kernel-arm64
FROM ghcr.io/talos-systems/liblzma:${PKGS} AS pkg-liblzma
FROM ghcr.io/talos-systems/ipxe:${PKGS} AS pkg-ipxe
FROM --platform=amd64 ghcr.io/talos-systems/ipxe:${PKGS} AS pkg-ipxe-amd64
FROM --platform=arm64 ghcr.io/talos-systems/ipxe:${PKGS} AS pkg-ipxe-arm64

# The base target provides the base for running various tasks against the source
# code
Expand Down Expand Up @@ -122,11 +126,6 @@ ARG TARGETARCH
RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=${TARGETARCH} go build -ldflags "-s -w" -o /manager ./app/metal-controller-manager
RUN chmod +x /manager

FROM scratch AS assets
ADD http://boot.ipxe.org/undionly.kpxe /amd64/undionly.kpxe
ADD http://boot.ipxe.org/ipxe.efi /amd64/ipxe.efi
ADD http://boot.ipxe.org/arm64-efi/ipxe.efi /arm64/ipxe.efi

FROM base AS agent-build-amd64
RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o /agent ./app/metal-controller-manager/cmd/agent
RUN chmod +x /agent
Expand All @@ -149,21 +148,23 @@ COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2 ./lib/firmware/bnx2
COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2x ./lib/firmware/bnx2x
RUN set -o pipefail && find . 2>/dev/null | cpio -H newc -o | xz -v -C crc32 -0 -e -T 0 -z >/initramfs.xz

FROM scratch AS metal-controller-manager
FROM scratch AS metal-controller-manager-image
COPY --from=pkg-ca-certificates / /
COPY --from=pkg-fhs / /
COPY --from=pkg-musl / /
COPY --from=pkg-libressl / /
COPY --from=pkg-liblzma / /
COPY --from=pkg-ipmitool / /
COPY --from=assets /amd64/undionly.kpxe /var/lib/sidero/tftp/undionly.kpxe
COPY --from=assets /amd64/undionly.kpxe /var/lib/sidero/tftp/undionly.kpxe.0
COPY --from=assets /amd64/ipxe.efi /var/lib/sidero/tftp/ipxe.efi
COPY --from=assets /arm64/ipxe.efi /var/lib/sidero/tftp/ipxe-arm64.efi
COPY --from=pkg-ipxe-amd64 /usr/libexec/ /var/lib/sidero/ipxe/amd64
COPY --from=pkg-ipxe-arm64 /usr/libexec/ /var/lib/sidero/ipxe/arm64
COPY --from=pkg-ipxe /usr/libexec/zbin /bin/zbin
COPY --from=initramfs-archive-amd64 /initramfs.xz /var/lib/sidero/env/agent-amd64/initramfs.xz
COPY --from=initramfs-archive-arm64 /initramfs.xz /var/lib/sidero/env/agent-arm64/initramfs.xz
COPY --from=pkg-kernel-amd64 /boot/vmlinuz /var/lib/sidero/env/agent-amd64/vmlinuz
COPY --from=pkg-kernel-arm64 /boot/vmlinuz /var/lib/sidero/env/agent-arm64/vmlinuz
COPY --from=build-metal-controller-manager /manager /manager

FROM metal-controller-manager-image AS metal-controller-manager
LABEL org.opencontainers.image.source https://github.com/talos-systems/sidero
ENTRYPOINT [ "/manager" ]

Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ TEST_PKGS ?= ./...
TALOS_RELEASE ?= v0.9.3

TOOLS ?= ghcr.io/talos-systems/tools:v0.5.0
PKGS ?= v0.5.0
PKGS ?= v0.5.0-8-gb0d9cd2

SFYRA_CLUSTERCTL_CONFIG ?= $(HOME)/.cluster-api/clusterctl.sfyra.yaml

Expand Down
30 changes: 28 additions & 2 deletions app/metal-controller-manager/internal/ipxe/ipxe_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"log"
"net"
"net/http"
"strconv"
"strings"
"text/template"

Expand All @@ -36,16 +37,28 @@ var (
ErrBootFromDisk = errors.New("boot from disk")
)

const iPXEPort = 8081

// bootFile is used when iPXE is booted without embedded script via iPXE request http://endpoint:8081/boot.ipxe.
const bootFile = `#!ipxe
chain ipxe?uuid=${uuid}&mac=${mac:hexhyp}&domain=${domain}&hostname=${hostname}&serial=${serial}&arch=${buildarch}
`

// bootTemplate is embedded into iPXE binary when that binary is sent to the node.
//
// bootTemplate should be kept in sync with the bootFile above.
var bootTemplate = template.Must(template.New("iPXE embedded").Parse(`dhcp
chain http://{{ .Endpoint }}:{{ .Port }}/ipxe?uuid=${uuid}&mac=${mac:hexhyp}&domain=${domain}&hostname=${hostname}&serial=${serial}&arch=${buildarch}
`))

// ipxeTemplate is returned as response to `chain` request from the bootFile/bootTemplate to boot actual OS (or Sidero agent).
var ipxeTemplate = template.Must(template.New("iPXE config").Parse(`#!ipxe
kernel /env/{{ .Env.Name }}/{{ .KernelAsset }} {{range $arg := .Env.Spec.Kernel.Args}} {{$arg}}{{end}}
initrd /env/{{ .Env.Name }}/{{ .InitrdAsset }}
boot
`))

// ipxeBootFromDisk script is used to skip PXE booting and boot from disk.
const ipxeBootFromDisk = `#!ipxe
exit
`
Expand Down Expand Up @@ -151,7 +164,7 @@ func ipxeHandler(w http.ResponseWriter, r *http.Request) {
return
}

if env.ObjectMeta.Name != "agent" {
if !strings.HasPrefix(env.ObjectMeta.Name, "agent") {
if err = markAsPXEBooted(server); err != nil {
log.Printf("error marking server as PXE booted: %s", err)
}
Expand All @@ -163,6 +176,19 @@ func ServeIPXE(endpoint, args string, mgrClient client.Client) error {
extraAgentKernelArgs = args
c = mgrClient

var embeddedScriptBuf bytes.Buffer

if err := bootTemplate.Execute(&embeddedScriptBuf, map[string]string{
"Endpoint": apiEndpoint,
"Port": strconv.Itoa(iPXEPort),
}); err != nil {
return err
}

if err := PatchBinaries(embeddedScriptBuf.Bytes()); err != nil {
return err
}

mux := http.NewServeMux()

mux.Handle("/boot.ipxe", logRequest(http.HandlerFunc(bootFileHandler)))
Expand All @@ -172,7 +198,7 @@ func ServeIPXE(endpoint, args string, mgrClient client.Client) error {

log.Println("Listening...")

return http.ListenAndServe(":8081", mux)
return http.ListenAndServe(fmt.Sprintf(":%d", iPXEPort), mux)
}

func logRequest(next http.Handler) http.Handler {
Expand Down
102 changes: 102 additions & 0 deletions app/metal-controller-manager/internal/ipxe/patch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package ipxe

import (
"bytes"
"fmt"
"os"
"os/exec"
"path/filepath"
)

// PatchBinaries patches iPXE binaries on the fly with the new embedded script.
//
// This relies on special build in `pkgs/ipxe` where a placeholder iPXE script is embedded.
// EFI iPXE binaries are uncompressed, so these are patched directly.
// BIOS amd64 undionly.pxe is compressed, so we instead patch uncompressed version and compress it back using zbin.
// (zbin is built with iPXE).
func PatchBinaries(script []byte) error {
if err := patchScript("/var/lib/sidero/ipxe/amd64/ipxe.efi", "/var/lib/sidero/tftp/ipxe.efi", script); err != nil {
return err
}

if err := patchScript("/var/lib/sidero/ipxe/arm64/ipxe.efi", "/var/lib/sidero/tftp/ipxe-arm64.efi", script); err != nil {
return err
}

if err := patchScript("/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin", "/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin.patched", script); err != nil {
return err
}

if err := compressKPXE("/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin.patched", "/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.zinfo", "/var/lib/sidero/tftp/undionly.kpxe"); err != nil {
return err
}

if err := compressKPXE("/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.bin.patched", "/var/lib/sidero/ipxe/amd64/kpxe/undionly.kpxe.zinfo", "/var/lib/sidero/tftp/undionly.kpxe.0"); err != nil {
return err
}

return nil
}

var (
placeholderStart = []byte("# *PLACEHOLDER START*")
placeholderEnd = []byte("# *PLACEHOLDER END*")
)

func patchScript(source, destination string, script []byte) error {
contents, err := os.ReadFile(source)
if err != nil {
return err
}

start := bytes.Index(contents, placeholderStart)
if start == -1 {
return fmt.Errorf("placeholder start not found in %q", source)
}

end := bytes.Index(contents, placeholderEnd)
if end == -1 {
return fmt.Errorf("placeholder end not found in %q", source)
}

if end < start {
return fmt.Errorf("placeholder end before start")
}

end += len(placeholderEnd)

length := end - start

if len(script) > length {
return fmt.Errorf("script size %d is larger than placeholder space %d", len(script), length)
}

script = append(script, bytes.Repeat([]byte{'\n'}, length-len(script))...)

copy(contents[start:end], script)

if err = os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
return err
}

return os.WriteFile(destination, contents, 0o644)
}

// compressPXE is equivalent to: ./util/zbin bin/undionly.kpxe.bin bin/undionly.kpxe.zinfo > bin/undionly.kpxe.zbin.
func compressKPXE(binFile, infoFile, outFile string) error {
out, err := os.Create(outFile)
if err != nil {
return err
}

defer out.Close()

cmd := exec.Command("/bin/zbin", binFile, infoFile)
cmd.Stdout = out

return cmd.Run()
}
9 changes: 1 addition & 8 deletions docs/website/content/docs/v0.3/Guides/bootstrapping.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,7 @@ allow bootp;
allow booting;

next-server 192.168.1.150;
if exists user-class and option user-class = "iPXE" {
filename "http://192.168.1.150:8081/boot.ipxe";
} elsif substring (option vendor-class-identifier, 0, 10) = "HTTPClient" {
option vendor-class-identifier "HTTPClient";
filename "http://192.168.1.150:8081/tftp/ipxe.efi";
} else {
filename "ipxe.efi";
}
filename "ipxe.efi";

host talos-mgmt-0 {
fixed-address 192.168.254.2;
Expand Down
6 changes: 6 additions & 0 deletions docs/website/content/docs/v0.3/Guides/first-cluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,12 @@ host talos-mgmt-0 {
}
```
There are multiple ways to boot the via iPXE:
* if the node has built-in iPXE, direct URL to the iPXE script can be used: `http://192.168.254.2:8081/boot.ipxe`.
* depending on the boot mode (BIOS or UEFI), either `ipxe.efi` or `undionly.kpxe` can be used (these images contain embedded iPXE scripts).
* iPXE binaries can be delivered either over TFTP or HTTP (HTTP support depends on node firmware).
## Register the Servers
At this point, any servers on the same network as Sidero should PXE boot using the Sidero PXE service.
Expand Down
2 changes: 0 additions & 2 deletions sfyra/pkg/bootstrap/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,6 @@ func (cluster *Cluster) findExisting(ctx context.Context) error {
return err
}

config.Context = cluster.options.Name

_, cidr, err := net.ParseCIDR(cluster.options.CIDR)
if err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion sfyra/pkg/vm/set.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func (set *Set) create(ctx context.Context) error {
},
PXEBooted: true,
TFTPServer: set.options.BootSource.String(),
IPXEBootFilename: fmt.Sprintf("http://%s:8081/boot.ipxe", set.options.BootSource),
IPXEBootFilename: "undionly.kpxe",
SkipInjectingConfig: true,
})
}
Expand Down

0 comments on commit 44eaa7d

Please sign in to comment.