Skip to content

Commit

Permalink
Cherry-pick the firmware implementation (#52)
Browse files Browse the repository at this point in the history 
* Load firmware from the module loader (#17)

It is common for kernel module to come with firmware(s). On immutable
operating systems like CoreOS, the standard firmware path
`/lib/firmware` is read-only. So, we need to let the kernel know where
to lookup firmware file(s).

This change adds the `FirmwarePath` attribute to the `ModprobeSpec`
definition and modify the `MakeLoadCommand` and `MakeUnloadCommand`
methods to leverage it.

It also adds a `Volume` and a `VolumeMount` to expose the
`/var/lib/firmware/<Module.Name>` host directory (created if it doesn't
exist) inside the pod. The `/var/lib/firmware` is [the recommended path
for firmwares in OpenShift with RHCOS
nodes](https://access.redhat.com/documentation/fr-fr/openshift_container_platform/4.10/html-single/post-installation_configuration/index#rhcos-load-firmware-blobs_post-install-machine-configuration-tasks).
Putting the firmware in a `Module.Name` subfolder avoids multiple
Modules to override each others firmware. It also simplifies the
removal of the firmware files.

The `MakeLoadCommand` adds a step to copy the `FirmwarePath` to
`/var/lib/firmware/<Module.Name>` to the `modprobe` command. The
`MakeUnloadCommand` removes the firmware folder previously copied after
unloading the module. The commands use `/usr/bin -c` to allow shell
control like `&&`.

Signed-off-by: Fabien Dupont <[email protected]>

Signed-off-by: Fabien Dupont <[email protected]>

* Use a strings.Builder in Make{Un,}LoadCommand (#46)

This change replaces the many `append` statement with a
`strings.Builder` stream to make the string creation more efficient.

We also use this change to allow loading the firware even when
`Modprobe.Args` is set, as it's not related to how modprobe itself is
invoked.

Signed-off-by: Fabien Dupont <[email protected]>

Signed-off-by: Fabien Dupont <[email protected]>

Signed-off-by: Fabien Dupont <[email protected]>
Co-authored-by: Fabien Dupont <[email protected]>
  • Loading branch information
@qbarrand @fabiendupont
qbarrand and fabiendupont authored Sep 26, 2022
1 parent 02c6d44 commit 17b7ef7
Show file tree
Hide file tree
Showing 4 changed files with 298 additions and 103 deletions.
5 changes: 5 additions & 0 deletions api/v1beta1/module_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,11 @@ type ModprobeSpec struct {
// The resulting commands will be: `modprobe ${RawArgs}`.
// +optional
RawArgs *ModprobeArgs `json:"rawArgs,omitempty"`

// FirmwarePath is the path of the firmware(s).
// The firmware(s) will be copied to the host for the kernel to find them.
// +optional
FirmwarePath string `json:"firmwarePath,omitempty"`
}

type ModuleLoaderContainerSpec struct {
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/kmm.sigs.k8s.io_modules.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2069,6 +2069,11 @@ spec:
description: DirName is the root directory for modules.
It adds `-d ${DirName}` to the modprobe command-line.
type: string
firmwarePath:
description: FirmwarePath is the path of the firmware(s).
The firmware(s) will be copied to the host for the kernel
to find them.
type: string
moduleName:
description: ModuleName is the name of the Module to be
loaded.
Expand Down
239 changes: 158 additions & 81 deletions internal/daemonset/daemonset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"

kmmv1beta1 "github.com/rh-ecosystem-edge/kernel-module-management/api/v1beta1"
"github.com/rh-ecosystem-edge/kernel-module-management/internal/constants"
Expand All @@ -24,6 +25,8 @@ const (
nodeLibModulesVolumeName = "node-lib-modules"
nodeUsrLibModulesPath = "/usr/lib/modules"
nodeUsrLibModulesVolumeName = "node-usr-lib-modules"
nodeVarLibFirmwarePath = "/var/lib/firmware"
nodeVarLibFirmwareVolumeName = "node-var-lib-firmware"
devicePluginKernelVersion = ""
)

Expand Down Expand Up @@ -115,6 +118,91 @@ func (dc *daemonSetGenerator) SetDriverContainerAsDesired(ctx context.Context, d
nodeSelector[dc.kernelLabel] = kernelVersion

hostPathDirectory := v1.HostPathDirectory
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate

container := v1.Container{
Command: []string{"sleep", "infinity"},
Name: "module-loader",
Image: image,
ImagePullPolicy: mod.Spec.ModuleLoader.Container.ImagePullPolicy,
Lifecycle: &v1.Lifecycle{
PostStart: &v1.LifecycleHandler{
Exec: &v1.ExecAction{
Command: MakeLoadCommand(mod.Spec.ModuleLoader.Container.Modprobe, mod.Name),
},
},
PreStop: &v1.LifecycleHandler{
Exec: &v1.ExecAction{
Command: MakeUnloadCommand(mod.Spec.ModuleLoader.Container.Modprobe, mod.Name),
},
},
},
SecurityContext: &v1.SecurityContext{
AllowPrivilegeEscalation: pointer.Bool(false),
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_MODULE"},
},
RunAsUser: pointer.Int64(0),
SELinuxOptions: &v1.SELinuxOptions{
Type: "spc_t",
},
},
VolumeMounts: []v1.VolumeMount{
{
Name: nodeLibModulesVolumeName,
ReadOnly: true,
MountPath: nodeLibModulesPath,
},
{
Name: nodeUsrLibModulesVolumeName,
ReadOnly: true,
MountPath: nodeUsrLibModulesPath,
},
},
}

volumes := []v1.Volume{
{
Name: nodeLibModulesVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: nodeLibModulesPath,
Type: &hostPathDirectory,
},
},
},
{
Name: nodeUsrLibModulesVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: nodeUsrLibModulesPath,
Type: &hostPathDirectory,
},
},
},
}

if fw := mod.Spec.ModuleLoader.Container.Modprobe.FirmwarePath; fw != "" {
moduleFirmwarePath := fmt.Sprintf("%s/%s", nodeVarLibFirmwarePath, mod.Name)

firmwareVolume := v1.Volume{
Name: nodeVarLibFirmwareVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: moduleFirmwarePath,
Type: &hostPathDirectoryOrCreate,
},
},
}
volumes = append(volumes, firmwareVolume)

firmwareVolumeMount := v1.VolumeMount{
Name: nodeVarLibFirmwareVolumeName,
MountPath: moduleFirmwarePath,
}

container.VolumeMounts = append(container.VolumeMounts, firmwareVolumeMount)
}

ds.Spec = appsv1.DaemonSetSpec{
Template: v1.PodTemplateSpec{
Expand All @@ -123,72 +211,12 @@ func (dc *daemonSetGenerator) SetDriverContainerAsDesired(ctx context.Context, d
Finalizers: []string{constants.NodeLabelerFinalizer},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Command: []string{"sleep", "infinity"},
Name: "module-loader",
Image: image,
ImagePullPolicy: mod.Spec.ModuleLoader.Container.ImagePullPolicy,
Lifecycle: &v1.Lifecycle{
PostStart: &v1.LifecycleHandler{
Exec: &v1.ExecAction{
Command: MakeLoadCommand(mod.Spec.ModuleLoader.Container.Modprobe),
},
},
PreStop: &v1.LifecycleHandler{
Exec: &v1.ExecAction{
Command: MakeUnloadCommand(mod.Spec.ModuleLoader.Container.Modprobe),
},
},
},
SecurityContext: &v1.SecurityContext{
AllowPrivilegeEscalation: pointer.Bool(false),
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_MODULE"},
},
RunAsUser: pointer.Int64(0),
SELinuxOptions: &v1.SELinuxOptions{
Type: "spc_t",
},
},
VolumeMounts: []v1.VolumeMount{
{
Name: nodeLibModulesVolumeName,
ReadOnly: true,
MountPath: nodeLibModulesPath,
},
{
Name: nodeUsrLibModulesVolumeName,
ReadOnly: true,
MountPath: nodeUsrLibModulesPath,
},
},
},
},
Containers: []v1.Container{container},
ImagePullSecrets: GetPodPullSecrets(mod.Spec.ImageRepoSecret),
NodeSelector: nodeSelector,
PriorityClassName: "system-node-critical",
ServiceAccountName: mod.Spec.ModuleLoader.ServiceAccountName,
Volumes: []v1.Volume{
{
Name: nodeLibModulesVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: nodeLibModulesPath,
Type: &hostPathDirectory,
},
},
},
{
Name: nodeUsrLibModulesVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: nodeUsrLibModulesPath,
Type: &hostPathDirectory,
},
},
},
},
Volumes: volumes,
},
},
Selector: &metav1.LabelSelector{MatchLabels: standardLabels},
Expand Down Expand Up @@ -335,43 +363,92 @@ func OverrideLabels(labels, overrides map[string]string) map[string]string {
return labels
}

func MakeLoadCommand(spec kmmv1beta1.ModprobeSpec) []string {
loadCommand := []string{"modprobe"}
func MakeLoadCommand(spec kmmv1beta1.ModprobeSpec, modName string) []string {
loadCommandShell := []string{
"/bin/sh",
"-c",
}

var loadCommand strings.Builder

if ra := spec.RawArgs; ra != nil && len(ra.Load) > 0 {
return append(loadCommand, ra.Load...)
if fw := spec.FirmwarePath; fw != "" {
fmt.Fprintf(&loadCommand, "cp -r %s %s/%s && ", fw, nodeVarLibFirmwarePath, modName)
}

if a := spec.Args; a != nil && len(a.Load) > 0 {
loadCommand = append(loadCommand, a.Load...)
loadCommand.WriteString("modprobe")

if rawArgs := spec.RawArgs; rawArgs != nil && len(rawArgs.Load) > 0 {
for _, arg := range rawArgs.Load {
loadCommand.WriteRune(' ')
loadCommand.WriteString(arg)
}
return append(loadCommandShell, loadCommand.String())
}

if args := spec.Args; args != nil && len(args.Load) > 0 {
for _, arg := range args.Load {
loadCommand.WriteRune(' ')
loadCommand.WriteString(arg)
}
} else {
loadCommand = append(loadCommand, "-v")
loadCommand.WriteString(" -v")
}

if dirName := spec.DirName; dirName != "" {
loadCommand = append(loadCommand, "-d", dirName)
loadCommand.WriteString(" -d " + dirName)
}

loadCommand = append(loadCommand, spec.ModuleName)
return append(loadCommand, spec.Parameters...)
loadCommand.WriteString(" " + spec.ModuleName)

if params := spec.Parameters; len(params) > 0 {
for _, param := range params {
loadCommand.WriteRune(' ')
loadCommand.WriteString(param)
}
}

return append(loadCommandShell, loadCommand.String())
}

func MakeUnloadCommand(spec kmmv1beta1.ModprobeSpec) []string {
unloadCommand := []string{"modprobe"}
func MakeUnloadCommand(spec kmmv1beta1.ModprobeSpec, modName string) []string {
unloadCommandShell := []string{
"/bin/sh",
"-c",
}

var unloadCommand strings.Builder
unloadCommand.WriteString("modprobe")

if ra := spec.RawArgs; ra != nil && len(ra.Unload) > 0 {
return append(unloadCommand, ra.Unload...)
fwUnloadCommand := ""
if fw := spec.FirmwarePath; fw != "" {
fwUnloadCommand = fmt.Sprintf(" && rm -rf %s/%s", nodeVarLibFirmwarePath, modName)
}

if a := spec.Args; a != nil && len(a.Unload) > 0 {
unloadCommand = append(unloadCommand, a.Unload...)
if rawArgs := spec.RawArgs; rawArgs != nil && len(rawArgs.Unload) > 0 {
for _, arg := range rawArgs.Unload {
unloadCommand.WriteRune(' ')
unloadCommand.WriteString(arg)
unloadCommand.WriteString(fwUnloadCommand)
}

return append(unloadCommandShell, unloadCommand.String())
}

if args := spec.Args; args != nil && len(args.Unload) > 0 {
for _, arg := range args.Unload {
unloadCommand.WriteRune(' ')
unloadCommand.WriteString(arg)
}
} else {
unloadCommand = append(unloadCommand, "-rv")
unloadCommand.WriteString(" -rv")
}

if dirName := spec.DirName; dirName != "" {
unloadCommand = append(unloadCommand, "-d", dirName)
unloadCommand.WriteString(" -d " + dirName)
}

return append(unloadCommand, spec.ModuleName)
unloadCommand.WriteString(" " + spec.ModuleName)
unloadCommand.WriteString(fwUnloadCommand)

return append(unloadCommandShell, unloadCommand.String())
}
Loading

0 comments on commit 17b7ef7

Please sign in to comment.