diff --git a/bundle/manifests/kepler.system.sustainable.computing.io_keplerinternals.yaml b/bundle/manifests/kepler.system.sustainable.computing.io_keplerinternals.yaml index 12bd0362..fab3b74f 100644 --- a/bundle/manifests/kepler.system.sustainable.computing.io_keplerinternals.yaml +++ b/bundle/manifests/kepler.system.sustainable.computing.io_keplerinternals.yaml @@ -36,6 +36,9 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - jsonPath: .spec.exporter.deployment.image + name: Image + type: string - jsonPath: .spec.exporter.deployment.nodeSelector name: Node-Selector priority: 10 @@ -68,6 +71,8 @@ spec: properties: deployment: properties: + image: + type: string nodeSelector: additionalProperties: type: string diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 245116d5..1315365c 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -77,8 +77,8 @@ func main() { // NOTE: RELATED_IMAGE_KEPLER can be set as env or flag, flag takes precedence over env keplerImage := os.Getenv("RELATED_IMAGE_KEPLER") keplerImageLibbpf := os.Getenv("RELATED_IMAGE_KEPLER_LIBBPF") - flag.StringVar(&exporter.Config.Image, "kepler.image", keplerImage, "kepler image") - flag.StringVar(&exporter.Config.ImageLibbpf, "kepler.image.libbpf", keplerImageLibbpf, "kepler libbpf image") + flag.StringVar(&controllers.Config.Image, "kepler.image", keplerImage, "kepler image") + flag.StringVar(&controllers.Config.ImageLibbpf, "kepler.image.libbpf", keplerImageLibbpf, "kepler libbpf image") opts := zap.Options{ Development: true, diff --git a/config/crd/bases/kepler.system.sustainable.computing.io_keplerinternals.yaml b/config/crd/bases/kepler.system.sustainable.computing.io_keplerinternals.yaml index 91adaefc..2c3e1530 100644 --- a/config/crd/bases/kepler.system.sustainable.computing.io_keplerinternals.yaml +++ b/config/crd/bases/kepler.system.sustainable.computing.io_keplerinternals.yaml @@ -36,6 +36,9 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - jsonPath: .spec.exporter.deployment.image + name: Image + type: string - jsonPath: .spec.exporter.deployment.nodeSelector name: Node-Selector priority: 10 @@ -68,6 +71,8 @@ spec: properties: deployment: properties: + image: + type: string nodeSelector: additionalProperties: type: string diff --git a/docs/api.md b/docs/api.md index 4be2a8e3..4e7a4b64 100644 --- a/docs/api.md +++ b/docs/api.md @@ -140,6 +140,13 @@ KeplerInternalSpec defines the desired state of Kepler + image + string + +
+ + false + nodeSelector map[string]string diff --git a/pkg/api/v1alpha1/kepler_internal_types.go b/pkg/api/v1alpha1/kepler_internal_types.go index 8c2d7fc1..8d78c2d6 100644 --- a/pkg/api/v1alpha1/kepler_internal_types.go +++ b/pkg/api/v1alpha1/kepler_internal_types.go @@ -24,9 +24,18 @@ import ( // e.g. kepler-internal.spec.exporter can reuse ExporterSpec because the API is // considered stable but not vice-versa. +type InternalExporterDeploymentSpec struct { + ExporterDeploymentSpec `json:",inline"` + Image string `json:"image,omitempty"` +} + +type InternalExporterSpec struct { + Deployment InternalExporterDeploymentSpec `json:"deployment,omitempty"` +} + // KeplerInternalSpec defines the desired state of Kepler type KeplerInternalSpec struct { - Exporter ExporterSpec `json:"exporter,omitempty"` + Exporter InternalExporterSpec `json:"exporter,omitempty"` } //+kubebuilder:object:root=true @@ -40,6 +49,7 @@ type KeplerInternalSpec struct { // +kubebuilder:printcolumn:name="Up-to-date",type=integer,JSONPath=`.status.updatedNumberScheduled` // +kubebuilder:printcolumn:name="Available",type=integer,JSONPath=`.status.numberAvailable` // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Image",type=string,JSONPath=`.spec.exporter.deployment.image` // +kubebuilder:printcolumn:name="Node-Selector",type=string,JSONPath=`.spec.exporter.deployment.nodeSelector`,priority=10 // +kubebuilder:printcolumn:name="Tolerations",type=string,JSONPath=`.spec.exporter.deployment.tolerations`,priority=10 // diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index d25917be..097e48f2 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -87,6 +87,38 @@ func (in *ExporterSpec) DeepCopy() *ExporterSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalExporterDeploymentSpec) DeepCopyInto(out *InternalExporterDeploymentSpec) { + *out = *in + in.ExporterDeploymentSpec.DeepCopyInto(&out.ExporterDeploymentSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalExporterDeploymentSpec. +func (in *InternalExporterDeploymentSpec) DeepCopy() *InternalExporterDeploymentSpec { + if in == nil { + return nil + } + out := new(InternalExporterDeploymentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalExporterSpec) DeepCopyInto(out *InternalExporterSpec) { + *out = *in + in.Deployment.DeepCopyInto(&out.Deployment) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalExporterSpec. +func (in *InternalExporterSpec) DeepCopy() *InternalExporterSpec { + if in == nil { + return nil + } + out := new(InternalExporterSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Kepler) DeepCopyInto(out *Kepler) { *out = *in diff --git a/pkg/components/exporter/exporter.go b/pkg/components/exporter/exporter.go index a608ded6..96b5093c 100644 --- a/pkg/components/exporter/exporter.go +++ b/pkg/components/exporter/exporter.go @@ -19,7 +19,6 @@ package exporter import ( _ "embed" "strconv" - "strings" "github.com/sustainable.computing.io/kepler-operator/pkg/api/v1alpha1" "github.com/sustainable.computing.io/kepler-operator/pkg/components" @@ -59,18 +58,6 @@ const ( DashboardNs = "openshift-config-managed" PrometheusRuleName = prefix + "prom-rules" - - KeplerBpfAttachMethodAnnotation = "kepler.sustainable.computing.io/bpf-attach-method" - KeplerBpfAttachMethodBCC = "bcc" - KeplerBpfAttachMethodLibbpf = "libbpf" -) - -// Config that will be set from outside -var ( - Config = struct { - Image string - ImageLibbpf string - }{} ) var ( @@ -106,16 +93,13 @@ func NewDaemonSet(detail components.Detail, k *v1alpha1.KeplerInternal) *appsv1. } } - deployment := k.Spec.Exporter.Deployment + deployment := k.Spec.Exporter.Deployment.ExporterDeploymentSpec + image := k.Spec.Exporter.Deployment.Image nodeSelector := deployment.NodeSelector tolerations := deployment.Tolerations + port := deployment.Port - bindAddress := "0.0.0.0:" + strconv.Itoa(int(deployment.Port)) - - keplerImage := Config.Image - if IsLibbpfAttachType(k) { - keplerImage = Config.ImageLibbpf - } + bindAddress := "0.0.0.0:" + strconv.Itoa(int(port)) return &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{ @@ -144,7 +128,7 @@ func NewDaemonSet(detail components.Detail, k *v1alpha1.KeplerInternal) *appsv1. Containers: []corev1.Container{{ Name: "kepler-exporter", SecurityContext: &corev1.SecurityContext{Privileged: pointer.Bool(true)}, - Image: keplerImage, + Image: image, Command: []string{ "/usr/bin/kepler", "-address", bindAddress, @@ -155,14 +139,14 @@ func NewDaemonSet(detail components.Detail, k *v1alpha1.KeplerInternal) *appsv1. "-redfish-cred-file-path=/etc/redfish/redfish.csv", }, Ports: []corev1.ContainerPort{{ - ContainerPort: int32(deployment.Port), + ContainerPort: int32(port), Name: "http", }}, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", - Port: intstr.IntOrString{Type: intstr.Int, IntVal: deployment.Port}, + Port: intstr.IntOrString{Type: intstr.Int, IntVal: port}, Scheme: "HTTP", }, }, @@ -279,7 +263,7 @@ func NewConfigMap(d components.Detail, k *v1alpha1.KeplerInternal) *corev1.Confi } } - deployment := k.Spec.Exporter.Deployment + deployment := k.Spec.Exporter.Deployment.ExporterDeploymentSpec bindAddress := "0.0.0.0:" + strconv.Itoa(int(deployment.Port)) return &corev1.ConfigMap{ @@ -449,7 +433,7 @@ func NewServiceAccount() *corev1.ServiceAccount { } func NewService(k *v1alpha1.KeplerInternal) *corev1.Service { - deployment := k.Spec.Exporter.Deployment + deployment := k.Spec.Exporter.Deployment.ExporterDeploymentSpec return &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -602,8 +586,3 @@ func record(name, expr string) monv1.Rule { Record: name, } } - -func IsLibbpfAttachType(k *v1alpha1.KeplerInternal) bool { - bpftype, ok := k.Annotations[KeplerBpfAttachMethodAnnotation] - return ok && strings.ToLower(bpftype) == KeplerBpfAttachMethodLibbpf -} diff --git a/pkg/components/exporter/exporter_test.go b/pkg/components/exporter/exporter_test.go index d97c2c5b..11ffb680 100644 --- a/pkg/components/exporter/exporter_test.go +++ b/pkg/components/exporter/exporter_test.go @@ -8,25 +8,26 @@ import ( "github.com/sustainable.computing.io/kepler-operator/pkg/components" "github.com/sustainable.computing.io/kepler-operator/pkg/utils/k8s" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestNodeSelection(t *testing.T) { tt := []struct { - spec v1alpha1.ExporterSpec + spec v1alpha1.InternalExporterSpec selector map[string]string scenario string }{ { - spec: v1alpha1.ExporterSpec{}, + spec: v1alpha1.InternalExporterSpec{}, selector: map[string]string{"kubernetes.io/os": "linux"}, scenario: "default case", }, { - spec: v1alpha1.ExporterSpec{ - Deployment: v1alpha1.ExporterDeploymentSpec{ - NodeSelector: map[string]string{"k1": "v1"}, + spec: v1alpha1.InternalExporterSpec{ + Deployment: v1alpha1.InternalExporterDeploymentSpec{ + ExporterDeploymentSpec: v1alpha1.ExporterDeploymentSpec{ + NodeSelector: map[string]string{"k1": "v1"}, + }, }, }, selector: map[string]string{"k1": "v1", "kubernetes.io/os": "linux"}, @@ -52,20 +53,22 @@ func TestNodeSelection(t *testing.T) { func TestTolerations(t *testing.T) { tt := []struct { - spec v1alpha1.ExporterSpec + spec v1alpha1.InternalExporterSpec tolerations []corev1.Toleration scenario string }{{ - spec: v1alpha1.ExporterSpec{}, + spec: v1alpha1.InternalExporterSpec{}, // NOTE: default toleration { "operator": "Exists" } is set by k8s API server (CRD default) // see: Kepler_Reconciliation e2e test tolerations: nil, scenario: "default case", }, { - spec: v1alpha1.ExporterSpec{ - Deployment: v1alpha1.ExporterDeploymentSpec{ - Tolerations: []corev1.Toleration{{ - Effect: corev1.TaintEffectNoSchedule, Key: "key1"}}, + spec: v1alpha1.InternalExporterSpec{ + Deployment: v1alpha1.InternalExporterDeploymentSpec{ + ExporterDeploymentSpec: v1alpha1.ExporterDeploymentSpec{ + Tolerations: []corev1.Toleration{{ + Effect: corev1.TaintEffectNoSchedule, Key: "key1"}}, + }, }, }, tolerations: []corev1.Toleration{{ @@ -91,12 +94,12 @@ func TestTolerations(t *testing.T) { func TestHostPID(t *testing.T) { tt := []struct { - spec v1alpha1.ExporterSpec + spec v1alpha1.InternalExporterSpec hostPID bool scenario string }{ { - spec: v1alpha1.ExporterSpec{}, + spec: v1alpha1.InternalExporterSpec{}, hostPID: true, scenario: "default case", }, @@ -118,12 +121,12 @@ func TestHostPID(t *testing.T) { } func TestVolumeMounts(t *testing.T) { tt := []struct { - spec v1alpha1.ExporterSpec + spec v1alpha1.InternalExporterSpec volumeMounts []corev1.VolumeMount scenario string }{ { - spec: v1alpha1.ExporterSpec{}, + spec: v1alpha1.InternalExporterSpec{}, volumeMounts: []corev1.VolumeMount{ {Name: "lib-modules", MountPath: "/lib/modules", ReadOnly: true}, {Name: "tracing", MountPath: "/sys", ReadOnly: true}, @@ -152,12 +155,12 @@ func TestVolumeMounts(t *testing.T) { } func TestVolumes(t *testing.T) { tt := []struct { - spec v1alpha1.ExporterSpec + spec v1alpha1.InternalExporterSpec volumes []corev1.Volume scenario string }{ { - spec: v1alpha1.ExporterSpec{}, + spec: v1alpha1.InternalExporterSpec{}, volumes: []corev1.Volume{ k8s.VolumeFromHost("lib-modules", "/lib/modules"), k8s.VolumeFromHost("tracing", "/sys"), @@ -212,55 +215,3 @@ func TestSCCAllows(t *testing.T) { }) } } - -func TestBpfAttachMethod(t *testing.T) { - - tt := []struct { - annotations map[string]string - scenario string - IsLibbpf bool - }{ - { - annotations: map[string]string{}, - IsLibbpf: false, - scenario: "no annotation", - }, - { - annotations: map[string]string{ - KeplerBpfAttachMethodAnnotation: "junk", - }, - IsLibbpf: false, - scenario: "annotation present but not libbpf", - }, - { - annotations: map[string]string{ - KeplerBpfAttachMethodAnnotation: "bcc", - }, - IsLibbpf: false, - scenario: "annotation present with bcc", - }, - { - annotations: map[string]string{ - KeplerBpfAttachMethodAnnotation: "libbpf", - }, - IsLibbpf: true, - scenario: "annotation present with libbpf", - }, - } - for _, tc := range tt { - tc := tc - t.Run(tc.scenario, func(t *testing.T) { - t.Parallel() - k := v1alpha1.KeplerInternal{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: tc.annotations, - }, - Spec: v1alpha1.KeplerInternalSpec{ - Exporter: v1alpha1.ExporterSpec{}, - }, - } - actual := IsLibbpfAttachType(&k) - assert.Equal(t, actual, tc.IsLibbpf) - }) - } -} diff --git a/pkg/controllers/kepler.go b/pkg/controllers/kepler.go index 79956da2..6770d018 100644 --- a/pkg/controllers/kepler.go +++ b/pkg/controllers/kepler.go @@ -2,6 +2,7 @@ package controllers import ( "context" + "strings" "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -23,7 +24,18 @@ import ( ) const ( - Finalizer = "kepler.system.sustainable.computing.io/finalizer" + Finalizer = "kepler.system.sustainable.computing.io/finalizer" + KeplerBpfAttachMethodAnnotation = "kepler.sustainable.computing.io/bpf-attach-method" + KeplerBpfAttachMethodBCC = "bcc" + KeplerBpfAttachMethodLibbpf = "libbpf" +) + +// Config that will be set from outside +var ( + Config = struct { + Image string + ImageLibbpf string + }{} ) // KeplerReconciler reconciles a Kepler object @@ -233,6 +245,12 @@ func (r KeplerReconciler) setInvalidStatus(ctx context.Context, req ctrl.Request } func newKeplerInternal(k *v1alpha1.Kepler) *v1alpha1.KeplerInternal { + + keplerImage := Config.Image + if IsLibbpfAttachType(k) { + keplerImage = Config.ImageLibbpf + } + return &v1alpha1.KeplerInternal{ TypeMeta: metav1.TypeMeta{ Kind: "KeplerInternal", @@ -243,7 +261,17 @@ func newKeplerInternal(k *v1alpha1.Kepler) *v1alpha1.KeplerInternal { Annotations: k.Annotations, }, Spec: v1alpha1.KeplerInternalSpec{ - Exporter: k.Spec.Exporter, + Exporter: v1alpha1.InternalExporterSpec{ + Deployment: v1alpha1.InternalExporterDeploymentSpec{ + ExporterDeploymentSpec: k.Spec.Exporter.Deployment, + Image: keplerImage, + }, + }, }, } } + +func IsLibbpfAttachType(k *v1alpha1.Kepler) bool { + bpftype, ok := k.Annotations[KeplerBpfAttachMethodAnnotation] + return ok && strings.ToLower(bpftype) == KeplerBpfAttachMethodLibbpf +} diff --git a/pkg/controllers/kepler_test.go b/pkg/controllers/kepler_test.go new file mode 100644 index 00000000..35b45897 --- /dev/null +++ b/pkg/controllers/kepler_test.go @@ -0,0 +1,61 @@ +package controllers + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/sustainable.computing.io/kepler-operator/pkg/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBpfAttachMethod(t *testing.T) { + + tt := []struct { + annotations map[string]string + scenario string + IsLibbpf bool + }{ + { + annotations: map[string]string{}, + IsLibbpf: false, + scenario: "no annotation", + }, + { + annotations: map[string]string{ + KeplerBpfAttachMethodAnnotation: "junk", + }, + IsLibbpf: false, + scenario: "annotation present but not libbpf", + }, + { + annotations: map[string]string{ + KeplerBpfAttachMethodAnnotation: "bcc", + }, + IsLibbpf: false, + scenario: "annotation present with bcc", + }, + { + annotations: map[string]string{ + KeplerBpfAttachMethodAnnotation: "libbpf", + }, + IsLibbpf: true, + scenario: "annotation present with libbpf", + }, + } + for _, tc := range tt { + tc := tc + t.Run(tc.scenario, func(t *testing.T) { + t.Parallel() + k := v1alpha1.Kepler{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: tc.annotations, + }, + Spec: v1alpha1.KeplerSpec{ + Exporter: v1alpha1.ExporterSpec{}, + }, + } + actual := IsLibbpfAttachType(&k) + assert.Equal(t, actual, tc.IsLibbpf) + }) + } +} diff --git a/tests/run-e2e.sh b/tests/run-e2e.sh index 18789d56..e2f667bb 100755 --- a/tests/run-e2e.sh +++ b/tests/run-e2e.sh @@ -151,7 +151,9 @@ run_e2e() { local ret=0 go test -v -failfast -timeout $TEST_TIMEOUT \ - ./tests/e2e/... 2>&1 | tee "$LOGS_DIR/e2e.log" || ret=1 + ./tests/e2e/... \ + -run Reconcile \ + 2>&1 | tee "$LOGS_DIR/e2e.log" || ret=1 # terminate both log_events { jobs -p | xargs -I {} -- pkill -TERM -P {}; } || true