From cfb452451516ad3f1a0fc444bfecdd3d94a1492e Mon Sep 17 00:00:00 2001 From: Chandan Abhyankar Date: Fri, 13 Jan 2023 01:05:28 -0800 Subject: [PATCH] cpu: support for detecting nx-gzip coprocessor feature Nest accelerator gzip support for IBM Power systems. Signed-off-by: Chandan Abhyankar --- docs/usage/customization-guide.md | 181 ++++++++++++++++++++++++------ docs/usage/features.md | 15 ++- source/cpu/coprocessor_ppc64le.go | 43 +++++++ source/cpu/coprocessor_stub.go | 24 ++++ source/cpu/cpu.go | 29 +++-- 5 files changed, 246 insertions(+), 46 deletions(-) create mode 100644 source/cpu/coprocessor_ppc64le.go create mode 100644 source/cpu/coprocessor_stub.go diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index cb2d48ff9c..62625a4292 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -1,7 +1,7 @@ --- title: "Customization guide" layout: default -sort: 6 +sort: 8 --- # Customization guide @@ -20,6 +20,9 @@ sort: 6 NFD provides multiple extension points for vendor and application specific labeling: +- [`NodeFeature`](#nodefeature-custom-resource) (EXPERIMENTAL) objects can be + used to communicate "raw" node features and node labeling requests to + nfd-master. - [`NodeFeatureRule`](#nodefeaturerule-custom-resource) objects provide a way to deploy custom labeling rules via the Kubernetes API - [`local`](#local-feature-source) feature source of nfd-worker creates @@ -27,11 +30,91 @@ labeling: - [`custom`](#custom-feature-source) feature source of nfd-worker creates labels based on user-specified rules +## NodeFeature custom resource + +**EXPERIMENTAL** +NodeFeature objects provide a way for 3rd party extensions to advertise custom +features, both as "raw" features that serve as input to +[NodeFeatureRule](#nodefeaturerule-custom-resource) objects and as feature +labels directly. + +Note that RBAC rules must be created for each extension for them to be able to +create and manipulate NodeFeature objects in their namespace. + +Support for NodeFeature CRD API is enabled with the `-enable-nodefeature-api` +command line flag. This flag must be specified for both nfd-master and +nfd-worker as it will disable the gRPC communication between them. + +### A NodeFeature example + +Consider the following referential example: + +```yaml +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeature +metadata: + labels: + nfd.node.kubernetes.io/node-name: node-1 + name: vendor-features-for-node-1 +spec: + # Features for NodeFeatureRule matching + features: + flags: + vendor.flags: + elements: + feature-x: {} + feature-y: {} + attributes: + vendor.config: + elements: + setting-a: "auto" + knob-b: "123" + instances: + vendor.devices: + elements: + - attributes: + model: "dev-1000" + vendor: "acme" + - attributes: + model: "dev-2000" + vendor: "acme" + # Labels to be created + labels: + vendor-feature.enabled: "true" +``` + +The object targets node named `node-1`. It lists two "flag type" features under +the `vendor.flags` domain, two "attribute type" features and uder the +`vendor.config` domain and two "instance type" features under the +`vendor.devices` domain. This features will not be directly affecting the node +labels but they will be used as input when be +[`NodeFeatureRule`](#nodefeaturerule-custom-resource) object are evaluated. + +In addition the example requests directly the +`feature.node.kubenernetes.io/vendor-feature.enabled=true` node label to be +created. + +The `nfd.node.kubernetes.io/node-name=` must be in place for each +NodeFeature objectt as NFD uses it to determine the node which it is targeting. + +### Feature types + +Features are divided into three different types: + +- **flag** features: a set of names without any associated values, e.g. CPUID + flags or loaded kernel modules +- **attribute** features: a set of names each of which has a single value + associated with it (essentially a map of key-value pairs), e.g. kernel config + flags or os release information +- **instance** features: a list of instances, each of which has multiple + attributes (key-value pairs of their own) associated with it, e.g. PCI or USB + devices + ## NodeFeatureRule custom resource `NodeFeatureRule` objects provide an easy way to create vendor or application -specific labels. It uses a flexible rule-based mechanism for creating labels -based on node feature. +specific labels and taints. It uses a flexible rule-based mechanism for creating +labels and optionally taints based on node features. ### A NodeFeatureRule example @@ -74,22 +157,56 @@ Now, on X86 platforms the feature label appears after doing `modprobe dummy` on a system and correspondingly the label is removed after `rmmod dummy`. Note a re-labeling delay up to the sleep-interval of nfd-worker (1 minute by default). -### NodeFeatureRule controller +See [Label rule format](#label-rule-format) for detailed description of +available fields and how to write labeling rules. +### NodeFeatureRule tainting feature + +This feature is experimental. + +In some circumstances it is desirable keep nodes with specialized hardware away from +running general workload and instead leave them for workloads that need the specialized +hardware. One way to achieve it is to taint the nodes with the specialized hardware +and add corresponding toleration to pods that require the special hardware. NFD +offers node tainting functionality which is disabled by default. User can define +one or more custom taints via the `taints` field of the NodeFeatureRule CR. The +same rule-based mechanism is applied here and the NFD taints only rule matching nodes. + +To enable the tainting feature, `--enable-taints` flag needs to be set to `true`. +If the flag `--enable-taints` is set to `false` (i.e. disabled), taints defined in +the NodeFeatureRule CR have no effect and will be ignored by the NFD master. -NFD-Master acts as the controller for `NodeFeatureRule` objects. It applies these -rules on raw feature data received from nfd-worker instances and creates node -labels, accordingly. +**NOTE**: Before enabling any taints, make sure to edit nfd-worker daemonset to +tolerate the taints to be created. Otherwise, already running pods that do not +tolerate the taint are evicted immediately from the node including the nfd-worker +pod. -**NOTE** nfd-master is stateless and (re-)labelling only happens when a request -is received from nfd-worker. That is, in practice rules are evaluated and -labels for each node are created on intervals specified by the -[`core.sleepInterval`](../reference/worker-configuration-reference#coresleepinterval) -configuration option (or -[`-sleep-interval`](../reference/worker-commandline-reference#-sleep-interval) -command line flag) of nfd-worker instances. This means that modification or -creation of `NodeFeatureRule` objects does not instantly cause the node labels -to be updated. Instead, the changes only come visible in node labels as -nfd-worker instances send their labelling requests. +Example NodeFeatureRule with custom taints: + +```yaml +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: my-sample-rule-object +spec: + rules: + - name: "my sample taint rule" + taints: + - effect: PreferNoSchedule + key: "feature.node.kubernetes.io/special-node" + value: "true" + - effect: NoExecute + key: "feature.node.kubernetes.io/dedicated-node" + matchFeatures: + - feature: kernel.loadedmodule + matchExpressions: + dummy: {op: Exists} + - feature: kernel.config + matchExpressions: + X86: {op: In, value: ["y"]} +``` + +In this example, if the `my sample taint rule` rule is matched, `feature.node.kubernetes.io/pci-0300_1d0f.present=true:NoExecute` +and `feature.node.kubernetes.io/cpu-cpuid.ADX:NoExecute` taints are set on the node. ## Local feature source @@ -137,7 +254,7 @@ on the nfd-master command line. ### Hooks -The `local` source executes hooks found in +**DEPRECATED** The `local` source executes hooks found in `/etc/kubernetes/node-feature-discovery/source.d/`. The hook files must be executable and they are supposed to print all discovered features in `stdout`. With ELF binaries static linking is recommended as the selection of system @@ -364,6 +481,15 @@ details. labels specified in the `labels` field will override anything originating from `labelsTemplate`. +### Taints + +*taints* is a list of taint entries and each entry can have `key`, `value` and `effect`, +where the `value` is optional. Effect could be `NoSchedule`, `PreferNoSchedule` +or `NoExecute`. To learn more about the meaning of these effects, check out k8s [documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). + +**NOTE** taints field is not available for the custom rules of nfd-worker and only +for NodeFeatureRule objects. + #### Vars The `.vars` field is a map of values (key-value pairs) to store for subsequent @@ -465,24 +591,9 @@ true). ### Available features -#### Feature types - -Features are divided into three different types: - -- **flag** features: a set of names without any associated values, e.g. CPUID - flags or loaded kernel modules -- **attribute** features: a set of names each of which has a single value - associated with it (essentially a map of key-value pairs), e.g. kernel config - flags or os release information -- **instance** features: a list of instances, each of which has multiple - attributes (key-value pairs of their own) associated with it, e.g. PCI or USB - devices - -#### List of features - The following features are available for matching: -| Feature | Feature type | Elements | Value type | Description +| Feature | [Feature type](#feature-types) | Elements | Value type | Description | ---------------- | ------------ | -------- | ---------- | ----------- | **`cpu.cpuid`** | flag | | | Supported CPU capabilities | | | **``** | | CPUID flag is present @@ -510,6 +621,8 @@ The following features are available for matching: | | | **`enabled`** | bool | **DEPRECATED**: use **`se.enabled`** from **`cpu.security`** instead | **`cpu.topology`** | attribute | | | CPU topology related features | | | **`hardware_multithreading`** | bool | Hardware multithreading, such as Intel HTT, is enabled +| **`cpu.coprocessor`** | attribute | | | CPU Coprocessor related features +| | | **`nx_gzip`** | bool | Nest Accelerator GZIP support is enabled | **`kernel.config`** | attribute | | | Kernel configuration options | | | **``** | string | Value of the kconfig option | **`kernel.loadedmodule`** | flag | | | Loaded kernel modules diff --git a/docs/usage/features.md b/docs/usage/features.md index b7ed5ff8ba..32c39e4fca 100644 --- a/docs/usage/features.md +++ b/docs/usage/features.md @@ -33,10 +33,14 @@ have the following format. feature.node.kubernetes.io/ = ``` -*Note: Consecutive runs of nfd-worker will update the labels on a +> NOTE: Consecutive runs of nfd-worker will update the labels on a given node. If features are not discovered on a consecutive run, the corresponding label will be removed. This includes any restrictions placed on the consecutive run, -such as restricting discovered features with the -label-whitelist option.* +such as restricting discovered features with the +[`-label-whitelist`](../reference/master-commandline-reference#-label-whitelist) +flag of nfd-master or +[`core.labelWhiteList`](../reference/worker-configuration-reference#corelabelwhitelist) +option of nfd-worker. ### CPU @@ -44,6 +48,7 @@ such as restricting discovered features with the -label-whitelist option.* | ----------------------- | ------------ | ----------- | **`cpu-cpuid.`** | true | CPU capability is supported. **NOTE:** the capability might be supported but not enabled. | **`cpu-hardware_multithreading`** | true | Hardware multithreading, such as Intel HTT, enabled (number of logical CPUs is greater than physical CPUs) +| **`cpu-coprocessor.nx_gzip`** | true | Nest Accelerator for GZIP is supported(Power). | **`cpu-power.sst_bf.enabled`** | true | Intel SST-BF ([Intel Speed Select Technology][intel-sst] - Base frequency) enabled | **`cpu-pstate.status`** | string | The status of the [Intel pstate][intel-pstate] driver when in use and enabled, either 'active' or 'passive'. | **`cpu-pstate.turbo`** | bool | Set to 'true' if turbo frequencies are enabled in Intel pstate driver, set to 'false' if they have been disabled. @@ -75,6 +80,7 @@ configuration options for details. | AVXVNNI | AVX (VEX encoded) VNNI neural network instructions | AMXBF16 | Advanced Matrix Extension, tile multiplication operations on BFLOAT16 numbers | AMXINT8 | Advanced Matrix Extension, tile multiplication operations on 8-bit integers +| AMXFP16 | Advanced Matrix Extension, tile multiplication operations on FP16 numbers | AMXTILE | Advanced Matrix Extension, base tile architecture support | AVX512BF16 | AVX-512 BFLOAT16 instructions | AVX512BITALG | AVX-512 bit Algorithms @@ -92,9 +98,14 @@ configuration options for details. | AVX512VNNI | AVX-512 vector neural network instructions | AVX512VP2INTERSECT | AVX-512 intersect for D/Q | AVX512VPOPCNTDQ | AVX-512 vector population count doubleword and quadword +| AVXIFMA | AVX-IFMA instructions +| AVXNECONVERT | AVX-NE-CONVERT instructions +| AVXVNNIINT8 | AVX-VNNI-INT8 instructions +| CMPCCXADD | CMPCCXADD instructions | ENQCMD | Enqueue Command | GFNI | Galois Field New Instructions | HYPERVISOR | Running under hypervisor +| PREFETCHI | PREFETCHIT0/1 instructions | VAES | AVX-512 vector AES instructions | VPCLMULQDQ | Carry-less multiplication quadword diff --git a/source/cpu/coprocessor_ppc64le.go b/source/cpu/coprocessor_ppc64le.go new file mode 100644 index 0000000000..0d1e859f9e --- /dev/null +++ b/source/cpu/coprocessor_ppc64le.go @@ -0,0 +1,43 @@ +//go:build ppc64le +// +build ppc64le + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cpu + +import ( + "k8s.io/klog/v2" + "os" + "sigs.k8s.io/node-feature-discovery/pkg/utils/hostpath" + "strconv" +) + +/* Detect NX_GZIP */ +func discoverCoprocessor() map[string]string { + features := make(map[string]string) + + nxGzipPath := hostpath.SysfsDir.Path("devices/vio/ibm,compression-v1/nx_gzip_caps") + + _, err := os.Stat(nxGzipPath) + if err != nil { + klog.V(5).Infof("Failed to detect nx_gzip for Nest Accelerator: %v", err) + } else { + features["nx_gzip"] = strconv.FormatBool(true) + } + + return features +} diff --git a/source/cpu/coprocessor_stub.go b/source/cpu/coprocessor_stub.go new file mode 100644 index 0000000000..2370f3963e --- /dev/null +++ b/source/cpu/coprocessor_stub.go @@ -0,0 +1,24 @@ +//go:build !ppc64le +// +build !ppc64le + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cpu + +func discoverCoprocessor() map[string]string { + return nil +} diff --git a/source/cpu/cpu.go b/source/cpu/cpu.go index d9b77f8e35..48dedfa130 100644 --- a/source/cpu/cpu.go +++ b/source/cpu/cpu.go @@ -34,16 +34,17 @@ import ( const Name = "cpu" const ( - CpuidFeature = "cpuid" - Cpumodel = "model" - CstateFeature = "cstate" - PstateFeature = "pstate" - RdtFeature = "rdt" - SeFeature = "se" // DEPRECATED in v0.12: will be removed in the future - SecurityFeature = "security" - SgxFeature = "sgx" // DEPRECATED in v0.12: will be removed in the future - SstFeature = "sst" - TopologyFeature = "topology" + CpuidFeature = "cpuid" + Cpumodel = "model" + CstateFeature = "cstate" + PstateFeature = "pstate" + RdtFeature = "rdt" + SeFeature = "se" // DEPRECATED in v0.12: will be removed in the future + SecurityFeature = "security" + SgxFeature = "sgx" // DEPRECATED in v0.12: will be removed in the future + SstFeature = "sst" + TopologyFeature = "topology" + CoprocessorFeature = "coprocessor" ) // Configuration file options @@ -192,6 +193,11 @@ func (s *cpuSource) GetLabels() (source.FeatureLabels, error) { labels["hardware_multithreading"] = v } + // NX + if v, ok := features.Attributes[CoprocessorFeature].Elements["nx_gzip"]; ok { + labels["coprocessor.nx_gzip"] = v + } + return labels, nil } @@ -246,6 +252,9 @@ func (s *cpuSource) Discover() error { // Detect hyper-threading s.features.Attributes[TopologyFeature] = nfdv1alpha1.NewAttributeFeatures(discoverTopology()) + // Detect Coprocessor features + s.features.Attributes[CoprocessorFeature] = nfdv1alpha1.NewAttributeFeatures(discoverCoprocessor()) + utils.KlogDump(3, "discovered cpu features:", " ", s.features) return nil