Add support for credential provider plugin

Makefile

@@ -48,7 +48,9 @@ lint: golangci-lint ## Run golangci-lint against code.
 
 test: generate fmt vet manifests envtest ## Run tests
 	echo $(ENVTEST)
+	go build -o pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin pkg/daemon/criruntime/imageruntime/fake_plugin/main.go && chmod +x pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./pkg/... -coverprofile cover.out
+	rm pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
 
 coverage-report: ## Generate cover.html from cover.out
 	go tool cover -html=cover.out -o cover.html

cmd/daemon/main.go

@@ -17,6 +17,10 @@ limitations under the License.
 package main
 
 import (
+	"os"
+
+	"k8s.io/kubernetes/pkg/credentialprovider/plugin"
+
 	"flag"
 	"math/rand"
 	"net/http"
@@ -34,12 +38,15 @@ import (
 	"github.com/openkruise/kruise/pkg/daemon"
 	"github.com/openkruise/kruise/pkg/features"
 	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
+	"github.com/openkruise/kruise/pkg/util/secret"
 )
 
 var (
-	bindAddr    = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.")
-	pprofAddr   = flag.String("pprof-addr", ":10222", "The address the pprof binds to.")
-	enablePprof = flag.Bool("enable-pprof", true, "Enable pprof for daemon.")
+	bindAddr         = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.")
+	pprofAddr        = flag.String("pprof-addr", ":10222", "The address the pprof binds to.")
+	enablePprof      = flag.Bool("enable-pprof", true, "Enable pprof for daemon.")
+	pluginConfigFile = flag.String("plugin-config-file", "/kruise/CredentialProviderPlugin.yaml", "The path of plugin config file.")
+	pluginBinDir     = flag.String("plugin-bin-dir", "/kruise/plugins", "The path of directory of plugin binaries.")
 )
 
 func main() {
@@ -68,6 +75,20 @@ func main() {
 	if err != nil {
 		klog.Fatalf("Failed to new daemon: %v", err)
 	}
+
+	if _, err := os.Stat(*pluginConfigFile); err == nil {
+		err = plugin.RegisterCredentialProviderPlugins(*pluginConfigFile, *pluginBinDir)
+		if err != nil {
+			klog.Errorf("Failed to register credential provider plugins: %v", err)
+		}
+	} else if os.IsNotExist(err) {
+		klog.Infof("No plugin config file found, skipping: %s", *pluginConfigFile)
+	} else {
+		klog.Errorf("Failed to check plugin config file: %v", err)
+	}
+	// make sure the new docker key ring is made and set after the credential plugins are registered
+	secret.MakeAndSetKeyring()
+
 	if err := d.Run(ctx); err != nil {
 		klog.Fatalf("Failed to start daemon: %v", err)
 	}

pkg/daemon/criruntime/imageruntime/cri.go

@@ -131,31 +131,31 @@
 	// for some runtime implementations.
 	pullImageReq.SandboxConfig.Annotations[pullingImageSandboxConfigAnno] = "kruise-daemon"
 
-	if len(pullSecrets) > 0 {
-		var authInfos []daemonutil.AuthInfo
-		authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, repoToPull)
-		if err == nil {
-			var pullErrs []error
-			for _, authInfo := range authInfos {
-				var pullErr error
-				klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
-				pullImageReq.Auth = &runtimeapi.AuthConfig{
-					Username: authInfo.Username,
-					Password: authInfo.Password,
-				}
-				_, pullErr = c.criImageClient.PullImage(ctx, pullImageReq)
-				if pullErr == nil {
-					pipeW.CloseWithError(io.EOF)
-					return newImagePullStatusReader(pipeR), nil
-				}
-				klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
-				pullErrs = append(pullErrs, pullErr)
-
+	var authInfos []daemonutil.AuthInfo
+	authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, repoToPull)
+	if err == nil {
+		var pullErrs []error
+		for _, authInfo := range authInfos {
+			var pullErr error
+			klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
+			pullImageReq.Auth = &runtimeapi.AuthConfig{
+				Username: authInfo.Username,
+				Password: authInfo.Password,
 			}
-			if len(pullErrs) > 0 {
-				err = utilerrors.NewAggregate(pullErrs)
+			_, pullErr = c.criImageClient.PullImage(ctx, pullImageReq)
+			if pullErr == nil {
+				pipeW.CloseWithError(io.EOF)
+				return newImagePullStatusReader(pipeR), nil
 			}
+			klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
+			pullErrs = append(pullErrs, pullErr)
+
+		}
+		if len(pullErrs) > 0 {
+			err = utilerrors.NewAggregate(pullErrs)
 		}
+	} else {
+		klog.Errorf("Failed to convert to auth info for registry, err %v", err)
 	}
 
 	// Try the default secret

pkg/daemon/criruntime/imageruntime/docker.go

@@ -81,26 +81,26 @@
 	fullName := imageName + ":" + tag
 	var ioReader io.ReadCloser
 
-	if len(pullSecrets) > 0 {
-		var authInfos []daemonutil.AuthInfo
-		authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry)
-		if err == nil {
-			var pullErrs []error
-			for _, authInfo := range authInfos {
-				var pullErr error
-				klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
-				ioReader, pullErr = d.client.ImagePull(ctx, fullName, dockertypes.ImagePullOptions{RegistryAuth: authInfo.EncodeToString()})
-				if pullErr == nil {
-					return newImagePullStatusReader(ioReader), nil
-				}
-				d.handleRuntimeError(pullErr)
-				klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
-				pullErrs = append(pullErrs, pullErr)
-			}
-			if len(pullErrs) > 0 {
-				err = utilerrors.NewAggregate(pullErrs)
+	var authInfos []daemonutil.AuthInfo
+	authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry)
+	if err == nil {
+		var pullErrs []error
+		for _, authInfo := range authInfos {
+			var pullErr error
+			klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
+			ioReader, pullErr = d.client.ImagePull(ctx, fullName, dockertypes.ImagePullOptions{RegistryAuth: authInfo.EncodeToString()})
+			if pullErr == nil {
+				return newImagePullStatusReader(ioReader), nil
 			}
+			d.handleRuntimeError(pullErr)
+			klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
+			pullErrs = append(pullErrs, pullErr)
+		}
+		if len(pullErrs) > 0 {
+			err = utilerrors.NewAggregate(pullErrs)
 		}
+	} else {
+		klog.Errorf("Failed to convert to auth info for registry, err %v", err)
 	}
 
 	// Try the default secret

pkg/daemon/criruntime/imageruntime/fake_plugin/main.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog/v2"
+	"k8s.io/kubelet/pkg/apis/credentialprovider/install"
+	v1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
+)
+
+var (
+	scheme = runtime.NewScheme()
+	codecs = serializer.NewCodecFactory(scheme)
+)
+
+func init() {
+	install.Install(scheme)
+}
+
+func getCredentials(ctx context.Context, image string, args []string) (*v1.CredentialProviderResponse, error) {
+	response := &v1.CredentialProviderResponse{
+		CacheKeyType: v1.RegistryPluginCacheKeyType,
+		Auth: map[string]v1.AuthConfig{
+			"registry.plugin.com/test": {
+				Username: "user",
+				Password: "password",
+			},
+		},
+	}
+	return response, nil
+}
+
+func runPlugin(ctx context.Context, r io.Reader, w io.Writer, args []string) error {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	_, err = json.DefaultMetaFactory.Interpret(data)
+	if err != nil {
+		return err
+	}
+
+	request, err := decodeRequest(data)
+	if err != nil {
+		return err
+	}
+
+	if request.Image == "" {
+		return errors.New("image in plugin request was empty")
+	}
+
+	// Deny all requests except for those where the image URL contains registry.plugin.com
+	// to test whether kruise could get expected auths if plugin fails to run
+	if !strings.Contains(request.Image, "registry.plugin.com") {
+		return errors.New("image in plugin request not supported: " + request.Image)
+	}
+
+	response, err := getCredentials(ctx, request.Image, args)
+	if err != nil {
+		return err
+	}
+
+	if response == nil {
+		return errors.New("CredentialProviderResponse from plugin was nil")
+	}
+
+	encodedResponse, err := encodeResponse(response)
+	if err != nil {
+		return err
+	}
+
+	writer := bufio.NewWriter(w)
+	defer writer.Flush()
+	if _, err := writer.Write(encodedResponse); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func decodeRequest(data []byte) (*v1.CredentialProviderRequest, error) {
+	obj, gvk, err := codecs.UniversalDecoder(v1.SchemeGroupVersion).Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if gvk.Kind != "CredentialProviderRequest" {
+		return nil, fmt.Errorf("kind was %q, expected CredentialProviderRequest", gvk.Kind)
+	}
+
+	if gvk.Group != v1.GroupName {
+		return nil, fmt.Errorf("group was %q, expected %s", gvk.Group, v1.GroupName)
+	}
+
+	request, ok := obj.(*v1.CredentialProviderRequest)
+	if !ok {
+		return nil, fmt.Errorf("unable to convert %T to *CredentialProviderRequest", obj)
+	}
+
+	return request, nil
+}
+
+func encodeResponse(response *v1.CredentialProviderResponse) ([]byte, error) {
+	mediaType := "application/json"
+	info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return nil, fmt.Errorf("unsupported media type %q", mediaType)
+	}
+
+	encoder := codecs.EncoderForVersion(info.Serializer, v1.SchemeGroupVersion)
+	data, err := runtime.Encode(encoder, response)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode response: %v", err)
+	}
+
+	return data, nil
+}
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if err := newCredentialProviderCommand().Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func newCredentialProviderCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "acr-credential-provider",
+		Short: "ACR credential provider for kubelet",
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := runPlugin(context.TODO(), os.Stdin, os.Stdout, os.Args[1:]); err != nil {
+				klog.Errorf("Error running credential provider plugin: %v", err)
+				os.Exit(1)
+			}
+		},
+	}
+	return cmd
+}

pkg/daemon/criruntime/imageruntime/fake_plugin/plugin-config.yaml

@@ -0,0 +1,9 @@
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: fake-credential-plugin
+    matchImages:
+      - "registry.plugin.com"
+      - "registry.private.com"
+    defaultCacheDuration: "12h"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1

pkg/daemon/criruntime/imageruntime/helpers_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/credentialprovider/plugin"
 	"k8s.io/kubernetes/pkg/util/parsers"
 
 	"github.com/openkruise/kruise/pkg/util/secret"
@@ -103,7 +105,26 @@ func TestMatchRegistryAuths(t *testing.T) {
 			},
 			ExpectMinValue: 0,
 		},
+		{
+			name:  "test credential plugin if matched",
+			Image: "registry.plugin.com/test/echoserver:v1",
+			GetSecrets: func() []v1.Secret {
+				return []v1.Secret{}
+			},
+			ExpectMinValue: 1,
+		},
+	}
+	pluginBinDir := "fake_plugin"
+	pluginConfigFile := "fake_plugin/plugin-config.yaml"
+	// credential plugin is configured for images with "registry.plugin.com" and "registry.private.com",
+	// however, only images with "registry.plugin.com" will return a fake credential,
+	// other images will be denied by the plugin and an error will be raised,
+	// this is to test whether kruise could get expected auths if plugin fails to run
+	err := plugin.RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir)
+	if err != nil {
+		klog.Errorf("Failed to register credential provider plugins: %v", err)
 	}
+	secret.MakeAndSetKeyring()
 	for _, cs := range cases {
 		t.Run(cs.name, func(t *testing.T) {
 			repoToPull, _, _, err := parsers.ParseImageName(cs.Image)