diff --git a/pkg/karmadactl/addons/search/manifests.go b/pkg/karmadactl/addons/search/manifests.go index b510b5ab45bc..818f9781709a 100644 --- a/pkg/karmadactl/addons/search/manifests.go +++ b/pkg/karmadactl/addons/search/manifests.go @@ -48,7 +48,8 @@ spec: - --audit-log-path=- - --feature-gates=APIPriorityAndFairness=false - --audit-log-maxage=0 - - --audit-log-maxbackup=0 + - --audit-log-maxbackup=0{{- if .KeyPrefix }} + - --etcd-prefix={{ .KeyPrefix }}{{- end }} livenessProbe: httpGet: path: /livez @@ -126,6 +127,7 @@ type DeploymentReplace struct { Replicas *int32 Image string ETCDSevers string + KeyPrefix string } // ServiceReplace is a struct to help to concrete diff --git a/pkg/karmadactl/addons/search/search.go b/pkg/karmadactl/addons/search/search.go index 580e95d9d6e8..9d488cc5599b 100644 --- a/pkg/karmadactl/addons/search/search.go +++ b/pkg/karmadactl/addons/search/search.go @@ -11,6 +11,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kuberuntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" clientsetscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" @@ -28,6 +29,8 @@ const ( // etcdStatefulSetAndServiceName define etcd statefulSet and serviceName installed by init command etcdStatefulSetAndServiceName = "etcd" + // karmadaAPIServerDeploymentAndServiceName defines the name of karmada-apiserver deployment and service installed by init command + karmadaAPIServerDeploymentAndServiceName = "karmada-apiserver" // etcdContainerClientPort define etcd pod installed by init command etcdContainerClientPort = 2379 @@ -134,7 +137,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e return fmt.Errorf("create karmada search service error: %v", err) } - etcdServers, err := etcdServers(opts) + etcdServers, keyPrefix, err := etcdServers(opts) if err != nil { return err } @@ -146,6 +149,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e Namespace: opts.Namespace, Replicas: &opts.KarmadaSearchReplicas, ETCDSevers: etcdServers, + KeyPrefix: keyPrefix, Image: addoninit.KarmadaSearchImage(opts), }) if err != nil { @@ -212,10 +216,56 @@ func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableO return nil } -func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, error) { - sts, err := opts.KubeClientSet.AppsV1().StatefulSets(opts.Namespace).Get(context.TODO(), etcdStatefulSetAndServiceName, metav1.GetOptions{}) +const ( + etcdServerArgPrefix = "--etcd-servers=" + etcdServerArgPrefixLength = len(etcdServerArgPrefix) + etcdKeyPrefixArgPrefix = "--etcd-prefix=" + etcdKeyPrefixArgPrefixLength = len(etcdKeyPrefixArgPrefix) +) + +func getExternalEtcdServerConfig(ctx context.Context, host kubernetes.Interface, namespace string) (servers, prefix string, err error) { + var apiserver *appsv1.Deployment + if apiserver, err = host.AppsV1().Deployments(namespace).Get( + ctx, karmadaAPIServerDeploymentAndServiceName, metav1.GetOptions{}); err != nil { + return + } + // should be only one container, but it may be injected others by mutating webhook of host cluster, + // anyway, a for can handle all cases. + var apiServerContainer *corev1.Container + for i, container := range apiserver.Spec.Template.Spec.Containers { + if container.Name == karmadaAPIServerDeploymentAndServiceName { + apiServerContainer = &apiserver.Spec.Template.Spec.Containers[i] + break + } + } + if apiServerContainer == nil { + return + } + for _, cmd := range apiServerContainer.Command { + if strings.HasPrefix(cmd, etcdServerArgPrefix) { + servers = cmd[etcdServerArgPrefixLength:] + } else if strings.HasPrefix(cmd, etcdKeyPrefixArgPrefix) { + prefix = cmd[etcdKeyPrefixArgPrefixLength:] + } + if servers != "" && prefix != "" { + break + } + } + return +} + +func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, string, error) { + ctx := context.TODO() + sts, err := opts.KubeClientSet.AppsV1().StatefulSets(opts.Namespace).Get(ctx, etcdStatefulSetAndServiceName, metav1.GetOptions{}) if err != nil { - return "", err + if apierrors.IsNotFound(err) { + if servers, prefix, cfgErr := getExternalEtcdServerConfig(ctx, opts.KubeClientSet, opts.Namespace); cfgErr != nil { + return "", "", cfgErr + } else if servers != "" { + return servers, prefix, nil + } + } + return "", "", err } etcdReplicas := *sts.Spec.Replicas @@ -225,5 +275,5 @@ func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, error) { etcdServers += fmt.Sprintf("https://%s-%v.%s.%s.svc.%s:%v", etcdStatefulSetAndServiceName, v, etcdStatefulSetAndServiceName, opts.Namespace, opts.HostClusterDomain, etcdContainerClientPort) + "," } - return strings.TrimRight(etcdServers, ","), nil + return strings.TrimRight(etcdServers, ","), "", nil } diff --git a/pkg/karmadactl/cmdinit/cert/cert.go b/pkg/karmadactl/cmdinit/cert/cert.go index 98f6bf0fbcbf..60c6fb5a96b7 100644 --- a/pkg/karmadactl/cmdinit/cert/cert.go +++ b/pkg/karmadactl/cmdinit/cert/cert.go @@ -285,6 +285,10 @@ func GenCerts(pkiPath string, etcdServerCertCfg, etcdClientCertCfg, karmadaCertC return err } + if etcdServerCertCfg == nil && etcdClientCertCfg == nil { + // use external etcd + return nil + } return genEtcdCerts(pkiPath, etcdServerCertCfg, etcdClientCertCfg) } diff --git a/pkg/karmadactl/cmdinit/cmdinit.go b/pkg/karmadactl/cmdinit/cmdinit.go index 1e5eb5585249..b2e793e283ab 100644 --- a/pkg/karmadactl/cmdinit/cmdinit.go +++ b/pkg/karmadactl/cmdinit/cmdinit.go @@ -122,6 +122,11 @@ func NewCmdInit(parentCommand string) *cobra.Command { flags.StringVarP(&opts.EtcdHostDataPath, "etcd-data", "", "/var/lib/karmada-etcd", "etcd data path,valid in hostPath mode.") flags.StringVarP(&opts.EtcdNodeSelectorLabels, "etcd-node-selector-labels", "", "", "etcd pod select the labels of the node. valid in hostPath mode ( e.g. --etcd-node-selector-labels karmada.io/etcd=true)") flags.StringVarP(&opts.EtcdPersistentVolumeSize, "etcd-pvc-size", "", "5Gi", "etcd data path,valid in pvc mode.") + flags.StringVar(&opts.ExternalEtcdCACertPath, "external-etcd-ca-cert-path", "", "The path of CA certificate of the external etcd cluster in pem format.") + flags.StringVar(&opts.ExternalEtcdClientCertPath, "external-etcd-client-cert-path", "", "The path of client side certificate to the external etcd cluster in pem format.") + flags.StringVar(&opts.ExternalEtcdClientKeyPath, "external-etcd-client-key-path", "", "The path of client side private key to the external etcd cluster in pem format.") + flags.StringVar(&opts.ExternalEtcdServers, "external-etcd-servers", "", "The server urls of external etcd cluster, to be used by kube-apiserver through --etcd-servers.") + flags.StringVar(&opts.ExternalEtcdKeyPrefix, "external-etcd-key-prefix", "", "The key prefix to be configured to kube-apiserver through --etcd-prefix.") // karmada flags.StringVar(&opts.CRDs, "crds", kubernetes.DefaultCrdURL, "Karmada crds resource.(local file e.g. --crds /root/crds.tar.gz)") flags.StringVarP(&opts.KarmadaAPIServerAdvertiseAddress, "karmada-apiserver-advertise-address", "", "", "The IP address the Karmada API Server will advertise it's listening on. If not set, the address on the master node will be used.") diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index 7fe9794bee6b..70d6ff19b746 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -45,6 +45,25 @@ var ( options.FrontProxyClientCertAndKeyName, } + emptyByteSlice = make([]byte, 0) + externalEtcdCertSpecialization = map[string]func(*CommandInitOption) ([]byte, []byte, error){ + options.EtcdCaCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) { + cert, err = os.ReadFile(option.ExternalEtcdCACertPath) + key = emptyByteSlice + return + }, + options.EtcdServerCertAndKeyName: func(_ *CommandInitOption) ([]byte, []byte, error) { + return emptyByteSlice, emptyByteSlice, nil + }, + options.EtcdClientCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) { + if cert, err = os.ReadFile(option.ExternalEtcdClientCertPath); err != nil { + return + } + key, err = os.ReadFile(option.ExternalEtcdClientKeyPath) + return + }, + } + karmadaRelease string defaultEtcdImage = "etcd:3.5.9-0" @@ -98,6 +117,11 @@ type CommandInitOption struct { EtcdHostDataPath string EtcdNodeSelectorLabels string EtcdPersistentVolumeSize string + ExternalEtcdCACertPath string + ExternalEtcdClientCertPath string + ExternalEtcdClientKeyPath string + ExternalEtcdServers string + ExternalEtcdKeyPrefix string KarmadaAPIServerImage string KarmadaAPIServerReplicas int32 KarmadaAPIServerAdvertiseAddress string @@ -131,16 +155,7 @@ type CommandInitOption struct { WaitComponentReadyTimeout int } -// Validate Check that there are enough flags to run the command. -// -//nolint:gocyclo -func (i *CommandInitOption) Validate(parentCommand string) error { - if i.KarmadaAPIServerAdvertiseAddress != "" { - if netutils.ParseIPSloppy(i.KarmadaAPIServerAdvertiseAddress) == nil { - return fmt.Errorf("karmada apiserver advertise address is not valid") - } - } - +func (i *CommandInitOption) validateLocalEtcd(parentCommand string) error { if i.EtcdStorageMode == etcdStorageModeHostPath && i.EtcdHostDataPath == "" { return fmt.Errorf("when etcd storage mode is hostPath, dataPath is not empty. See '%s init --help'", parentCommand) } @@ -173,6 +188,35 @@ func (i *CommandInitOption) Validate(parentCommand string) error { return nil } +func (i *CommandInitOption) validateExternalEtcd(_ string) error { + if (i.ExternalEtcdClientCertPath == "" && i.ExternalEtcdClientKeyPath != "") || + (i.ExternalEtcdClientCertPath != "" && i.ExternalEtcdClientKeyPath == "") { + return fmt.Errorf("etcd client cert and key should be specified both or none") + } + return nil +} + +func (i *CommandInitOption) isExternalEtcdProvided() bool { + return i.ExternalEtcdServers != "" +} + +// Validate Check that there are enough flags to run the command. +// +//nolint:gocyclo +func (i *CommandInitOption) Validate(parentCommand string) error { + if i.KarmadaAPIServerAdvertiseAddress != "" { + if netutils.ParseIPSloppy(i.KarmadaAPIServerAdvertiseAddress) == nil { + return fmt.Errorf("karmada apiserver advertise address is not valid") + } + } + + if i.isExternalEtcdProvided() { + return i.validateExternalEtcd(parentCommand) + } else { + return i.validateLocalEtcd(parentCommand) + } +} + // Complete Initialize k8s client func (i *CommandInitOption) Complete() error { restConfig, err := apiclient.RestConfig(i.Context, i.KubeConfig) @@ -192,7 +236,7 @@ func (i *CommandInitOption) Complete() error { return fmt.Errorf("nodePort of karmada apiserver %v already exist", i.KarmadaAPIServerNodePort) } - if i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels == "" { + if !i.isExternalEtcdProvided() && i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels == "" { if err := i.AddNodeSelectorLabels(); err != nil { return err } @@ -203,7 +247,7 @@ func (i *CommandInitOption) Complete() error { } klog.Infof("karmada apiserver ip: %s", i.KarmadaAPIServerIP) - if i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels != "" { + if !i.isExternalEtcdProvided() && i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels != "" { if !i.isNodeExist(i.EtcdNodeSelectorLabels) { return fmt.Errorf("no node found by label %s", i.EtcdNodeSelectorLabels) } @@ -230,19 +274,22 @@ func initializeDirectory(path string) error { func (i *CommandInitOption) genCerts() error { notAfter := time.Now().Add(i.CertValidity).UTC() - etcdServerCertDNS := []string{ - "localhost", - } - for number := int32(0); number < i.EtcdReplicas; number++ { - etcdServerCertDNS = append(etcdServerCertDNS, fmt.Sprintf("%s-%v.%s.%s.svc.%s", - etcdStatefulSetAndServiceName, number, etcdStatefulSetAndServiceName, i.Namespace, i.HostClusterDomain)) - } - etcdServerAltNames := certutil.AltNames{ - DNSNames: etcdServerCertDNS, - IPs: []net.IP{utils.StringToNetIP("127.0.0.1")}, + var etcdServerCertConfig, etcdClientCertCfg *cert.CertsConfig + if !i.isExternalEtcdProvided() { + etcdServerCertDNS := []string{ + "localhost", + } + for number := int32(0); number < i.EtcdReplicas; number++ { + etcdServerCertDNS = append(etcdServerCertDNS, fmt.Sprintf("%s-%v.%s.%s.svc.%s", + etcdStatefulSetAndServiceName, number, etcdStatefulSetAndServiceName, i.Namespace, i.HostClusterDomain)) + } + etcdServerAltNames := certutil.AltNames{ + DNSNames: etcdServerCertDNS, + IPs: []net.IP{utils.StringToNetIP("127.0.0.1")}, + } + etcdServerCertConfig = cert.NewCertConfig("karmada-etcd-server", []string{}, etcdServerAltNames, ¬After) + etcdClientCertCfg = cert.NewCertConfig("karmada-etcd-client", []string{}, certutil.AltNames{}, ¬After) } - etcdServerCertConfig := cert.NewCertConfig("karmada-etcd-server", []string{}, etcdServerAltNames, ¬After) - etcdClientCertCfg := cert.NewCertConfig("karmada-etcd-client", []string{}, certutil.AltNames{}, ¬After) karmadaDNS := []string{ "localhost", @@ -357,16 +404,18 @@ func (i *CommandInitOption) createCertsSecrets() error { } func (i *CommandInitOption) initKarmadaAPIServer() error { - if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeEtcdService(etcdStatefulSetAndServiceName)); err != nil { - return err - } - klog.Info("Create etcd StatefulSets") - etcdStatefulSet := i.makeETCDStatefulSet() - if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), etcdStatefulSet, metav1.CreateOptions{}); err != nil { - klog.Warning(err) - } - if err := util.WaitForStatefulSetRollout(i.KubeClientSet, etcdStatefulSet, i.WaitComponentReadyTimeout); err != nil { - klog.Warning(err) + if !i.isExternalEtcdProvided() { + if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeEtcdService(etcdStatefulSetAndServiceName)); err != nil { + return err + } + klog.Info("Create etcd StatefulSets") + etcdStatefulSet := i.makeETCDStatefulSet() + if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), etcdStatefulSet, metav1.CreateOptions{}); err != nil { + klog.Warning(err) + } + if err := util.WaitForStatefulSetRollout(i.KubeClientSet, etcdStatefulSet, i.WaitComponentReadyTimeout); err != nil { + klog.Warning(err) + } } klog.Info("Create karmada ApiServer Deployment") if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeKarmadaAPIServerService()); err != nil { @@ -388,7 +437,7 @@ func (i *CommandInitOption) initKarmadaAPIServer() error { klog.Exitln(err) } karmadaAggregatedAPIServerDeployment := i.makeKarmadaAggregatedAPIServerDeployment() - if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAggregatedAPIServerDeployment(), metav1.CreateOptions{}); err != nil { + if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), karmadaAggregatedAPIServerDeployment, metav1.CreateOptions{}); err != nil { klog.Warning(err) } if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaAggregatedAPIServerDeployment, i.WaitComponentReadyTimeout); err != nil { @@ -451,6 +500,22 @@ func (i *CommandInitOption) initKarmadaComponent() error { return nil } +func (i *CommandInitOption) readExternalEtcdCert(name string) (isExternalEtcdCert bool, err error) { + if !i.isExternalEtcdProvided() { + return + } + var getCertAndKey func(*CommandInitOption) ([]byte, []byte, error) + if getCertAndKey, isExternalEtcdCert = externalEtcdCertSpecialization[name]; isExternalEtcdCert { + var certs, key []byte + if certs, key, err = getCertAndKey(i); err != nil { + return + } + i.CertAndKeyFileData[fmt.Sprintf("%s.crt", name)] = certs + i.CertAndKeyFileData[fmt.Sprintf("%s.key", name)] = key + } + return +} + // RunInit Deploy karmada in kubernetes func (i *CommandInitOption) RunInit(parentCommand string) error { // generate certificate @@ -461,6 +526,11 @@ func (i *CommandInitOption) RunInit(parentCommand string) error { i.CertAndKeyFileData = map[string][]byte{} for _, v := range certList { + if isExternalEtcdCert, err := i.readExternalEtcdCert(v); err != nil { + return fmt.Errorf("read external etcd certificate failed, %s. %v", v, err) + } else if isExternalEtcdCert { + continue + } certs, err := utils.FileToBytes(i.KarmadaPkiPath, fmt.Sprintf("%s.crt", v)) if err != nil { return fmt.Errorf("'%s.crt' conversion failed. %v", v, err) diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index a83982a7a8b0..d3d4ff989fe2 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -59,7 +59,11 @@ func (i *CommandInitOption) etcdServers() string { } func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string { - return []string{ + var etcdServers string + if etcdServers = i.ExternalEtcdServers; etcdServers == "" { + etcdServers = strings.TrimRight(i.etcdServers(), ",") + } + command := []string{ "kube-apiserver", "--allow-privileged=true", "--authorization-mode=Node,RBAC", @@ -68,7 +72,7 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string { fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), - fmt.Sprintf("--etcd-servers=%s", strings.TrimRight(i.etcdServers(), ",")), + fmt.Sprintf("--etcd-servers=%s", etcdServers), "--bind-address=0.0.0.0", fmt.Sprintf("--kubelet-client-certificate=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), fmt.Sprintf("--kubelet-client-key=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), @@ -91,6 +95,10 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string { fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName), fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName), } + if i.ExternalEtcdKeyPrefix != "" { + command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix)) + } + return command } func (i *CommandInitOption) makeKarmadaAPIServerDeployment() *appsv1.Deployment { @@ -775,6 +783,29 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D TimeoutSeconds: 15, } + var etcdServers string + if etcdServers = i.ExternalEtcdServers; etcdServers == "" { + etcdServers = strings.TrimRight(i.etcdServers(), ",") + } + command := []string{ + "/bin/karmada-aggregated-apiserver", + "--kubeconfig=/etc/kubeconfig", + "--authentication-kubeconfig=/etc/kubeconfig", + "--authorization-kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--etcd-servers=%s", etcdServers), + fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), + fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), + fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), + fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), + fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), + "--audit-log-path=-", + "--feature-gates=APIPriorityAndFairness=false", + "--audit-log-maxage=0", + "--audit-log-maxbackup=0", + } + if i.ExternalEtcdKeyPrefix != "" { + command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix)) + } podSpec := corev1.PodSpec{ ImagePullSecrets: i.getImagePullSecrets(), Affinity: &corev1.Affinity{ @@ -798,24 +829,9 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D AutomountServiceAccountToken: pointer.Bool(false), Containers: []corev1.Container{ { - Name: karmadaAggregatedAPIServerDeploymentAndServiceName, - Image: i.karmadaAggregatedAPIServerImage(), - Command: []string{ - "/bin/karmada-aggregated-apiserver", - "--kubeconfig=/etc/kubeconfig", - "--authentication-kubeconfig=/etc/kubeconfig", - "--authorization-kubeconfig=/etc/kubeconfig", - fmt.Sprintf("--etcd-servers=%s", strings.TrimRight(i.etcdServers(), ",")), - fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), - fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), - fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), - fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), - fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), - "--audit-log-path=-", - "--feature-gates=APIPriorityAndFairness=false", - "--audit-log-maxage=0", - "--audit-log-maxbackup=0", - }, + Name: karmadaAggregatedAPIServerDeploymentAndServiceName, + Image: i.karmadaAggregatedAPIServerImage(), + Command: command, VolumeMounts: []corev1.VolumeMount{ { Name: KubeConfigSecretAndMountName,