From 994668b5849976837036031fb98e265dd6f854c3 Mon Sep 17 00:00:00 2001 From: wangdepeng Date: Wed, 22 May 2024 14:45:18 +0800 Subject: [PATCH 1/2] feat: anp support tcp mode Signed-off-by: wangdepeng (cherry picked from commit c6b3fc4e7c16838f7fa735cf8ee5bf2f43076c7b) --- cmd/kubenest/operator/app/operator.go | 11 +- cmd/kubenest/operator/app/options/options.go | 2 + pkg/kubenest/constants/constant.go | 1 + .../virtualcluster_execute_controller.go | 4 +- .../virtualcluster_init_controller.go | 20 ++-- pkg/kubenest/controlplane/service.go | 27 ++++- pkg/kubenest/init.go | 14 +++ .../apiserver/mainfests_deployment.go | 35 ++++-- .../apiserver/mainfests_service.go | 19 +++ .../apiserver/manifeats_configmap.go | 11 +- pkg/kubenest/tasks/anp.go | 108 ++++++++++++++++-- pkg/kubenest/tasks/data.go | 2 + pkg/kubenest/tasks/service.go | 3 +- pkg/kubenest/util/cert/certs.go | 47 ++++++++ pkg/kubenest/util/helper.go | 1 + 15 files changed, 263 insertions(+), 42 deletions(-) diff --git a/cmd/kubenest/operator/app/operator.go b/cmd/kubenest/operator/app/operator.go index 529565b5d..faab3d91e 100644 --- a/cmd/kubenest/operator/app/operator.go +++ b/cmd/kubenest/operator/app/operator.go @@ -89,11 +89,12 @@ func run(ctx context.Context, opts *options.Options) error { } VirtualClusterInitController := controller.VirtualClusterInitController{ - Client: mgr.GetClient(), - Config: mgr.GetConfig(), - EventRecorder: mgr.GetEventRecorderFor(constants.InitControllerName), - RootClientSet: hostKubeClient, - KosmosClient: kosmosClient, + Client: mgr.GetClient(), + Config: mgr.GetConfig(), + EventRecorder: mgr.GetEventRecorderFor(constants.InitControllerName), + RootClientSet: hostKubeClient, + KosmosClient: kosmosClient, + KubeNestOptions: &opts.KubeNestOptions, } if err = VirtualClusterInitController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting %s: %v", constants.InitControllerName, err) diff --git a/cmd/kubenest/operator/app/options/options.go b/cmd/kubenest/operator/app/options/options.go index 28ce9c289..9f5133566 100644 --- a/cmd/kubenest/operator/app/options/options.go +++ b/cmd/kubenest/operator/app/options/options.go @@ -25,6 +25,7 @@ type KubernetesOptions struct { type KubeNestOptions struct { ForceDestroy bool + AnpMode string } func NewOptions() *Options { @@ -53,4 +54,5 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.AllowNodeOwnbyMulticluster, "multiowner", false, "Allow node own by multicluster or not.") flags.BoolVar(&o.KosmosJoinController, "kosmos-join-controller", false, "Turn on or off kosmos-join-controller.") flags.BoolVar(&o.KubeNestOptions.ForceDestroy, "kube-nest-force-destroy", false, "Force destroy the node.If it set true.If set to true, Kubernetes will not evict the existing nodes on the node when joining nodes to the tenant's control plane, but will instead force destroy.") + flags.StringVar(&o.KubeNestOptions.AnpMode, "kube-nest-anp-mode", "tcp", "kube-apiserver network proxy mode, must be set to tcp or uds. uds mode the replicas for apiserver should be one, and tcp for multi apiserver replicas.") } diff --git a/pkg/kubenest/constants/constant.go b/pkg/kubenest/constants/constant.go index 87a46229b..bedf66d77 100644 --- a/pkg/kubenest/constants/constant.go +++ b/pkg/kubenest/constants/constant.go @@ -36,6 +36,7 @@ const ( EtcdClientCertAndKeyName = "etcd-client" FrontProxyCaCertAndKeyName = "front-proxy-ca" FrontProxyClientCertAndKeyName = "front-proxy-client" + ProxyServerCertAndKeyName = "proxy-server" //controlplane apiserver ApiServer = "apiserver" diff --git a/pkg/kubenest/controller/virtualcluster_execute_controller.go b/pkg/kubenest/controller/virtualcluster_execute_controller.go index 9a07bdc1a..085eadd12 100644 --- a/pkg/kubenest/controller/virtualcluster_execute_controller.go +++ b/pkg/kubenest/controller/virtualcluster_execute_controller.go @@ -8,6 +8,7 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/kubenest" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" @@ -21,12 +22,13 @@ type Executor struct { config *rest.Config } -func NewExecutor(virtualCluster *v1alpha1.VirtualCluster, c client.Client, config *rest.Config) (*Executor, error) { +func NewExecutor(virtualCluster *v1alpha1.VirtualCluster, c client.Client, config *rest.Config, kubeNestOptions *ko.KubeNestOptions) (*Executor, error) { var phase *workflow.Phase opts := []kubenest.InitOpt{ kubenest.NewInitOptWithVirtualCluster(virtualCluster), kubenest.NewInitOptWithKubeconfig(config), + kubenest.NewInitOptWithKubeNestOptions(kubeNestOptions), } options := kubenest.NewPhaseInitOptions(opts...) action := recognizeActionFor(virtualCluster) diff --git a/pkg/kubenest/controller/virtualcluster_init_controller.go b/pkg/kubenest/controller/virtualcluster_init_controller.go index 4718ba2d7..9ada6ba87 100644 --- a/pkg/kubenest/controller/virtualcluster_init_controller.go +++ b/pkg/kubenest/controller/virtualcluster_init_controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" @@ -35,11 +36,12 @@ import ( type VirtualClusterInitController struct { client.Client - Config *rest.Config - EventRecorder record.EventRecorder - RootClientSet kubernetes.Interface - KosmosClient versioned.Interface - lock sync.Mutex + Config *rest.Config + EventRecorder record.EventRecorder + RootClientSet kubernetes.Interface + KosmosClient versioned.Interface + lock sync.Mutex + KubeNestOptions *options.KubeNestOptions } type NodePool struct { @@ -118,7 +120,7 @@ func (c *VirtualClusterInitController) Reconcile(ctx context.Context, request re return reconcile.Result{RequeueAfter: RequeueTime}, nil } updatedCluster := originalCluster.DeepCopy() - err = c.createVirtualCluster(updatedCluster) + err = c.createVirtualCluster(updatedCluster, c.KubeNestOptions) if err != nil { klog.Errorf("Failed to create virtualcluster %s. err: %s", updatedCluster.Name, err.Error()) updatedCluster.Status.Reason = err.Error() @@ -230,7 +232,7 @@ func (c *VirtualClusterInitController) removeFinalizer(virtualCluster *v1alpha1. } // createVirtualCluster assign work nodes, create control plane and create compoennts from manifests -func (c *VirtualClusterInitController) createVirtualCluster(virtualCluster *v1alpha1.VirtualCluster) error { +func (c *VirtualClusterInitController) createVirtualCluster(virtualCluster *v1alpha1.VirtualCluster, kubeNestOptions *options.KubeNestOptions) error { klog.V(2).Infof("Reconciling virtual cluster", "name", virtualCluster.Name) //Assign host port @@ -239,7 +241,7 @@ func (c *VirtualClusterInitController) createVirtualCluster(virtualCluster *v1al return errors.Wrap(err, "Error in assign host port!") } - executer, err := NewExecutor(virtualCluster, c.Client, c.Config) + executer, err := NewExecutor(virtualCluster, c.Client, c.Config, kubeNestOptions) if err != nil { return err } @@ -267,7 +269,7 @@ func (c *VirtualClusterInitController) createVirtualCluster(virtualCluster *v1al func (c *VirtualClusterInitController) destroyVirtualCluster(virtualCluster *v1alpha1.VirtualCluster) error { klog.V(2).Infof("Destroying virtual cluster %s", virtualCluster.Name) - execute, err := NewExecutor(virtualCluster, c.Client, c.Config) + execute, err := NewExecutor(virtualCluster, c.Client, c.Config, c.KubeNestOptions) if err != nil { return err } diff --git a/pkg/kubenest/controlplane/service.go b/pkg/kubenest/controlplane/service.go index 63f5d76e4..44740d570 100644 --- a/pkg/kubenest/controlplane/service.go +++ b/pkg/kubenest/controlplane/service.go @@ -18,8 +18,8 @@ import ( "github.com/kosmos.io/kosmos/pkg/kubenest/util" ) -func EnsureVirtualClusterService(client clientset.Interface, name, namespace string, port int32) error { - if err := createServerService(client, name, namespace, port); err != nil { +func EnsureVirtualClusterService(client clientset.Interface, name, namespace string, portMap map[string]int32) error { + if err := createServerService(client, name, namespace, portMap); err != nil { return fmt.Errorf("failed to create virtual cluster apiserver-service, err: %w", err) } return nil @@ -46,7 +46,7 @@ func DeleteVirtualClusterService(client clientset.Interface, name, namespace str return nil } -func createServerService(client clientset.Interface, name, namespace string, port int32) error { +func createServerService(client clientset.Interface, name, namespace string, portMap map[string]int32) error { apiserverServiceBytes, err := util.ParseTemplate(apiserver.ApiserverService, struct { ServiceName, Namespace, ServiceType string ServicePort int32 @@ -54,11 +54,22 @@ func createServerService(client clientset.Interface, name, namespace string, por ServiceName: fmt.Sprintf("%s-%s", name, "apiserver"), Namespace: namespace, ServiceType: constants.ApiServerServiceType, - ServicePort: port, + ServicePort: portMap[constants.ApiServerPortKey], }) if err != nil { return fmt.Errorf("error when parsing virtualClusterApiserver serive template: %w", err) } + anpServiceBytes, err := util.ParseTemplate(apiserver.ApiserverAnpService, struct { + ServiceName, Namespace string + ProxyServerPort int32 + }{ + ServiceName: fmt.Sprintf("%s-%s", name, "konnectivity-server"), + Namespace: namespace, + ProxyServerPort: portMap[constants.ApiServerNetworkProxyServerPortKey], + }) + if err != nil { + return fmt.Errorf("error when parsing virtualClusterApiserver anp service template: %w", err) + } apiserverService := &corev1.Service{} if err := yaml.Unmarshal([]byte(apiserverServiceBytes), apiserverService); err != nil { @@ -68,6 +79,14 @@ func createServerService(client clientset.Interface, name, namespace string, por return fmt.Errorf("err when creating virtual cluster apiserver service for %s, err: %w", apiserverService.Name, err) } + anpService := &corev1.Service{} + if err := yaml.Unmarshal([]byte(anpServiceBytes), anpService); err != nil { + return fmt.Errorf("error when decoding virtual cluster anp service: %w", err) + } + if err := createOrUpdateService(client, anpService); err != nil { + return fmt.Errorf("err when creating virtual cluster anp service for %s, err: %w", anpService.Name, err) + } + etcdServicePeerBytes, err := util.ParseTemplate(etcd.EtcdPeerService, struct { ServiceName, Namespace string EtcdListenClientPort, EtcdListenPeerPort int32 diff --git a/pkg/kubenest/init.go b/pkg/kubenest/init.go index c01fc8684..298397dc2 100644 --- a/pkg/kubenest/init.go +++ b/pkg/kubenest/init.go @@ -10,6 +10,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/tasks" @@ -36,6 +37,7 @@ type initData struct { externalIP string hostPort int32 hostPortMap map[string]int32 + kubeNestOptions *ko.KubeNestOptions } type InitOptions struct { @@ -45,6 +47,7 @@ type InitOptions struct { virtualClusterVersion string virtualClusterDataDir string virtualCluster *v1alpha1.VirtualCluster + KubeNestOptions *ko.KubeNestOptions } func NewInitPhase(opts *InitOptions) *workflow.Phase { @@ -122,6 +125,12 @@ func NewInitOptWithKubeconfig(config *rest.Config) InitOpt { } } +func NewInitOptWithKubeNestOptions(options *ko.KubeNestOptions) InitOpt { + return func(o *InitOptions) { + o.KubeNestOptions = options + } +} + func newRunData(opt *InitOptions) (*initData, error) { if err := opt.Validate(); err != nil { return nil, err @@ -174,6 +183,7 @@ func newRunData(opt *InitOptions) (*initData, error) { externalIP: opt.virtualCluster.Spec.ExternalIP, hostPort: opt.virtualCluster.Status.Port, hostPortMap: opt.virtualCluster.Status.PortMap, + kubeNestOptions: opt.KubeNestOptions, }, nil } @@ -244,3 +254,7 @@ func (i initData) HostPortMap() map[string]int32 { func (i initData) DynamicClient() *dynamic.DynamicClient { return i.dynamicClient } + +func (i initData) KubeNestOpt() *ko.KubeNestOptions { + return i.kubeNestOptions +} diff --git a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go index 5b02760c9..14c958cb3 100644 --- a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go @@ -22,6 +22,7 @@ spec: spec: automountServiceAccountToken: false hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" @@ -150,6 +151,7 @@ spec: spec: automountServiceAccountToken: false hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" @@ -265,19 +267,27 @@ spec: runAsUser: 0 command: [ "/proxy-server"] args: [ - "--log-file=/var/log/konnectivity-server.log", + "--log-file=/var/log/{{ .Namespace }}/{{ .Name }}/konnectivity-server.log", "--logtostderr=true", "--log-file-max-size=0", - "--uds-name=/etc/kubernetes/konnectivity-server/{{ .Namespace }}/{{ .Name }}/konnectivity-server.socket", - "--delete-existing-uds-file", "--cluster-cert=/etc/virtualcluster/pki/apiserver.crt", "--cluster-key=/etc/virtualcluster/pki/apiserver.key", + {{ if eq .AnpMode "uds" }} "--server-port=0", + "--mode=grpc", + "--uds-name=/etc/kubernetes/konnectivity-server/{{ .Namespace }}/{{ .Name }}/konnectivity-server.socket", + "--delete-existing-uds-file", + {{ else }} + "--server-port={{ .ServerPort }}", + "--mode=http-connect", + "--server-cert=/etc/virtualcluster/pki/proxy-server.crt", + "--server-ca-cert=/etc/virtualcluster/pki/ca.crt", + "--server-key=/etc/virtualcluster/pki/proxy-server.key", + {{ end }} "--agent-port={{ .AgentPort }}", "--health-port={{ .HealthPort }}", "--admin-port={{ .AdminPort }}", "--keepalive-time=1h", - "--mode=grpc", "--agent-namespace=kube-system", "--agent-service-account=konnectivity-agent", "--kubeconfig=/etc/apiserver/kubeconfig", @@ -309,7 +319,7 @@ spec: name: apiserver-cert readOnly: true - name: varlogkonnectivityserver - mountPath: /var/log/konnectivity-server.log + mountPath: /var/log/{{ .Namespace }}/{{ .Name }} readOnly: false - name: konnectivity-home mountPath: /etc/kubernetes/konnectivity-server/{{ .Namespace }}/{{ .Name }} @@ -324,8 +334,8 @@ spec: secretName: {{ .KubeconfigSecret }} - name: varlogkonnectivityserver hostPath: - path: /var/log/{{ .Namespace }}/{{ .Name }}/konnectivity-server.log - type: FileOrCreate + path: /var/log/{{ .Namespace }}/{{ .Name }} + type: DirectoryOrCreate - name: konnectivity-home hostPath: path: /etc/kubernetes/konnectivity-server/{{ .Namespace }}/{{ .Name }} @@ -413,7 +423,11 @@ spec: "--sync-interval-cap=30s", "--probe-interval=5s", "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", - "--agent-identifiers=ipv4=$(HOST_IP)" + "--agent-identifiers=ipv4=$(HOST_IP)", + {{ if ne .AnpMode "uds" }} + "--agent-cert=/etc/virtualcluster/pki/apiserver.crt", + "--agent-key=/etc/virtualcluster/pki/apiserver.key", + {{ end }} ] env: - name: POD_NAME @@ -443,10 +457,15 @@ spec: initialDelaySeconds: 15 timeoutSeconds: 15 volumeMounts: + - name: agent-cert + mountPath: /etc/virtualcluster/pki - mountPath: /var/run/secrets/tokens name: konnectivity-agent-token serviceAccountName: konnectivity-agent volumes: + - name: agent-cert + secret: + secretName: {{ .AgentCertName }} - name: konnectivity-agent-token projected: sources: diff --git a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_service.go b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_service.go index 5cafbcf53..7c0dd7966 100644 --- a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_service.go +++ b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_service.go @@ -19,5 +19,24 @@ spec: selector: virtualCluster-app: apiserver type: {{ .ServiceType }} +` + ApiserverAnpService = ` +apiVersion: v1 +kind: Service +metadata: + labels: + virtualCluster-app: apiserver + app.kubernetes.io/managed-by: virtual-cluster-controller + name: {{ .ServiceName }} + namespace: {{ .Namespace }} +spec: + ports: + - name: proxy-server + port: {{ .ProxyServerPort }} + protocol: TCP + targetPort: {{ .ProxyServerPort }} + selector: + virtualCluster-app: apiserver + type: ClusterIP ` ) diff --git a/pkg/kubenest/manifest/controlplane/apiserver/manifeats_configmap.go b/pkg/kubenest/manifest/controlplane/apiserver/manifeats_configmap.go index b69a97522..3096aab02 100644 --- a/pkg/kubenest/manifest/controlplane/apiserver/manifeats_configmap.go +++ b/pkg/kubenest/manifest/controlplane/apiserver/manifeats_configmap.go @@ -10,10 +10,19 @@ data: egressSelections: - name: cluster connection: - proxyProtocol: GRPC + proxyProtocol: {{ if eq .AnpMode "uds" }}GRPC{{ else }}HTTPConnect{{ end }} transport: + {{ if eq .AnpMode "uds" }} uds: udsName: /etc/kubernetes/konnectivity-server/{{ .Namespace }}/{{ .Name }}/konnectivity-server.socket + {{ else }} + tcp: + url: https://{{ .SvcName }}:{{ .ProxyServerPort }} + tlsConfig: + caBundle: /etc/virtualcluster/pki/ca.crt + clientKey: /etc/virtualcluster/pki/proxy-server.key + clientCert: /etc/virtualcluster/pki/proxy-server.crt + {{ end }} - name: master connection: proxyProtocol: Direct diff --git a/pkg/kubenest/tasks/anp.go b/pkg/kubenest/tasks/anp.go index f22405285..451434c3f 100644 --- a/pkg/kubenest/tasks/anp.go +++ b/pkg/kubenest/tasks/anp.go @@ -3,6 +3,7 @@ package tasks import ( "context" "fmt" + "strings" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" @@ -15,6 +16,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" "github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/apiserver" "github.com/kosmos.io/kosmos/pkg/kubenest/util" @@ -31,6 +33,10 @@ func NewAnpTask() workflow.Task { Name: "deploy-anp-server", Run: runAnpServer, }, + { + Name: "Upload-ProxyAgentCert", + Run: runUploadProxyAgentCert, + }, { Name: "deploy-anp-agent", Run: runAnpAgent, @@ -54,11 +60,22 @@ func runAnpServer(r workflow.RunData) error { if !ok { return errors.New("Virtual cluster anp task invoked with an invalid data struct") } + name, namespace := data.GetName(), data.GetNamespace() + kubeNestOpt := data.KubeNestOpt() + portMap := data.HostPortMap() // install egress_selector_configuration config map egressSelectorConfig, err := util.ParseTemplate(apiserver.EgressSelectorConfiguration, struct { - Namespace string + Namespace string + Name string + AnpMode string + ProxyServerPort int32 + SvcName string }{ - Namespace: data.GetNamespace(), + Namespace: namespace, + Name: name, + ProxyServerPort: portMap[constants.ApiServerNetworkProxyServerPortKey], + SvcName: fmt.Sprintf("%s-konnectivity-server.%s.svc.cluster.local", name, namespace), + AnpMode: kubeNestOpt.AnpMode, }) if err != nil { return fmt.Errorf("failed to parse egress_selector_configuration config map template, err: %w", err) @@ -73,7 +90,7 @@ func runAnpServer(r workflow.RunData) error { if err != nil { return fmt.Errorf("failed to create egress_selector_configuration config map, err: %w", err) } - err = installAnpServer(data.RemoteClient(), data.GetName(), data.GetNamespace(), data.HostPortMap()) + err = installAnpServer(data.RemoteClient(), name, namespace, portMap, kubeNestOpt) if err != nil { return fmt.Errorf("failed to install virtual cluster anp component, err: %w", err) } @@ -87,7 +104,7 @@ func runAnpAgent(r workflow.RunData) error { if !ok { return errors.New("check-VirtualClusterAnp task invoked with an invalid data struct") } - return installAnpAgent(data.RemoteClient(), data.GetName(), data.GetNamespace(), data.HostPortMap()) + return installAnpAgent(data) } func UninstallAnpTask() workflow.Task { @@ -109,8 +126,11 @@ func uninstallAnp(r workflow.RunData) error { if !ok { return errors.New("Virtual cluster anp task invoked with an invalid data struct") } - - anpManifest, vcClient, err := getAnpAgentManifest(data.RemoteClient(), data.GetName(), data.GetNamespace(), data.HostPortMap()) + name, namespace := data.GetName(), data.GetNamespace() + client := data.RemoteClient() + portMap := data.HostPortMap() + kubeNestOpt := data.KubeNestOpt() + anpManifest, vcClient, err := getAnpAgentManifest(client, name, namespace, portMap, kubeNestOpt) if err != nil { return fmt.Errorf("failed to uninstall anp agent when get anp manifest, err: %w", err) } @@ -122,7 +142,7 @@ func uninstallAnp(r workflow.RunData) error { klog.V(2).InfoS("[VirtualClusterAnp] Successfully uninstalled virtual cluster anp component", "virtual cluster", klog.KObj(data)) return util.ForEachObjectInYAML(context.TODO(), vcClient, []byte(anpManifest), "", actionFunc) } -func installAnpServer(client clientset.Interface, name, namespace string, portMap map[string]int32) error { +func installAnpServer(client clientset.Interface, name, namespace string, portMap map[string]int32, kubeNestOpt *ko.KubeNestOptions) error { imageRepository, imageVersion := util.GetImageMessage() clusterIp, err := util.GetEtcdServiceClusterIp(namespace, name+constants.EtcdSuffix, client) if err != nil { @@ -141,6 +161,7 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa AdminPort int32 KubeconfigSecret string Name string + AnpMode string }{ DeploymentName: fmt.Sprintf("%s-%s", name, "apiserver"), Namespace: namespace, @@ -159,6 +180,7 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa AdminPort: portMap[constants.ApiServerNetworkProxyAdminPortKey], KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"), Name: name, + AnpMode: kubeNestOpt.AnpMode, }) if err != nil { return fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err) @@ -176,8 +198,13 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa return nil } -func installAnpAgent(client clientset.Interface, name, namespace string, portMap map[string]int32) error { - anpAgentManifestBytes, vcClient, err2 := getAnpAgentManifest(client, name, namespace, portMap) +func installAnpAgent(data InitData) error { + client := data.RemoteClient() + name := data.GetName() + namespace := data.GetNamespace() + portMap := data.HostPortMap() + kubeNestOpt := data.KubeNestOpt() + anpAgentManifestBytes, vcClient, err2 := getAnpAgentManifest(client, name, namespace, portMap, kubeNestOpt) if err2 != nil { return err2 } @@ -188,7 +215,7 @@ func installAnpAgent(client clientset.Interface, name, namespace string, portMap return util.ForEachObjectInYAML(context.TODO(), vcClient, []byte(anpAgentManifestBytes), "", actionFunc) } -func getAnpAgentManifest(client clientset.Interface, name string, namespace string, portMap map[string]int32) (string, dynamic.Interface, error) { +func getAnpAgentManifest(client clientset.Interface, name string, namespace string, portMap map[string]int32, kubeNestOpt *ko.KubeNestOptions) (string, dynamic.Interface, error) { imageRepository, imageVersion := util.GetImageMessage() // get apiServer hostIp proxyServerHost, err := getDeploymentPodIPs(client, namespace, fmt.Sprintf("%s-%s", name, "apiserver")) @@ -201,17 +228,21 @@ func getAnpAgentManifest(client clientset.Interface, name string, namespace stri Version string AgentPort int32 ProxyServerHost []string + AnpMode string + AgentCertName string }{ ImageRepository: imageRepository, Version: imageVersion, AgentPort: portMap[constants.ApiServerNetworkProxyAgentPortKey], ProxyServerHost: proxyServerHost, + AnpMode: kubeNestOpt.AnpMode, + AgentCertName: fmt.Sprintf("%s-%s", name, "cert"), }) if err != nil { return "", nil, fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err) } klog.V(4).InfoS("[anp] apply anp agent", "agent manifest", anpAgentManifeattBytes) - vcClient, err := getVcClient(client, name, namespace) + vcClient, err := getVcDynamicClient(client, name, namespace) if err != nil { return "", nil, fmt.Errorf("error when get vcClient, err: %v", err) } @@ -243,7 +274,7 @@ func getDeploymentPodIPs(clientset clientset.Interface, namespace, deploymentNam return podIPs, nil } -func getVcClient(client clientset.Interface, name, namespace string) (dynamic.Interface, error) { +func getVcDynamicClient(client clientset.Interface, name, namespace string) (dynamic.Interface, error) { secret, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), fmt.Sprintf("%s-%s", name, constants.AdminConfig), metav1.GetOptions{}) if err != nil { @@ -259,3 +290,56 @@ func getVcClient(client clientset.Interface, name, namespace string) (dynamic.In } return dynamicClient, nil } +func getVcClientset(client clientset.Interface, name, namespace string) (clientset.Interface, error) { + secret, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), + fmt.Sprintf("%s-%s", name, constants.AdminConfig), metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "Get virtualcluster kubeconfig secret error") + } + + config, err := clientcmd.RESTConfigFromKubeConfig(secret.Data[constants.KubeConfig]) + if err != nil { + return nil, err + } + + vcClient, err := clientset.NewForConfig(config) + if err != nil { + return nil, err + } + + return vcClient, nil +} + +func runUploadProxyAgentCert(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("upload proxy agent cert task invoked with an invalid data struct") + } + name, namespace := data.GetName(), data.GetNamespace() + certList := data.CertList() + certsData := make(map[string][]byte, len(certList)) + for _, c := range certList { + if strings.Contains(c.KeyName(), "apiserver") { + certsData[c.KeyName()] = c.KeyData() + certsData[c.CertName()] = c.CertData() + } + } + vcClient, err := getVcClientset(data.RemoteClient(), name, namespace) + if err != nil { + return fmt.Errorf("failed to get virtual cluster client, err: %w", err) + } + err = createOrUpdateSecret(vcClient, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", data.GetName(), "cert"), + Namespace: "kube-system", + Labels: VirtualClusterControllerLabel, + }, + Data: certsData, + }) + if err != nil { + return fmt.Errorf("failed to upload agent cert to tenant, err: %w", err) + } + + klog.V(2).InfoS("[Upload-ProxyAgentCert] Successfully uploaded virtual cluster agent certs to secret", "virtual cluster", klog.KObj(data)) + return nil +} diff --git a/pkg/kubenest/tasks/data.go b/pkg/kubenest/tasks/data.go index eea44e9da..7b9de7cae 100644 --- a/pkg/kubenest/tasks/data.go +++ b/pkg/kubenest/tasks/data.go @@ -4,6 +4,7 @@ import ( "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" + ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/util/cert" ) @@ -22,4 +23,5 @@ type InitData interface { HostPort() int32 HostPortMap() map[string]int32 DynamicClient() *dynamic.DynamicClient + KubeNestOpt() *ko.KubeNestOptions } diff --git a/pkg/kubenest/tasks/service.go b/pkg/kubenest/tasks/service.go index 6fb6a99ea..84e4116d4 100644 --- a/pkg/kubenest/tasks/service.go +++ b/pkg/kubenest/tasks/service.go @@ -6,7 +6,6 @@ import ( "k8s.io/klog/v2" - "github.com/kosmos.io/kosmos/pkg/kubenest/constants" "github.com/kosmos.io/kosmos/pkg/kubenest/controlplane" "github.com/kosmos.io/kosmos/pkg/kubenest/workflow" ) @@ -45,7 +44,7 @@ func runVirtualClusterService(r workflow.RunData) error { data.RemoteClient(), data.GetName(), data.GetNamespace(), - data.HostPortMap()[constants.ApiServerPortKey], + data.HostPortMap(), ) if err != nil { return fmt.Errorf("failed to install virtual cluster service , err: %w", err) diff --git a/pkg/kubenest/util/cert/certs.go b/pkg/kubenest/util/cert/certs.go index 9aa7f40a3..44fbfc9b4 100644 --- a/pkg/kubenest/util/cert/certs.go +++ b/pkg/kubenest/util/cert/certs.go @@ -70,6 +70,20 @@ func GetDefaultCertList() []*CertConfig { VirtualClusterCertEtcdCA(), VirtualClusterCertEtcdServer(), VirtualClusterCertEtcdClient(), + // proxy server cert config. + VirtualClusterProxyServer(), + } +} + +func VirtualClusterProxyServer() *CertConfig { + return &CertConfig{ + Name: constants.ProxyServerCertAndKeyName, + CAName: constants.CaCertAndKeyName, + Config: certutil.Config{ + CommonName: "virtualCluster-proxy-server", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + }, + AltNamesMutatorFunc: makeAltNamesMutator(proxyServerAltNamesMutator), } } @@ -188,6 +202,39 @@ func makeAltNamesMutator(f func(cfg *AltNamesMutatorConfig) (*certutil.AltNames, } } +func proxyServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, error) { + altNames := &certutil.AltNames{ + DNSNames: []string{ + "localhost", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + }, + IPs: []net.IP{ + net.IPv4(127, 0, 0, 1), + net.IPv4(10, 237, 6, 17), + net.IPv4(10, 237, 0, 1), + }, + } + + if cfg.Namespace != constants.VirtualClusterSystemNamespace { + appendSANsToAltNames(altNames, []string{fmt.Sprintf("*.%s.svc.cluster.local", cfg.Namespace), + fmt.Sprintf("*.%s.svc", cfg.Namespace)}) + } + if len(cfg.ControlplaneAddr) > 0 { + appendSANsToAltNames(altNames, []string{cfg.ControlplaneAddr}) + } + if len(cfg.ExternalIP) > 0 { + appendSANsToAltNames(altNames, []string{cfg.ExternalIP}) + } + if len(cfg.ClusterIps) > 0 { + for _, clusterIp := range cfg.ClusterIps { + appendSANsToAltNames(altNames, []string{clusterIp}) + } + } + return altNames, nil +} + func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, error) { altNames := &certutil.AltNames{ DNSNames: []string{ diff --git a/pkg/kubenest/util/helper.go b/pkg/kubenest/util/helper.go index fc9c3da72..a442c98a8 100644 --- a/pkg/kubenest/util/helper.go +++ b/pkg/kubenest/util/helper.go @@ -240,6 +240,7 @@ func ApplyObject(dynamicClient dynamic.Interface, obj *unstructured.Unstructured Force: true, }) if err != nil { + klog.V(2).Infof("Failed to apply changes to %s %s: %v", gvr.String(), name, err) return fmt.Errorf("failed to apply changes to %s %s: %v", gvr.String(), name, err) } From b2d3ba2dff2d66c68ec3ddc6c22329a5eb449e9a Mon Sep 17 00:00:00 2001 From: wangdepeng Date: Thu, 23 May 2024 14:04:45 +0800 Subject: [PATCH 2/2] fix: uninstall anp failed and failed to upload agent cert to tenant Signed-off-by: wangdepeng (cherry picked from commit 653c1debadf36277c77f4de761ebd6273c01c023) --- pkg/kubenest/tasks/anp.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/kubenest/tasks/anp.go b/pkg/kubenest/tasks/anp.go index 451434c3f..e257a3811 100644 --- a/pkg/kubenest/tasks/anp.go +++ b/pkg/kubenest/tasks/anp.go @@ -29,14 +29,14 @@ func NewAnpTask() workflow.Task { Run: runAnp, RunSubTasks: true, Tasks: []workflow.Task{ - { - Name: "deploy-anp-server", - Run: runAnpServer, - }, { Name: "Upload-ProxyAgentCert", Run: runUploadProxyAgentCert, }, + { + Name: "deploy-anp-server", + Run: runAnpServer, + }, { Name: "deploy-anp-agent", Run: runAnpAgent, @@ -220,7 +220,9 @@ func getAnpAgentManifest(client clientset.Interface, name string, namespace stri // get apiServer hostIp proxyServerHost, err := getDeploymentPodIPs(client, namespace, fmt.Sprintf("%s-%s", name, "apiserver")) if err != nil { - return "", nil, fmt.Errorf("error when get apiserver hostIp, err: %w", err) + klog.Warningf("Failed to get apiserver hostIp, err: %v", err) + // ignore if can't get the hostIp when uninstall the deployment + proxyServerHost = []string{"127.0.0.1"} } anpAgentManifeattBytes, err := util.ParseTemplate(apiserver.AnpAgentManifest, struct { @@ -319,6 +321,7 @@ func runUploadProxyAgentCert(r workflow.RunData) error { certList := data.CertList() certsData := make(map[string][]byte, len(certList)) for _, c := range certList { + // only upload apisever cert if strings.Contains(c.KeyName(), "apiserver") { certsData[c.KeyName()] = c.KeyData() certsData[c.CertName()] = c.CertData()