diff --git a/pkg/yurtctl/cmd/join/join.go b/pkg/yurtctl/cmd/join/join.go index 9f8e80b7acb..fc9adfd74a4 100644 --- a/pkg/yurtctl/cmd/join/join.go +++ b/pkg/yurtctl/cmd/join/join.go @@ -67,6 +67,7 @@ type joinOptions struct { kustomizeDir string nodeType string yurthubImage string + markAutonomous bool } // newJoinOptions returns a struct ready for being used for creating cmd join flags. @@ -116,8 +117,8 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { joinRunner.AppendPhase(yurtphase.NewPreparePhase()) joinRunner.AppendPhase(kubeadmPhase.NewPreflightPhase()) - joinRunner.AppendPhase(yurtphase.NewEdgeNodePhase()) joinRunner.AppendPhase(yurtphase.NewCloudNodePhase()) + joinRunner.AppendPhase(yurtphase.NewConvertPhase()) joinRunner.AppendPhase(yurtphase.NewPostcheckPhase()) joinRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { @@ -157,6 +158,10 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { &joinOptions.yurthubImage, "yurthub-image", "", "Sets the image version of yurthub component", ) + flagSet.BoolVar( + &joinOptions.markAutonomous, "mark-autonomous", true, + "Annotate node autonomous if set true, default true.", + ) cmdutil.AddCRISocketFlag(flagSet, &joinOptions.externalcfg.NodeRegistration.CRISocket) } @@ -170,6 +175,7 @@ type joinData struct { kustomizeDir string nodeType string yurthubImage string + markAutonomous bool } // newJoinData returns a new joinData struct to be used for the execution of the kubeadm join workflow. @@ -276,6 +282,7 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri kustomizeDir: opt.kustomizeDir, nodeType: opt.nodeType, yurthubImage: opt.yurthubImage, + markAutonomous: opt.markAutonomous, }, nil } @@ -400,3 +407,8 @@ func (j *joinData) NodeType() string { func (j *joinData) YurtHubImage() string { return j.yurthubImage } + +//MarkAutonomous returns markAutonomous setting. +func (j *joinData) MarkAutonomous() bool { + return j.markAutonomous +} diff --git a/pkg/yurtctl/cmd/join/phases/constants.go b/pkg/yurtctl/cmd/join/phases/constants.go deleted file mode 100644 index 158e25c1568..00000000000 --- a/pkg/yurtctl/cmd/join/phases/constants.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -const ( - defaultYurthubStaticPodFileName = "yurthub.yaml" -) - -const ( - kubeletConfForEdgeNode = ` -apiVersion: v1 -clusters: -- cluster: - server: http://127.0.0.1:10261 - name: default-cluster -contexts: -- context: - cluster: default-cluster - namespace: default - user: default-auth - name: default-context -current-context: default-context -kind: Config -preferences: {} -` -) diff --git a/pkg/yurtctl/cmd/join/phases/convert.go b/pkg/yurtctl/cmd/join/phases/convert.go new file mode 100644 index 00000000000..d1fd1dee39e --- /dev/null +++ b/pkg/yurtctl/cmd/join/phases/convert.go @@ -0,0 +1,91 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + "time" + + "github.com/spf13/pflag" + "k8s.io/klog" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + + nodeconverter "github.com/openyurtio/openyurt/pkg/node-servant/convert" + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +const ( + defaultYurthubHealthCheckTimeout = 2 * time.Minute +) + +// NewConvertPhase creates a yurtctl workflow phase that convert native k8s node to openyurt node. +func NewConvertPhase() workflow.Phase { + return workflow.Phase{ + Name: "Convert node to OpenYurt node. ", + Short: "Convert node", + Run: runConvertNode, + } +} + +// if comes to this phase, means node is up to running as a k8s node +// then we convert it into a edge-node or cloud-node +func runConvertNode(c workflow.RunData) error { + data, ok := c.(YurtJoinData) + if !ok { + return fmt.Errorf("Join edge-node phase invoked with an invalid data struct. ") + } + + // convert node + o := nodeconverter.NewConvertOptions() + f := constructFlagSet(data) + if err := o.Complete(f); err != nil { + return fmt.Errorf("fail to convert the kubernetes node to a yurt node: %s", err) + } + + klog.Infof("convert with options:%v ", o) + converter := nodeconverter.NewConverterWithOptions(o) + if err := converter.Do(); err != nil { + return fmt.Errorf("fail to convert the kubernetes node to a yurt node: %s", err) + } + + return nil +} + +func constructFlagSet(data YurtJoinData) *pflag.FlagSet { + f := pflag.FlagSet{} + + if data.NodeType() == constants.CloudNode { + f.String("working-mode", string(util.WorkingModeCloud), "") + } else if data.NodeType() == constants.EdgeNode { + f.String("working-mode", string(util.WorkingModeEdge), "") + } + + yurtHubImg := data.YurtHubImage() + if len(yurtHubImg) == 0 { + yurtHubImg = fmt.Sprintf("%s/%s:%s", constants.DefaultOpenYurtImageRegistry, constants.Yurthub, constants.DefaultOpenYurtVersion) + } + f.String("yurthub-image", yurtHubImg, "") + + token := data.Cfg().Discovery.TLSBootstrapToken + f.String("join-token", token, "") + + f.Duration("yurthub-healthcheck-timeout", defaultYurthubHealthCheckTimeout, "") + f.String("kubeadm-conf-path", "", "") + + return &f +} diff --git a/pkg/yurtctl/cmd/join/phases/join-cloud-node.go b/pkg/yurtctl/cmd/join/phases/join-cloud-node.go index b34a06c1151..b7f2e88e73b 100644 --- a/pkg/yurtctl/cmd/join/phases/join-cloud-node.go +++ b/pkg/yurtctl/cmd/join/phases/join-cloud-node.go @@ -40,8 +40,6 @@ import ( patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - - "github.com/openyurtio/openyurt/pkg/yurtctl/constants" ) var ( @@ -81,11 +79,7 @@ func NewCloudNodePhase() workflow.Phase { } //getCloudNodeJoinData get node configuration for cloud-node. -func getCloudNodeJoinData(c workflow.RunData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { - data, ok := c.(YurtJoinData) - if !ok { - return nil, nil, nil, errors.New("kubelet-start phase invoked with an invalid data struct") - } +func getCloudNodeJoinData(data YurtJoinData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { cfg := data.Cfg() initCfg, err := data.InitCfg() if err != nil { @@ -106,10 +100,10 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { if !ok { return errors.New("kubelet-start phase invoked with an invalid data struct") } - if data.NodeType() != constants.CloudNode { - return - } - cfg, initCfg, tlsBootstrapCfg, err := getCloudNodeJoinData(c) + //if data.NodeType() != constants.CloudNode { + // return + //} + cfg, initCfg, tlsBootstrapCfg, err := getCloudNodeJoinData(data) if err != nil { return err } diff --git a/pkg/yurtctl/cmd/join/phases/join-edge-node.go b/pkg/yurtctl/cmd/join/phases/join-edge-node.go deleted file mode 100644 index 78d83e7facb..00000000000 --- a/pkg/yurtctl/cmd/join/phases/join-edge-node.go +++ /dev/null @@ -1,238 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" - "k8s.io/klog" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" - kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" - utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" - - "github.com/openyurtio/openyurt/pkg/yurtctl/constants" - "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" - "github.com/openyurtio/openyurt/pkg/yurthub/util" - "github.com/pkg/errors" -) - -// NewEdgeNodePhase creates a yurtctl workflow phase that start kubelet on a edge node. -func NewEdgeNodePhase() workflow.Phase { - return workflow.Phase{ - Name: "Join edge-node to OpenYurt cluster. ", - Short: "Join edge-node", - Run: runJoinEdgeNode, - } -} - -//runJoinEdgeNode executes the edge node join process. -func runJoinEdgeNode(c workflow.RunData) error { - data, ok := c.(YurtJoinData) - if !ok { - return fmt.Errorf("Join edge-node phase invoked with an invalid data struct. ") - } - if data.NodeType() != constants.EdgeNode { - return nil - } - cfg, initCfg, tlsBootstrapCfg, err := getEdgeNodeJoinData(data) - if err != nil { - return err - } - - if err := setKubeletConfigForEdgeNode(); err != nil { - return err - } - clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) - if err := certutil.WriteCert(edgenode.KubeCaFile, clusterinfo.CertificateAuthorityData); err != nil { - return err - } - - tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) - if err != nil { - return err - } - kc, err := getKubeletConfig(cfg, initCfg, tlsClient) - if err != nil { - return err - } - if err := addYurthubStaticYaml(cfg, kc.StaticPodPath, data.YurtHubImage()); err != nil { - return err - } - klog.Info("[kubelet-start] Starting the kubelet") - kubeletphase.TryStartKubelet() - return nil -} - -//setKubeleConfigForEdgeNode write kubelet.conf for edge-node. -func setKubeletConfigForEdgeNode() error { - kubeletConfigDir := filepath.Dir(edgenode.KubeCondfigPath) - if _, err := os.Stat(kubeletConfigDir); err != nil { - if os.IsNotExist(err) { - if err := os.MkdirAll(kubeletConfigDir, os.ModePerm); err != nil { - klog.Errorf("Create dir %s fail: %v", kubeletConfigDir, err) - return err - } - } else { - klog.Errorf("Describe dir %s fail: %v", kubeletConfigDir, err) - return err - } - } - if err := ioutil.WriteFile(edgenode.KubeCondfigPath, []byte(kubeletConfForEdgeNode), 0755); err != nil { - return err - } - return nil -} - -//addYurthubStaticYaml generate YurtHub static yaml for edge-node. -func addYurthubStaticYaml(cfg *kubeadmapi.JoinConfiguration, podManifestPath string, yurthubImage string) error { - klog.Info("[join-node] Adding edge hub static yaml") - if len(yurthubImage) == 0 { - yurthubImage = fmt.Sprintf("%s/%s:%s", constants.DefaultOpenYurtImageRegistry, constants.Yurthub, constants.DefaultOpenYurtVersion) - } - if _, err := os.Stat(podManifestPath); err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(podManifestPath, os.ModePerm) - if err != nil { - return err - } - } else { - klog.Errorf("Describe dir %s fail: %v", podManifestPath, err) - return err - } - } - - yurthubTemplate := edgenode.ReplaceRegularExpression(edgenode.YurthubTemplate, - map[string]string{ - "__kubernetes_service_addr__": fmt.Sprintf("https://%s", cfg.Discovery.BootstrapToken.APIServerEndpoint), - "__yurthub_image__": yurthubImage, - "__join_token__": cfg.Discovery.BootstrapToken.Token, - "__working_mode__": string(util.WorkingModeEdge), - }) - - if err := ioutil.WriteFile(filepath.Join(podManifestPath, defaultYurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil { - return err - } - klog.Info("[join-node] Add edge hub static yaml is ok") - return nil -} - -//getKubeletConfig get kubelet configure from master. -func getKubeletConfig(cfg *kubeadmapi.JoinConfiguration, initCfg *kubeadmapi.InitConfiguration, tlsClient *clientset.Clientset) (*kubeletconfig.KubeletConfiguration, error) { - kubeletVersion, err := version.ParseSemantic(initCfg.ClusterConfiguration.KubernetesVersion) - if err != nil { - return nil, err - } - - // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start - kc, err := downloadConfig(tlsClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory) - if err != nil { - return nil, err - } - if err := kubeletphase.WriteKubeletDynamicEnvFile(&initCfg.ClusterConfiguration, &cfg.NodeRegistration, false, kubeadmconstants.KubeletRunDirectory); err != nil { - return kc, err - } - return kc, nil -} - -// downloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk. -// Used at "kubeadm join" time -func downloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) (*kubeletconfig.KubeletConfiguration, error) { - - // Download the ConfigMap from the cluster based on what version the kubelet is - configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion) - - fmt.Printf("[kubelet-start] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n", - configMapName, metav1.NamespaceSystem) - - kubeletCfg, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, configMapName) - // If the ConfigMap wasn't found and the kubelet version is v1.10.x, where we didn't support the config file yet - // just return, don't error out - if apierrors.IsNotFound(err) && kubeletVersion.Minor() == 10 { - return nil, nil - } - if err != nil { - return nil, err - } - _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() - if err != nil { - return nil, err - } - kc, err := utilcodec.DecodeKubeletConfiguration(kubeletCodecs, []byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])) - if err != nil { - return nil, err - } - if kc.StaticPodPath == "" { - kc.StaticPodPath = constants.StaticPodPath - } - encoder, err := utilcodec.NewKubeletconfigYAMLEncoder(kubeletconfigv1beta1.SchemeGroupVersion) - if err != nil { - return nil, err - } - data, err := runtime.Encode(encoder, kc) - if err != nil { - return nil, err - } - return kc, writeConfigBytesToDisk(data, kubeletDir) -} - -//getEdgeNodeJoinData get edge-node join configuration. -func getEdgeNodeJoinData(data YurtJoinData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { - cfg := data.Cfg() - initCfg, err := data.InitCfg() - if err != nil { - return nil, nil, nil, err - } - tlsBootstrapCfg, err := data.TLSBootstrapCfg() - if err != nil { - return nil, nil, nil, err - } - return cfg, initCfg, tlsBootstrapCfg, nil -} - -// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file -func writeConfigBytesToDisk(b []byte, kubeletDir string) error { - configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName) - fmt.Printf("[kubelet-start] Writing kubelet configuration to file %q\n", configFile) - - // creates target folder if not already exists - if err := os.MkdirAll(kubeletDir, 0700); err != nil { - return errors.Wrapf(err, "failed to create directory %q", kubeletDir) - } - - if err := ioutil.WriteFile(configFile, b, 0644); err != nil { - return errors.Wrapf(err, "failed to write kubelet configuration to the file %q", configFile) - } - return nil -} diff --git a/pkg/yurtctl/cmd/join/phases/postcheck.go b/pkg/yurtctl/cmd/join/phases/postcheck.go index af8ed085832..c906cef7f9b 100644 --- a/pkg/yurtctl/cmd/join/phases/postcheck.go +++ b/pkg/yurtctl/cmd/join/phases/postcheck.go @@ -65,7 +65,7 @@ func runPostcheck(c workflow.RunData) error { if err := checkYurthubHealthz(); err != nil { return err } - return patchEdgeNode(cfg) + return patchEdgeNode(cfg, j.MarkAutonomous()) } return patchCloudNode(cfg) } @@ -103,7 +103,7 @@ func checkYurthubHealthz() error { } //patchEdgeNode patch labels and annotations for edge-node. -func patchEdgeNode(cfg *kubeadm.JoinConfiguration) error { +func patchEdgeNode(cfg *kubeadm.JoinConfiguration, markAutonomous bool) error { client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) if err != nil { return err @@ -111,6 +111,13 @@ func patchEdgeNode(cfg *kubeadm.JoinConfiguration) error { if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { return err } + + if markAutonomous { + if err := apiclient.PatchNode(client, cfg.NodeRegistration.Name, annotateNodeWithAutonomousNode); err != nil { + return err + } + } + if err := apiclient.PatchNode(client, cfg.NodeRegistration.Name, func(n *v1.Node) { n.Labels[projectinfo.GetEdgeWorkerLabelKey()] = "true" }); err != nil { @@ -135,3 +142,12 @@ func patchCloudNode(cfg *kubeadm.JoinConfiguration) error { } return nil } + +func annotateNodeWithAutonomousNode(n *v1.Node) { + klog.V(1).Infof("[patchnode] mark autonomous to the Node API object %q as an annotation\n", n.Name) + + if n.ObjectMeta.Annotations == nil { + n.ObjectMeta.Annotations = make(map[string]string) + } + n.ObjectMeta.Annotations[constants.AnnotationAutonomy] = "true" +} diff --git a/pkg/yurtctl/cmd/join/phases/type.go b/pkg/yurtctl/cmd/join/phases/type.go index d9ad6c5dcdb..9d88280dec9 100644 --- a/pkg/yurtctl/cmd/join/phases/type.go +++ b/pkg/yurtctl/cmd/join/phases/type.go @@ -24,4 +24,5 @@ type YurtJoinData interface { joinphases.JoinData NodeType() string YurtHubImage() string + MarkAutonomous() bool }