read podManifestPath from kubeadm file (#473)

Co-authored-by: zhoulei5 <[email protected]>

pkg/yurtctl/cmd/convert/convert.go

@@ -24,12 +24,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-
-	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	"github.com/openyurtio/openyurt/pkg/yurtctl/lock"
-	kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes"
-	strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings"
-
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -39,6 +33,12 @@ import (
 	"k8s.io/klog"
 	clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
 	nodeutil "k8s.io/kubernetes/pkg/controller/util/node"
+
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	"github.com/openyurtio/openyurt/pkg/yurtctl/lock"
+	enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode"
+	kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes"
+	strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings"
 )
 
 // Provider signifies the provider type
@@ -134,9 +134,6 @@ func NewConvertCmd() *cobra.Command {
 		"The yurt-tunnel-agent image.")
 	cmd.Flags().BoolP("deploy-yurttunnel", "t", false,
 		"if set, yurttunnel will be deployed.")
-	cmd.Flags().String("pod-manifest-path",
-		"/etc/kubernetes/manifests",
-		"Path to the directory on edge node containing static pod files.")
 	cmd.Flags().BoolP("enable-app-manager", "e", false,
 		"if set, yurtappmanager will be deployed.")
 	cmd.Flags().String("yurt-app-manager-image",
@@ -221,17 +218,17 @@ func (co *ConvertOptions) Complete(flags *pflag.FlagSet) error {
 	}
 	co.YurtAppManagerImage = yami
 
-	pmp, err := flags.GetString("pod-manifest-path")
+	kcp, err := flags.GetString("kubeadm-conf-path")
 	if err != nil {
 		return err
 	}
-	co.PodMainfestPath = pmp
+	co.KubeadmConfPath = kcp
 
-	kcp, err := flags.GetString("kubeadm-conf-path")
+	podManifestPath, err := enutil.GetPodManifestPath(co.KubeadmConfPath)
 	if err != nil {
 		return err
 	}
-	co.KubeadmConfPath = kcp
+	co.PodMainfestPath = podManifestPath
 
 	sa, err := flags.GetString("system-architecture")
 	if err != nil {
@@ -399,7 +396,6 @@ func (co *ConvertOptions) RunConvert() (err error) {
 		"yurtctl_servant_image": co.YurctlServantImage,
 		"yurthub_image":         co.YurhubImage,
 		"joinToken":             joinToken,
-		"pod_manifest_path":     co.PodMainfestPath,
 		"kubeadm_conf_path":     co.KubeadmConfPath,
 	}
 

pkg/yurtctl/cmd/convert/edgenode.go

@@ -59,7 +59,6 @@ type ConvertEdgeNodeOptions struct {
 	YurthubImage              string
 	YurthubHealthCheckTimeout time.Duration
 	YurctlServantImage        string
-	PodMainfestPath           string
 	JoinToken                 string
 	KubeadmConfPath           string
 	openyurtDir               string
@@ -94,8 +93,6 @@ func NewConvertEdgeNodeCmd() *cobra.Command {
 		"The timeout for yurthub health check.")
 	cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest",
 		"The yurtctl-servant image.")
-	cmd.Flags().String("pod-manifest-path", "",
-		"Path to the directory on edge node containing static pod files.")
 	cmd.Flags().String("kubeadm-conf-path", "",
 		"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")
 	cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.")
@@ -131,18 +128,6 @@ func (c *ConvertEdgeNodeOptions) Complete(flags *pflag.FlagSet) error {
 	}
 	c.YurctlServantImage = ycsi
 
-	podMainfestPath, err := flags.GetString("pod-manifest-path")
-	if err != nil {
-		return err
-	}
-	if podMainfestPath == "" {
-		podMainfestPath = os.Getenv("STATIC_POD_PATH")
-	}
-	if podMainfestPath == "" {
-		podMainfestPath = enutil.StaticPodPath
-	}
-	c.PodMainfestPath = podMainfestPath
-
 	kubeadmConfPath, err := flags.GetString("kubeadm-conf-path")
 	if err != nil {
 		return err
@@ -230,7 +215,6 @@ func (c *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) {
 			"yurtctl_servant_image": c.YurctlServantImage,
 			"yurthub_image":         c.YurthubImage,
 			"joinToken":             c.JoinToken,
-			"pod_manifest_path":     c.PodMainfestPath,
 			"kubeadm_conf_path":     c.KubeadmConfPath,
 		}
 
@@ -248,9 +232,6 @@ func (c *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) {
 		if _, err := enutil.FileExists(c.KubeadmConfPath); err != nil {
 			return err
 		}
-		if ok, err := enutil.DirExists(c.PodMainfestPath); !ok {
-			return err
-		}
 
 		// 3.2. check the state of EdgeNodes
 		node, err := c.clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
@@ -335,15 +316,18 @@ func (c *ConvertEdgeNodeOptions) SetupYurthub() error {
 		})
 
 	// 1-3. create yurthub.yaml
-	_, err = enutil.DirExists(c.PodMainfestPath)
+	podManifestPath, err := enutil.GetPodManifestPath(c.KubeadmConfPath)
 	if err != nil {
 		return err
 	}
-	err = ioutil.WriteFile(c.getYurthubYaml(), []byte(yurthubTemplate), filemode)
+	if err = enutil.EnsureDir(podManifestPath); err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), filemode)
 	if err != nil {
 		return err
 	}
-	klog.Infof("create the %s/yurt-hub.yaml", c.PodMainfestPath)
+	klog.Infof("create the %s/yurt-hub.yaml", podManifestPath)
 
 	// 2. wait yurthub pod to be ready
 	err = hubHealthcheck(c.YurthubHealthCheckTimeout)
@@ -403,10 +387,6 @@ func (c *ConvertEdgeNodeOptions) ResetKubelet() error {
 	return nil
 }
 
-func (c *ConvertEdgeNodeOptions) getYurthubYaml() string {
-	return filepath.Join(c.PodMainfestPath, enutil.YurthubYamlName)
-}
-
 func (c *ConvertEdgeNodeOptions) getYurthubKubeletConf() string {
 	return filepath.Join(c.openyurtDir, enutil.KubeletConfName)
 }
@@ -415,6 +395,10 @@ func (c *ConvertEdgeNodeOptions) getKubeletSvcBackup() string {
 	return fmt.Sprintf(enutil.KubeletSvcBackup, c.KubeadmConfPath)
 }
 
+func getYurthubYaml(podManifestPath string) string {
+	return filepath.Join(podManifestPath, enutil.YurthubYamlName)
+}
+
 // hubHealthcheck will check the status of yurthub pod
 func hubHealthcheck(timeout time.Duration) error {
 	serverHealthzURL, err := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer))

pkg/yurtctl/cmd/revert/edgenode.go

@@ -45,7 +45,6 @@ type RevertEdgeNodeOptions struct {
 	clientSet           *kubernetes.Clientset
 	EdgeNodes           []string
 	YurtctlServantImage string
-	PodMainfestPath     string
 	KubeadmConfPath     string
 	openyurtDir         string
 }
@@ -75,8 +74,6 @@ func NewRevertEdgeNodeCmd() *cobra.Command {
 		"The list of edge nodes wanted to be revert.(e.g. -e edgenode1,edgenode2)")
 	cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest",
 		"The yurtctl-servant image.")
-	cmd.Flags().String("pod-manifest-path", "",
-		"Path to the directory on edge node containing static pod files.")
 	cmd.Flags().String("kubeadm-conf-path", "",
 		"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")
 
@@ -99,18 +96,6 @@ func (r *RevertEdgeNodeOptions) Complete(flags *pflag.FlagSet) (err error) {
 	}
 	r.YurtctlServantImage = ycsi
 
-	podMainfestPath, err := flags.GetString("pod-manifest-path")
-	if err != nil {
-		return err
-	}
-	if podMainfestPath == "" {
-		podMainfestPath = os.Getenv("STATIC_POD_PATH")
-	}
-	if podMainfestPath == "" {
-		podMainfestPath = enutil.StaticPodPath
-	}
-	r.PodMainfestPath = podMainfestPath
-
 	kubeadmConfPath, err := flags.GetString("kubeadm-conf-path")
 	if err != nil {
 		return err
@@ -187,7 +172,6 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) {
 			map[string]string{
 				"action":                "revert",
 				"yurtctl_servant_image": r.YurtctlServantImage,
-				"pod_manifest_path":     r.PodMainfestPath,
 				"kubeadm_conf_path":     r.KubeadmConfPath,
 			},
 			r.EdgeNodes); err != nil {
@@ -206,8 +190,13 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) {
 			klog.Errorf("fail to get file %s, should revise the %s directly", kubeletSvcBk, r.KubeadmConfPath)
 			return err
 		}
-		yurthubYaml := r.getYurthubYaml()
-		if ok, err := enutil.FileExists(yurthubYaml); !ok {
+		podManifestPath, err := enutil.GetPodManifestPath(r.KubeadmConfPath)
+		if err != nil {
+			klog.Errorf("get podManifestPath fail: %v", err)
+			return err
+		}
+		yurthubYamlPath := getYurthubYaml(podManifestPath)
+		if ok, err := enutil.FileExists(yurthubYamlPath); !ok {
 			return err
 		}
 
@@ -232,7 +221,7 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) {
 		if err := r.RevertKubelet(); err != nil {
 			return fmt.Errorf("fail to revert kubelet: %v", err)
 		}
-		if err := r.RemoveYurthub(); err != nil {
+		if err := r.RemoveYurthub(yurthubYamlPath); err != nil {
 			return err
 		}
 
@@ -289,16 +278,15 @@ func (r *RevertEdgeNodeOptions) RevertKubelet() error {
 }
 
 // RemoveYurthub deletes the yurt-hub pod
-func (r *RevertEdgeNodeOptions) RemoveYurthub() error {
+func (r *RevertEdgeNodeOptions) RemoveYurthub(yurthubYamlPath string) error {
 	// 1. remove the yurt-hub.yaml to delete the yurt-hub
-	yurthubYaml := r.getYurthubYaml()
-	err := os.Remove(yurthubYaml)
+	err := os.Remove(yurthubYamlPath)
 	if err != nil {
 		return err
 	}
 
 	// 2. remove yurt-hub config directory and certificates in it
-	yurthubConf := r.getYurthubConf()
+	yurthubConf := getYurthubConf()
 	err = os.RemoveAll(yurthubConf)
 	if err != nil {
 		return err
@@ -315,10 +303,10 @@ func (r *RevertEdgeNodeOptions) getKubeletSvcBackup() string {
 	return fmt.Sprintf(enutil.KubeletSvcBackup, r.KubeadmConfPath)
 }
 
-func (r *RevertEdgeNodeOptions) getYurthubYaml() string {
-	return filepath.Join(r.PodMainfestPath, enutil.YurthubYamlName)
+func getYurthubYaml(podManifestPath string) string {
+	return filepath.Join(podManifestPath, enutil.YurthubYamlName)
 }
 
-func (r *RevertEdgeNodeOptions) getYurthubConf() string {
+func getYurthubConf() string {
 	return filepath.Join(hubself.HubRootDir, hubself.HubName)
 }

pkg/yurtctl/cmd/revert/revert.go

@@ -32,6 +32,7 @@ import (
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/yurtctl/constants"
 	"github.com/openyurtio/openyurt/pkg/yurtctl/lock"
+	enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode"
 	kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes"
 	strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings"
 )
@@ -70,9 +71,6 @@ func NewRevertCmd() *cobra.Command {
 	cmd.Flags().String("yurtctl-servant-image",
 		"openyurt/yurtctl-servant:latest",
 		"The yurtctl-servant image.")
-	cmd.Flags().String("pod-manifest-path",
-		"/etc/kubernetes/manifests",
-		"Path to the directory on edge node containing static pod files.")
 	cmd.Flags().String("kubeadm-conf-path",
 		"/etc/systemd/system/kubelet.service.d/10-kubeadm.conf",
 		"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")
@@ -88,17 +86,17 @@ func (ro *RevertOptions) Complete(flags *pflag.FlagSet) error {
 	}
 	ro.YurtctlServantImage = ycsi
 
-	pmp, err := flags.GetString("pod-manifest-path")
+	kcp, err := flags.GetString("kubeadm-conf-path")
 	if err != nil {
 		return err
 	}
-	ro.PodMainfestPath = pmp
+	ro.KubeadmConfPath = kcp
 
-	kcp, err := flags.GetString("kubeadm-conf-path")
+	podManifestPath, err := enutil.GetPodManifestPath(ro.KubeadmConfPath)
 	if err != nil {
 		return err
 	}
-	ro.KubeadmConfPath = kcp
+	ro.PodMainfestPath = podManifestPath
 
 	ro.clientSet, err = kubeutil.GenClientSet(flags)
 	if err != nil {
@@ -237,7 +235,6 @@ func (ro *RevertOptions) RunRevert() (err error) {
 		map[string]string{
 			"action":                "revert",
 			"yurtctl_servant_image": ro.YurtctlServantImage,
-			"pod_manifest_path":     ro.PodMainfestPath,
 			"kubeadm_conf_path":     ro.KubeadmConfPath,
 		},
 		edgeNodeNames); err != nil {

pkg/yurtctl/constants/constants.go

@@ -282,8 +282,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        - name: STATIC_POD_PATH
-          value: {{.pod_manifest_path}}
           {{if  .kubeadm_conf_path }}
         - name: KUBELET_SVC
           value: {{.kubeadm_conf_path}}
@@ -327,8 +325,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        - name: STATIC_POD_PATH
-          value: {{.pod_manifest_path}}
           {{if  .kubeadm_conf_path }}
         - name: KUBELET_SVC
           value: {{.kubeadm_conf_path}}

pkg/yurtctl/util/edgenode/util.go

@@ -17,6 +17,7 @@ limitations under the License.
 package edgenode
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -71,16 +72,14 @@ func GetSingleContentFromFile(filename string, regularExpression string) (string
 	return contents[0], nil
 }
 
-// DirExists determines whether the directory exists
-func DirExists(dirname string) (bool, error) {
+// EnsureDir make sure dir is exists, if not create
+func EnsureDir(dirname string) error {
 	s, err := os.Stat(dirname)
-	if err != nil {
-		return false, err
-	}
-	if !s.IsDir() {
-		return false, fmt.Errorf("%s is not a dir", dirname)
+	if err == nil && s.IsDir() {
+		return nil
 	}
-	return true, nil
+
+	return os.MkdirAll(dirname, 0755)
 }
 
 // CopyFile copys sourceFile to destinationFile
@@ -201,3 +200,31 @@ func Exec(cmd *exec.Cmd) error {
 	}
 	return nil
 }
+
+// GetPodManifestPath get path string from kubeadmConf file
+func GetPodManifestPath(kubeadmConfPath string) (string, error) {
+	configRegularExpression := "\\-\\-config=.*config.yaml"
+	podManifestPathRegularExpression := "staticPodPath: .*"
+
+	configPath, err := GetSingleContentFromFile(kubeadmConfPath, configRegularExpression)
+	if err != nil {
+		return "", err
+	}
+	arr := strings.Split(configPath, "=")
+	if len(arr) != 2 {
+		return "", errors.New("kubelet config path format incorrect: " + configPath)
+	}
+	configPath = arr[1]
+
+	podManifestPath, err := GetSingleContentFromFile(configPath, podManifestPathRegularExpression)
+	if err != nil {
+		return "", err
+	}
+	arr = strings.Split(podManifestPath, " ")
+	if len(arr) != 2 {
+		return "", errors.New("podManifestPath format incorrect: " + podManifestPath)
+	}
+	podManifestPath = arr[1]
+
+	return podManifestPath, nil
+}