From 797c43d730a11a6cd9bd491f3d3838917e2a49cb Mon Sep 17 00:00:00 2001 From: adamzhoul <770822772@qq.com> Date: Fri, 17 Sep 2021 16:56:51 +0800 Subject: [PATCH] read podManifestPath from kubeadm file (#473) Co-authored-by: zhoulei5 --- pkg/yurtctl/cmd/convert/convert.go | 24 +++++------ pkg/yurtctl/cmd/convert/edgenode.go | 36 +++++------------ pkg/yurtctl/cmd/revert/edgenode.go | 40 +++++++----------- pkg/yurtctl/cmd/revert/revert.go | 13 +++--- pkg/yurtctl/constants/constants.go | 4 -- pkg/yurtctl/util/edgenode/util.go | 43 ++++++++++++++++---- pkg/yurtctl/util/edgenode/util_test.go | 56 ++++++++++++++++++++++++++ 7 files changed, 130 insertions(+), 86 deletions(-) create mode 100644 pkg/yurtctl/util/edgenode/util_test.go diff --git a/pkg/yurtctl/cmd/convert/convert.go b/pkg/yurtctl/cmd/convert/convert.go index 52067183f9f..3066eb69cd2 100644 --- a/pkg/yurtctl/cmd/convert/convert.go +++ b/pkg/yurtctl/cmd/convert/convert.go @@ -24,12 +24,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" - - "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurtctl/lock" - kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" - strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" - v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,6 +33,12 @@ import ( "k8s.io/klog" clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" nodeutil "k8s.io/kubernetes/pkg/controller/util/node" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurtctl/lock" + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" ) // Provider signifies the provider type @@ -134,9 +134,6 @@ func NewConvertCmd() *cobra.Command { "The yurt-tunnel-agent image.") cmd.Flags().BoolP("deploy-yurttunnel", "t", false, "if set, yurttunnel will be deployed.") - cmd.Flags().String("pod-manifest-path", - "/etc/kubernetes/manifests", - "Path to the directory on edge node containing static pod files.") cmd.Flags().BoolP("enable-app-manager", "e", false, "if set, yurtappmanager will be deployed.") cmd.Flags().String("yurt-app-manager-image", @@ -221,17 +218,17 @@ func (co *ConvertOptions) Complete(flags *pflag.FlagSet) error { } co.YurtAppManagerImage = yami - pmp, err := flags.GetString("pod-manifest-path") + kcp, err := flags.GetString("kubeadm-conf-path") if err != nil { return err } - co.PodMainfestPath = pmp + co.KubeadmConfPath = kcp - kcp, err := flags.GetString("kubeadm-conf-path") + podManifestPath, err := enutil.GetPodManifestPath(co.KubeadmConfPath) if err != nil { return err } - co.KubeadmConfPath = kcp + co.PodMainfestPath = podManifestPath sa, err := flags.GetString("system-architecture") if err != nil { @@ -399,7 +396,6 @@ func (co *ConvertOptions) RunConvert() (err error) { "yurtctl_servant_image": co.YurctlServantImage, "yurthub_image": co.YurhubImage, "joinToken": joinToken, - "pod_manifest_path": co.PodMainfestPath, "kubeadm_conf_path": co.KubeadmConfPath, } diff --git a/pkg/yurtctl/cmd/convert/edgenode.go b/pkg/yurtctl/cmd/convert/edgenode.go index 613405a62e1..66702d5259c 100644 --- a/pkg/yurtctl/cmd/convert/edgenode.go +++ b/pkg/yurtctl/cmd/convert/edgenode.go @@ -59,7 +59,6 @@ type ConvertEdgeNodeOptions struct { YurthubImage string YurthubHealthCheckTimeout time.Duration YurctlServantImage string - PodMainfestPath string JoinToken string KubeadmConfPath string openyurtDir string @@ -94,8 +93,6 @@ func NewConvertEdgeNodeCmd() *cobra.Command { "The timeout for yurthub health check.") cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest", "The yurtctl-servant image.") - cmd.Flags().String("pod-manifest-path", "", - "Path to the directory on edge node containing static pod files.") cmd.Flags().String("kubeadm-conf-path", "", "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.") @@ -131,18 +128,6 @@ func (c *ConvertEdgeNodeOptions) Complete(flags *pflag.FlagSet) error { } c.YurctlServantImage = ycsi - podMainfestPath, err := flags.GetString("pod-manifest-path") - if err != nil { - return err - } - if podMainfestPath == "" { - podMainfestPath = os.Getenv("STATIC_POD_PATH") - } - if podMainfestPath == "" { - podMainfestPath = enutil.StaticPodPath - } - c.PodMainfestPath = podMainfestPath - kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") if err != nil { return err @@ -230,7 +215,6 @@ func (c *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) { "yurtctl_servant_image": c.YurctlServantImage, "yurthub_image": c.YurthubImage, "joinToken": c.JoinToken, - "pod_manifest_path": c.PodMainfestPath, "kubeadm_conf_path": c.KubeadmConfPath, } @@ -248,9 +232,6 @@ func (c *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) { if _, err := enutil.FileExists(c.KubeadmConfPath); err != nil { return err } - if ok, err := enutil.DirExists(c.PodMainfestPath); !ok { - return err - } // 3.2. check the state of EdgeNodes node, err := c.clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) @@ -335,15 +316,18 @@ func (c *ConvertEdgeNodeOptions) SetupYurthub() error { }) // 1-3. create yurthub.yaml - _, err = enutil.DirExists(c.PodMainfestPath) + podManifestPath, err := enutil.GetPodManifestPath(c.KubeadmConfPath) if err != nil { return err } - err = ioutil.WriteFile(c.getYurthubYaml(), []byte(yurthubTemplate), filemode) + if err = enutil.EnsureDir(podManifestPath); err != nil { + return err + } + err = ioutil.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), filemode) if err != nil { return err } - klog.Infof("create the %s/yurt-hub.yaml", c.PodMainfestPath) + klog.Infof("create the %s/yurt-hub.yaml", podManifestPath) // 2. wait yurthub pod to be ready err = hubHealthcheck(c.YurthubHealthCheckTimeout) @@ -403,10 +387,6 @@ func (c *ConvertEdgeNodeOptions) ResetKubelet() error { return nil } -func (c *ConvertEdgeNodeOptions) getYurthubYaml() string { - return filepath.Join(c.PodMainfestPath, enutil.YurthubYamlName) -} - func (c *ConvertEdgeNodeOptions) getYurthubKubeletConf() string { return filepath.Join(c.openyurtDir, enutil.KubeletConfName) } @@ -415,6 +395,10 @@ func (c *ConvertEdgeNodeOptions) getKubeletSvcBackup() string { return fmt.Sprintf(enutil.KubeletSvcBackup, c.KubeadmConfPath) } +func getYurthubYaml(podManifestPath string) string { + return filepath.Join(podManifestPath, enutil.YurthubYamlName) +} + // hubHealthcheck will check the status of yurthub pod func hubHealthcheck(timeout time.Duration) error { serverHealthzURL, err := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer)) diff --git a/pkg/yurtctl/cmd/revert/edgenode.go b/pkg/yurtctl/cmd/revert/edgenode.go index b07e90a608c..31798f78881 100644 --- a/pkg/yurtctl/cmd/revert/edgenode.go +++ b/pkg/yurtctl/cmd/revert/edgenode.go @@ -45,7 +45,6 @@ type RevertEdgeNodeOptions struct { clientSet *kubernetes.Clientset EdgeNodes []string YurtctlServantImage string - PodMainfestPath string KubeadmConfPath string openyurtDir string } @@ -75,8 +74,6 @@ func NewRevertEdgeNodeCmd() *cobra.Command { "The list of edge nodes wanted to be revert.(e.g. -e edgenode1,edgenode2)") cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest", "The yurtctl-servant image.") - cmd.Flags().String("pod-manifest-path", "", - "Path to the directory on edge node containing static pod files.") cmd.Flags().String("kubeadm-conf-path", "", "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") @@ -99,18 +96,6 @@ func (r *RevertEdgeNodeOptions) Complete(flags *pflag.FlagSet) (err error) { } r.YurtctlServantImage = ycsi - podMainfestPath, err := flags.GetString("pod-manifest-path") - if err != nil { - return err - } - if podMainfestPath == "" { - podMainfestPath = os.Getenv("STATIC_POD_PATH") - } - if podMainfestPath == "" { - podMainfestPath = enutil.StaticPodPath - } - r.PodMainfestPath = podMainfestPath - kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") if err != nil { return err @@ -187,7 +172,6 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) { map[string]string{ "action": "revert", "yurtctl_servant_image": r.YurtctlServantImage, - "pod_manifest_path": r.PodMainfestPath, "kubeadm_conf_path": r.KubeadmConfPath, }, r.EdgeNodes); err != nil { @@ -206,8 +190,13 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) { klog.Errorf("fail to get file %s, should revise the %s directly", kubeletSvcBk, r.KubeadmConfPath) return err } - yurthubYaml := r.getYurthubYaml() - if ok, err := enutil.FileExists(yurthubYaml); !ok { + podManifestPath, err := enutil.GetPodManifestPath(r.KubeadmConfPath) + if err != nil { + klog.Errorf("get podManifestPath fail: %v", err) + return err + } + yurthubYamlPath := getYurthubYaml(podManifestPath) + if ok, err := enutil.FileExists(yurthubYamlPath); !ok { return err } @@ -232,7 +221,7 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) { if err := r.RevertKubelet(); err != nil { return fmt.Errorf("fail to revert kubelet: %v", err) } - if err := r.RemoveYurthub(); err != nil { + if err := r.RemoveYurthub(yurthubYamlPath); err != nil { return err } @@ -289,16 +278,15 @@ func (r *RevertEdgeNodeOptions) RevertKubelet() error { } // RemoveYurthub deletes the yurt-hub pod -func (r *RevertEdgeNodeOptions) RemoveYurthub() error { +func (r *RevertEdgeNodeOptions) RemoveYurthub(yurthubYamlPath string) error { // 1. remove the yurt-hub.yaml to delete the yurt-hub - yurthubYaml := r.getYurthubYaml() - err := os.Remove(yurthubYaml) + err := os.Remove(yurthubYamlPath) if err != nil { return err } // 2. remove yurt-hub config directory and certificates in it - yurthubConf := r.getYurthubConf() + yurthubConf := getYurthubConf() err = os.RemoveAll(yurthubConf) if err != nil { return err @@ -315,10 +303,10 @@ func (r *RevertEdgeNodeOptions) getKubeletSvcBackup() string { return fmt.Sprintf(enutil.KubeletSvcBackup, r.KubeadmConfPath) } -func (r *RevertEdgeNodeOptions) getYurthubYaml() string { - return filepath.Join(r.PodMainfestPath, enutil.YurthubYamlName) +func getYurthubYaml(podManifestPath string) string { + return filepath.Join(podManifestPath, enutil.YurthubYamlName) } -func (r *RevertEdgeNodeOptions) getYurthubConf() string { +func getYurthubConf() string { return filepath.Join(hubself.HubRootDir, hubself.HubName) } diff --git a/pkg/yurtctl/cmd/revert/revert.go b/pkg/yurtctl/cmd/revert/revert.go index 4146e3507c4..8813af51251 100644 --- a/pkg/yurtctl/cmd/revert/revert.go +++ b/pkg/yurtctl/cmd/revert/revert.go @@ -32,6 +32,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurtctl/constants" "github.com/openyurtio/openyurt/pkg/yurtctl/lock" + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" ) @@ -70,9 +71,6 @@ func NewRevertCmd() *cobra.Command { cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest", "The yurtctl-servant image.") - cmd.Flags().String("pod-manifest-path", - "/etc/kubernetes/manifests", - "Path to the directory on edge node containing static pod files.") cmd.Flags().String("kubeadm-conf-path", "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") @@ -88,17 +86,17 @@ func (ro *RevertOptions) Complete(flags *pflag.FlagSet) error { } ro.YurtctlServantImage = ycsi - pmp, err := flags.GetString("pod-manifest-path") + kcp, err := flags.GetString("kubeadm-conf-path") if err != nil { return err } - ro.PodMainfestPath = pmp + ro.KubeadmConfPath = kcp - kcp, err := flags.GetString("kubeadm-conf-path") + podManifestPath, err := enutil.GetPodManifestPath(ro.KubeadmConfPath) if err != nil { return err } - ro.KubeadmConfPath = kcp + ro.PodMainfestPath = podManifestPath ro.clientSet, err = kubeutil.GenClientSet(flags) if err != nil { @@ -237,7 +235,6 @@ func (ro *RevertOptions) RunRevert() (err error) { map[string]string{ "action": "revert", "yurtctl_servant_image": ro.YurtctlServantImage, - "pod_manifest_path": ro.PodMainfestPath, "kubeadm_conf_path": ro.KubeadmConfPath, }, edgeNodeNames); err != nil { diff --git a/pkg/yurtctl/constants/constants.go b/pkg/yurtctl/constants/constants.go index 764bd83a2d3..78e18946567 100644 --- a/pkg/yurtctl/constants/constants.go +++ b/pkg/yurtctl/constants/constants.go @@ -282,8 +282,6 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - - name: STATIC_POD_PATH - value: {{.pod_manifest_path}} {{if .kubeadm_conf_path }} - name: KUBELET_SVC value: {{.kubeadm_conf_path}} @@ -327,8 +325,6 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - - name: STATIC_POD_PATH - value: {{.pod_manifest_path}} {{if .kubeadm_conf_path }} - name: KUBELET_SVC value: {{.kubeadm_conf_path}} diff --git a/pkg/yurtctl/util/edgenode/util.go b/pkg/yurtctl/util/edgenode/util.go index 899db26d83d..6af885ee529 100644 --- a/pkg/yurtctl/util/edgenode/util.go +++ b/pkg/yurtctl/util/edgenode/util.go @@ -17,6 +17,7 @@ limitations under the License. package edgenode import ( + "errors" "fmt" "io/ioutil" "os" @@ -71,16 +72,14 @@ func GetSingleContentFromFile(filename string, regularExpression string) (string return contents[0], nil } -// DirExists determines whether the directory exists -func DirExists(dirname string) (bool, error) { +// EnsureDir make sure dir is exists, if not create +func EnsureDir(dirname string) error { s, err := os.Stat(dirname) - if err != nil { - return false, err - } - if !s.IsDir() { - return false, fmt.Errorf("%s is not a dir", dirname) + if err == nil && s.IsDir() { + return nil } - return true, nil + + return os.MkdirAll(dirname, 0755) } // CopyFile copys sourceFile to destinationFile @@ -201,3 +200,31 @@ func Exec(cmd *exec.Cmd) error { } return nil } + +// GetPodManifestPath get path string from kubeadmConf file +func GetPodManifestPath(kubeadmConfPath string) (string, error) { + configRegularExpression := "\\-\\-config=.*config.yaml" + podManifestPathRegularExpression := "staticPodPath: .*" + + configPath, err := GetSingleContentFromFile(kubeadmConfPath, configRegularExpression) + if err != nil { + return "", err + } + arr := strings.Split(configPath, "=") + if len(arr) != 2 { + return "", errors.New("kubelet config path format incorrect: " + configPath) + } + configPath = arr[1] + + podManifestPath, err := GetSingleContentFromFile(configPath, podManifestPathRegularExpression) + if err != nil { + return "", err + } + arr = strings.Split(podManifestPath, " ") + if len(arr) != 2 { + return "", errors.New("podManifestPath format incorrect: " + podManifestPath) + } + podManifestPath = arr[1] + + return podManifestPath, nil +} diff --git a/pkg/yurtctl/util/edgenode/util_test.go b/pkg/yurtctl/util/edgenode/util_test.go new file mode 100644 index 00000000000..194b70295c2 --- /dev/null +++ b/pkg/yurtctl/util/edgenode/util_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edgenode + +import ( + "io/ioutil" + "os" + "testing" +) + +var kubeadmConf = ` +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/tmp/openyurt-test/config.yaml" +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS` + +var confYaml = `serializeImagePulls: true +staticPodPath: /etc/kubernetes/manifests +streamingConnectionIdleTimeout: 4h0m0s +syncFrequency: 1m0s` + +func Test_GetPodManifestPath(t *testing.T) { + err := EnsureDir("/tmp/openyurt-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll("/tmp/openyurt-test") + + _ = ioutil.WriteFile("/tmp/openyurt-test/kubeadm.conf", []byte(kubeadmConf), 0755) + _ = ioutil.WriteFile("/tmp/openyurt-test/config.yaml", []byte(confYaml), 0755) + + path, err := GetPodManifestPath("/tmp/openyurt-test/kubeadm.conf") + if err != nil { + t.Fatal(err) + } + if path != "/etc/kubernetes/manifests" { + t.Fatal("get path err: " + path) + } +}