Skip to content

Commit

Permalink
read podManifestPath from kubeadm file (openyurtio#473)
Browse files Browse the repository at this point in the history
Co-authored-by: zhoulei5 <[email protected]>
  • Loading branch information
adamzhoul and zhoulei5 authored Sep 17, 2021
1 parent 3dc8a1f commit 3ee1139
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 86 deletions.
24 changes: 10 additions & 14 deletions pkg/yurtctl/cmd/convert/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,6 @@ import (

"github.com/spf13/cobra"
"github.com/spf13/pflag"

"github.com/openyurtio/openyurt/pkg/projectinfo"
"github.com/openyurtio/openyurt/pkg/yurtctl/lock"
kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes"
strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings"

v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -39,6 +33,12 @@ import (
"k8s.io/klog"
clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
nodeutil "k8s.io/kubernetes/pkg/controller/util/node"

"github.com/openyurtio/openyurt/pkg/projectinfo"
"github.com/openyurtio/openyurt/pkg/yurtctl/lock"
enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode"
kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes"
strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings"
)

// Provider signifies the provider type
Expand Down Expand Up @@ -134,9 +134,6 @@ func NewConvertCmd() *cobra.Command {
"The yurt-tunnel-agent image.")
cmd.Flags().BoolP("deploy-yurttunnel", "t", false,
"if set, yurttunnel will be deployed.")
cmd.Flags().String("pod-manifest-path",
"/etc/kubernetes/manifests",
"Path to the directory on edge node containing static pod files.")
cmd.Flags().BoolP("enable-app-manager", "e", false,
"if set, yurtappmanager will be deployed.")
cmd.Flags().String("yurt-app-manager-image",
Expand Down Expand Up @@ -221,17 +218,17 @@ func (co *ConvertOptions) Complete(flags *pflag.FlagSet) error {
}
co.YurtAppManagerImage = yami

pmp, err := flags.GetString("pod-manifest-path")
kcp, err := flags.GetString("kubeadm-conf-path")
if err != nil {
return err
}
co.PodMainfestPath = pmp
co.KubeadmConfPath = kcp

kcp, err := flags.GetString("kubeadm-conf-path")
podManifestPath, err := enutil.GetPodManifestPath(co.KubeadmConfPath)
if err != nil {
return err
}
co.KubeadmConfPath = kcp
co.PodMainfestPath = podManifestPath

sa, err := flags.GetString("system-architecture")
if err != nil {
Expand Down Expand Up @@ -399,7 +396,6 @@ func (co *ConvertOptions) RunConvert() (err error) {
"yurtctl_servant_image": co.YurctlServantImage,
"yurthub_image": co.YurhubImage,
"joinToken": joinToken,
"pod_manifest_path": co.PodMainfestPath,
"kubeadm_conf_path": co.KubeadmConfPath,
}

Expand Down
36 changes: 10 additions & 26 deletions pkg/yurtctl/cmd/convert/edgenode.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ type ConvertEdgeNodeOptions struct {
YurthubImage string
YurthubHealthCheckTimeout time.Duration
YurctlServantImage string
PodMainfestPath string
JoinToken string
KubeadmConfPath string
openyurtDir string
Expand Down Expand Up @@ -94,8 +93,6 @@ func NewConvertEdgeNodeCmd() *cobra.Command {
"The timeout for yurthub health check.")
cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest",
"The yurtctl-servant image.")
cmd.Flags().String("pod-manifest-path", "",
"Path to the directory on edge node containing static pod files.")
cmd.Flags().String("kubeadm-conf-path", "",
"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")
cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.")
Expand Down Expand Up @@ -131,18 +128,6 @@ func (c *ConvertEdgeNodeOptions) Complete(flags *pflag.FlagSet) error {
}
c.YurctlServantImage = ycsi

podMainfestPath, err := flags.GetString("pod-manifest-path")
if err != nil {
return err
}
if podMainfestPath == "" {
podMainfestPath = os.Getenv("STATIC_POD_PATH")
}
if podMainfestPath == "" {
podMainfestPath = enutil.StaticPodPath
}
c.PodMainfestPath = podMainfestPath

kubeadmConfPath, err := flags.GetString("kubeadm-conf-path")
if err != nil {
return err
Expand Down Expand Up @@ -230,7 +215,6 @@ func (c *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) {
"yurtctl_servant_image": c.YurctlServantImage,
"yurthub_image": c.YurthubImage,
"joinToken": c.JoinToken,
"pod_manifest_path": c.PodMainfestPath,
"kubeadm_conf_path": c.KubeadmConfPath,
}

Expand All @@ -248,9 +232,6 @@ func (c *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) {
if _, err := enutil.FileExists(c.KubeadmConfPath); err != nil {
return err
}
if ok, err := enutil.DirExists(c.PodMainfestPath); !ok {
return err
}

// 3.2. check the state of EdgeNodes
node, err := c.clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
Expand Down Expand Up @@ -335,15 +316,18 @@ func (c *ConvertEdgeNodeOptions) SetupYurthub() error {
})

// 1-3. create yurthub.yaml
_, err = enutil.DirExists(c.PodMainfestPath)
podManifestPath, err := enutil.GetPodManifestPath(c.KubeadmConfPath)
if err != nil {
return err
}
err = ioutil.WriteFile(c.getYurthubYaml(), []byte(yurthubTemplate), filemode)
if err = enutil.EnsureDir(podManifestPath); err != nil {
return err
}
err = ioutil.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), filemode)
if err != nil {
return err
}
klog.Infof("create the %s/yurt-hub.yaml", c.PodMainfestPath)
klog.Infof("create the %s/yurt-hub.yaml", podManifestPath)

// 2. wait yurthub pod to be ready
err = hubHealthcheck(c.YurthubHealthCheckTimeout)
Expand Down Expand Up @@ -403,10 +387,6 @@ func (c *ConvertEdgeNodeOptions) ResetKubelet() error {
return nil
}

func (c *ConvertEdgeNodeOptions) getYurthubYaml() string {
return filepath.Join(c.PodMainfestPath, enutil.YurthubYamlName)
}

func (c *ConvertEdgeNodeOptions) getYurthubKubeletConf() string {
return filepath.Join(c.openyurtDir, enutil.KubeletConfName)
}
Expand All @@ -415,6 +395,10 @@ func (c *ConvertEdgeNodeOptions) getKubeletSvcBackup() string {
return fmt.Sprintf(enutil.KubeletSvcBackup, c.KubeadmConfPath)
}

func getYurthubYaml(podManifestPath string) string {
return filepath.Join(podManifestPath, enutil.YurthubYamlName)
}

// hubHealthcheck will check the status of yurthub pod
func hubHealthcheck(timeout time.Duration) error {
serverHealthzURL, err := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer))
Expand Down
40 changes: 14 additions & 26 deletions pkg/yurtctl/cmd/revert/edgenode.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ type RevertEdgeNodeOptions struct {
clientSet *kubernetes.Clientset
EdgeNodes []string
YurtctlServantImage string
PodMainfestPath string
KubeadmConfPath string
openyurtDir string
}
Expand Down Expand Up @@ -75,8 +74,6 @@ func NewRevertEdgeNodeCmd() *cobra.Command {
"The list of edge nodes wanted to be revert.(e.g. -e edgenode1,edgenode2)")
cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest",
"The yurtctl-servant image.")
cmd.Flags().String("pod-manifest-path", "",
"Path to the directory on edge node containing static pod files.")
cmd.Flags().String("kubeadm-conf-path", "",
"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")

Expand All @@ -99,18 +96,6 @@ func (r *RevertEdgeNodeOptions) Complete(flags *pflag.FlagSet) (err error) {
}
r.YurtctlServantImage = ycsi

podMainfestPath, err := flags.GetString("pod-manifest-path")
if err != nil {
return err
}
if podMainfestPath == "" {
podMainfestPath = os.Getenv("STATIC_POD_PATH")
}
if podMainfestPath == "" {
podMainfestPath = enutil.StaticPodPath
}
r.PodMainfestPath = podMainfestPath

kubeadmConfPath, err := flags.GetString("kubeadm-conf-path")
if err != nil {
return err
Expand Down Expand Up @@ -187,7 +172,6 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) {
map[string]string{
"action": "revert",
"yurtctl_servant_image": r.YurtctlServantImage,
"pod_manifest_path": r.PodMainfestPath,
"kubeadm_conf_path": r.KubeadmConfPath,
},
r.EdgeNodes); err != nil {
Expand All @@ -206,8 +190,13 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) {
klog.Errorf("fail to get file %s, should revise the %s directly", kubeletSvcBk, r.KubeadmConfPath)
return err
}
yurthubYaml := r.getYurthubYaml()
if ok, err := enutil.FileExists(yurthubYaml); !ok {
podManifestPath, err := enutil.GetPodManifestPath(r.KubeadmConfPath)
if err != nil {
klog.Errorf("get podManifestPath fail: %v", err)
return err
}
yurthubYamlPath := getYurthubYaml(podManifestPath)
if ok, err := enutil.FileExists(yurthubYamlPath); !ok {
return err
}

Expand All @@ -232,7 +221,7 @@ func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) {
if err := r.RevertKubelet(); err != nil {
return fmt.Errorf("fail to revert kubelet: %v", err)
}
if err := r.RemoveYurthub(); err != nil {
if err := r.RemoveYurthub(yurthubYamlPath); err != nil {
return err
}

Expand Down Expand Up @@ -289,16 +278,15 @@ func (r *RevertEdgeNodeOptions) RevertKubelet() error {
}

// RemoveYurthub deletes the yurt-hub pod
func (r *RevertEdgeNodeOptions) RemoveYurthub() error {
func (r *RevertEdgeNodeOptions) RemoveYurthub(yurthubYamlPath string) error {
// 1. remove the yurt-hub.yaml to delete the yurt-hub
yurthubYaml := r.getYurthubYaml()
err := os.Remove(yurthubYaml)
err := os.Remove(yurthubYamlPath)
if err != nil {
return err
}

// 2. remove yurt-hub config directory and certificates in it
yurthubConf := r.getYurthubConf()
yurthubConf := getYurthubConf()
err = os.RemoveAll(yurthubConf)
if err != nil {
return err
Expand All @@ -315,10 +303,10 @@ func (r *RevertEdgeNodeOptions) getKubeletSvcBackup() string {
return fmt.Sprintf(enutil.KubeletSvcBackup, r.KubeadmConfPath)
}

func (r *RevertEdgeNodeOptions) getYurthubYaml() string {
return filepath.Join(r.PodMainfestPath, enutil.YurthubYamlName)
func getYurthubYaml(podManifestPath string) string {
return filepath.Join(podManifestPath, enutil.YurthubYamlName)
}

func (r *RevertEdgeNodeOptions) getYurthubConf() string {
func getYurthubConf() string {
return filepath.Join(hubself.HubRootDir, hubself.HubName)
}
13 changes: 5 additions & 8 deletions pkg/yurtctl/cmd/revert/revert.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/openyurtio/openyurt/pkg/projectinfo"
"github.com/openyurtio/openyurt/pkg/yurtctl/constants"
"github.com/openyurtio/openyurt/pkg/yurtctl/lock"
enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode"
kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes"
strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings"
)
Expand Down Expand Up @@ -70,9 +71,6 @@ func NewRevertCmd() *cobra.Command {
cmd.Flags().String("yurtctl-servant-image",
"openyurt/yurtctl-servant:latest",
"The yurtctl-servant image.")
cmd.Flags().String("pod-manifest-path",
"/etc/kubernetes/manifests",
"Path to the directory on edge node containing static pod files.")
cmd.Flags().String("kubeadm-conf-path",
"/etc/systemd/system/kubelet.service.d/10-kubeadm.conf",
"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")
Expand All @@ -88,17 +86,17 @@ func (ro *RevertOptions) Complete(flags *pflag.FlagSet) error {
}
ro.YurtctlServantImage = ycsi

pmp, err := flags.GetString("pod-manifest-path")
kcp, err := flags.GetString("kubeadm-conf-path")
if err != nil {
return err
}
ro.PodMainfestPath = pmp
ro.KubeadmConfPath = kcp

kcp, err := flags.GetString("kubeadm-conf-path")
podManifestPath, err := enutil.GetPodManifestPath(ro.KubeadmConfPath)
if err != nil {
return err
}
ro.KubeadmConfPath = kcp
ro.PodMainfestPath = podManifestPath

ro.clientSet, err = kubeutil.GenClientSet(flags)
if err != nil {
Expand Down Expand Up @@ -237,7 +235,6 @@ func (ro *RevertOptions) RunRevert() (err error) {
map[string]string{
"action": "revert",
"yurtctl_servant_image": ro.YurtctlServantImage,
"pod_manifest_path": ro.PodMainfestPath,
"kubeadm_conf_path": ro.KubeadmConfPath,
},
edgeNodeNames); err != nil {
Expand Down
4 changes: 0 additions & 4 deletions pkg/yurtctl/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,8 +282,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: STATIC_POD_PATH
value: {{.pod_manifest_path}}
{{if .kubeadm_conf_path }}
- name: KUBELET_SVC
value: {{.kubeadm_conf_path}}
Expand Down Expand Up @@ -327,8 +325,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: STATIC_POD_PATH
value: {{.pod_manifest_path}}
{{if .kubeadm_conf_path }}
- name: KUBELET_SVC
value: {{.kubeadm_conf_path}}
Expand Down
43 changes: 35 additions & 8 deletions pkg/yurtctl/util/edgenode/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package edgenode

import (
"errors"
"fmt"
"io/ioutil"
"os"
Expand Down Expand Up @@ -71,16 +72,14 @@ func GetSingleContentFromFile(filename string, regularExpression string) (string
return contents[0], nil
}

// DirExists determines whether the directory exists
func DirExists(dirname string) (bool, error) {
// EnsureDir make sure dir is exists, if not create
func EnsureDir(dirname string) error {
s, err := os.Stat(dirname)
if err != nil {
return false, err
}
if !s.IsDir() {
return false, fmt.Errorf("%s is not a dir", dirname)
if err == nil && s.IsDir() {
return nil
}
return true, nil

return os.MkdirAll(dirname, 0755)
}

// CopyFile copys sourceFile to destinationFile
Expand Down Expand Up @@ -201,3 +200,31 @@ func Exec(cmd *exec.Cmd) error {
}
return nil
}

// GetPodManifestPath get path string from kubeadmConf file
func GetPodManifestPath(kubeadmConfPath string) (string, error) {
configRegularExpression := "\\-\\-config=.*config.yaml"
podManifestPathRegularExpression := "staticPodPath: .*"

configPath, err := GetSingleContentFromFile(kubeadmConfPath, configRegularExpression)
if err != nil {
return "", err
}
arr := strings.Split(configPath, "=")
if len(arr) != 2 {
return "", errors.New("kubelet config path format incorrect: " + configPath)
}
configPath = arr[1]

podManifestPath, err := GetSingleContentFromFile(configPath, podManifestPathRegularExpression)
if err != nil {
return "", err
}
arr = strings.Split(podManifestPath, " ")
if len(arr) != 2 {
return "", errors.New("podManifestPath format incorrect: " + podManifestPath)
}
podManifestPath = arr[1]

return podManifestPath, nil
}
Loading

0 comments on commit 3ee1139

Please sign in to comment.