diff --git a/cmd/yurt-node-servant/convert/convert.go b/cmd/yurt-node-servant/convert/convert.go index 78002fc7c88..b60cee230c0 100644 --- a/cmd/yurt-node-servant/convert/convert.go +++ b/cmd/yurt-node-servant/convert/convert.go @@ -59,8 +59,10 @@ func setFlags(cmd *cobra.Command) { "The yurthub image.") cmd.Flags().Duration("yurthub-healthcheck-timeout", defaultYurthubHealthCheckTimeout, "The timeout for yurthub health check.") - cmd.Flags().String("kubeadm-conf-path", "", - "The path to kubelet service conf that is used by kubelet component to join the cluster on the work node.") + cmd.Flags().StringP("kubeadm-conf-path", "k", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the work node."+ + "Support multiple values, will search in order until get the file.(e.g -k kbcfg1,kbcfg2)", + ) cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.") cmd.Flags().String("working-mode", "edge", "The node type cloud/edge, effect yurthub workingMode.") } diff --git a/cmd/yurt-node-servant/node-servant.go b/cmd/yurt-node-servant/node-servant.go index 62e7da87998..5b8313cd84c 100644 --- a/cmd/yurt-node-servant/node-servant.go +++ b/cmd/yurt-node-servant/node-servant.go @@ -25,6 +25,7 @@ import ( "github.com/spf13/cobra" "github.com/openyurtio/openyurt/cmd/yurt-node-servant/convert" + preflightconvert "github.com/openyurtio/openyurt/cmd/yurt-node-servant/preflight-convert" "github.com/openyurtio/openyurt/cmd/yurt-node-servant/revert" "github.com/openyurtio/openyurt/pkg/projectinfo" ) @@ -38,12 +39,13 @@ func main() { version := fmt.Sprintf("%#v", projectinfo.Get()) rootCmd := &cobra.Command{ Use: "node-servant", - Short: "node-servant do convert/revert specific node", + Short: "node-servant do preflight-convert/convert/revert specific node", Version: version, } rootCmd.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file") rootCmd.AddCommand(convert.NewConvertCmd()) rootCmd.AddCommand(revert.NewRevertCmd()) + rootCmd.AddCommand(preflightconvert.NewxPreflightConvertCmd()) if err := rootCmd.Execute(); err != nil { // run command os.Exit(1) diff --git a/cmd/yurt-node-servant/preflight-convert/preflight.go b/cmd/yurt-node-servant/preflight-convert/preflight.go new file mode 100644 index 00000000000..946cf9c9ace --- /dev/null +++ b/cmd/yurt-node-servant/preflight-convert/preflight.go @@ -0,0 +1,68 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preflight_convert + +import ( + "os" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + + preflightconvert "github.com/openyurtio/openyurt/pkg/node-servant/preflight-convert" +) + +const ( + latestYurtHubImage = "openyurt/yurthub:latest" + latestYurtTunnelAgentImage = "openyurt/yurt-tunnel-agent:latest" +) + +// NewxPreflightConvertCmd generates a new preflight-convert check command +func NewxPreflightConvertCmd() *cobra.Command { + o := preflightconvert.NewPreflightConvertOptions() + cmd := &cobra.Command{ + Use: "preflight-convert", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + if err := o.Complete(cmd.Flags()); err != nil { + klog.Errorf("Fail to complete the preflight-convert option: %s", err) + os.Exit(1) + } + preflighter := preflightconvert.NewPreflighterWithOptions(o) + if err := preflighter.Do(); err != nil { + klog.Errorf("Fail to run pre-flight checks: %s", err) + os.Exit(1) + } + klog.Info("convert pre-flight checks success") + }, + } + setFlags(cmd) + + return cmd +} + +func setFlags(cmd *cobra.Command) { + cmd.Flags().StringP("kubeadm-conf-path", "k", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the work node."+ + "Support multiple values, will search in order until get the file.(e.g -k kbcfg1,kbcfg2)", + ) + cmd.Flags().String("yurthub-image", latestYurtHubImage, "The yurthub image.") + cmd.Flags().String("yurt-tunnel-agent-image", latestYurtTunnelAgentImage, "The yurt-tunnel-agent image.") + cmd.Flags().BoolP("deploy-yurttunnel", "t", false, "If set, yurt-tunnel-agent will be deployed.") + cmd.Flags().String("ignore-preflight-errors", "", "A list of checks whose errors will be shown as warnings. "+ + "And value needs to be lowercase. Example: 'isprivilegeduser,imagepull'.Value 'all' ignores errors from all checks.", + ) +} diff --git a/pkg/node-servant/components/kubelet.go b/pkg/node-servant/components/kubelet.go index a7ddc2b28d6..261e31ec3c6 100644 --- a/pkg/node-servant/components/kubelet.go +++ b/pkg/node-servant/components/kubelet.go @@ -39,15 +39,13 @@ const ( ) type kubeletOperator struct { - openyurtDir string - kubeadmConfPath string + openyurtDir string } // NewKubeletOperator create kubeletOperator -func NewKubeletOperator(openyurtDir, kubeadmConfPath string) *kubeletOperator { +func NewKubeletOperator(openyurtDir string) *kubeletOperator { return &kubeletOperator{ - openyurtDir: openyurtDir, - kubeadmConfPath: kubeadmConfPath, + openyurtDir: openyurtDir, } } @@ -187,8 +185,19 @@ func restartKubeletService() error { } // GetApiServerAddress parse apiServer address from conf file -func GetApiServerAddress(kubeadmConfPath string) (string, error) { - kubeletConfPath, err := enutil.GetSingleContentFromFile(kubeadmConfPath, kubeletConfigRegularExpression) +func GetApiServerAddress(kubeadmConfPaths []string) (string, error) { + var kbcfg string + for _, path := range kubeadmConfPaths { + if exist, _ := enutil.FileExists(path); exist { + kbcfg = path + break + } + } + if kbcfg == "" { + return "", fmt.Errorf("get apiserverAddr err: no file exists in list %s", kubeadmConfPaths) + } + + kubeletConfPath, err := enutil.GetSingleContentFromFile(kbcfg, kubeletConfigRegularExpression) if err != nil { return "", err } diff --git a/pkg/node-servant/components/runtime.go b/pkg/node-servant/components/runtime.go new file mode 100644 index 00000000000..aad1e6172e8 --- /dev/null +++ b/pkg/node-servant/components/runtime.go @@ -0,0 +1,180 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "os" + "path/filepath" + goruntime "runtime" + "strings" + + "github.com/pkg/errors" + utilsexec "k8s.io/utils/exec" +) + +const ( + dockerSocket = "/var/run/docker.sock" // The Docker socket is not CRI compatible + containerdSocket = "/run/containerd/containerd.sock" + // DefaultDockerCRISocket defines the default Docker CRI socket + DefaultDockerCRISocket = "/var/run/dockershim.sock" + + // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed + PullImageRetry = 5 +) + +// ContainerRuntime is an interface for working with container runtimes +type ContainerRuntimeForImage interface { + IsDocker() bool + PullImage(image string) error + ImageExists(image string) (bool, error) +} + +// CRIRuntime is a struct that interfaces with the CRI +type CRIRuntime struct { + exec utilsexec.Interface + criSocket string +} + +// DockerRuntime is a struct that interfaces with the Docker daemon +type DockerRuntime struct { + exec utilsexec.Interface +} + +// NewContainerRuntime sets up and returns a ContainerRuntime struct +func NewContainerRuntimeForImage(execer utilsexec.Interface, criSocket string) (ContainerRuntimeForImage, error) { + var toolName string + var runtime ContainerRuntimeForImage + + if criSocket != DefaultDockerCRISocket { + toolName = "crictl" + // !!! temporary work around crictl warning: + // Using "/var/run/crio/crio.sock" as endpoint is deprecated, + // please consider using full url format "unix:///var/run/crio/crio.sock" + if filepath.IsAbs(criSocket) && goruntime.GOOS != "windows" { + criSocket = "unix://" + criSocket + } + runtime = &CRIRuntime{execer, criSocket} + } else { + toolName = "docker" + runtime = &DockerRuntime{execer} + } + + if _, err := execer.LookPath(toolName); err != nil { + return nil, errors.Wrapf(err, "%s is required for container runtime", toolName) + } + return runtime, nil +} + +// IsDocker returns true if the runtime is docker +func (runtime *CRIRuntime) IsDocker() bool { + return false +} + +// IsDocker returns true if the runtime is docker +func (runtime *DockerRuntime) IsDocker() bool { + return true +} + +// PullImage pulls the image +func (runtime *CRIRuntime) PullImage(image string) error { + var err error + var out []byte + for i := 0; i < PullImageRetry; i++ { + out, err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).CombinedOutput() + if err == nil { + return nil + } + } + return errors.Wrapf(err, "output: %s, error", out) +} + +// PullImage pulls the image +func (runtime *DockerRuntime) PullImage(image string) error { + var err error + var out []byte + for i := 0; i < PullImageRetry; i++ { + out, err = runtime.exec.Command("docker", "pull", image).CombinedOutput() + if err == nil { + return nil + } + } + return errors.Wrapf(err, "output: %s, error", out) +} + +// ImageExists checks to see if the image exists on the system +func (runtime *CRIRuntime) ImageExists(image string) (bool, error) { + err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "inspecti", image).Run() + return err == nil, nil +} + +// ImageExists checks to see if the image exists on the system +func (runtime *DockerRuntime) ImageExists(image string) (bool, error) { + err := runtime.exec.Command("docker", "inspect", image).Run() + return err == nil, nil +} + +// detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead +func detectCRISocketImpl(isSocket func(string) bool) (string, error) { + foundCRISockets := []string{} + knownCRISockets := []string{ + // Docker and containerd sockets are special cased below, hence not to be included here + "/var/run/crio/crio.sock", + } + + if isSocket(dockerSocket) { + // the path in dockerSocket is not CRI compatible, hence we should replace it with a CRI compatible socket + foundCRISockets = append(foundCRISockets, DefaultDockerCRISocket) + } else if isSocket(containerdSocket) { + // Docker 18.09 gets bundled together with containerd, thus having both dockerSocket and containerdSocket present. + // For compatibility reasons, we use the containerd socket only if Docker is not detected. + foundCRISockets = append(foundCRISockets, containerdSocket) + } + + for _, socket := range knownCRISockets { + if isSocket(socket) { + foundCRISockets = append(foundCRISockets, socket) + } + } + + switch len(foundCRISockets) { + case 0: + // Fall back to Docker if no CRI is detected, we can error out later on if we need it + return DefaultDockerCRISocket, nil + case 1: + // Precisely one CRI found, use that + return foundCRISockets[0], nil + default: + // Multiple CRIs installed? + return "", errors.Errorf("Found multiple CRI sockets, please use --cri-socket to select one: %s", strings.Join(foundCRISockets, ", ")) + } + +} + +// isExistingSocket checks if path exists and is domain socket +func isExistingSocket(path string) bool { + fileInfo, err := os.Stat(path) + if err != nil { + return false + } + + return fileInfo.Mode()&os.ModeSocket != 0 +} + +// DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned. +func DetectCRISocket() (string, error) { + return detectCRISocketImpl(isExistingSocket) +} diff --git a/pkg/node-servant/components/util.go b/pkg/node-servant/components/util.go new file mode 100644 index 00000000000..eefa566d87d --- /dev/null +++ b/pkg/node-servant/components/util.go @@ -0,0 +1,37 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "os" +) + +const ( + KubeletSvcEnv = "KUBELET_SVC" + KubeletSvcPathSystemUsr = "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf" + KubelerSvcPathSystemEtc = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" +) + +func GetDefaultKubeadmConfPath() []string { + kubeadmConfPath := []string{} + path := os.Getenv(KubeletSvcEnv) + if path != "" && path != KubeletSvcPathSystemUsr && path != KubelerSvcPathSystemEtc { + kubeadmConfPath = append(kubeadmConfPath, path) + } + kubeadmConfPath = append(kubeadmConfPath, KubeletSvcPathSystemUsr, KubelerSvcPathSystemEtc) + return kubeadmConfPath +} diff --git a/pkg/node-servant/constant.go b/pkg/node-servant/constant.go index 1cbef51c1b5..ac609f014b1 100644 --- a/pkg/node-servant/constant.go +++ b/pkg/node-servant/constant.go @@ -23,6 +23,9 @@ const ( // RevertJobNameBase is the prefix of the revert ServantJob name RevertJobNameBase = "node-servant-revert" + //ConvertPreflightJobNameBase is the prefix of the preflight-convert ServantJob name + ConvertPreflightJobNameBase = "node-servant-preflight-convert" + // ConvertServantJobTemplate defines the yurtctl convert servant job in yaml format ConvertServantJobTemplate = ` apiVersion: batch/v1 @@ -108,5 +111,48 @@ spec: - name: KUBELET_SVC value: {{.kubeadm_conf_path}} {{end}} +` + // ConvertPreflightJobTemplate defines the yurtctl convert preflight checks servant job in yaml format + ConvertPreflightJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: {{.jobName}} + namespace: kube-system +spec: + template: + spec: + hostPID: true + hostNetwork: true + restartPolicy: OnFailure + nodeName: {{.nodeName}} + volumes: + - name: host-root + hostPath: + path: / + type: Directory + containers: + - name: node-servant + image: {{.node_servant_image}} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - "/usr/local/bin/entry.sh preflight-convert" + securityContext: + privileged: true + volumeMounts: + - mountPath: /openyurt + name: host-root + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{if .kubeadm_conf_path }} + - name: KUBELET_SVC + value: {{.kubeadm_conf_path}} + {{end}} ` ) diff --git a/pkg/node-servant/convert/convert.go b/pkg/node-servant/convert/convert.go index f634c2c80e8..a588252bf2e 100644 --- a/pkg/node-servant/convert/convert.go +++ b/pkg/node-servant/convert/convert.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/openyurtio/openyurt/pkg/node-servant/components" - enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -42,9 +41,6 @@ func (n *nodeConverter) Do() error { if err := n.validateOptions(); err != nil { return err } - if err := n.preflightCheck(); err != nil { - return err - } if err := n.installYurtHub(); err != nil { return err @@ -64,17 +60,8 @@ func (n *nodeConverter) validateOptions() error { return nil } -func (n *nodeConverter) preflightCheck() error { - // 1. check if critical files exist - if _, err := enutil.FileExists(n.kubeadmConfPath); err != nil { - return err - } - - return nil -} - func (n *nodeConverter) installYurtHub() error { - apiServerAddress, err := components.GetApiServerAddress(n.kubeadmConfPath) + apiServerAddress, err := components.GetApiServerAddress(n.kubeadmConfPaths) if err != nil { return err } @@ -87,6 +74,6 @@ func (n *nodeConverter) installYurtHub() error { } func (n *nodeConverter) convertKubelet() error { - op := components.NewKubeletOperator(n.openyurtDir, n.kubeadmConfPath) + op := components.NewKubeletOperator(n.openyurtDir) return op.RedirectTrafficToYurtHub() } diff --git a/pkg/node-servant/convert/options.go b/pkg/node-servant/convert/options.go index cfd5fee6c33..5c62bf27380 100644 --- a/pkg/node-servant/convert/options.go +++ b/pkg/node-servant/convert/options.go @@ -19,29 +19,32 @@ package convert import ( "fmt" "os" + "strings" "time" "github.com/spf13/pflag" + "github.com/openyurtio/openyurt/pkg/node-servant/components" enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" - "github.com/openyurtio/openyurt/pkg/yurthub/util" + hubutil "github.com/openyurtio/openyurt/pkg/yurthub/util" ) // Options has the information that required by convert operation type Options struct { yurthubImage string yurthubHealthCheckTimeout time.Duration - workingMode util.WorkingMode + workingMode hubutil.WorkingMode - joinToken string - kubeadmConfPath string - openyurtDir string - nodeName string + joinToken string + kubeadmConfPaths []string + openyurtDir string } // NewConvertOptions creates a new Options func NewConvertOptions() *Options { - return &Options{} + return &Options{ + kubeadmConfPaths: components.GetDefaultKubeadmConfPath(), + } } // Complete completes all the required options. @@ -58,23 +61,13 @@ func (o *Options) Complete(flags *pflag.FlagSet) error { } o.yurthubHealthCheckTimeout = yurthubHealthCheckTimeout - kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") + kubeadmConfPaths, err := flags.GetString("kubeadm-conf-path") if err != nil { return err } - if kubeadmConfPath == "" { - kubeadmConfPath = os.Getenv("KUBELET_SVC") - } - if kubeadmConfPath == "" { - kubeadmConfPath = enutil.KubeletSvcPath - } - o.kubeadmConfPath = kubeadmConfPath - - nodeName, err := enutil.GetNodeName(kubeadmConfPath) - if err != nil { - return err + if kubeadmConfPaths != "" { + o.kubeadmConfPaths = strings.Split(kubeadmConfPaths, ",") } - o.nodeName = nodeName joinToken, err := flags.GetString("join-token") if err != nil { @@ -96,8 +89,8 @@ func (o *Options) Complete(flags *pflag.FlagSet) error { return err } - wm := util.WorkingMode(workingMode) - if !util.IsSupportedWorkingMode(wm) { + wm := hubutil.WorkingMode(workingMode) + if !hubutil.IsSupportedWorkingMode(wm) { return fmt.Errorf("invalid working mode: %s", workingMode) } o.workingMode = wm diff --git a/pkg/node-servant/job.go b/pkg/node-servant/job.go index e8a40489656..8575a2e3f66 100644 --- a/pkg/node-servant/job.go +++ b/pkg/node-servant/job.go @@ -42,6 +42,9 @@ func RenderNodeServantJob(action string, tmplCtx map[string]string, nodeName str case "revert": servantJobTemplate = RevertServantJobTemplate jobBaseName = RevertJobNameBase + case "preflight-convert": + servantJobTemplate = ConvertPreflightJobTemplate + jobBaseName = ConvertPreflightJobNameBase } tmplCtx["jobName"] = jobBaseName + "-" + nodeName @@ -85,6 +88,9 @@ func validate(action string, tmplCtx map[string]string, nodeName string) error { case "revert": keysMustHave := []string{"node_servant_image"} return checkKeys(keysMustHave, tmplCtx) + case "preflight-convert": + keysMustHave := []string{"node_servant_image"} + return checkKeys(keysMustHave, tmplCtx) default: return fmt.Errorf("action invalied: %s ", action) } diff --git a/pkg/node-servant/preflight-convert/options.go b/pkg/node-servant/preflight-convert/options.go new file mode 100644 index 00000000000..e146e90950f --- /dev/null +++ b/pkg/node-servant/preflight-convert/options.go @@ -0,0 +1,125 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preflight_convert + +import ( + "strings" + + "github.com/spf13/pflag" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" +) + +const ( + kubeAdmFlagsEnvFile = "/var/lib/kubelet/kubeadm-flags.env" +) + +// Options has the information that required by preflight-convert operation +type Options struct { + KubeadmConfPaths []string + YurthubImage string + YurttunnelAgentImage string + DeployTunnel bool + IgnorePreflightErrors sets.String + + KubeAdmFlagsEnvFile string + ImagePullPolicy v1.PullPolicy + CRISocket string +} + +func (o *Options) GetCRISocket() string { + return o.CRISocket +} + +func (o *Options) GetImageList() []string { + imgs := []string{} + + imgs = append(imgs, o.YurthubImage) + if o.DeployTunnel { + imgs = append(imgs, o.YurttunnelAgentImage) + } + return imgs +} + +func (o *Options) GetImagePullPolicy() v1.PullPolicy { + return o.ImagePullPolicy +} + +func (o *Options) GetKubeadmConfPaths() []string { + return o.KubeadmConfPaths +} + +func (o *Options) GetKubeAdmFlagsEnvFile() string { + return o.KubeAdmFlagsEnvFile +} + +// NewPreflightConvertOptions creates a new Options +func NewPreflightConvertOptions() *Options { + return &Options{ + KubeadmConfPaths: components.GetDefaultKubeadmConfPath(), + IgnorePreflightErrors: sets.NewString(), + KubeAdmFlagsEnvFile: kubeAdmFlagsEnvFile, + ImagePullPolicy: v1.PullIfNotPresent, + } +} + +// Complete completes all the required options. +func (o *Options) Complete(flags *pflag.FlagSet) error { + + kubeadmConfPaths, err := flags.GetString("kubeadm-conf-path") + if err != nil { + return err + } + if kubeadmConfPaths != "" { + o.KubeadmConfPaths = strings.Split(kubeadmConfPaths, ",") + } + + yurthubImage, err := flags.GetString("yurthub-image") + if err != nil { + return err + } + o.YurthubImage = yurthubImage + + yurttunnelAgentImage, err := flags.GetString("yurt-tunnel-agent-image") + if err != nil { + return err + } + o.YurttunnelAgentImage = yurttunnelAgentImage + + dt, err := flags.GetBool("deploy-yurttunnel") + if err != nil { + return err + } + o.DeployTunnel = dt + + ipStr, err := flags.GetString("ignore-preflight-errors") + if err != nil { + return err + } + if ipStr != "" { + o.IgnorePreflightErrors = sets.NewString(strings.Split(ipStr, ",")...) + } + + CRISocket, err := components.DetectCRISocket() + if err != nil { + return err + } + o.CRISocket = CRISocket + return nil +} diff --git a/pkg/node-servant/preflight-convert/preflight.go b/pkg/node-servant/preflight-convert/preflight.go new file mode 100644 index 00000000000..5f33dcf535a --- /dev/null +++ b/pkg/node-servant/preflight-convert/preflight.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preflight_convert + +import ( + "fmt" + + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/preflight" +) + +// ConvertPreflighter do the preflight-convert-convert job +type ConvertPreflighter struct { + Options +} + +// NewPreflighterWithOptions create nodePreflighter +func NewPreflighterWithOptions(o *Options) *ConvertPreflighter { + return &ConvertPreflighter{ + *o, + } +} + +func (n *ConvertPreflighter) Do() error { + klog.Infof("[preflight-convert] Running node-servant pre-flight checks") + if err := preflight.RunConvertNodeChecks(n, n.IgnorePreflightErrors, n.DeployTunnel); err != nil { + return err + } + + fmt.Println("[preflight-convert] Pulling images required for converting a Kubernetes cluster to an OpenYurt cluster") + fmt.Println("[preflight-convert] This might take a minute or two, depending on the speed of your internet connection") + if err := preflight.RunPullImagesCheck(n, n.IgnorePreflightErrors); err != nil { + return err + } + + return nil +} diff --git a/pkg/node-servant/revert/revert.go b/pkg/node-servant/revert/revert.go index 22416f319ff..483f4e58c1a 100644 --- a/pkg/node-servant/revert/revert.go +++ b/pkg/node-servant/revert/revert.go @@ -50,7 +50,7 @@ func (n *nodeReverter) Do() error { } func (n *nodeReverter) revertKubelet() error { - op := components.NewKubeletOperator(n.openyurtDir, n.kubeadmConfPath) + op := components.NewKubeletOperator(n.openyurtDir) return op.UndoRedirectTrafficToYurtHub() } diff --git a/pkg/preflight/checks.go b/pkg/preflight/checks.go new file mode 100644 index 00000000000..0a1c3d68b61 --- /dev/null +++ b/pkg/preflight/checks.go @@ -0,0 +1,309 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preflight + +import ( + "bytes" + "fmt" + "io" + "net" + "os" + "strings" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + utilsexec "k8s.io/utils/exec" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" +) + +// Error defines struct for communicating error messages generated by preflight-convert-convert checks +type Error struct { + Msg string +} + +// Error implements the standard error interface +func (e *Error) Error() string { + return fmt.Sprintf("[preflight] Some fatal errors occurred:\n%s%s", e.Msg, "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`\n") +} + +// Preflight identifies this error as a preflight-convert-convert error +func (e *Error) Preflight() bool { + return true +} + +// Checker validates the state of the system to ensure kubeadm will be +// successful as often as possible. +type Checker interface { + Check() (warnings, errorList []error) + Name() string +} + +// IsPrivilegedUserCheck verifies user is privileged (linux - root). +// The check under windows environment has not been implemented yet. +type IsPrivilegedUserCheck struct{} + +func (IsPrivilegedUserCheck) Name() string { + return "IsPrivilegedUser" +} + +// Check validates if an user has elevated (root) privileges. +func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { + if os.Getuid() != 0 { + return nil, []error{errors.New("user is not running as root")} + } + + return nil, nil +} + +// FileExistingCheck checks that the given file already exist. +type FileExistingCheck struct { + Path string + Label string +} + +// Name returns label for individual FileExistingChecks. If not known, will return based on path. +func (fac FileExistingCheck) Name() string { + if fac.Label != "" { + return fac.Label + } + return fmt.Sprintf("FileExisting-%s", strings.Replace(fac.Path, "/", "-", -1)) +} + +func (fac FileExistingCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating the existence of file %s", fac.Path) + + if _, err := os.Stat(fac.Path); err != nil { + return nil, []error{errors.Errorf("%s doesn't exist", fac.Path)} + } + return nil, nil +} + +// FileAtLeastOneExistingCheck checks if at least one file exists in the file list. +// After a file is found, the remaining files will not be checked. +type FileAtLeastOneExistingCheck struct { + Paths []string + Label string +} + +func (foc FileAtLeastOneExistingCheck) Name() string { + if foc.Label != "" { + return foc.Label + } + return fmt.Sprintf("FileAtLeastOneExistingCheck-%s", foc.Paths[0]) +} + +func (foc FileAtLeastOneExistingCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating if at least one file exists in the file list: %s", foc.Paths) + for _, path := range foc.Paths { + if _, err := os.Stat(path); err == nil { + return nil, nil + } + } + return nil, []error{errors.Errorf("no file in list %s exists", foc.Paths)} + +} + +// DirExistingCheck checks if the given directory either exist, or is not empty. +type DirExistingCheck struct { + Path string + Label string +} + +// Name returns label for individual DirExistingChecks. If not known, will return based on path. +func (dac DirExistingCheck) Name() string { + if dac.Label != "" { + return dac.Label + } + return fmt.Sprintf("DirExisting-%s", strings.Replace(dac.Path, "/", "-", -1)) +} + +// Check validates if a directory exists or does not empty. +func (dac DirExistingCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating the existence of directory %s", dac.Path) + + if _, err := os.Stat(dac.Path); os.IsNotExist(err) { + return nil, []error{errors.Errorf("%s doesn't exist", dac.Path)} + } + + f, err := os.Open(dac.Path) + if err != nil { + return nil, []error{errors.Wrapf(err, "unable to check if %s is empty", dac.Path)} + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return nil, []error{errors.Errorf("%s is empty", dac.Path)} + } + return nil, nil +} + +// PortOpenCheck ensures the given port is available for use. +type PortOpenCheck struct { + port int + label string +} + +func (poc PortOpenCheck) Name() string { + if poc.label != "" { + return poc.label + } + return fmt.Sprintf("Port-%d", poc.port) +} + +func (poc PortOpenCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating availability of port %d", poc.port) + + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", poc.port)) + if err != nil { + errorList = []error{errors.Errorf("Port %d is in use", poc.port)} + } + if ln != nil { + if err = ln.Close(); err != nil { + warnings = append(warnings, errors.Errorf("when closing port %d, encountered %v", poc.port, err)) + } + } + return warnings, errorList +} + +// ImagePullCheck will pull container images used by node-servant +type ImagePullCheck struct { + //runtime utilruntime.ContainerRuntime + runtime components.ContainerRuntimeForImage + imageList []string + imagePullPolicy v1.PullPolicy +} + +func (ImagePullCheck) Name() string { + return "ImagePull" +} + +func (ipc ImagePullCheck) Check() (warnings, errorList []error) { + policy := ipc.imagePullPolicy + klog.V(1).Infof("using image pull policy: %s", policy) + for _, image := range ipc.imageList { + switch policy { + case v1.PullNever: + klog.V(1).Infof("skipping pull of image: %s", image) + continue + case v1.PullIfNotPresent: + ret, err := ipc.runtime.ImageExists(image) + if ret && err == nil { + klog.V(1).Infof("image exists: %s", image) + continue + } + if err != nil { + errorList = append(errorList, errors.Wrapf(err, "failed to check if image %s exists", image)) + } + fallthrough // Proceed with pulling the image if it does not exist + case v1.PullAlways: + klog.V(1).Infof("pulling: %s", image) + if err := ipc.runtime.PullImage(image); err != nil { + errorList = append(errorList, errors.Wrapf(err, "failed to pull image %s", image)) + } + default: + // If the policy is unknown return early with an error + errorList = append(errorList, errors.Errorf("unsupported pull policy %q", policy)) + return warnings, errorList + } + } + return warnings, errorList +} + +func RunConvertNodeChecks(o KubePathOperator, ignorePreflightErrors sets.String, deployTunnel bool) error { + // First, check if we're root separately from the other preflight-convert-convert checks and fail fast + if err := RunRootCheckOnly(ignorePreflightErrors); err != nil { + return err + } + + checks := []Checker{ + FileAtLeastOneExistingCheck{Paths: o.GetKubeadmConfPaths(), Label: "KubeadmConfig"}, + FileExistingCheck{Path: o.GetKubeAdmFlagsEnvFile(), Label: "KubeAdmFlagsEnv"}, + DirExistingCheck{Path: KubernetesDir}, + DirExistingCheck{Path: KubeletPkiDir}, + PortOpenCheck{port: YurtHubProxySecurePort}, + PortOpenCheck{port: YurtHubProxyPort}, + PortOpenCheck{port: YurtHubPort}, + } + + if deployTunnel { + checks = append(checks, PortOpenCheck{port: YurttunnelAgentPort}) + } + return RunChecks(checks, os.Stderr, ignorePreflightErrors) + +} + +// RunRootCheckOnly initializes checks slice of structs and call RunChecks +func RunRootCheckOnly(ignorePreflightErrors sets.String) error { + checks := []Checker{ + IsPrivilegedUserCheck{}, + } + + return RunChecks(checks, os.Stderr, ignorePreflightErrors) +} + +// RunPullImagesCheck will pull images convert needs if they are not found on the system +func RunPullImagesCheck(o ImageOperator, ignorePreflightErrors sets.String) error { + containerRuntime, err := components.NewContainerRuntimeForImage(utilsexec.New(), o.GetCRISocket()) + if err != nil { + return err + } + + checks := []Checker{ + ImagePullCheck{runtime: containerRuntime, imageList: o.GetImageList(), imagePullPolicy: o.GetImagePullPolicy()}, + } + return RunChecks(checks, os.Stderr, ignorePreflightErrors) +} + +// RunChecks runs each check, displays it's warnings/errors, and once all +// are processed will exit if any errors occurred. +func RunChecks(checks []Checker, ww io.Writer, ignorePreflightErrors sets.String) error { + var errsBuffer bytes.Buffer + + for _, c := range checks { + name := c.Name() + warnings, errs := c.Check() + + if setHasItemOrAll(ignorePreflightErrors, name) { + // Decrease severity of errors to warnings for this check + warnings = append(warnings, errs...) + errs = []error{} + } + + for _, w := range warnings { + io.WriteString(ww, fmt.Sprintf("\t[WARNING %s]: %v\n", name, w)) + } + for _, i := range errs { + errsBuffer.WriteString(fmt.Sprintf("\t[ERROR %s]: %v\n", name, i.Error())) + } + } + if errsBuffer.Len() > 0 { + return &Error{Msg: errsBuffer.String()} + } + return nil +} + +// setHasItemOrAll is helper function that return true if item is present in the set (case insensitive) or special key 'all' is present +func setHasItemOrAll(s sets.String, item string) bool { + if s.Has("all") || s.Has(strings.ToLower(item)) { + return true + } + return false +} diff --git a/pkg/preflight/constants.go b/pkg/preflight/constants.go new file mode 100644 index 00000000000..7b3c2cf8299 --- /dev/null +++ b/pkg/preflight/constants.go @@ -0,0 +1,27 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preflight + +const ( + KubernetesDir = "/etc/kubernetes" + KubeletPkiDir = "/var/lib/kubelet/pki" + + YurtHubProxySecurePort = 10268 + YurtHubProxyPort = 10261 + YurtHubPort = 10267 + YurttunnelAgentPort = 10266 +) diff --git a/pkg/preflight/interface.go b/pkg/preflight/interface.go new file mode 100644 index 00000000000..3d93ec6ba71 --- /dev/null +++ b/pkg/preflight/interface.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preflight + +import v1 "k8s.io/api/core/v1" + +type ImageOperator interface { + GetCRISocket() string + GetImageList() []string + GetImagePullPolicy() v1.PullPolicy +} + +type KubePathOperator interface { + GetKubeadmConfPaths() []string + GetKubeAdmFlagsEnvFile() string +} diff --git a/pkg/yurtctl/util/edgenode/common.go b/pkg/yurtctl/util/edgenode/common.go index fe24d321db1..384f632e2ed 100644 --- a/pkg/yurtctl/util/edgenode/common.go +++ b/pkg/yurtctl/util/edgenode/common.go @@ -17,7 +17,7 @@ limitations under the License. package edgenode const ( - KubeletSvcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + KubeletSvcPath = "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf" OpenyurtDir = "/var/lib/openyurt" StaticPodPath = "/etc/kubernetes/manifests" KubeCondfigPath = "/etc/kubernetes/kubelet.conf"