diff --git a/cmd/yurt-node-servant/convert/convert.go b/cmd/yurt-node-servant/convert/convert.go new file mode 100644 index 00000000000..5ec1acff646 --- /dev/null +++ b/cmd/yurt-node-servant/convert/convert.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "time" + + "github.com/spf13/cobra" + "k8s.io/klog" + + nodeconverter "github.com/openyurtio/openyurt/pkg/node-servant/convert" +) + +const ( + // defaultYurthubHealthCheckTimeout defines the default timeout for yurthub health check phase + defaultYurthubHealthCheckTimeout = 2 * time.Minute +) + +// NewConvertCmd generates a new convert command +func NewConvertCmd() *cobra.Command { + o := nodeconverter.NewConvertOptions() + cmd := &cobra.Command{ + Use: "convert --working-mode", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + if err := o.Complete(cmd.Flags()); err != nil { + klog.Fatalf("fail to complete the convert option: %s", err) + } + + converter := nodeconverter.NewConverterWithOptions(o) + if err := converter.Do(); err != nil { + klog.Fatalf("fail to convert the kubernetes node to a yurt node: %s", err) + } + klog.Info("convert success") + }, + } + setFlags(cmd) + + return cmd +} + +// setFlags sets flags. +func setFlags(cmd *cobra.Command) { + cmd.Flags().String("yurthub-image", "openyurt/yurthub:latest", + "The yurthub image.") + cmd.Flags().Duration("yurthub-healthcheck-timeout", defaultYurthubHealthCheckTimeout, + "The timeout for yurthub health check.") + cmd.Flags().String("kubeadm-conf-path", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the work node.") + cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.") + cmd.Flags().String("working-mode", "edge", "The node type cloud/edge, effect yurthub workingMode.") +} diff --git a/cmd/yurt-node-servant/node-servant.go b/cmd/yurt-node-servant/node-servant.go new file mode 100644 index 00000000000..3f36d94c9be --- /dev/null +++ b/cmd/yurt-node-servant/node-servant.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "math/rand" + "os" + "time" + + "github.com/openyurtio/openyurt/cmd/yurt-node-servant/convert" + "github.com/openyurtio/openyurt/cmd/yurt-node-servant/revert" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/spf13/cobra" +) + +// node-servant +// running on specific node, do convert/revert job +// yurtctl convert/revert join/reset, yurtcluster operator shall start a k8s job to run this. +func main() { + rand.Seed(time.Now().UnixNano()) + + version := fmt.Sprintf("%#v", projectinfo.Get()) + rootCmd := &cobra.Command{ + Use: "node-servant", + Short: "node-servant do convert/revert specific node", + Version: version, + } + rootCmd.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file") + rootCmd.AddCommand(convert.NewConvertCmd()) + rootCmd.AddCommand(revert.NewRevertCmd()) + + if err := rootCmd.Execute(); err != nil { // run command + os.Exit(1) + } +} diff --git a/cmd/yurt-node-servant/revert/revert.go b/cmd/yurt-node-servant/revert/revert.go new file mode 100644 index 00000000000..67d599bd21f --- /dev/null +++ b/cmd/yurt-node-servant/revert/revert.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "github.com/openyurtio/openyurt/pkg/node-servant/revert" + "github.com/spf13/cobra" + "k8s.io/klog" +) + +// NewRevertCmd generates a new revert command +func NewRevertCmd() *cobra.Command { + o := revert.NewRevertOptions() + cmd := &cobra.Command{ + Use: "revert", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + if err := o.Complete(cmd.Flags()); err != nil { + klog.Fatalf("fail to complete the revert option: %s", err) + } + + r := revert.NewReverterWithOptions(o) + if err := r.Do(); err != nil { + klog.Fatalf("fail to revert the yurt node to a kubernetes node: %s", err) + } + klog.Info("revert success") + }, + } + setFlags(cmd) + + return cmd +} + +// setFlags sets flags. +func setFlags(cmd *cobra.Command) { + cmd.Flags().String("kubeadm-conf-path", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") +} diff --git a/hack/lib/build.sh b/hack/lib/build.sh index 6e67230be91..083b0383667 100644 --- a/hack/lib/build.sh +++ b/hack/lib/build.sh @@ -18,6 +18,7 @@ set -x readonly YURT_ALL_TARGETS=( yurtctl + yurt-node-servant yurthub yurt-controller-manager yurt-tunnel-server diff --git a/hack/lib/node-servant-entry.sh b/hack/lib/node-servant-entry.sh new file mode 100644 index 00000000000..403ef6af820 --- /dev/null +++ b/hack/lib/node-servant-entry.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env sh + +# Copyright 2021 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +if [ -d "/openyurt" ]; then + rm -rf /openyurt/usr/local/servant +fi + +mkdir -p /openyurt/usr/local/servant +cp /usr/local/bin/node-servant /openyurt/usr/local/servant/ + +chmod -R +x /openyurt/usr/local/servant/ +chroot /openyurt /usr/local/servant/node-servant "$@" +rm -rf /openyurt/usr/local/servant + diff --git a/hack/lib/release-images.sh b/hack/lib/release-images.sh index bf341e64fdf..95a3ecd3e17 100644 --- a/hack/lib/release-images.sh +++ b/hack/lib/release-images.sh @@ -27,6 +27,7 @@ readonly -a YURT_BIN_TARGETS=( yurthub yurt-controller-manager yurtctl + yurt-node-servant yurt-tunnel-server yurt-tunnel-agent ) @@ -126,6 +127,30 @@ function build_docker_image() { cat << EOF > $docker_file_path FROM ${base_image} ADD ${binary_name} /usr/local/bin/yurtctl +EOF + elif [[ ${binary} =~ yurt-node-servant ]]; + then + yurt_component_name="node-servant" + case $arch in + amd64) + base_image="amd64/alpine:3.9" + ;; + arm64) + base_image="arm64v8/alpine:3.9" + ;; + arm) + base_image="arm32v7/alpine:3.9" + ;; + *) + echo unknown arch $arch + exit 1 + esac + ln ./hack/lib/node-servant-entry.sh "${docker_build_path}/entry.sh" + cat << EOF > $docker_file_path +FROM ${base_image} +ADD entry.sh /usr/local/bin/entry.sh +RUN chmod +x /usr/local/bin/entry.sh +ADD ${binary_name} /usr/local/bin/node-servant EOF else yurt_component_name=${binary_name} diff --git a/pkg/node-servant/components/kubelet.go b/pkg/node-servant/components/kubelet.go new file mode 100644 index 00000000000..fb296791d42 --- /dev/null +++ b/pkg/node-servant/components/kubelet.go @@ -0,0 +1,212 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "k8s.io/klog" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" +) + +const ( + kubeletConfigRegularExpression = "\\-\\-kubeconfig=.*kubelet.conf" + apiserverAddrRegularExpression = "server: (http(s)?:\\/\\/)?[\\w][-\\w]{0,62}(\\.[\\w][-\\w]{0,62})*(:[\\d]{1,5})?" + + kubeAdmFlagsEnvFile = "/var/lib/kubelet/kubeadm-flags.env" + dirMode = 0755 +) + +type kubeletOperator struct { + openyurtDir string + kubeadmConfPath string +} + +// NewKubeletOperator create kubeletOperator +func NewKubeletOperator(openyurtDir, kubeadmConfPath string) *kubeletOperator { + return &kubeletOperator{ + openyurtDir: openyurtDir, + kubeadmConfPath: kubeadmConfPath, + } +} + +// RedirectTrafficToYurtHub +// add env config leads kubelet to visit yurtHub as apiServer +func (op *kubeletOperator) RedirectTrafficToYurtHub() error { + // 1. create a working dir to store revised kubelet.conf + _, err := op.writeYurthubKubeletConfig() + if err != nil { + return err + } + + // 2. append /var/lib/kubelet/kubeadm-flags.env + if err := op.appendConfig(); err != nil { + return err + } + + // 3. restart + return restartKubeletService() +} + +// UndoRedirectTrafficToYurtHub +// undo what's done to kubelet and restart +// to compatible the old convert way for a while , so do renameSvcBk +func (op *kubeletOperator) UndoRedirectTrafficToYurtHub() error { + if err := op.undoAppendConfig(); err != nil { + return err + } + + if err := restartKubeletService(); err != nil { + return err + } + + if err := op.undoWriteYurthubKubeletConfig(); err != nil { + return err + } + klog.Info("revertKubelet: undoWriteYurthubKubeletConfig finished") + + return nil +} + +func (op *kubeletOperator) writeYurthubKubeletConfig() (string, error) { + err := os.MkdirAll(op.openyurtDir, dirMode) + if err != nil { + return "", err + } + fullPath := op.getYurthubKubeletConf() + err = ioutil.WriteFile(fullPath, []byte(enutil.OpenyurtKubeletConf), fileMode) + if err != nil { + return "", err + } + klog.Infof("revised kubeconfig %s is generated", fullPath) + return fullPath, nil +} + +func (op *kubeletOperator) undoWriteYurthubKubeletConfig() error { + yurtKubeletConf := op.getYurthubKubeletConf() + if _, err := enutil.FileExists(yurtKubeletConf); err != nil && os.IsNotExist(err) { + return nil + } + + return os.Remove(yurtKubeletConf) +} + +func (op *kubeletOperator) appendConfig() error { + // set env KUBELET_KUBEADM_ARGS, args set later will override before + // ExecStart: kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS + // append setup: " --kubeconfig=$yurthubKubeletConf -bootstrap-kubeconfig= " + kubeConfigSetup := op.getAppendSetting() + + // if wrote, return + content, err := ioutil.ReadFile(kubeAdmFlagsEnvFile) + if err != nil { + return err + } + args := string(content) + if strings.Contains(args, kubeConfigSetup) { + klog.Info("kubeConfigSetup has wrote before") + return nil + } + + // append KUBELET_KUBEADM_ARGS + argsRegexp := regexp.MustCompile(`KUBELET_KUBEADM_ARGS="(.+)"`) + finding := argsRegexp.FindStringSubmatch(args) + if len(finding) != 2 { + return fmt.Errorf("kubeadm-flags.env error format. %s", args) + } + + r := strings.Replace(args, finding[1], fmt.Sprintf("%s %s", finding[1], kubeConfigSetup), 1) + err = ioutil.WriteFile(kubeAdmFlagsEnvFile, []byte(r), fileMode) + if err != nil { + return err + } + + return nil +} + +func (op *kubeletOperator) undoAppendConfig() error { + kubeConfigSetup := op.getAppendSetting() + contentbyte, err := ioutil.ReadFile(kubeAdmFlagsEnvFile) + if err != nil { + return err + } + + content := strings.Replace(string(contentbyte), kubeConfigSetup, "", -1) + err = ioutil.WriteFile(kubeAdmFlagsEnvFile, []byte(content), fileMode) + if err != nil { + return err + } + klog.Info("revertKubelet: undoAppendConfig finished") + + return nil +} + +func (op *kubeletOperator) getAppendSetting() string { + configPath := op.getYurthubKubeletConf() + return fmt.Sprintf(" --kubeconfig=%s --bootstrap-kubeconfig= ", configPath) +} + +func (op *kubeletOperator) getYurthubKubeletConf() string { + return filepath.Join(op.openyurtDir, enutil.KubeletConfName) +} + +func restartKubeletService() error { + klog.Info("restartKubelet: " + enutil.DaemonReload) + cmd := exec.Command("bash", "-c", enutil.DaemonReload) + if err := enutil.Exec(cmd); err != nil { + return err + } + klog.Info("restartKubelet: " + enutil.RestartKubeletSvc) + cmd = exec.Command("bash", "-c", enutil.RestartKubeletSvc) + if err := enutil.Exec(cmd); err != nil { + return err + } + klog.Infof("restartKubelet: kubelet has been restarted") + return nil +} + +// GetApiServerAddress parse apiServer address from conf file +func GetApiServerAddress(kubeadmConfPath string) (string, error) { + kubeletConfPath, err := enutil.GetSingleContentFromFile(kubeadmConfPath, kubeletConfigRegularExpression) + if err != nil { + return "", err + } + + confArr := strings.Split(kubeletConfPath, "=") + if len(confArr) != 2 { + return "", fmt.Errorf("get kubeletConfPath format err:%s", kubeletConfPath) + } + kubeletConfPath = confArr[1] + apiserverAddr, err := enutil.GetSingleContentFromFile(kubeletConfPath, apiserverAddrRegularExpression) + if err != nil { + return "", err + } + + addrArr := strings.Split(apiserverAddr, " ") + if len(addrArr) != 2 { + return "", fmt.Errorf("get apiserverAddr format err:%s", apiserverAddr) + } + apiserverAddr = addrArr[1] + return apiserverAddr, nil +} diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go new file mode 100644 index 00000000000..4607a82ea11 --- /dev/null +++ b/pkg/node-servant/components/yurthub.go @@ -0,0 +1,212 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" +) + +const ( + hubHealthzCheckFrequency = 10 * time.Second + fileMode = 0666 +) + +type yurtHubOperator struct { + apiServerAddr string + yurthubImage string + joinToken string + workingMode util.WorkingMode + yurthubHealthCheckTimeout time.Duration +} + +// NewYurthubOperator new yurtHubOperator struct +func NewYurthubOperator(apiServerAddr string, yurthubImage string, joinToken string, + workingMode util.WorkingMode, yurthubHealthCheckTimeout time.Duration) *yurtHubOperator { + return &yurtHubOperator{ + apiServerAddr: apiServerAddr, + yurthubImage: yurthubImage, + joinToken: joinToken, + workingMode: workingMode, + yurthubHealthCheckTimeout: yurthubHealthCheckTimeout, + } +} + +// Install set yurthub yaml to static path to start pod +func (op *yurtHubOperator) Install() error { + + // 1. put yurt-hub yaml into /etc/kubernetes/manifests + klog.Infof("setting up yurthub on node") + + // 1-1. replace variables in yaml file + klog.Infof("setting up yurthub apiServer addr") + yurthubTemplate := enutil.ReplaceRegularExpression(enutil.YurthubTemplate, + map[string]string{ + "__kubernetes_service_addr__": op.apiServerAddr, + "__yurthub_image__": op.yurthubImage, + "__join_token__": op.joinToken, + "__working_mode__": string(op.workingMode), + }) + + // 1-2. create yurthub.yaml + podManifestPath := enutil.GetPodManifestPath() + if err := enutil.EnsureDir(podManifestPath); err != nil { + return err + } + err := ioutil.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), fileMode) + if err != nil { + return err + } + klog.Infof("create the %s/yurt-hub.yaml", podManifestPath) + + // 2. wait yurthub pod to be ready + return hubHealthcheck(op.yurthubHealthCheckTimeout) +} + +// UnInstall remove yaml and configs of yurthub +func (op *yurtHubOperator) UnInstall() error { + // 1. remove the yurt-hub.yaml to delete the yurt-hub + podManifestPath := enutil.GetPodManifestPath() + yurthubYamlPath := getYurthubYaml(podManifestPath) + if _, err := enutil.FileExists(yurthubYamlPath); os.IsNotExist(err) { + klog.Infof("UnInstallYurthub: %s is not exists, skip delete", yurthubYamlPath) + } else { + err := os.Remove(yurthubYamlPath) + if err != nil { + return err + } + klog.Infof("UnInstallYurthub: %s has been removed", yurthubYamlPath) + } + + // 2. remove yurt-hub config directory and certificates in it + yurthubConf := getYurthubConf() + if _, err := enutil.FileExists(yurthubConf); os.IsNotExist(err) { + klog.Infof("UnInstallYurthub: dir %s is not exists, skip delete", yurthubConf) + return nil + } + err := os.RemoveAll(yurthubConf) + if err != nil { + return err + } + klog.Infof("UnInstallYurthub: config dir %s has been removed", yurthubConf) + + // 3. remove yurthub cache dir + // since k8s may takes a while to notice and remove yurthub pod, we have to wait for that. + // because, if we delete dir before yurthub exit, yurthub may recreate cache/kubelet dir before exit. + err = waitUntilYurthubExit(time.Duration(60)*time.Second, time.Duration(1)*time.Second) + if err != nil { + return err + } + cacheDir := getYurthubCacheDir() + err = os.RemoveAll(cacheDir) + if err != nil { + return err + } + klog.Infof("UnInstallYurthub: cache dir %s has been removed", cacheDir) + + return nil +} + +func getYurthubYaml(podManifestPath string) string { + return filepath.Join(podManifestPath, enutil.YurthubYamlName) +} + +func getYurthubConf() string { + return filepath.Join(hubself.HubRootDir, hubself.HubName) +} + +func getYurthubCacheDir() string { + // get default dir + return disk.CacheBaseDir +} + +func waitUntilYurthubExit(timeout time.Duration, period time.Duration) error { + klog.Info("wait for yurt-hub exit") + serverHealthzURL, _ := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer)) + serverHealthzURL.Path = enutil.ServerHealthzURLPath + + return wait.PollImmediate(period, timeout, func() (bool, error) { + _, err := pingClusterHealthz(http.DefaultClient, serverHealthzURL.String()) + if err != nil { // means yurthub has exited + klog.Infof("yurt-hub is not running, with ping result: %v", err) + return true, nil + } + klog.Infof("yurt-hub is still running") + return false, nil + }) +} + +// hubHealthcheck will check the status of yurthub pod +func hubHealthcheck(timeout time.Duration) error { + serverHealthzURL, err := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer)) + if err != nil { + return err + } + serverHealthzURL.Path = enutil.ServerHealthzURLPath + + start := time.Now() + return wait.PollImmediate(hubHealthzCheckFrequency, timeout, func() (bool, error) { + _, err := pingClusterHealthz(http.DefaultClient, serverHealthzURL.String()) + if err != nil { + klog.Infof("yurt-hub is not ready, ping cluster healthz with result: %v", err) + return false, nil + } + klog.Infof("yurt-hub healthz is OK after %f seconds", time.Since(start).Seconds()) + return true, nil + }) +} + +func pingClusterHealthz(client *http.Client, addr string) (bool, error) { + if client == nil { + return false, fmt.Errorf("http client is invalid") + } + + resp, err := client.Get(addr) + if err != nil { + return false, err + } + + b, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return false, fmt.Errorf("failed to read response of cluster healthz, %v", err) + } + + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("response status code is %d", resp.StatusCode) + } + + if strings.ToLower(string(b)) != "ok" { + return false, fmt.Errorf("cluster healthz is %s", string(b)) + } + + return true, nil +} diff --git a/pkg/node-servant/constant.go b/pkg/node-servant/constant.go new file mode 100644 index 00000000000..1cbef51c1b5 --- /dev/null +++ b/pkg/node-servant/constant.go @@ -0,0 +1,112 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node_servant + +const ( + + // ConvertJobNameBase is the prefix of the convert ServantJob name + ConvertJobNameBase = "node-servant-convert" + // RevertJobNameBase is the prefix of the revert ServantJob name + RevertJobNameBase = "node-servant-revert" + + // ConvertServantJobTemplate defines the yurtctl convert servant job in yaml format + ConvertServantJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: {{.jobName}} + namespace: kube-system +spec: + template: + spec: + hostPID: true + hostNetwork: true + restartPolicy: OnFailure + nodeName: {{.nodeName}} + volumes: + - name: host-root + hostPath: + path: / + type: Directory + containers: + - name: node-servant-servant + image: {{.node_servant_image}} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - "/usr/local/bin/entry.sh convert --working-mode {{.working_mode}} --yurthub-image {{.yurthub_image}} {{if .yurthub_healthcheck_timeout}}--yurthub-healthcheck-timeout {{.yurthub_healthcheck_timeout}} {{end}}--join-token {{.joinToken}}" + securityContext: + privileged: true + volumeMounts: + - mountPath: /openyurt + name: host-root + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{if .kubeadm_conf_path }} + - name: KUBELET_SVC + value: {{.kubeadm_conf_path}} + {{end}} +` + // RevertServantJobTemplate defines the yurtctl revert servant job in yaml format + RevertServantJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: {{.jobName}} + namespace: kube-system +spec: + template: + spec: + hostPID: true + hostNetwork: true + restartPolicy: OnFailure + nodeName: {{.nodeName}} + volumes: + - name: host-root + hostPath: + path: / + type: Directory + containers: + - name: node-servant + image: {{.node_servant_image}} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - "/usr/local/bin/entry.sh revert" + securityContext: + privileged: true + volumeMounts: + - mountPath: /openyurt + name: host-root + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{if .kubeadm_conf_path }} + - name: KUBELET_SVC + value: {{.kubeadm_conf_path}} + {{end}} +` +) diff --git a/pkg/node-servant/convert/convert.go b/pkg/node-servant/convert/convert.go new file mode 100644 index 00000000000..f634c2c80e8 --- /dev/null +++ b/pkg/node-servant/convert/convert.go @@ -0,0 +1,92 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "fmt" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +// NodeConverter do the convert job +type nodeConverter struct { + Options +} + +// NewConverterWithOptions create nodeConverter +func NewConverterWithOptions(o *Options) *nodeConverter { + return &nodeConverter{ + *o, + } +} + +// Do, do the convert job. +// shall be implemented as idempotent, can execute multiple times with no side-affect. +func (n *nodeConverter) Do() error { + if err := n.validateOptions(); err != nil { + return err + } + if err := n.preflightCheck(); err != nil { + return err + } + + if err := n.installYurtHub(); err != nil { + return err + } + if err := n.convertKubelet(); err != nil { + return err + } + + return nil +} + +func (n *nodeConverter) validateOptions() error { + if !util.IsSupportedWorkingMode(n.workingMode) { + return fmt.Errorf("workingMode must be pointed out as cloud or edge. got %s", n.workingMode) + } + + return nil +} + +func (n *nodeConverter) preflightCheck() error { + // 1. check if critical files exist + if _, err := enutil.FileExists(n.kubeadmConfPath); err != nil { + return err + } + + return nil +} + +func (n *nodeConverter) installYurtHub() error { + apiServerAddress, err := components.GetApiServerAddress(n.kubeadmConfPath) + if err != nil { + return err + } + if apiServerAddress == "" { + return fmt.Errorf("get apiServerAddress empty") + } + op := components.NewYurthubOperator(apiServerAddress, n.yurthubImage, n.joinToken, + n.workingMode, n.yurthubHealthCheckTimeout) + return op.Install() +} + +func (n *nodeConverter) convertKubelet() error { + op := components.NewKubeletOperator(n.openyurtDir, n.kubeadmConfPath) + return op.RedirectTrafficToYurtHub() +} diff --git a/pkg/node-servant/convert/options.go b/pkg/node-servant/convert/options.go new file mode 100644 index 00000000000..511be154835 --- /dev/null +++ b/pkg/node-servant/convert/options.go @@ -0,0 +1,101 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "fmt" + "os" + "time" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + "github.com/spf13/pflag" +) + +// Options has the information that required by convert operation +type Options struct { + yurthubImage string + yurthubHealthCheckTimeout time.Duration + workingMode util.WorkingMode + + joinToken string + kubeadmConfPath string + openyurtDir string + nodeName string +} + +// NewConvertOptions creates a new Options +func NewConvertOptions() *Options { + return &Options{} +} + +// Complete completes all the required options. +func (o *Options) Complete(flags *pflag.FlagSet) error { + yurthubImage, err := flags.GetString("yurthub-image") + if err != nil { + return err + } + o.yurthubImage = yurthubImage + + yurthubHealthCheckTimeout, err := flags.GetDuration("yurthub-healthcheck-timeout") + if err != nil { + return err + } + o.yurthubHealthCheckTimeout = yurthubHealthCheckTimeout + + kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") + if err != nil { + return err + } + if kubeadmConfPath == "" { + kubeadmConfPath = os.Getenv("KUBELET_SVC") + } + if kubeadmConfPath == "" { + kubeadmConfPath = enutil.KubeletSvcPath + } + o.kubeadmConfPath = kubeadmConfPath + + nodeName, err := enutil.GetNodeName(kubeadmConfPath) + if err != nil { + return err + } + o.nodeName = nodeName + + joinToken, err := flags.GetString("join-token") + if err != nil { + return err + } + if joinToken == "" { + return fmt.Errorf("get joinToken empty") + } + o.joinToken = joinToken + + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir + } + o.openyurtDir = openyurtDir + + workingMode, err := flags.GetString("working-mode") + if err != nil { + return err + } + o.workingMode = util.WorkingMode(workingMode) + + return nil +} diff --git a/pkg/node-servant/job.go b/pkg/node-servant/job.go new file mode 100644 index 00000000000..73274efa073 --- /dev/null +++ b/pkg/node-servant/job.go @@ -0,0 +1,99 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node_servant + +import ( + "fmt" + + tmplutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/templates" + batchv1 "k8s.io/api/batch/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes/scheme" +) + +// RenderNodeServantJob return k8s job +// to start k8s job to run convert/revert on specific node +func RenderNodeServantJob(action string, tmplCtx map[string]string, nodeName string) (*batchv1.Job, error) { + if err := validate(action, tmplCtx, nodeName); err != nil { + return nil, err + } + + var servantJobTemplate, jobBaseName string + switch action { + case "convert": + servantJobTemplate = ConvertServantJobTemplate + jobBaseName = ConvertJobNameBase + case "revert": + servantJobTemplate = RevertServantJobTemplate + jobBaseName = RevertJobNameBase + } + + tmplCtx["jobName"] = jobBaseName + "-" + nodeName + tmplCtx["nodeName"] = nodeName + jobYaml, err := tmplutil.SubsituteTemplate(servantJobTemplate, tmplCtx) + if err != nil { + return nil, err + } + + srvJobObj, err := YamlToObject([]byte(jobYaml)) + if err != nil { + return nil, err + } + srvJob, ok := srvJobObj.(*batchv1.Job) + if !ok { + return nil, fmt.Errorf("fail to assert yurtctl-servant job") + } + + return srvJob, nil +} + +// YamlToObject deserializes object in yaml format to a runtime.Object +func YamlToObject(yamlContent []byte) (k8sruntime.Object, error) { + decode := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer().Decode + obj, _, err := decode(yamlContent, nil, nil) + if err != nil { + return nil, err + } + return obj, nil +} + +func validate(action string, tmplCtx map[string]string, nodeName string) error { + if nodeName == "" { + return fmt.Errorf("nodeName empty") + } + + switch action { + case "convert": + keysMustHave := []string{"node_servant_image", "yurthub_image", "joinToken"} + return checkKeys(keysMustHave, tmplCtx) + case "revert": + keysMustHave := []string{"node_servant_image"} + return checkKeys(keysMustHave, tmplCtx) + default: + return fmt.Errorf("action invalied: %s ", action) + } +} + +func checkKeys(arr []string, tmplCtx map[string]string) error { + for _, k := range arr { + if _, ok := tmplCtx[k]; !ok { + return fmt.Errorf("key %s not found", k) + } + } + return nil +} diff --git a/pkg/node-servant/revert/options.go b/pkg/node-servant/revert/options.go new file mode 100644 index 00000000000..07dde436aeb --- /dev/null +++ b/pkg/node-servant/revert/options.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "os" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + + "github.com/spf13/pflag" +) + +// Options has the information that required by revert operation +type Options struct { + kubeadmConfPath string + openyurtDir string + nodeName string +} + +// NewRevertOptions creates a new Options +func NewRevertOptions() *Options { + return &Options{} +} + +// Complete completes all the required options. +func (o *Options) Complete(flags *pflag.FlagSet) error { + + kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") + if err != nil { + return err + } + if kubeadmConfPath == "" { + kubeadmConfPath = os.Getenv("KUBELET_SVC") + } + if kubeadmConfPath == "" { + kubeadmConfPath = enutil.KubeletSvcPath + } + o.kubeadmConfPath = kubeadmConfPath + + nodeName, err := enutil.GetNodeName(kubeadmConfPath) + if err != nil { + return err + } + o.nodeName = nodeName + + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir + } + o.openyurtDir = openyurtDir + + return nil +} diff --git a/pkg/node-servant/revert/revert.go b/pkg/node-servant/revert/revert.go new file mode 100644 index 00000000000..22416f319ff --- /dev/null +++ b/pkg/node-servant/revert/revert.go @@ -0,0 +1,61 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "time" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +// NodeReverter do the revert job +type nodeReverter struct { + Options +} + +// NewReverterWithOptions creates nodeReverter +func NewReverterWithOptions(o *Options) *nodeReverter { + return &nodeReverter{ + *o, + } +} + +// Do, do the convert job +// shall be implemented as idempotent, can execute multiple times with no side-affect. +func (n *nodeReverter) Do() error { + + if err := n.revertKubelet(); err != nil { + return err + } + if err := n.unInstallYurtHub(); err != nil { + return err + } + + return nil +} + +func (n *nodeReverter) revertKubelet() error { + op := components.NewKubeletOperator(n.openyurtDir, n.kubeadmConfPath) + return op.UndoRedirectTrafficToYurtHub() +} + +func (n *nodeReverter) unInstallYurtHub() error { + op := components.NewYurthubOperator("", "", "", + util.WorkingModeCloud, time.Duration(1)) // params is not important here + return op.UnInstall() +}