add app to support upgrade static pod

cmd/yurt-node-servant/node-servant.go

@@ -28,6 +28,7 @@ import (
 	"github.com/openyurtio/openyurt/cmd/yurt-node-servant/convert"
 	preflightconvert "github.com/openyurtio/openyurt/cmd/yurt-node-servant/preflight-convert"
 	"github.com/openyurtio/openyurt/cmd/yurt-node-servant/revert"
+	upgrade "github.com/openyurtio/openyurt/cmd/yurt-node-servant/static-pod-upgrade"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 )
 
@@ -49,6 +50,7 @@ func main() {
 	rootCmd.AddCommand(revert.NewRevertCmd())
 	rootCmd.AddCommand(preflightconvert.NewxPreflightConvertCmd())
 	rootCmd.AddCommand(config.NewConfigCmd())
+	rootCmd.AddCommand(upgrade.NewUpgradeCmd())
 
 	if err := rootCmd.Execute(); err != nil { // run command
 		os.Exit(1)

cmd/yurt-node-servant/static-pod-upgrade/upgrade.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"k8s.io/klog/v2"
+
+	upgrade "github.com/openyurtio/openyurt/pkg/node-servant/static-pod-upgrade"
+)
+
+// NewUpgradeCmd generates a new upgrade command
+func NewUpgradeCmd() *cobra.Command {
+	o := upgrade.NewUpgradeOptions()
+	cmd := &cobra.Command{
+		Use:   "static-pod-upgrade",
+		Short: "",
+		Run: func(cmd *cobra.Command, args []string) {
+			cmd.Flags().VisitAll(func(flag *pflag.Flag) {
+				klog.Infof("FLAG: --%s=%q", flag.Name, flag.Value)
+			})
+
+			if err := o.Validate(); err != nil {
+				klog.Fatalf("Fail to validate static pod upgrade args, %v", err)
+			}
+
+			ctrl, err := upgrade.NewWithOptions(o)
+			if err != nil {
+				klog.Fatalf("Fail to create static-pod-upgrade controller, %v", err)
+			}
+
+			if err = ctrl.Upgrade(); err != nil {
+				klog.Fatalf("Fail to upgrade static pod, %v", err)
+			}
+
+			klog.Info("Static pod upgrade Success")
+		},
+		Args: cobra.NoArgs,
+	}
+	o.AddFlags(cmd.Flags())
+
+	return cmd
+}

pkg/node-servant/static-pod-upgrade/options.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/pflag"
+)
+
+const (
+	DefaultStaticPodRunningCheckTimeout = 2 * time.Minute
+)
+
+// Options has the information that required by static-pod-upgrade operation
+type Options struct {
+	name      string
+	namespace string
+	manifest  string
+	hash      string
+	mode      string
+	timeout   time.Duration
+}
+
+// NewUpgradeOptions creates a new Options
+func NewUpgradeOptions() *Options {
+	return &Options{
+		timeout: DefaultStaticPodRunningCheckTimeout,
+	}
+}
+
+// AddFlags sets flags.
+func (o *Options) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.name, "name", o.name, "The name of static pod which needs be upgraded")
+	fs.StringVar(&o.namespace, "namespace", o.namespace, "The namespace of static pod which needs be upgraded")
+	fs.StringVar(&o.manifest, "manifest", o.manifest, "The manifest file name of static pod which needs be upgraded")
+	fs.StringVar(&o.hash, "hash", o.hash, "The hash value of new static pod specification")
+	fs.StringVar(&o.mode, "mode", o.mode, "The upgrade mode which is used")
+	fs.DurationVar(&o.timeout, "timeout", o.timeout, "The timeout for upgrade success check.")
+}
+
+// Validate validates Options
+func (o *Options) Validate() error {
+	if len(o.name) == 0 || len(o.namespace) == 0 || len(o.manifest) == 0 || len(o.hash) == 0 || len(o.mode) == 0 {
+		return fmt.Errorf("args can not be empty, name is %s, namespace is %s,manifest is %s, hash is %s,mode is %s",
+			o.name, o.namespace, o.manifest, o.hash, o.mode)
+	}
+
+	return nil
+}

pkg/node-servant/static-pod-upgrade/upgrade.go

@@ -0,0 +1,184 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"github.com/openyurtio/openyurt/pkg/node-servant/static-pod-upgrade/util"
+)
+
+const (
+	// TODO: use constant value of static-pod controller
+	OTA  = "ota"
+	Auto = "auto"
+)
+
+var (
+	DefaultConfigmapPath = "/data"
+	DefaultManifestPath  = "/etc/kubernetes/manifests"
+	DefaultUpgradePath   = "/tmp/manifests"
+)
+
+type Controller struct {
+	// Name of static pod
+	name string
+	// Namespace of static pod
+	namespace string
+	// Manifest file name of static pod
+	manifest string
+	// The latest static pod hash
+	hash string
+	// Only support `OTA` and `Auto`
+	upgradeMode string
+	// Timeout for upgrade success check
+	timeout time.Duration
+
+	// Manifest path of static pod, default `/etc/kubernetes/manifests/manifestName.yaml`
+	manifestPath string
+	// The backup manifest path, default `/etc/kubernetes/manifests/openyurtio-upgrade/manifestName.bak`
+	bakManifestPath string
+	// Default is `/data/podName`
+	configMapDataPath string
+	// The latest manifest path, default `/etc/kubernetes/manifests/openyurtio-upgrade/manifestName.upgrade`
+	upgradeManifestPath string
+}
+
+func NewWithOptions(o *Options) (*Controller, error) {
+	ctrl := &Controller{
+		name:        o.name,
+		namespace:   o.namespace,
+		manifest:    o.manifest,
+		hash:        o.hash,
+		upgradeMode: o.mode,
+		timeout:     o.timeout,
+	}
+
+	ctrl.manifestPath = filepath.Join(DefaultManifestPath, util.WithYamlSuffix(ctrl.manifest))
+	ctrl.bakManifestPath = filepath.Join(DefaultUpgradePath, util.WithBackupSuffix(ctrl.manifest))
+	ctrl.configMapDataPath = filepath.Join(DefaultConfigmapPath, ctrl.manifest)
+	ctrl.upgradeManifestPath = filepath.Join(DefaultUpgradePath, util.WithUpgradeSuffix(ctrl.manifest))
+
+	return ctrl, nil
+}
+
+func (ctrl *Controller) Upgrade() error {
+	// 1. Check old manifest and the latest manifest exist
+	if err := ctrl.checkManifestFileExist(); err != nil {
+		return err
+	}
+	klog.Info("Check old manifest and new manifest files existence success")
+
+	// 2. prepare the latest manifest
+	if err := ctrl.prepareManifest(); err != nil {
+		return err
+	}
+	klog.Info("Prepare upgrade manifest success")
+
+	// 3. execute upgrade operations
+	switch ctrl.upgradeMode {
+	case Auto:
+		return ctrl.AutoUpgrade()
+	}
+
+	return nil
+}
+
+func (ctrl *Controller) AutoUpgrade() error {
+	// (1) Back up the old manifest in case of upgrade failure
+	if err := ctrl.backupManifest(); err != nil {
+		return err
+	}
+	klog.Info("Auto upgrade backupManifest success")
+
+	// (2) Replace manifest and kubelet will upgrade the static pod automatically
+	if err := ctrl.replaceManifest(); err != nil {
+		return err
+	}
+	klog.Info("Auto upgrade replaceManifest success")
+
+	// (3) Verify the new static pod is running
+	ok, err := ctrl.verify()
+	if err != nil {
+		if err := ctrl.rollbackManifest(); err != nil {
+			klog.Errorf("Fail to rollback manifest when upgrade failed, %v", err)
+		}
+		return err
+	}
+	if !ok {
+		if err := ctrl.rollbackManifest(); err != nil {
+			klog.Errorf("Fail to rollback manifest when upgrade failed, %v", err)
+		}
+		return fmt.Errorf("the latest static pod is not running")
+	}
+	klog.Info("Auto upgrade verify success")
+
+	return nil
+}
+
+// checkManifestFileExist check if the specified files exist
+func (ctrl *Controller) checkManifestFileExist() error {
+	check := []string{ctrl.manifestPath, ctrl.configMapDataPath}
+	for _, c := range check {
+		_, err := os.Stat(c)
+		if os.IsNotExist(err) {
+			return fmt.Errorf("manifest %s does not exist", c)
+		}
+	}
+
+	return nil
+}
+
+// prepareManifest move the latest manifest to DefaultUpgradePath and set `.upgrade` suffix
+// TODO: In kubernetes when mount configmap file to the sub path of hostpath mount, it will not be persistent
+// TODO: Init configmap(latest manifest) to a default place and move it to `DefaultUpgradePath` to save it persistent
+func (ctrl *Controller) prepareManifest() error {
+	// Make sure upgrade dir exist
+	if _, err := os.Stat(DefaultUpgradePath); os.IsNotExist(err) {
+		if err = os.Mkdir(DefaultUpgradePath, 0755); err != nil {
+			return err
+		}
+	}
+
+	return util.CopyFile(ctrl.configMapDataPath, ctrl.upgradeManifestPath)
+}
+
+// backUpManifest backup the old manifest in order to roll back when errors occur
+func (ctrl *Controller) backupManifest() error {
+	return util.CopyFile(ctrl.manifestPath, ctrl.bakManifestPath)
+}
+
+// replaceManifest replace old manifest with the latest one, it achieves static pod upgrade
+func (ctrl *Controller) replaceManifest() error {
+	return util.CopyFile(ctrl.upgradeManifestPath, ctrl.manifestPath)
+}
+
+// rollbackManifest replace new manifest with the backup
+func (ctrl *Controller) rollbackManifest() error {
+	return util.CopyFile(ctrl.bakManifestPath, ctrl.manifestPath)
+}
+
+// verify make sure the latest static pod is running
+// return false when the latest static pod failed or check status time out
+func (ctrl *Controller) verify() (bool, error) {
+	return util.WaitForPodRunning(ctrl.namespace, ctrl.name, ctrl.hash, ctrl.timeout)
+}