diff --git a/Makefile b/Makefile index bee30ced7d2..b8714f829c0 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,9 @@ docker-build-yurt-tunnel-agent: docker-build-node-servant: docker buildx build --no-cache --load ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-node-servant . -t ${IMAGE_REPO}/node-servant:${GIT_VERSION} +docker-build-yurt-static-pod-upgrade: + docker buildx build --no-cache --load ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-static-pod-upgrade . -t ${IMAGE_REPO}/yurt-static-pod-upgrade:${GIT_VERSION} + # Build and Push the docker images with multi-arch docker-push: docker-push-yurthub docker-push-yurt-controller-manager docker-push-yurt-tunnel-server docker-push-yurt-tunnel-agent docker-push-node-servant @@ -169,3 +172,12 @@ docker-push-node-servant: docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64 docker run --rm --privileged tonistiigi/binfmt --install all docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-node-servant . -t ${IMAGE_REPO}/node-servant:${GIT_VERSION} + +docker-yurt-static-pod-upgrade: + docker buildx rm yspu-container-builder || true + docker buildx create --use --name=yspu-container-builder + # enable qemu for arm64 build + # https://github.com/docker/buildx/issues/464#issuecomment-741507760 + docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64 + docker run --rm --privileged tonistiigi/binfmt --install all + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-static-pod-upgrade . -t ${IMAGE_REPO}/yurt-static-pod-upgrade:${GIT_VERSION} \ No newline at end of file diff --git a/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go b/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go new file mode 100644 index 00000000000..581722a09a3 --- /dev/null +++ b/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "math/rand" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + upgrade "github.com/openyurtio/openyurt/pkg/static-pod-upgrade" +) + +func main() { + rand.Seed(time.Now().UnixNano()) + version := fmt.Sprintf("%#v", projectinfo.Get()) + cmd := &cobra.Command{ + Use: "yurt-static-pod-upgrade", + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("yurt-static-pod-upgrade version: %#v\n", version) + + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + klog.Infof("FLAG: --%s=%q", flag.Name, flag.Value) + }) + + if err := upgrade.Validate(); err != nil { + klog.Fatalf("Fail to validate yurt static pod upgrade args, %v", err) + } + + c, err := upgrade.GetClient() + if err != nil { + klog.Fatalf("Fail to get kubernetes client, %v", err) + } + + ctrl, err := upgrade.New(c) + if err != nil { + klog.Fatal("Fail to create static-pod-upgrade controller, %v", err) + } + + if err := ctrl.Upgrade(); err != nil { + klog.Fatalf("Fail to upgrade static pod, %v", err) + } + klog.Info("Static pod upgrade Success") + }, + Version: version, + } + + addFlags(cmd) + + if err := viper.BindPFlags(cmd.Flags()); err != nil { + os.Exit(1) + } + + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} + +func addFlags(cmd *cobra.Command) { + cmd.Flags().String("kubeconfig", "", "The path to the kubeconfig file") + cmd.Flags().String("name", "", "The name of static pod which needs be upgraded") + cmd.Flags().String("namespace", "", "The namespace of static pod which needs be upgraded") + cmd.Flags().String("manifest", "", "The manifest file name of static pod which needs be upgraded") + cmd.Flags().String("hash", "", "The hash value of new static pod specification") + cmd.Flags().String("mode", "", "The upgrade mode which is used") +} diff --git a/go.mod b/go.mod index fd6966e5938..f4a8abe2d1d 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/prometheus/client_golang v1.11.0 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 diff --git a/go.sum b/go.sum index 55761cbc7f2..56e2a508dd3 100644 --- a/go.sum +++ b/go.sum @@ -393,6 +393,7 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -447,6 +448,7 @@ github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffkt github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -476,6 +478,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -533,6 +536,7 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -597,6 +601,7 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -604,6 +609,7 @@ github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSW github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -612,6 +618,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -624,6 +631,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/hack/dockerfiles/Dockerfile.yurt-static-pod-upgrade b/hack/dockerfiles/Dockerfile.yurt-static-pod-upgrade new file mode 100644 index 00000000000..b4499045da1 --- /dev/null +++ b/hack/dockerfiles/Dockerfile.yurt-static-pod-upgrade @@ -0,0 +1,13 @@ +# multi-arch image building for yurt-static-pod-upgrade + +FROM --platform=${BUILDPLATFORM} golang:1.16 as builder +ADD . /build +ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO +WORKDIR /build/ +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GIT_VERSION=${GIT_VERSION} make build WHAT=cmd/yurt-static-pod-upgrade + +FROM --platform=${TARGETPLATFORM} alpine:3.14 +ARG TARGETOS TARGETARCH MIRROR_REPO +RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \ + apk add ca-certificates bash libc6-compat && update-ca-certificates && rm /var/cache/apk/* +COPY --from=builder /build/_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-static-pod-upgrade /usr/local/bin/yurt-static-pod-upgrade diff --git a/hack/make-rules/build.sh b/hack/make-rules/build.sh index 7cddd634ae7..829a39bb827 100755 --- a/hack/make-rules/build.sh +++ b/hack/make-rules/build.sh @@ -28,6 +28,7 @@ readonly YURT_ALL_TARGETS=( yurt-controller-manager yurt-tunnel-server yurt-tunnel-agent + yurt-static-pod-upgrade ) build_binaries "$@" diff --git a/pkg/static-pod-upgrade/upgrade.go b/pkg/static-pod-upgrade/upgrade.go new file mode 100644 index 00000000000..a2d99787c49 --- /dev/null +++ b/pkg/static-pod-upgrade/upgrade.go @@ -0,0 +1,254 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/spf13/viper" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" +) + +const ( + DefaultUpgradeDir = "openyurtio-upgrade" + defaultStaticPodRunningCheckTimeout = 2 * time.Minute + + // TODO: use static-pod-upgrade's constant value + OTA = "ota" + Auto = "auto" +) + +var ( + DefaultConfigmapPath = "/data" + DefaultManifestPath = "/etc/kubernetes/manifests" + + RequiredField = []string{"name", "namespace", "hash", "mode"} +) + +type UpgradeController struct { + client kubernetes.Interface + + // Name of static pod + name string + // Namespace of static pod + namespace string + // Manifest file name of static pod + manifest string + // The latest static pod hash + hash string + // Only support `OTA` and `Auto` + upgradeMode string + + // Manifest path of static pod, default `/etc/kubernetes/manifests/manifestName.yaml` + manifestPath string + // The backup manifest path, default `/etc/kubernetes/manifests/openyurtio-upgrade/manifestName.bak` + bakManifestPath string + // Default is `/data/podName` + configMapDataPath string + // The latest manifest path, default `/etc/kubernetes/manifests/openyurtio-upgrade/manifestName.upgrade` + upgradeManifestPath string +} + +func New(client kubernetes.Interface) (*UpgradeController, error) { + ctrl := &UpgradeController{ + client: client, + } + + ctrl.name = viper.GetString("name") + ctrl.namespace = viper.GetString("namespace") + ctrl.manifest = viper.GetString("manifest") + ctrl.hash = viper.GetString("hash") + ctrl.upgradeMode = viper.GetString("mode") + + // Manifest file name is optional. If not set, default is static pod name + if len(ctrl.manifest) == 0 { + ctrl.manifest = ctrl.name + } + + ctrl.manifestPath = filepath.Join(DefaultManifestPath, WithYamlSuffix(ctrl.manifest)) + ctrl.bakManifestPath = filepath.Join(DefaultManifestPath, DefaultUpgradeDir, WithBackupSuffix(ctrl.manifest)) + ctrl.configMapDataPath = filepath.Join(DefaultConfigmapPath, ctrl.manifest) + ctrl.upgradeManifestPath = filepath.Join(DefaultManifestPath, DefaultUpgradeDir, WithUpgradeSuffix(ctrl.manifest)) + + return ctrl, nil +} + +func (ctrl *UpgradeController) Upgrade() error { + // 1. Check the target static pod exist + if err := ctrl.checkStaticPodExist(); err != nil { + return err + } + klog.Info("Check static pod existence success") + + // 2. Check old manifest and the latest manifest exist + if err := ctrl.checkManifestFileExist(); err != nil { + return err + } + klog.Info("Check old manifest and new manifest files existence success") + + // 3. prepare the latest manifest + if err := ctrl.prepareManifest(); err != nil { + return err + } + klog.Info("Prepare upgrade manifest success") + + // 4. execute upgrade operations + switch ctrl.upgradeMode { + case Auto: + return ctrl.AutoUpgrade() + + case OTA: + return ctrl.OTAUpgrade() + } + + return nil +} + +func (ctrl *UpgradeController) AutoUpgrade() error { + // (1) Back up the old manifest in case of upgrade failure + if err := ctrl.backupManifest(); err != nil { + return err + } + klog.Info("Auto upgrade backupManifest success") + + // (2) Replace manifest and kubelet will upgrade the static pod automatically + if err := ctrl.replaceManifest(); err != nil { + return err + } + klog.Info("Auto upgrade replaceManifest success") + + // (3) Verify the new static pod is running + ok, err := ctrl.verify() + if err != nil { + return err + } + if !ok { + return fmt.Errorf("the latest static pod is not running") + } + klog.Info("Auto upgrade verify success") + + return nil +} + +// In ota mode, just need to set the latest upgrade manifest version +func (ctrl *UpgradeController) OTAUpgrade() error { + if err := ctrl.setLatestManifestHash(); err != nil { + return err + } + klog.Info("OTA upgrade set latest manifest hash success") + return nil +} + +// checkStaticPodExist check if the target static pod exist in cluster +func (ctrl *UpgradeController) checkStaticPodExist() error { + if errs := validation.IsDNS1123Subdomain(ctrl.name); len(errs) > 0 { + return fmt.Errorf("pod name %s is invalid: %v", ctrl.name, errs) + } + _, err := ctrl.client.CoreV1().Pods(ctrl.namespace).Get(context.TODO(), ctrl.name, metav1.GetOptions{}) + if err != nil { + return err + } + return nil +} + +// checkManifestFileExist check if the specified files exist +func (ctrl *UpgradeController) checkManifestFileExist() error { + check := []string{ctrl.manifestPath, ctrl.configMapDataPath} + for _, c := range check { + _, err := os.Stat(c) + if os.IsNotExist(err) { + return fmt.Errorf("manifest %s does not exist", c) + } + } + + return nil +} + +// prepareManifest move the latest manifest to DefaultUpgradeDir and set `.upgrade` suffix +// TODO: In kubernetes when mount configmap file to the sub path of hostpath mount, it will not be persistent +// TODO: Init configmap(latest manifest) to a default place and move it to `DefaultUpgradeDir` to save it persistent +func (ctrl *UpgradeController) prepareManifest() error { + // Make sure upgrade dir exist + if _, err := os.Stat(filepath.Join(DefaultManifestPath, DefaultUpgradeDir)); os.IsNotExist(err) { + if err = os.Mkdir(filepath.Join(DefaultManifestPath, DefaultUpgradeDir), 0755); err != nil { + return err + } + } + + return CopyFile(ctrl.configMapDataPath, ctrl.upgradeManifestPath) +} + +// backUpManifest backup the old manifest in order to roll back when errors occur +func (ctrl *UpgradeController) backupManifest() error { + return CopyFile(ctrl.manifestPath, ctrl.bakManifestPath) +} + +// replaceManifest replace old manifest with the latest one, it achieves static pod upgrade +func (ctrl *UpgradeController) replaceManifest() error { + return CopyFile(ctrl.upgradeManifestPath, ctrl.manifestPath) +} + +// verify make sure the latest static pod is running +// return false when the latest static pod failed or check status time out +func (ctrl *UpgradeController) verify() (bool, error) { + return WaitForPodRunning(ctrl.client, ctrl.name, ctrl.namespace, ctrl.hash, defaultStaticPodRunningCheckTimeout) +} + +// setLatestManifestHash set the latest manifest hash value to target static pod annotation +// TODO: In ota mode, it's hard for controller to check whether the latest manifest file has been issued to nodes +// TODO: Use annotation `openyurt.io/ota-manifest-version` to indicate the version of manifest issued to nodes +func (ctrl *UpgradeController) setLatestManifestHash() error { + pod, err := ctrl.client.CoreV1().Pods(ctrl.namespace).Get(context.TODO(), ctrl.name, metav1.GetOptions{}) + if err != nil { + return err + } + metav1.SetMetaDataAnnotation(&pod.ObjectMeta, OTALatestManifestAnnotation, ctrl.hash) + _, err = ctrl.client.CoreV1().Pods(ctrl.namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}) + return err +} + +// Validate check if all the required arguments are valid +func Validate() error { + for _, r := range RequiredField { + if v := viper.GetString(r); len(v) == 0 { + return fmt.Errorf("arg %s is empty", r) + } + } + return nil +} + +func GetClient() (kubernetes.Interface, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + c, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + return c, nil +} diff --git a/pkg/static-pod-upgrade/upgrade_test.go b/pkg/static-pod-upgrade/upgrade_test.go new file mode 100644 index 00000000000..7e6204c43ed --- /dev/null +++ b/pkg/static-pod-upgrade/upgrade_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/spf13/viper" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +func Test(t *testing.T) { + // Temporarily modify the manifest path in order to test + DefaultManifestPath = t.TempDir() + DefaultConfigmapPath = t.TempDir() + _, _ = os.Create(filepath.Join(DefaultManifestPath, WithYamlSuffix("nginxManifest"))) + _, _ = os.Create(filepath.Join(DefaultConfigmapPath, "nginxManifest")) + + viper.Set("name", "nginx-node") + viper.Set("namespace", "default") + viper.Set("manifest", "nginxManifest") + viper.Set("hash", "789c7f9f47") + + runningStaticPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-node", + Namespace: "default", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + modes := []string{"ota", "auto"} + + for _, mode := range modes { + /* + 1. Prepare the test environment + */ + if mode == "auto" { + runningStaticPod.Annotations = map[string]string{ + StaticPodHashAnnotation: "789c7f9f47", + } + } + c := fake.NewSimpleClientset(runningStaticPod) + // Add watch event for verify + watcher := watch.NewFake() + c.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(watcher, nil)) + go func() { + watcher.Add(runningStaticPod) + }() + + viper.Set("mode", mode) + + /* + 2. Test + */ + ctrl, err := New(c) + if err != nil { + t.Errorf("Fail to get upgrade controller, %v", err) + } + + if err := ctrl.Upgrade(); err != nil { + t.Errorf("Fail to upgrade, %v", err) + } + + /* + 3. Verify OTA upgrade mode + */ + if mode == "ota" { + ok, err := util.FileExists(ctrl.upgradeManifestPath) + if err != nil { + t.Errorf("Fail to check manifest existence for ota upgrade, %v", err) + } + if !ok { + t.Errorf("Manifest for ota upgrade does not exist") + } + + pod, err := ctrl.client.CoreV1().Pods(runningStaticPod.Namespace). + Get(context.TODO(), runningStaticPod.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Fail to get the running static pod, %v", err) + } + + if pod.Annotations[OTALatestManifestAnnotation] != "789c7f9f47" { + t.Errorf("Fail to verify hash annotation for ota upgrade, %v", err) + } + } + /* + 4. Verify Auto upgrade mode + */ + if mode == "auto" { + checkFiles := []string{ctrl.upgradeManifestPath, ctrl.bakManifestPath} + for _, file := range checkFiles { + ok, err := util.FileExists(file) + if err != nil { + t.Errorf("Fail to check %s manifest existence for auto upgrade, %v", file, err) + } + if !ok { + t.Errorf("Manifest %s for auto upgrade does not exist", file) + } + } + } + } +} diff --git a/pkg/static-pod-upgrade/util.go b/pkg/static-pod-upgrade/util.go new file mode 100644 index 00000000000..87a1aee8f00 --- /dev/null +++ b/pkg/static-pod-upgrade/util.go @@ -0,0 +1,123 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "fmt" + "io" + "os" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +const ( + YamlSuffix string = ".yaml" + BackupSuffix = ".bak" + UpgradeSuffix string = ".upgrade" + + StaticPodHashAnnotation = "openyurt.io/static-pod-hash" + OTALatestManifestAnnotation = "openyurt.io/ota-latest-version" +) + +func WithYamlSuffix(path string) string { + return path + YamlSuffix +} + +func WithBackupSuffix(path string) string { + return path + BackupSuffix +} + +func WithUpgradeSuffix(path string) string { + return path + UpgradeSuffix +} + +// CopyFile copy file content from src to dst, if destination file not exist, then create it +func CopyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return nil +} + +// WaitForPodRunning waits static pod to run +// Success: Static pod annotation `StaticPodHashAnnotation` value equals to function argument hash +// Failed: Receive PodFailed event +func WaitForPodRunning(c kubernetes.Interface, name, namespace, hash string, timeout time.Duration) (bool, error) { + klog.Infof("WaitForPodRuning name is %s, namespace is %s", name, namespace) + // Create a watcher to watch the pod's status + watcher, err := c.CoreV1().Pods(namespace).Watch(context.TODO(), metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return false, err + } + defer watcher.Stop() + + // Create a channel to receive updates from the watcher + ch := watcher.ResultChan() + + // Start a goroutine to monitor the pod's status + running := make(chan struct{}) + failed := make(chan struct{}) + + go func() { + for event := range ch { + obj, ok := event.Object.(*corev1.Pod) + if !ok { + continue + } + + h := obj.Annotations[StaticPodHashAnnotation] + + if obj.Status.Phase == corev1.PodRunning && h == hash { + close(running) + return + } + + if obj.Status.Phase == corev1.PodFailed { + close(failed) + return + } + } + }() + + // Wait for watch event to finish or the timeout to expire + select { + case <-running: + return true, nil + case <-failed: + return false, nil + case <-time.After(timeout): + return false, fmt.Errorf("timeout waiting for static pod %s/%s to be running", namespace, name) + } +}