From 3ada9ba78a0aca47f95173d7a3738e255a60fd6b Mon Sep 17 00:00:00 2001 From: hxcGit Date: Thu, 19 Jan 2023 23:11:33 +0200 Subject: [PATCH] add app to support upgrade static pod Signed-off-by: hxcGit --- Makefile | 5 +- Makefile1 | 194 +++++++++++++ .../static-pod-upgrade.go | 86 ++++++ .../Dockerfile.yurt-static-pod-upgrade | 13 + hack/make-rules/build.sh | 1 + pkg/static-pod-upgrade/upgrade.go | 254 ++++++++++++++++++ pkg/static-pod-upgrade/upgrade_test.go | 128 +++++++++ pkg/static-pod-upgrade/util.go | 123 +++++++++ 8 files changed, 803 insertions(+), 1 deletion(-) create mode 100644 Makefile1 create mode 100644 cmd/yurt-static-pod-upgrade/static-pod-upgrade.go create mode 100644 hack/dockerfiles/release/Dockerfile.yurt-static-pod-upgrade create mode 100644 pkg/static-pod-upgrade/upgrade.go create mode 100644 pkg/static-pod-upgrade/upgrade_test.go create mode 100644 pkg/static-pod-upgrade/util.go diff --git a/Makefile b/Makefile index 0e38e74563d..bf3f4febc7f 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ docker-build: # Build and Push the docker images with multi-arch -docker-push: docker-push-yurthub docker-push-node-servant docker-push-yurt-manager +docker-push: docker-push-yurthub docker-push-node-servant docker-push-yurt-manager docker-push-yurt-static-pod-upgrade docker-buildx-builder: @@ -149,6 +149,9 @@ docker-push-node-servant: docker-buildx-builder docker-push-yurt-manager: manifests docker-buildx-builder docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-manager . -t ${IMAGE_REPO}/yurt-manager:${GIT_VERSION} +docker-push-yurt-static-pod-upgrade: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-static-pod-upgrade . -t ${IMAGE_REPO}/yurt-static-pod-upgrade:${GIT_VERSION} + generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. # hack/make-rule/generate_openapi.sh // TODO by kadisi $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./pkg/apis/..." diff --git a/Makefile1 b/Makefile1 new file mode 100644 index 00000000000..ea902915f40 --- /dev/null +++ b/Makefile1 @@ -0,0 +1,194 @@ +# Copyright 2020 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KUBERNETESVERSION ?=v1.22 +TARGET_PLATFORMS ?= linux/amd64 +IMAGE_REPO ?= openyurt +IMAGE_TAG ?= $(shell git describe --abbrev=0 --tags) +GIT_COMMIT = $(shell git rev-parse HEAD) +ENABLE_AUTONOMY_TESTS ?=true +CRD_OPTIONS ?= "crd:crdVersions=v1" +BUILD_KUSTOMIZE ?= _output/manifest + +ifeq ($(shell git tag --points-at ${GIT_COMMIT}),) +GIT_VERSION=$(IMAGE_TAG)-$(shell echo ${GIT_COMMIT} | cut -c 1-7) +else +GIT_VERSION=$(IMAGE_TAG) +endif + +ifneq ($(IMAGE_TAG), $(shell git describe --abbrev=0 --tags)) +GIT_VERSION=$(IMAGE_TAG) +endif + +DOCKER_BUILD_ARGS = --build-arg GIT_VERSION=${GIT_VERSION} + +ifeq (${REGION}, cn) +DOCKER_BUILD_ARGS += --build-arg GOPROXY=https://goproxy.cn --build-arg MIRROR_REPO=mirrors.aliyun.com +endif + +ifneq (${http_proxy},) +DOCKER_BUILD_ARGS += --build-arg http_proxy='${http_proxy}' +endif + +ifneq (${https_proxy},) +DOCKER_BUILD_ARGS += --build-arg https_proxy='${https_proxy}' +endif + +.PHONY: clean all build test + +all: test build + +# Build binaries in the host environment +build: + bash hack/make-rules/build.sh $(WHAT) + +# Run test +test: + go test -v -short ./pkg/... ./cmd/... -coverprofile cover.out + go test -v -coverpkg=./pkg/yurttunnel/... -coverprofile=yurttunnel-cover.out ./test/integration/yurttunnel_test.go + +clean: + -rm -Rf _output + +# verify will verify the code. +verify: verify-mod verify-license + +# verify-license will check if license has been added to files. +verify-license: + hack/make-rules/check_license.sh + +# verify-mod will check if go.mod has beed tidied. +verify-mod: + hack/make-rules/verify_mod.sh + +# Start up OpenYurt cluster on local machine based on a Kind cluster +local-up-openyurt: + KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh + +# Build all OpenYurt components images and then start up OpenYurt cluster on local machine based on a Kind cluster +# And you can run the following command on different env by specify TARGET_PLATFORMS, default platform is linux/amd64 +# - on centos env: make docker-build-and-up-openyurt +# - on MACBook Pro M1: make docker-build-and-up-openyurt TARGET_PLATFORMS=linux/arm64 +docker-build-and-up-openyurt: docker-build + KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh + +# Start up e2e tests for OpenYurt +# And you can run the following command on different env by specify TARGET_PLATFORMS, default platform is linux/amd64 +# - on centos env: make e2e-tests +# - on MACBook Pro M1: make e2e-tests TARGET_PLATFORMS=linux/arm64 +e2e-tests: + ENABLE_AUTONOMY_TESTS=${ENABLE_AUTONOMY_TESTS} TARGET_PLATFORMS=${TARGET_PLATFORMS} hack/make-rules/run-e2e-tests.sh + +install-golint: ## check golint if not exist install golint tools +ifeq (, $(shell which golangci-lint)) + @{ \ + set -e ;\ + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2 ;\ + } +GOLINT_BIN=$(shell go env GOPATH)/bin/golangci-lint +else +GOLINT_BIN=$(shell which golangci-lint) +endif + +lint: install-golint ## Run go lint against code. + $(GOLINT_BIN) run -v + +# Build the docker images only one arch(specify arch by TARGET_PLATFORMS env) +# otherwise the platform of host will be used. +# e.g. +# - build linux/amd64 docker images: +# $# make docker-build TARGET_PLATFORMS=linux/amd64 +# - build linux/arm64 docker images: +# $# make docker-build TARGET_PLATFORMS=linux/arm64 +# - build a specific image: +# $# make docker-build WHAT=yurthub +# - build with proxy, maybe useful for Chinese users +# $# REGION=cn make docker-build +docker-build: + TARGET_PLATFORMS=${TARGET_PLATFORMS} hack/make-rules/image_build.sh $(WHAT) + + +# Build and Push the docker images with multi-arch +docker-push: docker-push-yurthub docker-push-yurt-controller-manager docker-push-yurt-tunnel-server docker-push-yurt-tunnel-agent docker-push-node-servant docker-push-yurt-manager docker-push-yurt-static-pod-upgrade + + +docker-buildx-builder: + if ! docker buildx ls | grep -q container-builder; then\ + docker buildx create --name container-builder --use;\ + fi + # enable qemu for arm64 build + # https://github.com/docker/buildx/issues/464#issuecomment-741507760 + docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64 + docker run --rm --privileged tonistiigi/binfmt --install all + +docker-push-yurthub: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurthub . -t ${IMAGE_REPO}/yurthub:${GIT_VERSION} + +docker-push-yurt-controller-manager: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-controller-manager . -t ${IMAGE_REPO}/yurt-controller-manager:${GIT_VERSION} + +docker-push-yurt-tunnel-server: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-tunnel-server . -t ${IMAGE_REPO}/yurt-tunnel-server:${GIT_VERSION} + +docker-push-yurt-tunnel-agent: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent . -t ${IMAGE_REPO}/yurt-tunnel-agent:${GIT_VERSION} + +docker-push-node-servant: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.node-servant . -t ${IMAGE_REPO}/node-servant:${GIT_VERSION} + +docker-push-yurt-manager: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-manager . -t ${IMAGE_REPO}/yurt-manager:${GIT_VERSION} + +docker-push-yurt-static-pod-upgrade: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-static-pod-upgrade . -t ${IMAGE_REPO}/yurt-static-pod-upgrade:${GIT_VERSION} + +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. +# hack/make-rule/generate_openapi.sh // TODO by kadisi + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./pkg/apis/..." + +manifests: generate ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + rm -rf $(BUILD_KUSTOMIZE) + $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=role webhook paths="./pkg/..." output:crd:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/crd output:rbac:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/rbac output:webhook:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/webhook + hack/make-rules/kustomize_to_chart.sh --crd $(BUILD_KUSTOMIZE)/auto_generate/crd --webhook $(BUILD_KUSTOMIZE)/auto_generate/webhook --rbac $(BUILD_KUSTOMIZE)/auto_generate/rbac --output $(BUILD_KUSTOMIZE)/kustomize --templateDir charts/openyurt/templates + + +# newcontroller +# .e.g +# make newcontroller GROUP=apps VERSION=v1beta1 KIND=example SHORTNAME=examples SCOPE=Namespaced +# make newcontroller GROUP=apps VERSION=v1beta1 KIND=example SHORTNAME=examples SCOPE=Cluster +newcontroller: + hack/make-rules/add_controller.sh --group $(GROUP) --version $(VERSION) --kind $(KIND) --shortname $(SHORTNAME) --scope $(SCOPE) + + +CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +controller-gen: ## Download controller-gen locally if necessary. +ifeq ("$(shell $(CONTROLLER_GEN) --version 2> /dev/null)", "Version: v0.7.0") +else + rm -rf $(CONTROLLER_GEN) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0) +endif + +# go-get-tool will 'go get' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-get-tool +@[ -f $(1) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +go mod init tmp ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ +rm -rf $$TMP_DIR ;\ +} +endef \ No newline at end of file diff --git a/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go b/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go new file mode 100644 index 00000000000..581722a09a3 --- /dev/null +++ b/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "math/rand" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + upgrade "github.com/openyurtio/openyurt/pkg/static-pod-upgrade" +) + +func main() { + rand.Seed(time.Now().UnixNano()) + version := fmt.Sprintf("%#v", projectinfo.Get()) + cmd := &cobra.Command{ + Use: "yurt-static-pod-upgrade", + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("yurt-static-pod-upgrade version: %#v\n", version) + + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + klog.Infof("FLAG: --%s=%q", flag.Name, flag.Value) + }) + + if err := upgrade.Validate(); err != nil { + klog.Fatalf("Fail to validate yurt static pod upgrade args, %v", err) + } + + c, err := upgrade.GetClient() + if err != nil { + klog.Fatalf("Fail to get kubernetes client, %v", err) + } + + ctrl, err := upgrade.New(c) + if err != nil { + klog.Fatal("Fail to create static-pod-upgrade controller, %v", err) + } + + if err := ctrl.Upgrade(); err != nil { + klog.Fatalf("Fail to upgrade static pod, %v", err) + } + klog.Info("Static pod upgrade Success") + }, + Version: version, + } + + addFlags(cmd) + + if err := viper.BindPFlags(cmd.Flags()); err != nil { + os.Exit(1) + } + + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} + +func addFlags(cmd *cobra.Command) { + cmd.Flags().String("kubeconfig", "", "The path to the kubeconfig file") + cmd.Flags().String("name", "", "The name of static pod which needs be upgraded") + cmd.Flags().String("namespace", "", "The namespace of static pod which needs be upgraded") + cmd.Flags().String("manifest", "", "The manifest file name of static pod which needs be upgraded") + cmd.Flags().String("hash", "", "The hash value of new static pod specification") + cmd.Flags().String("mode", "", "The upgrade mode which is used") +} diff --git a/hack/dockerfiles/release/Dockerfile.yurt-static-pod-upgrade b/hack/dockerfiles/release/Dockerfile.yurt-static-pod-upgrade new file mode 100644 index 00000000000..2c866cd2c72 --- /dev/null +++ b/hack/dockerfiles/release/Dockerfile.yurt-static-pod-upgrade @@ -0,0 +1,13 @@ +# multi-arch image building for yurt-static-pod-upgrade + +FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +ADD . /build +ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO +WORKDIR /build/ +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GIT_VERSION=${GIT_VERSION} make build WHAT=cmd/yurt-static-pod-upgrade + +FROM --platform=${TARGETPLATFORM} alpine:3.17 +ARG TARGETOS TARGETARCH MIRROR_REPO +RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \ + apk add ca-certificates bash libc6-compat && update-ca-certificates && rm /var/cache/apk/* +COPY --from=builder /build/_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-static-pod-upgrade /usr/local/bin/yurt-static-pod-upgrade diff --git a/hack/make-rules/build.sh b/hack/make-rules/build.sh index 1422dd5df7d..6b580c4c50c 100755 --- a/hack/make-rules/build.sh +++ b/hack/make-rules/build.sh @@ -25,6 +25,7 @@ readonly YURT_ALL_TARGETS=( yurt-node-servant yurthub yurt-manager + yurt-static-pod-upgrade ) # clean old binaries at GOOS and GOARCH diff --git a/pkg/static-pod-upgrade/upgrade.go b/pkg/static-pod-upgrade/upgrade.go new file mode 100644 index 00000000000..a2d99787c49 --- /dev/null +++ b/pkg/static-pod-upgrade/upgrade.go @@ -0,0 +1,254 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/spf13/viper" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" +) + +const ( + DefaultUpgradeDir = "openyurtio-upgrade" + defaultStaticPodRunningCheckTimeout = 2 * time.Minute + + // TODO: use static-pod-upgrade's constant value + OTA = "ota" + Auto = "auto" +) + +var ( + DefaultConfigmapPath = "/data" + DefaultManifestPath = "/etc/kubernetes/manifests" + + RequiredField = []string{"name", "namespace", "hash", "mode"} +) + +type UpgradeController struct { + client kubernetes.Interface + + // Name of static pod + name string + // Namespace of static pod + namespace string + // Manifest file name of static pod + manifest string + // The latest static pod hash + hash string + // Only support `OTA` and `Auto` + upgradeMode string + + // Manifest path of static pod, default `/etc/kubernetes/manifests/manifestName.yaml` + manifestPath string + // The backup manifest path, default `/etc/kubernetes/manifests/openyurtio-upgrade/manifestName.bak` + bakManifestPath string + // Default is `/data/podName` + configMapDataPath string + // The latest manifest path, default `/etc/kubernetes/manifests/openyurtio-upgrade/manifestName.upgrade` + upgradeManifestPath string +} + +func New(client kubernetes.Interface) (*UpgradeController, error) { + ctrl := &UpgradeController{ + client: client, + } + + ctrl.name = viper.GetString("name") + ctrl.namespace = viper.GetString("namespace") + ctrl.manifest = viper.GetString("manifest") + ctrl.hash = viper.GetString("hash") + ctrl.upgradeMode = viper.GetString("mode") + + // Manifest file name is optional. If not set, default is static pod name + if len(ctrl.manifest) == 0 { + ctrl.manifest = ctrl.name + } + + ctrl.manifestPath = filepath.Join(DefaultManifestPath, WithYamlSuffix(ctrl.manifest)) + ctrl.bakManifestPath = filepath.Join(DefaultManifestPath, DefaultUpgradeDir, WithBackupSuffix(ctrl.manifest)) + ctrl.configMapDataPath = filepath.Join(DefaultConfigmapPath, ctrl.manifest) + ctrl.upgradeManifestPath = filepath.Join(DefaultManifestPath, DefaultUpgradeDir, WithUpgradeSuffix(ctrl.manifest)) + + return ctrl, nil +} + +func (ctrl *UpgradeController) Upgrade() error { + // 1. Check the target static pod exist + if err := ctrl.checkStaticPodExist(); err != nil { + return err + } + klog.Info("Check static pod existence success") + + // 2. Check old manifest and the latest manifest exist + if err := ctrl.checkManifestFileExist(); err != nil { + return err + } + klog.Info("Check old manifest and new manifest files existence success") + + // 3. prepare the latest manifest + if err := ctrl.prepareManifest(); err != nil { + return err + } + klog.Info("Prepare upgrade manifest success") + + // 4. execute upgrade operations + switch ctrl.upgradeMode { + case Auto: + return ctrl.AutoUpgrade() + + case OTA: + return ctrl.OTAUpgrade() + } + + return nil +} + +func (ctrl *UpgradeController) AutoUpgrade() error { + // (1) Back up the old manifest in case of upgrade failure + if err := ctrl.backupManifest(); err != nil { + return err + } + klog.Info("Auto upgrade backupManifest success") + + // (2) Replace manifest and kubelet will upgrade the static pod automatically + if err := ctrl.replaceManifest(); err != nil { + return err + } + klog.Info("Auto upgrade replaceManifest success") + + // (3) Verify the new static pod is running + ok, err := ctrl.verify() + if err != nil { + return err + } + if !ok { + return fmt.Errorf("the latest static pod is not running") + } + klog.Info("Auto upgrade verify success") + + return nil +} + +// In ota mode, just need to set the latest upgrade manifest version +func (ctrl *UpgradeController) OTAUpgrade() error { + if err := ctrl.setLatestManifestHash(); err != nil { + return err + } + klog.Info("OTA upgrade set latest manifest hash success") + return nil +} + +// checkStaticPodExist check if the target static pod exist in cluster +func (ctrl *UpgradeController) checkStaticPodExist() error { + if errs := validation.IsDNS1123Subdomain(ctrl.name); len(errs) > 0 { + return fmt.Errorf("pod name %s is invalid: %v", ctrl.name, errs) + } + _, err := ctrl.client.CoreV1().Pods(ctrl.namespace).Get(context.TODO(), ctrl.name, metav1.GetOptions{}) + if err != nil { + return err + } + return nil +} + +// checkManifestFileExist check if the specified files exist +func (ctrl *UpgradeController) checkManifestFileExist() error { + check := []string{ctrl.manifestPath, ctrl.configMapDataPath} + for _, c := range check { + _, err := os.Stat(c) + if os.IsNotExist(err) { + return fmt.Errorf("manifest %s does not exist", c) + } + } + + return nil +} + +// prepareManifest move the latest manifest to DefaultUpgradeDir and set `.upgrade` suffix +// TODO: In kubernetes when mount configmap file to the sub path of hostpath mount, it will not be persistent +// TODO: Init configmap(latest manifest) to a default place and move it to `DefaultUpgradeDir` to save it persistent +func (ctrl *UpgradeController) prepareManifest() error { + // Make sure upgrade dir exist + if _, err := os.Stat(filepath.Join(DefaultManifestPath, DefaultUpgradeDir)); os.IsNotExist(err) { + if err = os.Mkdir(filepath.Join(DefaultManifestPath, DefaultUpgradeDir), 0755); err != nil { + return err + } + } + + return CopyFile(ctrl.configMapDataPath, ctrl.upgradeManifestPath) +} + +// backUpManifest backup the old manifest in order to roll back when errors occur +func (ctrl *UpgradeController) backupManifest() error { + return CopyFile(ctrl.manifestPath, ctrl.bakManifestPath) +} + +// replaceManifest replace old manifest with the latest one, it achieves static pod upgrade +func (ctrl *UpgradeController) replaceManifest() error { + return CopyFile(ctrl.upgradeManifestPath, ctrl.manifestPath) +} + +// verify make sure the latest static pod is running +// return false when the latest static pod failed or check status time out +func (ctrl *UpgradeController) verify() (bool, error) { + return WaitForPodRunning(ctrl.client, ctrl.name, ctrl.namespace, ctrl.hash, defaultStaticPodRunningCheckTimeout) +} + +// setLatestManifestHash set the latest manifest hash value to target static pod annotation +// TODO: In ota mode, it's hard for controller to check whether the latest manifest file has been issued to nodes +// TODO: Use annotation `openyurt.io/ota-manifest-version` to indicate the version of manifest issued to nodes +func (ctrl *UpgradeController) setLatestManifestHash() error { + pod, err := ctrl.client.CoreV1().Pods(ctrl.namespace).Get(context.TODO(), ctrl.name, metav1.GetOptions{}) + if err != nil { + return err + } + metav1.SetMetaDataAnnotation(&pod.ObjectMeta, OTALatestManifestAnnotation, ctrl.hash) + _, err = ctrl.client.CoreV1().Pods(ctrl.namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}) + return err +} + +// Validate check if all the required arguments are valid +func Validate() error { + for _, r := range RequiredField { + if v := viper.GetString(r); len(v) == 0 { + return fmt.Errorf("arg %s is empty", r) + } + } + return nil +} + +func GetClient() (kubernetes.Interface, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + c, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + return c, nil +} diff --git a/pkg/static-pod-upgrade/upgrade_test.go b/pkg/static-pod-upgrade/upgrade_test.go new file mode 100644 index 00000000000..7e6204c43ed --- /dev/null +++ b/pkg/static-pod-upgrade/upgrade_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/spf13/viper" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +func Test(t *testing.T) { + // Temporarily modify the manifest path in order to test + DefaultManifestPath = t.TempDir() + DefaultConfigmapPath = t.TempDir() + _, _ = os.Create(filepath.Join(DefaultManifestPath, WithYamlSuffix("nginxManifest"))) + _, _ = os.Create(filepath.Join(DefaultConfigmapPath, "nginxManifest")) + + viper.Set("name", "nginx-node") + viper.Set("namespace", "default") + viper.Set("manifest", "nginxManifest") + viper.Set("hash", "789c7f9f47") + + runningStaticPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-node", + Namespace: "default", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + modes := []string{"ota", "auto"} + + for _, mode := range modes { + /* + 1. Prepare the test environment + */ + if mode == "auto" { + runningStaticPod.Annotations = map[string]string{ + StaticPodHashAnnotation: "789c7f9f47", + } + } + c := fake.NewSimpleClientset(runningStaticPod) + // Add watch event for verify + watcher := watch.NewFake() + c.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(watcher, nil)) + go func() { + watcher.Add(runningStaticPod) + }() + + viper.Set("mode", mode) + + /* + 2. Test + */ + ctrl, err := New(c) + if err != nil { + t.Errorf("Fail to get upgrade controller, %v", err) + } + + if err := ctrl.Upgrade(); err != nil { + t.Errorf("Fail to upgrade, %v", err) + } + + /* + 3. Verify OTA upgrade mode + */ + if mode == "ota" { + ok, err := util.FileExists(ctrl.upgradeManifestPath) + if err != nil { + t.Errorf("Fail to check manifest existence for ota upgrade, %v", err) + } + if !ok { + t.Errorf("Manifest for ota upgrade does not exist") + } + + pod, err := ctrl.client.CoreV1().Pods(runningStaticPod.Namespace). + Get(context.TODO(), runningStaticPod.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Fail to get the running static pod, %v", err) + } + + if pod.Annotations[OTALatestManifestAnnotation] != "789c7f9f47" { + t.Errorf("Fail to verify hash annotation for ota upgrade, %v", err) + } + } + /* + 4. Verify Auto upgrade mode + */ + if mode == "auto" { + checkFiles := []string{ctrl.upgradeManifestPath, ctrl.bakManifestPath} + for _, file := range checkFiles { + ok, err := util.FileExists(file) + if err != nil { + t.Errorf("Fail to check %s manifest existence for auto upgrade, %v", file, err) + } + if !ok { + t.Errorf("Manifest %s for auto upgrade does not exist", file) + } + } + } + } +} diff --git a/pkg/static-pod-upgrade/util.go b/pkg/static-pod-upgrade/util.go new file mode 100644 index 00000000000..87a1aee8f00 --- /dev/null +++ b/pkg/static-pod-upgrade/util.go @@ -0,0 +1,123 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "fmt" + "io" + "os" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +const ( + YamlSuffix string = ".yaml" + BackupSuffix = ".bak" + UpgradeSuffix string = ".upgrade" + + StaticPodHashAnnotation = "openyurt.io/static-pod-hash" + OTALatestManifestAnnotation = "openyurt.io/ota-latest-version" +) + +func WithYamlSuffix(path string) string { + return path + YamlSuffix +} + +func WithBackupSuffix(path string) string { + return path + BackupSuffix +} + +func WithUpgradeSuffix(path string) string { + return path + UpgradeSuffix +} + +// CopyFile copy file content from src to dst, if destination file not exist, then create it +func CopyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return nil +} + +// WaitForPodRunning waits static pod to run +// Success: Static pod annotation `StaticPodHashAnnotation` value equals to function argument hash +// Failed: Receive PodFailed event +func WaitForPodRunning(c kubernetes.Interface, name, namespace, hash string, timeout time.Duration) (bool, error) { + klog.Infof("WaitForPodRuning name is %s, namespace is %s", name, namespace) + // Create a watcher to watch the pod's status + watcher, err := c.CoreV1().Pods(namespace).Watch(context.TODO(), metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return false, err + } + defer watcher.Stop() + + // Create a channel to receive updates from the watcher + ch := watcher.ResultChan() + + // Start a goroutine to monitor the pod's status + running := make(chan struct{}) + failed := make(chan struct{}) + + go func() { + for event := range ch { + obj, ok := event.Object.(*corev1.Pod) + if !ok { + continue + } + + h := obj.Annotations[StaticPodHashAnnotation] + + if obj.Status.Phase == corev1.PodRunning && h == hash { + close(running) + return + } + + if obj.Status.Phase == corev1.PodFailed { + close(failed) + return + } + } + }() + + // Wait for watch event to finish or the timeout to expire + select { + case <-running: + return true, nil + case <-failed: + return false, nil + case <-time.After(timeout): + return false, fmt.Errorf("timeout waiting for static pod %s/%s to be running", namespace, name) + } +}