diff --git a/Makefile1 b/Makefile1 deleted file mode 100644 index ea902915f40..00000000000 --- a/Makefile1 +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright 2020 The OpenYurt Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -KUBERNETESVERSION ?=v1.22 -TARGET_PLATFORMS ?= linux/amd64 -IMAGE_REPO ?= openyurt -IMAGE_TAG ?= $(shell git describe --abbrev=0 --tags) -GIT_COMMIT = $(shell git rev-parse HEAD) -ENABLE_AUTONOMY_TESTS ?=true -CRD_OPTIONS ?= "crd:crdVersions=v1" -BUILD_KUSTOMIZE ?= _output/manifest - -ifeq ($(shell git tag --points-at ${GIT_COMMIT}),) -GIT_VERSION=$(IMAGE_TAG)-$(shell echo ${GIT_COMMIT} | cut -c 1-7) -else -GIT_VERSION=$(IMAGE_TAG) -endif - -ifneq ($(IMAGE_TAG), $(shell git describe --abbrev=0 --tags)) -GIT_VERSION=$(IMAGE_TAG) -endif - -DOCKER_BUILD_ARGS = --build-arg GIT_VERSION=${GIT_VERSION} - -ifeq (${REGION}, cn) -DOCKER_BUILD_ARGS += --build-arg GOPROXY=https://goproxy.cn --build-arg MIRROR_REPO=mirrors.aliyun.com -endif - -ifneq (${http_proxy},) -DOCKER_BUILD_ARGS += --build-arg http_proxy='${http_proxy}' -endif - -ifneq (${https_proxy},) -DOCKER_BUILD_ARGS += --build-arg https_proxy='${https_proxy}' -endif - -.PHONY: clean all build test - -all: test build - -# Build binaries in the host environment -build: - bash hack/make-rules/build.sh $(WHAT) - -# Run test -test: - go test -v -short ./pkg/... ./cmd/... -coverprofile cover.out - go test -v -coverpkg=./pkg/yurttunnel/... -coverprofile=yurttunnel-cover.out ./test/integration/yurttunnel_test.go - -clean: - -rm -Rf _output - -# verify will verify the code. -verify: verify-mod verify-license - -# verify-license will check if license has been added to files. -verify-license: - hack/make-rules/check_license.sh - -# verify-mod will check if go.mod has beed tidied. -verify-mod: - hack/make-rules/verify_mod.sh - -# Start up OpenYurt cluster on local machine based on a Kind cluster -local-up-openyurt: - KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh - -# Build all OpenYurt components images and then start up OpenYurt cluster on local machine based on a Kind cluster -# And you can run the following command on different env by specify TARGET_PLATFORMS, default platform is linux/amd64 -# - on centos env: make docker-build-and-up-openyurt -# - on MACBook Pro M1: make docker-build-and-up-openyurt TARGET_PLATFORMS=linux/arm64 -docker-build-and-up-openyurt: docker-build - KUBERNETESVERSION=${KUBERNETESVERSION} YURT_VERSION=$(GIT_VERSION) bash hack/make-rules/local-up-openyurt.sh - -# Start up e2e tests for OpenYurt -# And you can run the following command on different env by specify TARGET_PLATFORMS, default platform is linux/amd64 -# - on centos env: make e2e-tests -# - on MACBook Pro M1: make e2e-tests TARGET_PLATFORMS=linux/arm64 -e2e-tests: - ENABLE_AUTONOMY_TESTS=${ENABLE_AUTONOMY_TESTS} TARGET_PLATFORMS=${TARGET_PLATFORMS} hack/make-rules/run-e2e-tests.sh - -install-golint: ## check golint if not exist install golint tools -ifeq (, $(shell which golangci-lint)) - @{ \ - set -e ;\ - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2 ;\ - } -GOLINT_BIN=$(shell go env GOPATH)/bin/golangci-lint -else -GOLINT_BIN=$(shell which golangci-lint) -endif - -lint: install-golint ## Run go lint against code. - $(GOLINT_BIN) run -v - -# Build the docker images only one arch(specify arch by TARGET_PLATFORMS env) -# otherwise the platform of host will be used. -# e.g. -# - build linux/amd64 docker images: -# $# make docker-build TARGET_PLATFORMS=linux/amd64 -# - build linux/arm64 docker images: -# $# make docker-build TARGET_PLATFORMS=linux/arm64 -# - build a specific image: -# $# make docker-build WHAT=yurthub -# - build with proxy, maybe useful for Chinese users -# $# REGION=cn make docker-build -docker-build: - TARGET_PLATFORMS=${TARGET_PLATFORMS} hack/make-rules/image_build.sh $(WHAT) - - -# Build and Push the docker images with multi-arch -docker-push: docker-push-yurthub docker-push-yurt-controller-manager docker-push-yurt-tunnel-server docker-push-yurt-tunnel-agent docker-push-node-servant docker-push-yurt-manager docker-push-yurt-static-pod-upgrade - - -docker-buildx-builder: - if ! docker buildx ls | grep -q container-builder; then\ - docker buildx create --name container-builder --use;\ - fi - # enable qemu for arm64 build - # https://github.com/docker/buildx/issues/464#issuecomment-741507760 - docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64 - docker run --rm --privileged tonistiigi/binfmt --install all - -docker-push-yurthub: docker-buildx-builder - docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurthub . -t ${IMAGE_REPO}/yurthub:${GIT_VERSION} - -docker-push-yurt-controller-manager: docker-buildx-builder - docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-controller-manager . -t ${IMAGE_REPO}/yurt-controller-manager:${GIT_VERSION} - -docker-push-yurt-tunnel-server: docker-buildx-builder - docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-tunnel-server . -t ${IMAGE_REPO}/yurt-tunnel-server:${GIT_VERSION} - -docker-push-yurt-tunnel-agent: docker-buildx-builder - docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent . -t ${IMAGE_REPO}/yurt-tunnel-agent:${GIT_VERSION} - -docker-push-node-servant: docker-buildx-builder - docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.node-servant . -t ${IMAGE_REPO}/node-servant:${GIT_VERSION} - -docker-push-yurt-manager: docker-buildx-builder - docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-manager . -t ${IMAGE_REPO}/yurt-manager:${GIT_VERSION} - -docker-push-yurt-static-pod-upgrade: docker-buildx-builder - docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-static-pod-upgrade . -t ${IMAGE_REPO}/yurt-static-pod-upgrade:${GIT_VERSION} - -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. -# hack/make-rule/generate_openapi.sh // TODO by kadisi - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./pkg/apis/..." - -manifests: generate ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - rm -rf $(BUILD_KUSTOMIZE) - $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=role webhook paths="./pkg/..." output:crd:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/crd output:rbac:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/rbac output:webhook:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/webhook - hack/make-rules/kustomize_to_chart.sh --crd $(BUILD_KUSTOMIZE)/auto_generate/crd --webhook $(BUILD_KUSTOMIZE)/auto_generate/webhook --rbac $(BUILD_KUSTOMIZE)/auto_generate/rbac --output $(BUILD_KUSTOMIZE)/kustomize --templateDir charts/openyurt/templates - - -# newcontroller -# .e.g -# make newcontroller GROUP=apps VERSION=v1beta1 KIND=example SHORTNAME=examples SCOPE=Namespaced -# make newcontroller GROUP=apps VERSION=v1beta1 KIND=example SHORTNAME=examples SCOPE=Cluster -newcontroller: - hack/make-rules/add_controller.sh --group $(GROUP) --version $(VERSION) --kind $(KIND) --shortname $(SHORTNAME) --scope $(SCOPE) - - -CONTROLLER_GEN = $(shell pwd)/bin/controller-gen -controller-gen: ## Download controller-gen locally if necessary. -ifeq ("$(shell $(CONTROLLER_GEN) --version 2> /dev/null)", "Version: v0.7.0") -else - rm -rf $(CONTROLLER_GEN) - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0) -endif - -# go-get-tool will 'go get' any package $2 and install it to $1. -PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) -define go-get-tool -@[ -f $(1) ] || { \ -set -e ;\ -TMP_DIR=$$(mktemp -d) ;\ -cd $$TMP_DIR ;\ -go mod init tmp ;\ -echo "Downloading $(2)" ;\ -GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ -rm -rf $$TMP_DIR ;\ -} -endef \ No newline at end of file diff --git a/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go b/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go index a32241bf738..3d5ce28cd44 100644 --- a/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go +++ b/cmd/yurt-static-pod-upgrade/static-pod-upgrade.go @@ -20,7 +20,6 @@ import ( "fmt" "math/rand" "os" - "reflect" "time" "github.com/spf13/cobra" @@ -31,16 +30,12 @@ import ( upgrade "github.com/openyurtio/openyurt/pkg/static-pod-upgrade" ) -type Config struct { - Name string - Namespace string - Manifest string - Hash string - Mode string -} +var ( + manifest string + mode string +) func main() { - cfg := &Config{} rand.Seed(time.Now().UnixNano()) version := fmt.Sprintf("%#v", projectinfo.Get()) cmd := &cobra.Command{ @@ -52,16 +47,11 @@ func main() { klog.Infof("FLAG: --%s=%q", flag.Name, flag.Value) }) - if err := cfg.validate(); err != nil { + if err := validate(); err != nil { klog.Fatalf("Fail to validate yurt static pod upgrade args, %v", err) } - c, err := upgrade.GetClient() - if err != nil { - klog.Fatalf("Fail to get kubernetes client, %v", err) - } - - ctrl, err := upgrade.New(c, cfg.Name, cfg.Namespace, cfg.Manifest, cfg.Hash, cfg.Mode) + ctrl, err := upgrade.New(manifest, mode) if err != nil { klog.Fatalf("Fail to create static-pod-upgrade controller, %v", err) } @@ -74,34 +64,26 @@ func main() { Version: version, } - cfg.addFlags(cmd) + addFlags(cmd) if err := cmd.Execute(); err != nil { os.Exit(1) } } -func (c *Config) addFlags(cmd *cobra.Command) { - cmd.Flags().StringVar(&c.Name, "name", "", "The name of static pod which needs be upgraded") - cmd.Flags().StringVar(&c.Namespace, "namespace", "", "The namespace of static pod which needs be upgraded") - cmd.Flags().StringVar(&c.Manifest, "manifest", "", "The manifest file name of static pod which needs be upgraded") - cmd.Flags().StringVar(&c.Hash, "hash", "", "The hash value of new static pod specification") - cmd.Flags().StringVar(&c.Mode, "mode", "", "The upgrade mode which is used") +func addFlags(cmd *cobra.Command) { + cmd.Flags().StringVar(&manifest, "manifest", "", "The manifest file name of static pod which needs be upgraded") + cmd.Flags().StringVar(&mode, "mode", "", "The upgrade mode which is used") } // Validate check if all the required arguments are valid -func (c *Config) validate() error { - v := reflect.ValueOf(*c) - - for i := 0; i < v.NumField(); i++ { - field := v.Field(i) - if field.Len() == 0 { - return fmt.Errorf("arg %s is empty", v.Type().Field(i).Name) - } +func validate() error { + if manifest == "" || mode == "" { + return fmt.Errorf("args can not be empty, manifest is %s, mode is %s", manifest, mode) } // TODO: use constant value of static-pod controller - if c.Mode != "auto" && c.Mode != "ota" { + if mode != "auto" && mode != "ota" { return fmt.Errorf("only support auto or ota upgrade mode") } diff --git a/pkg/static-pod-upgrade/upgrade.go b/pkg/static-pod-upgrade/upgrade.go index 246f61e8b7b..f9eebcbc5bd 100644 --- a/pkg/static-pod-upgrade/upgrade.go +++ b/pkg/static-pod-upgrade/upgrade.go @@ -17,22 +17,15 @@ limitations under the License. package upgrade import ( - "context" "fmt" "os" "path/filepath" - "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/klog/v2" ) const ( - DefaultUpgradeDir = "openyurtio-upgrade" - defaultStaticPodRunningCheckTimeout = 2 * time.Minute + DefaultUpgradeDir = "openyurtio-upgrade" // TODO: use constant value of static-pod controller OTA = "ota" @@ -45,16 +38,8 @@ var ( ) type Controller struct { - client kubernetes.Interface - - // Name of static pod - name string - // Namespace of static pod - namespace string // Manifest file name of static pod manifest string - // The latest static pod hash - hash string // Only support `OTA` and `Auto` upgradeMode string @@ -68,13 +53,9 @@ type Controller struct { upgradeManifestPath string } -func New(client kubernetes.Interface, name, namespace, manifest, hash, mode string) (*Controller, error) { +func New(manifest, mode string) (*Controller, error) { ctrl := &Controller{ - client: client, - name: name, - namespace: namespace, manifest: manifest, - hash: hash, upgradeMode: mode, } @@ -87,31 +68,22 @@ func New(client kubernetes.Interface, name, namespace, manifest, hash, mode stri } func (ctrl *Controller) Upgrade() error { - // 1. Check the target static pod exist - if err := ctrl.checkStaticPodExist(); err != nil { - return err - } - klog.Info("Check static pod existence success") - - // 2. Check old manifest and the latest manifest exist + // 1. Check old manifest and the latest manifest exist if err := ctrl.checkManifestFileExist(); err != nil { return err } klog.Info("Check old manifest and new manifest files existence success") - // 3. prepare the latest manifest + // 2. prepare the latest manifest if err := ctrl.prepareManifest(); err != nil { return err } klog.Info("Prepare upgrade manifest success") - // 4. execute upgrade operations + // 3. execute upgrade operations switch ctrl.upgradeMode { case Auto: return ctrl.AutoUpgrade() - - case OTA: - return ctrl.OTAUpgrade() } return nil @@ -130,37 +102,6 @@ func (ctrl *Controller) AutoUpgrade() error { } klog.Info("Auto upgrade replaceManifest success") - // (3) Verify the new static pod is running - ok, err := ctrl.verify() - if err != nil { - return err - } - if !ok { - return fmt.Errorf("the latest static pod is not running") - } - klog.Info("Auto upgrade verify success") - - return nil -} - -// In ota mode, just need to set the latest upgrade manifest version -func (ctrl *Controller) OTAUpgrade() error { - if err := ctrl.setLatestManifestHash(); err != nil { - return err - } - klog.Info("OTA upgrade set latest manifest hash success") - return nil -} - -// checkStaticPodExist check if the target static pod exist in cluster -func (ctrl *Controller) checkStaticPodExist() error { - if errs := validation.IsDNS1123Subdomain(ctrl.name); len(errs) > 0 { - return fmt.Errorf("pod name %s is invalid: %v", ctrl.name, errs) - } - _, err := ctrl.client.CoreV1().Pods(ctrl.namespace).Get(context.TODO(), ctrl.name, metav1.GetOptions{}) - if err != nil { - return err - } return nil } @@ -200,36 +141,3 @@ func (ctrl *Controller) backupManifest() error { func (ctrl *Controller) replaceManifest() error { return CopyFile(ctrl.upgradeManifestPath, ctrl.manifestPath) } - -// verify make sure the latest static pod is running -// return false when the latest static pod failed or check status time out -func (ctrl *Controller) verify() (bool, error) { - return WaitForPodRunning(ctrl.client, ctrl.name, ctrl.namespace, ctrl.hash, defaultStaticPodRunningCheckTimeout) -} - -// setLatestManifestHash set the latest manifest hash value to target static pod annotation -// TODO: In ota mode, it's hard for controller to check whether the latest manifest file has been issued to nodes -// TODO: Use annotation `openyurt.io/ota-manifest-version` to indicate the version of manifest issued to nodes -func (ctrl *Controller) setLatestManifestHash() error { - pod, err := ctrl.client.CoreV1().Pods(ctrl.namespace).Get(context.TODO(), ctrl.name, metav1.GetOptions{}) - if err != nil { - return err - } - metav1.SetMetaDataAnnotation(&pod.ObjectMeta, OTALatestManifestAnnotation, ctrl.hash) - _, err = ctrl.client.CoreV1().Pods(ctrl.namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}) - return err -} - -func GetClient() (kubernetes.Interface, error) { - config, err := rest.InClusterConfig() - if err != nil { - return nil, err - } - - c, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - - return c, nil -} diff --git a/pkg/static-pod-upgrade/upgrade_test.go b/pkg/static-pod-upgrade/upgrade_test.go index deb1567b5a4..6c6022ab5b5 100644 --- a/pkg/static-pod-upgrade/upgrade_test.go +++ b/pkg/static-pod-upgrade/upgrade_test.go @@ -17,7 +17,6 @@ limitations under the License. package upgrade import ( - "context" "os" "path/filepath" "testing" @@ -76,7 +75,7 @@ func Test(t *testing.T) { /* 2. Test */ - ctrl, err := New(c, TestPodName, metav1.NamespaceDefault, TestManifest, TestHashValue, mode) + ctrl, err := New(TestManifest, mode) if err != nil { t.Errorf("Fail to get upgrade controller, %v", err) } @@ -96,16 +95,6 @@ func Test(t *testing.T) { if !ok { t.Errorf("Manifest for ota upgrade does not exist") } - - pod, err := ctrl.client.CoreV1().Pods(runningStaticPod.Namespace). - Get(context.TODO(), runningStaticPod.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Fail to get the running static pod, %v", err) - } - - if pod.Annotations[OTALatestManifestAnnotation] != TestHashValue { - t.Errorf("Fail to verify hash annotation for ota upgrade, %v", err) - } } /* 4. Verify Auto upgrade mode