From c028c477e2f22b3c4f0e1a11558c185af5e95d7e Mon Sep 17 00:00:00 2001
From: rambohe <linbo.hlb@alibaba-inc.com>
Date: Tue, 21 Mar 2023 18:08:50 +0800
Subject: [PATCH 01/14] adjust ports for yurt-manager component to 10270~10279
 (#1325)

---
 test/e2e/cmd/init/constants/constants.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/e2e/cmd/init/constants/constants.go b/test/e2e/cmd/init/constants/constants.go
index 6642a122325..fc38c78f9b7 100644
--- a/test/e2e/cmd/init/constants/constants.go
+++ b/test/e2e/cmd/init/constants/constants.go
@@ -106,6 +106,9 @@ spec:
           image: {{.image}}
           imagePullPolicy: IfNotPresent
           name: yurt-manager
+          env:
+            - name: WEBHOOK_PORT
+              value: "10270"
           ports:
             - containerPort: 10270
               name: webhook-server

From 283d25dd5d995a0187c3a561b0ef1a8215778b69 Mon Sep 17 00:00:00 2001
From: rambohe <linbo.hlb@alibaba-inc.com>
Date: Tue, 21 Mar 2023 20:17:49 +0800
Subject: [PATCH 02/14] print logs of yurt-manager component in
 local-up-openyurt (#1326)

---
 test/e2e/cmd/init/converter.go            | 8 +++-----
 test/e2e/cmd/init/util/kubernetes/util.go | 7 ++++++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/test/e2e/cmd/init/converter.go b/test/e2e/cmd/init/converter.go
index 028f7177f66..80c00c7e92a 100644
--- a/test/e2e/cmd/init/converter.go
+++ b/test/e2e/cmd/init/converter.go
@@ -153,18 +153,16 @@ func (c *ClusterConverter) deployYurthub() error {
 			}
 
 			// print logs of yurt-manager
-			labelSelector, logErr := metav1.ParseToLabelSelector("app.kubernetes.io/name=yurt-manager")
-			if logErr != nil {
-				return err
-			}
 			podList, logErr := c.ClientSet.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{
-				LabelSelector: labelSelector.String(),
+				LabelSelector: labels.SelectorFromSet(map[string]string{"app.kubernetes.io/name": "yurt-manager"}).String(),
 			})
 			if logErr != nil {
+				klog.Errorf("failed to get yurt-manager pod, %v", logErr)
 				return err
 			}
 
 			if len(podList.Items) == 0 {
+				klog.Errorf("yurt-manager pod doesn't exist")
 				return err
 			}
 			if logErr = kubeutil.PrintPodLog(c.ClientSet, &podList.Items[0], os.Stderr); logErr != nil {
diff --git a/test/e2e/cmd/init/util/kubernetes/util.go b/test/e2e/cmd/init/util/kubernetes/util.go
index c2b638957fc..2c158c2b110 100644
--- a/test/e2e/cmd/init/util/kubernetes/util.go
+++ b/test/e2e/cmd/init/util/kubernetes/util.go
@@ -256,7 +256,12 @@ func RunJobAndCleanup(cliSet kubeclientset.Interface, job *batchv1.Job, timeout,
 func PrintPodLog(client kubeclientset.Interface, pod *corev1.Pod, w io.Writer) error {
 	klog.Infof("start to print logs for pod(%s/%s):", pod.Namespace, pod.Name)
 	req := client.CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.Name, &corev1.PodLogOptions{})
-	return kubectllogs.DefaultConsumeRequest(req, w)
+	if err := kubectllogs.DefaultConsumeRequest(req, w); err != nil {
+		klog.Errorf("failed to print logs for pod(%s/%s), %v", pod.Namespace, pod.Name, err)
+		return err
+	}
+
+	return nil
 }
 
 // RunServantJobs launch servant jobs on specified nodes and wait all jobs to finish.

From 7e41a07a59b1edf6e4b25df333d3b66048a76ae8 Mon Sep 17 00:00:00 2001
From: Yifei Zhang <chpzhangyifei@zju.edu.cn>
Date: Sat, 25 Mar 2023 22:52:50 +0800
Subject: [PATCH 03/14] enable to build binary with cn GOPROXY (#1331)

Signed-off-by: Congrool <chpzhangyifei@zju.edu.cn>
---
 Makefile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 0e38e74563d..105dd2683cf 100644
--- a/Makefile
+++ b/Makefile
@@ -24,6 +24,7 @@ GIT_COMMIT = $(shell git rev-parse HEAD)
 ENABLE_AUTONOMY_TESTS ?=true
 CRD_OPTIONS ?= "crd:crdVersions=v1"
 BUILD_KUSTOMIZE ?= _output/manifest
+GOPROXY ?= $(shell go env GOPROXY)
 
 ifeq ($(shell git tag --points-at ${GIT_COMMIT}),)
 GIT_VERSION=$(IMAGE_TAG)-$(shell echo ${GIT_COMMIT} | cut -c 1-7)
@@ -38,7 +39,8 @@ endif
 DOCKER_BUILD_ARGS = --build-arg GIT_VERSION=${GIT_VERSION}
 
 ifeq (${REGION}, cn)
-DOCKER_BUILD_ARGS += --build-arg GOPROXY=https://goproxy.cn --build-arg MIRROR_REPO=mirrors.aliyun.com
+GOPROXY=https://goproxy.cn
+DOCKER_BUILD_ARGS += --build-arg GOPROXY=$(GOPROXY) --build-arg MIRROR_REPO=mirrors.aliyun.com
 endif
 
 ifneq (${http_proxy},)
@@ -55,7 +57,7 @@ all: test build
 
 # Build binaries in the host environment
 build:
-	bash hack/make-rules/build.sh $(WHAT)
+	GOPROXY=$(GOPROXY) bash hack/make-rules/build.sh $(WHAT)
 
 # Run test
 test:

From 9d0f3165038fb58873d29ecbb4614d48e26256a9 Mon Sep 17 00:00:00 2001
From: AndyEWang <34705558+AndyEWang@users.noreply.github.com>
Date: Sun, 26 Mar 2023 10:16:21 +0800
Subject: [PATCH 04/14] feat: yurthub automatically recognize nftables or
 iptables (#1135) (#1327)

---
 hack/dockerfiles/build/Dockerfile.yurthub   |   2 +
 hack/dockerfiles/release/Dockerfile.yurthub |   2 +
 hack/iptables-wrapper-installer.sh          | 208 ++++++++++++++++++++
 3 files changed, 212 insertions(+)
 create mode 100644 hack/iptables-wrapper-installer.sh

diff --git a/hack/dockerfiles/build/Dockerfile.yurthub b/hack/dockerfiles/build/Dockerfile.yurthub
index b233165c2ca..688b5384518 100644
--- a/hack/dockerfiles/build/Dockerfile.yurthub
+++ b/hack/dockerfiles/build/Dockerfile.yurthub
@@ -5,4 +5,6 @@ ARG TARGETOS TARGETARCH MIRROR_REPO
 RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \
     apk add ca-certificates bash libc6-compat iptables ip6tables && update-ca-certificates && rm /var/cache/apk/*
 COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurthub /usr/local/bin/yurthub
+COPY hack/iptables-wrapper-installer.sh /
+RUN chmod +x /iptables-wrapper-installer.sh && /iptables-wrapper-installer.sh
 ENTRYPOINT ["/usr/local/bin/yurthub"]
\ No newline at end of file
diff --git a/hack/dockerfiles/release/Dockerfile.yurthub b/hack/dockerfiles/release/Dockerfile.yurthub
index 53b83d1e117..d6f8cfbe9d3 100644
--- a/hack/dockerfiles/release/Dockerfile.yurthub
+++ b/hack/dockerfiles/release/Dockerfile.yurthub
@@ -11,4 +11,6 @@ ARG TARGETOS TARGETARCH MIRROR_REPO
 RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \
     apk add ca-certificates bash libc6-compat iptables ip6tables && update-ca-certificates && rm /var/cache/apk/*
 COPY --from=builder /build/_output/local/bin/${TARGETOS}/${TARGETARCH}/yurthub /usr/local/bin/yurthub
+COPY hack/iptables-wrapper-installer.sh /
+RUN chmod +x /iptables-wrapper-installer.sh && /iptables-wrapper-installer.sh
 ENTRYPOINT ["/usr/local/bin/yurthub"]
\ No newline at end of file
diff --git a/hack/iptables-wrapper-installer.sh b/hack/iptables-wrapper-installer.sh
new file mode 100644
index 00000000000..e6eb7d8210d
--- /dev/null
+++ b/hack/iptables-wrapper-installer.sh
@@ -0,0 +1,208 @@
+#!/bin/sh
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Usage:
+#
+#   iptables-wrapper-installer.sh [--no-sanity-check]
+#
+# Installs a wrapper iptables script in a container that will figure out
+# whether iptables-legacy or iptables-nft is in use on the host and then
+# replaces itself with the correct underlying iptables version.
+#
+# Unless "--no-sanity-check" is passed, it will first verify that the
+# container already contains a suitable version of iptables.
+
+# NOTE: This can only use POSIX /bin/sh features; the build container
+# might not contain bash.
+
+set -eu
+
+# Find iptables binary location
+if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then
+    sbin="/usr/sbin"
+elif [ -d /sbin -a -e /sbin/iptables ]; then
+    sbin="/sbin"
+else
+    echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2
+    exit 1
+fi
+
+# Determine how the system selects between iptables-legacy and iptables-nft
+if [ -x /usr/sbin/alternatives ]; then
+    # Fedora/SUSE style alternatives
+    altstyle="fedora"
+elif [ -x /usr/sbin/update-alternatives ]; then
+    # Debian style alternatives
+    altstyle="debian"
+else
+    # No alternatives system
+    altstyle="none"
+fi
+
+if [ "${1:-}" != "--no-sanity-check" ]; then
+    # Ensure dependencies are installed
+    if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then
+        echo "ERROR: iptables-nft is not installed" 1>&2
+        exit 1
+    fi
+    if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then
+        echo "ERROR: iptables-legacy is not installed" 1>&2
+        exit 1
+    fi
+
+    case "${version}" in
+    *v1.8.[0123]\ *)
+        echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2
+        echo "       Upgrade to 1.8.4 or newer." 1>&2
+        exit 1
+        ;;
+    *)
+        # 1.8.4+ are OK
+        ;;
+    esac
+fi
+
+# Start creating the wrapper...
+rm -f "${sbin}/iptables-wrapper"
+cat > "${sbin}/iptables-wrapper" <<EOF
+#!/bin/sh
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This can only use POSIX /bin/sh features; the container image
+# might not contain bash.
+
+set -eu
+
+# In kubernetes 1.17 and later, kubelet will have created at least
+# one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
+# "KUBE-KUBELET-CANARY"), so check that first, against
+# iptables-nft, because we can check that more efficiently and
+# it's more common these days.
+nft_kubelet_rules=\$( (iptables-nft-save -t mangle || true; ip6tables-nft-save -t mangle || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
+if [ "\${nft_kubelet_rules}" -ne 0 ]; then
+    mode=nft
+else
+    # Check for kubernetes 1.17-or-later with iptables-legacy. We
+    # can't pass "-t mangle" to iptables-legacy-save because it would
+    # cause the kernel to create that table if it didn't already
+    # exist, which we don't want. So we have to grab all the rules
+    legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
+    if [ "\${legacy_kubelet_rules}" -ne 0 ]; then
+        mode=legacy
+    else
+        # With older kubernetes releases there may not be any _specific_
+        # rules we can look for, but we assume that some non-containerized process
+        # (possibly kubelet) will have created _some_ iptables rules.
+        num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
+        num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
+        if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
+            mode=legacy
+        else
+            mode=nft
+        fi
+    fi
+fi
+
+EOF
+
+# Write out the appropriate alternatives-selection commands
+case "${altstyle}" in
+    fedora)
+cat >> "${sbin}/iptables-wrapper" <<EOF
+# Update links to point to the selected binaries
+alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
+EOF
+    ;;
+
+    debian)
+cat >> "${sbin}/iptables-wrapper" <<EOF
+# Update links to point to the selected binaries
+update-alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
+update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1
+EOF
+    ;;
+
+    *)
+cat >> "${sbin}/iptables-wrapper" <<EOF
+# Update links to point to the selected binaries
+for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
+    rm -f "${sbin}/\${cmd}"
+    ln -s "${sbin}/xtables-\${mode}-multi" "${sbin}/\${cmd}"
+done 2>/dev/null || failed=1
+EOF
+    ;;
+esac
+
+# Write out the post-alternatives-selection error checking and final wrap-up
+cat >> "${sbin}/iptables-wrapper" <<EOF
+if [ "\${failed:-0}" = 1 ]; then
+    echo "Unable to redirect iptables binaries. (Are you running in an unprivileged pod?)" 1>&2
+    # fake it, though this will probably also fail if they aren't root
+    exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@"
+fi
+
+# Now re-exec the original command with the newly-selected alternative
+exec "\$0" "\$@"
+EOF
+chmod +x "${sbin}/iptables-wrapper"
+
+# Now back in the installer script, point the iptables binaries at our
+# wrapper
+case "${altstyle}" in
+    fedora)
+	alternatives \
+            --install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
+            --slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
+            --slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \
+            --slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \
+            --slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \
+            --slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper
+	;;
+
+    debian)
+	update-alternatives \
+            --install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
+            --slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
+            --slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper
+	update-alternatives \
+            --install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \
+            --slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \
+            --slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper
+	;;
+
+    *)
+	for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
+            rm -f "${sbin}/${cmd}"
+            ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}"
+	done
+	;;
+esac
+
+# Cleanup
+rm -f "$0"

From 4826713270a7c784ef3a60ce74f2e9e12c386b63 Mon Sep 17 00:00:00 2001
From: rambohe <linbo.hlb@alibaba-inc.com>
Date: Mon, 27 Mar 2023 17:03:50 +0800
Subject: [PATCH 05/14] improve prefilght into node-servant (#1332)

---
 pkg/node-servant/preflight-convert/preflight.go | 8 +++-----
 pkg/{ => node-servant}/preflight/checks.go      | 0
 pkg/{ => node-servant}/preflight/constants.go   | 0
 pkg/{ => node-servant}/preflight/interface.go   | 0
 4 files changed, 3 insertions(+), 5 deletions(-)
 rename pkg/{ => node-servant}/preflight/checks.go (100%)
 rename pkg/{ => node-servant}/preflight/constants.go (100%)
 rename pkg/{ => node-servant}/preflight/interface.go (100%)

diff --git a/pkg/node-servant/preflight-convert/preflight.go b/pkg/node-servant/preflight-convert/preflight.go
index 5f33dcf535a..d404e40fc5e 100644
--- a/pkg/node-servant/preflight-convert/preflight.go
+++ b/pkg/node-servant/preflight-convert/preflight.go
@@ -17,11 +17,9 @@ limitations under the License.
 package preflight_convert
 
 import (
-	"fmt"
-
 	"k8s.io/klog/v2"
 
-	"github.com/openyurtio/openyurt/pkg/preflight"
+	"github.com/openyurtio/openyurt/pkg/node-servant/preflight"
 )
 
 // ConvertPreflighter do the preflight-convert-convert job
@@ -42,8 +40,8 @@ func (n *ConvertPreflighter) Do() error {
 		return err
 	}
 
-	fmt.Println("[preflight-convert] Pulling images required for converting a Kubernetes cluster to an OpenYurt cluster")
-	fmt.Println("[preflight-convert] This might take a minute or two, depending on the speed of your internet connection")
+	klog.Infof("[preflight-convert] Pulling images required for converting a Kubernetes cluster to an OpenYurt cluster")
+	klog.Infof("[preflight-convert] This might take a minute or two, depending on the speed of your internet connection")
 	if err := preflight.RunPullImagesCheck(n, n.IgnorePreflightErrors); err != nil {
 		return err
 	}
diff --git a/pkg/preflight/checks.go b/pkg/node-servant/preflight/checks.go
similarity index 100%
rename from pkg/preflight/checks.go
rename to pkg/node-servant/preflight/checks.go
diff --git a/pkg/preflight/constants.go b/pkg/node-servant/preflight/constants.go
similarity index 100%
rename from pkg/preflight/constants.go
rename to pkg/node-servant/preflight/constants.go
diff --git a/pkg/preflight/interface.go b/pkg/node-servant/preflight/interface.go
similarity index 100%
rename from pkg/preflight/interface.go
rename to pkg/node-servant/preflight/interface.go

From 6c3cb34ab7d790ca941c2da2feb89900102d997d Mon Sep 17 00:00:00 2001
From: rambohe <linbo.hlb@alibaba-inc.com>
Date: Tue, 28 Mar 2023 10:47:50 +0800
Subject: [PATCH 06/14] support pass bootstrap-file to yurthub (#1333)

---
 cmd/yurthub/app/config/config.go       |  1 +
 cmd/yurthub/app/options/options.go     |  9 ++--
 pkg/yurthub/certificate/token/token.go | 61 +++++++++++++++++++++++++-
 pkg/yurthub/server/server.go           | 15 +++++++
 4 files changed, 81 insertions(+), 5 deletions(-)

diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go
index b95b50303f7..f6a8847b005 100644
--- a/cmd/yurthub/app/config/config.go
+++ b/cmd/yurthub/app/config/config.go
@@ -343,6 +343,7 @@ func createCertManager(options *options.YurtHubOptions, remoteServers []*url.URL
 		RootDir:                  options.RootDir,
 		NodeName:                 options.NodeName,
 		JoinToken:                options.JoinToken,
+		BootstrapFile:            options.BootstrapFile,
 		CaCertHashes:             options.CACertHashes,
 		YurtHubCertOrganizations: options.YurtHubCertOrganizations,
 		CertIPs:                  certIPs,
diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go
index db34ca60e01..32f90ed85d8 100644
--- a/cmd/yurthub/app/options/options.go
+++ b/cmd/yurthub/app/options/options.go
@@ -61,6 +61,7 @@ type YurtHubOptions struct {
 	HeartbeatIntervalSeconds  int
 	MaxRequestInFlight        int
 	JoinToken                 string
+	BootstrapFile             string
 	RootDir                   string
 	Version                   bool
 	EnableProfiling           bool
@@ -143,8 +144,8 @@ func (options *YurtHubOptions) Validate() error {
 		return fmt.Errorf("server-address is empty")
 	}
 
-	if len(options.JoinToken) == 0 {
-		return fmt.Errorf("bootstrap token is empty")
+	if len(options.JoinToken) == 0 && len(options.BootstrapFile) == 0 {
+		return fmt.Errorf("bootstrap token and bootstrap file are empty, one of them must be set")
 	}
 
 	if !util.IsSupportedLBMode(options.LBMode) {
@@ -183,7 +184,9 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&o.HeartbeatTimeoutSeconds, "heartbeat-timeout-seconds", o.HeartbeatTimeoutSeconds, " number of seconds after which the heartbeat times out.")
 	fs.IntVar(&o.HeartbeatIntervalSeconds, "heartbeat-interval-seconds", o.HeartbeatIntervalSeconds, " number of seconds for omitting one time heartbeat to remote server.")
 	fs.IntVar(&o.MaxRequestInFlight, "max-requests-in-flight", o.MaxRequestInFlight, "the maximum number of parallel requests.")
-	fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent when --cert-mgr-mode=hubself.")
+	fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent.")
+	fs.MarkDeprecated("join-token", "It is planned to be removed from OpenYurt in the version v1.5. Please use --bootstrap-file to bootstrap hub agent.")
+	fs.StringVar(&o.BootstrapFile, "bootstrap-file", o.BootstrapFile, "the bootstrap file for bootstrapping hub agent.")
 	fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).")
 	fs.BoolVar(&o.Version, "version", o.Version, "print the version information.")
 	fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling, "enable profiling via web interface host:port/debug/pprof/")
diff --git a/pkg/yurthub/certificate/token/token.go b/pkg/yurthub/certificate/token/token.go
index ca7bd075478..4920ca14dd3 100644
--- a/pkg/yurthub/certificate/token/token.go
+++ b/pkg/yurthub/certificate/token/token.go
@@ -68,6 +68,7 @@ type CertificateManagerConfiguration struct {
 	RootDir                  string
 	NodeName                 string
 	JoinToken                string
+	BootstrapFile            string
 	CaCertHashes             []string
 	YurtHubCertOrganizations []string
 	CertIPs                  []net.IP
@@ -86,6 +87,7 @@ type yurtHubCertManager struct {
 	hubRunDir                  string
 	hubName                    string
 	joinToken                  string
+	bootstrapFile              string
 	dialer                     *util.Dialer
 }
 
@@ -104,6 +106,7 @@ func NewYurtHubCertManager(cfg *CertificateManagerConfiguration) (hubCert.YurtCe
 		hubRunDir:     hubRunDir,
 		hubName:       projectinfo.GetHubName(),
 		joinToken:     cfg.JoinToken,
+		bootstrapFile: cfg.BootstrapFile,
 		caCertHashes:  cfg.CaCertHashes,
 		dialer:        util.NewDialer("hub certificate manager"),
 	}
@@ -150,10 +153,10 @@ func removeDirContents(dir string) error {
 }
 
 func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup(servers []*url.URL) {
-	if cfg, err := clientcmd.LoadFromFile(ycm.getBootstrapConfFile()); err == nil {
+	if cfg, err := clientcmd.LoadFromFile(ycm.GetHubConfFile()); err == nil {
 		cluster := kubeconfigutil.GetClusterFromKubeConfig(cfg)
 		if serverURL, err := url.Parse(cluster.Server); err != nil {
-			klog.Errorf("couldn't get server info from %s, %v", ycm.getBootstrapConfFile(), err)
+			klog.Errorf("couldn't get server info from %s, %v", ycm.GetHubConfFile(), err)
 		} else {
 			for i := range servers {
 				if servers[i].Host == serverURL.Host {
@@ -187,7 +190,58 @@ func (ycm *yurtHubCertManager) Start() {
 // if these files already exist, just reuse them.
 func (ycm *yurtHubCertManager) prepareConfigAndCaFile() error {
 	var tlsBootstrapCfg *clientcmdapi.Config
+	var hubKubeConfig *clientcmdapi.Config
+	var err error
+
+	// A bootstrap-file is prepared by yurtadm join command and configured as parameter for yurthub,
+	// yurthub only need to use it.
+	if len(ycm.bootstrapFile) != 0 {
+		// 1. load bootstrap config
+		if tlsBootstrapCfg, err = clientcmd.LoadFromFile(ycm.getBootstrapConfFile()); err != nil {
+			klog.Errorf("maybe hub agent restarted, failed to load bootstrap config file(%s), %v.", ycm.getBootstrapConfFile())
+		} else {
+			klog.V(2).Infof("%s file is configured, just use it", ycm.getBootstrapConfFile())
+		}
+
+		// 2. prepare kubeconfig file(/var/lib/yurthub/yurthub.conf) for yurthub
+		if exist, err := util.FileExists(ycm.GetHubConfFile()); err != nil {
+			return errors.Wrap(err, "couldn't stat hub kubeconfig file")
+		} else if exist {
+			klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetHubConfFile())
+			if hubKubeConfig, err = clientcmd.LoadFromFile(ycm.GetHubConfFile()); err != nil {
+				return errors.Wrapf(err, "couldn't load hub kubeconfig file(%s)", ycm.GetHubConfFile())
+			}
+		} else if tlsBootstrapCfg == nil {
+			return errors.Errorf("neither boostrap file(%s) nor kubeconfig file(%s) exist when hub agent started", ycm.bootstrapFile, ycm.GetHubConfFile())
+		} else {
+			// hub kubeconfig file doesn't exist, but bootstrap file is ready, so create hub.conf by bootstrap config
+			hubKubeConfig = createHubConfig(tlsBootstrapCfg, ycm.apiServerClientCertStore.CurrentPath())
+			if err = kubeconfigutil.WriteToDisk(ycm.GetHubConfFile(), hubKubeConfig); err != nil {
+				return errors.Wrapf(err, "couldn't save %s to disk", hubConfigFileName)
+			}
+		}
 
+		// 3. prepare ca.crt file(/var/lib/yurthub/pki/ca.crt) for yurthub
+		if exist, err := util.FileExists(ycm.GetCaFile()); err != nil {
+			return errors.Wrap(err, "couldn't stat ca.crt file")
+		} else if !exist {
+			cluster := kubeconfigutil.GetClusterFromKubeConfig(hubKubeConfig)
+			if cluster != nil {
+				if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil {
+					return errors.Wrap(err, "couldn't save the CA certificate to disk")
+				}
+			} else {
+				return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile())
+			}
+		} else {
+			klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile())
+		}
+		return nil
+	}
+
+	// in order to keep consistency with old version(with join token),
+	// if join token instead of bootstrap-file is set, we will use join token to create boostrap-hub.conf
+	// use join token to create bootstrap-hub.conf and will be removed in the future version
 	// 1. prepare bootstrap config file(/var/lib/yurthub/bootstrap-hub.conf) for yurthub
 	if exist, err := util.FileExists(ycm.getBootstrapConfFile()); err != nil {
 		return errors.Wrap(err, "couldn't stat bootstrap config file")
@@ -274,6 +328,9 @@ func (ycm *yurtHubCertManager) getPkiDir() string {
 
 // getBootstrapConfFile returns the path of yurthub bootstrap conf file
 func (ycm *yurtHubCertManager) getBootstrapConfFile() string {
+	if len(ycm.bootstrapFile) != 0 {
+		return ycm.bootstrapFile
+	}
 	return filepath.Join(ycm.hubRunDir, bootstrapConfigFileName)
 }
 
diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go
index d7b9b6e98ed..fead050d390 100644
--- a/pkg/yurthub/server/server.go
+++ b/pkg/yurthub/server/server.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
 	"github.com/openyurtio/openyurt/pkg/profile"
+	"github.com/openyurtio/openyurt/pkg/yurthub/certificate"
 	"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest"
 	ota "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate"
 	"github.com/openyurtio/openyurt/pkg/yurthub/util"
@@ -77,6 +78,7 @@ func registerHandlers(c *mux.Router, cfg *config.YurtHubConfiguration, rest *res
 
 	// register handler for health check
 	c.HandleFunc("/v1/healthz", healthz).Methods("GET")
+	c.Handle("/v1/readyz", readyz(cfg.CertManager)).Methods("GET")
 
 	// register handler for profile
 	if cfg.EnableProfiling {
@@ -97,3 +99,16 @@ func healthz(w http.ResponseWriter, _ *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	fmt.Fprintf(w, "OK")
 }
+
+// readyz is used for checking certificates are ready or not
+func readyz(certificateMgr certificate.YurtCertificateManager) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ready := certificateMgr.Ready()
+		if ready {
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprintf(w, "OK")
+		} else {
+			http.Error(w, "certificates are not ready", http.StatusInternalServerError)
+		}
+	})
+}

From 3ee1eff2dc7361497b8a699463dc611f3d28da61 Mon Sep 17 00:00:00 2001
From: gnunu <shaoqiang.chen@intel.com>
Date: Wed, 29 Mar 2023 09:53:51 +0800
Subject: [PATCH 07/14] add system proxy for docker run (#1335)

Signed-off-by: nunu <shaoqiang.chen@intel.com>
---
 hack/make-rules/image_build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hack/make-rules/image_build.sh b/hack/make-rules/image_build.sh
index 3c16d5ebc07..b069cfcbc0b 100755
--- a/hack/make-rules/image_build.sh
+++ b/hack/make-rules/image_build.sh
@@ -34,6 +34,7 @@ REGION=${REGION:-}
 IMAGE_REPO=${IMAGE_REPO:-"openyurt"}
 IMAGE_TAG=${IMAGE_TAG:-$(get_image_tag)}
 DOCKER_BUILD_ARGS=""
+DOCKER_EXTRA_ENVS=""
 BUILD_BASE_IMAGE="golang:1.18"
 BUILD_GOPROXY=$(go env GOPROXY)
 GOPROXY_CN="https://goproxy.cn"
@@ -46,10 +47,12 @@ fi
 
 if [[ ! -z ${http_proxy} ]]; then 
     DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg http_proxy=${http_proxy}"
+    DOCKER_EXTRA_ENVS="--env http_proxy=${http_proxy}"
 fi
 
 if [[ ! -z ${https_proxy} ]]; then
     DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg https_proxy=${https_proxy}"
+    DOCKER_EXTRA_ENVS="${DOCKER_EXTRA_ENVS=} --env https_proxy=${https_proxy}"
 fi
 
 if [[ ! -z ${TARGET_PLATFORMS} ]]; then
@@ -74,6 +77,7 @@ docker run \
     --env GOOS=${TARGETOS} \
     --env GOARCH=${TARGETARCH} \
     --env GOCACHE=/tmp/ \
+    ${DOCKER_EXTRA_ENVS} \
     --user $(id -u ${USER}):$(id -g ${USER}) \
     ${BUILD_BASE_IMAGE} \
     ./hack/make-rules/build.sh ${targets[@]}

From 65b65ef8a1dfa8ed2f686ff0b295dcddf263bcd2 Mon Sep 17 00:00:00 2001
From: rambohe <linbo.hlb@alibaba-inc.com>
Date: Wed, 29 Mar 2023 13:40:51 +0800
Subject: [PATCH 08/14] improve csr approver controller with injector (#1334)

---
 .../csrapprover/csrapprover_controller.go     | 63 +++++++++++--------
 1 file changed, 37 insertions(+), 26 deletions(-)

diff --git a/pkg/controller/csrapprover/csrapprover_controller.go b/pkg/controller/csrapprover/csrapprover_controller.go
index f7a9ce121ff..21e051ac2ce 100644
--- a/pkg/controller/csrapprover/csrapprover_controller.go
+++ b/pkg/controller/csrapprover/csrapprover_controller.go
@@ -28,10 +28,12 @@ import (
 	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -41,11 +43,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config"
-	registryClient "github.com/openyurtio/openyurt/pkg/client"
 	poolcoordinatorCert "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/cert"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	utilclient "github.com/openyurtio/openyurt/pkg/util/client"
-	utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery"
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
 )
@@ -56,7 +55,7 @@ func init() {
 
 var (
 	concurrentReconciles = 3
-	csrV1Kind            = certificatesv1.SchemeGroupVersion.WithKind("CertificateSigningRequest")
+	csrV1Resource        = certificatesv1.SchemeGroupVersion.WithResource("certificatesigningrequests")
 
 	yurtCsr                  = fmt.Sprintf("%s-csr", strings.TrimRightFunc(projectinfo.GetProjectPrefix(), func(c rune) bool { return c == '-' }))
 	yurtHubNodeCertOrgPrefix = "openyurt:tenant:"
@@ -105,8 +104,22 @@ type csrRecognizer struct {
 
 // Add creates a new CsrApprover Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error {
-	return add(mgr, newReconciler(c, mgr))
+func Add(_ *appconfig.CompletedConfig, mgr manager.Manager) error {
+	r := &ReconcileCsrApprover{}
+	// Create a new controller
+	c, err := controller.New(controllerName, mgr, controller.Options{
+		Reconciler: r, MaxConcurrentReconciles: concurrentReconciles,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Watch for csr changes
+	if r.csrV1Supported {
+		return c.Watch(&source.Kind{Type: &certificatesv1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{})
+	} else {
+		return c.Watch(&source.Kind{Type: &certificatesv1beta1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{})
+	}
 }
 
 var _ reconcile.Reconciler = &ReconcileCsrApprover{}
@@ -118,33 +131,31 @@ type ReconcileCsrApprover struct {
 	csrApproverClient kubernetes.Interface
 }
 
-// newReconciler returns a new reconcile.Reconciler
-func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler {
-	genericClient := registryClient.GetGenericClientWithName(controllerName)
+func (r *ReconcileCsrApprover) InjectClient(c client.Client) error {
+	r.Client = c
+	return nil
+}
 
-	return &ReconcileCsrApprover{
-		Client:            utilclient.NewClientFromManager(mgr, controllerName),
-		csrV1Supported:    utildiscovery.DiscoverGVK(csrV1Kind),
-		csrApproverClient: genericClient.KubeClient,
+func (r *ReconcileCsrApprover) InjectMapper(mapper meta.RESTMapper) error {
+	if gvk, err := mapper.KindFor(csrV1Resource); err != nil {
+		klog.Errorf("v1.CertificateSigningRequest is not supported, %v", err)
+		r.csrV1Supported = false
+	} else {
+		klog.Infof("%s is supported", gvk.String())
+		r.csrV1Supported = true
 	}
+
+	return nil
 }
 
-// add adds a new Controller to mgr with r as the reconcile.Reconciler
-func add(mgr manager.Manager, r reconcile.Reconciler) error {
-	// Create a new controller
-	c, err := controller.New(controllerName, mgr, controller.Options{
-		Reconciler: r, MaxConcurrentReconciles: concurrentReconciles,
-	})
+func (r *ReconcileCsrApprover) InjectConfig(cfg *rest.Config) error {
+	client, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
+		klog.Errorf("failed to create kube client, %v", err)
 		return err
 	}
-
-	// Watch for changes to CsrApprover
-	if utildiscovery.DiscoverGVK(csrV1Kind) {
-		return c.Watch(&source.Kind{Type: &certificatesv1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{})
-	} else {
-		return c.Watch(&source.Kind{Type: &certificatesv1beta1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{})
-	}
+	r.csrApproverClient = client
+	return nil
 }
 
 // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=get;list;watch

From 5b4cfd81db8d1e0baaf602edf7052bf8298fd431 Mon Sep 17 00:00:00 2001
From: Zhen Zhao <70508195+JameKeal@users.noreply.github.com>
Date: Wed, 29 Mar 2023 14:13:51 +0800
Subject: [PATCH 09/14] fix yurt-controller-manager clusterrole missing pod
 delete (#1338)

---
 charts/openyurt/templates/yurt-controller-manager.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/charts/openyurt/templates/yurt-controller-manager.yaml b/charts/openyurt/templates/yurt-controller-manager.yaml
index df521373d84..0540fc3e36d 100644
--- a/charts/openyurt/templates/yurt-controller-manager.yaml
+++ b/charts/openyurt/templates/yurt-controller-manager.yaml
@@ -29,6 +29,7 @@ rules:
       - list
       - update
       - watch
+      - delete
   - apiGroups:
       - ""
     resources:

From 0896832d67a2508749a4ccd2ba3fc48ce8543382 Mon Sep 17 00:00:00 2001
From: AndyEWang <34705558+AndyEWang@users.noreply.github.com>
Date: Wed, 29 Mar 2023 16:16:51 +0800
Subject: [PATCH 10/14] fix: yurt-controller-manager reboot cannot remove taint
 node.openyurt.io/unschedulable (#1233) (#1337)

---
 .../delegatelease/poolcoordinator_controller.go        | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go b/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go
index 27a3f050025..d6b224f2438 100644
--- a/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go
+++ b/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go
@@ -55,7 +55,8 @@ type Controller struct {
 	leaseLister     leaselisterv1.LeaseNamespaceLister
 	nodeUpdateQueue workqueue.Interface
 
-	ldc *utils.LeaseDelegatedCounter
+	ldc    *utils.LeaseDelegatedCounter
+	delLdc *utils.LeaseDelegatedCounter
 }
 
 func (c *Controller) onLeaseCreate(n interface{}) {
@@ -87,6 +88,7 @@ func NewController(kc client.Interface, informerFactory informers.SharedInformer
 		client:          kc,
 		nodeUpdateQueue: workqueue.NewNamed("poolcoordinator_node"),
 		ldc:             utils.NewLeaseDelegatedCounter(),
+		delLdc:          utils.NewLeaseDelegatedCounter(),
 	}
 
 	if informerFactory != nil {
@@ -152,6 +154,7 @@ func (c *Controller) doDeTaintNodeNotSchedulable(node *corev1.Node) *corev1.Node
 	taints := node.Spec.Taints
 	taints, deleted := utils.DeleteTaintsByKey(taints, constant.NodeNotSchedulableTaint)
 	if !deleted {
+		c.delLdc.Inc(node.Name)
 		klog.Infof("detaint %s: no key %s exists, nothing to do\n", node.Name, constant.NodeNotSchedulableTaint)
 		return node
 	}
@@ -162,6 +165,8 @@ func (c *Controller) doDeTaintNodeNotSchedulable(node *corev1.Node) *corev1.Node
 		nn, err = c.client.CoreV1().Nodes().Update(context.TODO(), nn, metav1.UpdateOptions{})
 		if err != nil {
 			klog.Error(err)
+		} else {
+			c.delLdc.Inc(node.Name)
 		}
 	}
 	return nn
@@ -186,9 +191,10 @@ func (c *Controller) syncHandler(key string) error {
 		c.ldc.Inc(nl.Name)
 		if c.ldc.Counter(nl.Name) >= constant.LeaseDelegationThreshold {
 			c.taintNodeNotSchedulable(nl.Name)
+			c.delLdc.Reset(nl.Name)
 		}
 	} else {
-		if c.ldc.Counter(nl.Name) >= constant.LeaseDelegationThreshold {
+		if c.delLdc.Counter(nl.Name) == 0 {
 			c.deTaintNodeNotSchedulable(nl.Name)
 		}
 		c.ldc.Reset(nl.Name)

From 384238f0eb904a3be6c8bc060f12e885986d0f9c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 29 Mar 2023 16:45:51 +0800
Subject: [PATCH 11/14] build(deps): bump google.golang.org/grpc from 1.40.0 to
 1.54.0 (#1328)

Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.40.0 to 1.54.0.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.40.0...v1.54.0)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 17 +++++-----
 go.sum | 98 ++++++++++++----------------------------------------------
 2 files changed, 28 insertions(+), 87 deletions(-)

diff --git a/go.mod b/go.mod
index ce9a8fc870a..df8b9b19361 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0
 	golang.org/x/sys v0.6.0
-	google.golang.org/grpc v1.40.0
+	google.golang.org/grpc v1.54.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gopkg.in/square/go-jose.v2 v2.6.0
 	k8s.io/api v0.22.3
@@ -64,7 +64,8 @@ require (
 )
 
 require (
-	cloud.google.com/go v0.81.0 // indirect
+	cloud.google.com/go/compute v1.15.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
@@ -72,7 +73,7 @@ require (
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
@@ -89,7 +90,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
@@ -131,15 +132,15 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.5.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/term v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
diff --git a/go.sum b/go.sum
index 5c28c9732fb..ca5676cc97b 100644
--- a/go.sum
+++ b/go.sum
@@ -13,18 +13,16 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE=
+cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
@@ -105,8 +103,9 @@ github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -114,9 +113,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
@@ -163,10 +160,8 @@ github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQm
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
@@ -258,9 +253,8 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -293,18 +287,16 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -312,10 +304,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
@@ -377,7 +365,6 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
@@ -670,8 +657,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE=
@@ -749,7 +734,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@@ -760,8 +744,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -802,33 +784,22 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
@@ -895,20 +866,13 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -927,8 +891,8 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -939,8 +903,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -999,12 +963,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
@@ -1031,11 +990,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1075,20 +1029,11 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1102,17 +1047,12 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
+google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

From 08bd281897af48ff9883c3a426302bbef7d5b389 Mon Sep 17 00:00:00 2001
From: Liang Deng <283304489@qq.com>
Date: Wed, 29 Mar 2023 20:09:51 +0800
Subject: [PATCH 12/14] feat: add yurtadm renew certificate command (#1314)

Signed-off-by: Liang Deng <283304489@qq.com>
---
 go.mod                                        |   1 +
 go.sum                                        |   6 +-
 pkg/node-servant/components/yurthub.go        |  44 ++++-
 pkg/yurtadm/cmd/cmd.go                        |   2 +
 pkg/yurtadm/cmd/join/phases/postcheck.go      |  49 +----
 pkg/yurtadm/cmd/join/phases/prepare.go        |  53 +-----
 .../cmd/renew/certificate/certificate.go      | 164 +++++++++++++++++
 pkg/yurtadm/cmd/renew/renew.go                |  67 +++++++
 pkg/yurtadm/constants/constants.go            |   7 +-
 pkg/yurtadm/util/error/error.go               | 114 ++++++++++++
 pkg/yurtadm/util/error/error_test.go          |  59 ++++++
 pkg/yurtadm/util/kubernetes/kubernetes.go     |  13 ++
 pkg/yurtadm/util/yurthub/yurthub.go           | 169 ++++++++++++++++++
 13 files changed, 652 insertions(+), 96 deletions(-)
 create mode 100644 pkg/yurtadm/cmd/renew/certificate/certificate.go
 create mode 100644 pkg/yurtadm/cmd/renew/renew.go
 create mode 100644 pkg/yurtadm/util/error/error.go
 create mode 100644 pkg/yurtadm/util/error/error_test.go
 create mode 100644 pkg/yurtadm/util/yurthub/yurthub.go

diff --git a/go.mod b/go.mod
index df8b9b19361..41b7e416f68 100644
--- a/go.mod
+++ b/go.mod
@@ -115,6 +115,7 @@ require (
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
 	go.opentelemetry.io/contrib v0.20.0 // indirect
diff --git a/go.sum b/go.sum
index ca5676cc97b..6626920d985 100644
--- a/go.sum
+++ b/go.sum
@@ -400,6 +400,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -508,6 +509,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -572,8 +574,9 @@ github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
 github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
@@ -705,6 +708,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go
index f363deda7d1..7114ee87a11 100644
--- a/pkg/node-servant/components/yurthub.go
+++ b/pkg/node-servant/components/yurthub.go
@@ -27,14 +27,16 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig"
 	"github.com/openyurtio/openyurt/pkg/util/templates"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
 	enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode"
-	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
 	"github.com/openyurtio/openyurt/pkg/yurthub/storage/disk"
 	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
@@ -42,6 +44,8 @@ import (
 const (
 	hubHealthzCheckFrequency = 10 * time.Second
 	fileMode                 = 0666
+	DefaultRootDir           = "/var/lib"
+	DefaultCaPath            = "/etc/kubernetes/pki/ca.crt"
 )
 
 type yurtHubOperator struct {
@@ -79,7 +83,7 @@ func (op *yurtHubOperator) Install() error {
 		"yurthubServerAddr":    constants.DefaultYurtHubServerAddr,
 		"kubernetesServerAddr": op.apiServerAddr,
 		"image":                op.yurthubImage,
-		"joinToken":            op.joinToken,
+		"bootstrapFile":        constants.YurtHubBootstrapConfig,
 		"workingMode":          string(op.workingMode),
 		"enableDummyIf":        strconv.FormatBool(op.enableDummyIf),
 		"enableNodePool":       strconv.FormatBool(op.enableNodePool),
@@ -88,7 +92,16 @@ func (op *yurtHubOperator) Install() error {
 		return err
 	}
 
-	// 1-2. create yurthub.yaml
+	// 1-2. create /var/lib/yurthub/bootstrap-hub.conf
+	if err := enutil.EnsureDir(constants.YurtHubWorkdir); err != nil {
+		return err
+	}
+	if err := setHubBootstrapConfig(op.apiServerAddr, op.joinToken); err != nil {
+		return err
+	}
+	klog.Infof("create the %s", constants.YurtHubBootstrapConfig)
+
+	// 1-3. create yurthub.yaml
 	podManifestPath := enutil.GetPodManifestPath()
 	if err := enutil.EnsureDir(podManifestPath); err != nil {
 		return err
@@ -151,7 +164,7 @@ func getYurthubYaml(podManifestPath string) string {
 }
 
 func getYurthubConf() string {
-	return filepath.Join(token.DefaultRootDir, projectinfo.GetHubName())
+	return filepath.Join(DefaultRootDir, projectinfo.GetHubName())
 }
 
 func getYurthubCacheDir() string {
@@ -221,3 +234,26 @@ func pingClusterHealthz(client *http.Client, addr string) (bool, error) {
 
 	return true, nil
 }
+
+func setHubBootstrapConfig(serverAddr string, joinToken string) error {
+	caData, err := os.ReadFile(DefaultCaPath)
+	if err != nil {
+		return err
+	}
+	tlsBootstrapCfg := kubeconfigutil.CreateWithToken(
+		serverAddr,
+		"openyurt-e2e-test",
+		"token-bootstrap-client",
+		caData,
+		joinToken,
+	)
+	content, err := clientcmd.Write(*tlsBootstrapCfg)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(constants.YurtHubBootstrapConfig, content, fileMode); err != nil {
+		return errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk")
+	}
+
+	return nil
+}
diff --git a/pkg/yurtadm/cmd/cmd.go b/pkg/yurtadm/cmd/cmd.go
index 2af8ffc80af..761310725a6 100644
--- a/pkg/yurtadm/cmd/cmd.go
+++ b/pkg/yurtadm/cmd/cmd.go
@@ -28,6 +28,7 @@ import (
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/docs"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/renew"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/reset"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/token"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/yurtinit"
@@ -50,6 +51,7 @@ func NewYurtadmCommand() *cobra.Command {
 	cmds.AddCommand(reset.NewCmdReset(os.Stdin, os.Stdout, os.Stderr))
 	cmds.AddCommand(token.NewCmdToken(os.Stdin, os.Stdout, os.Stderr))
 	cmds.AddCommand(docs.NewDocsCmd(cmds))
+	cmds.AddCommand(renew.NewCmdRenew(os.Stdin, os.Stdout, os.Stderr))
 
 	klog.InitFlags(nil)
 	// goflag.Parse()
diff --git a/pkg/yurtadm/cmd/join/phases/postcheck.go b/pkg/yurtadm/cmd/join/phases/postcheck.go
index aaa77e17dcc..a70cae1cb52 100644
--- a/pkg/yurtadm/cmd/join/phases/postcheck.go
+++ b/pkg/yurtadm/cmd/join/phases/postcheck.go
@@ -17,64 +17,31 @@ limitations under the License.
 package phases
 
 import (
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
-	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
-	"github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
 )
 
-// RunPostCheck executes the node health check process.
+// RunPostCheck executes the node health check and clean process.
 func RunPostCheck(data joindata.YurtJoinData) error {
 	klog.V(1).Infof("check kubelet status.")
-	if err := checkKubeletStatus(); err != nil {
+	if err := kubernetes.CheckKubeletStatus(); err != nil {
 		return err
 	}
 	klog.V(1).Infof("kubelet service is active")
 
 	klog.V(1).Infof("waiting hub agent ready.")
-	if err := checkYurthubHealthz(data); err != nil {
+	if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil {
 		return err
 	}
 	klog.V(1).Infof("hub agent is ready")
 
-	return nil
-}
-
-// checkKubeletStatus check if kubelet is healthy.
-func checkKubeletStatus() error {
-	initSystem, err := initsystem.GetInitSystem()
-	if err != nil {
+	if err := yurthub.CleanHubBootstrapConfig(); err != nil {
 		return err
 	}
-	if ok := initSystem.ServiceIsActive("kubelet"); !ok {
-		return fmt.Errorf("kubelet is not active. ")
-	}
-	return nil
-}
+	klog.V(1).Infof("clean yurthub bootstrap config file success")
 
-// checkYurthubHealthz check if YurtHub is healthy.
-func checkYurthubHealthz(joinData joindata.YurtJoinData) error {
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", joinData.YurtHubServer()), constants.ServerHealthzURLPath), nil)
-	if err != nil {
-		return err
-	}
-	client := &http.Client{}
-	return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) {
-		resp, err := client.Do(req)
-		if err != nil {
-			return false, nil
-		}
-		ok, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return false, nil
-		}
-		return string(ok) == "OK", nil
-	})
+	return nil
 }
diff --git a/pkg/yurtadm/cmd/join/phases/prepare.go b/pkg/yurtadm/cmd/join/phases/prepare.go
index acc46c4f2a1..6d21954b642 100644
--- a/pkg/yurtadm/cmd/join/phases/prepare.go
+++ b/pkg/yurtadm/cmd/join/phases/prepare.go
@@ -17,18 +17,16 @@ limitations under the License.
 package phases
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"k8s.io/klog/v2"
 
-	"github.com/openyurtio/openyurt/pkg/util/templates"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
 	yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/util/system"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
 )
 
 // RunPrepare executes the node initialization process.
@@ -66,58 +64,17 @@ func RunPrepare(data joindata.YurtJoinData) error {
 	if err := yurtadmutil.SetKubeletConfigForNode(); err != nil {
 		return err
 	}
-	if err := addYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil {
+	if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil {
 		return err
 	}
-	if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
+	if err := yurthub.AddYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil {
 		return err
 	}
-	if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil {
-		return err
-	}
-	return nil
-}
-
-// addYurthubStaticYaml generate YurtHub static yaml for worker node.
-func addYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) error {
-	klog.Info("[join-node] Adding edge hub static yaml")
-	if _, err := os.Stat(podManifestPath); err != nil {
-		if os.IsNotExist(err) {
-			err = os.MkdirAll(podManifestPath, os.ModePerm)
-			if err != nil {
-				return err
-			}
-		} else {
-			klog.Errorf("Describe dir %s fail: %v", podManifestPath, err)
-			return err
-		}
-	}
-
-	// There can be multiple master IP addresses
-	serverAddrs := strings.Split(data.ServerAddr(), ",")
-	for i := 0; i < len(serverAddrs); i++ {
-		serverAddrs[i] = fmt.Sprintf("https://%s", serverAddrs[i])
-	}
-
-	kubernetesServerAddrs := strings.Join(serverAddrs, ",")
-
-	ctx := map[string]string{
-		"kubernetesServerAddr": kubernetesServerAddrs,
-		"image":                data.YurtHubImage(),
-		"joinToken":            data.JoinToken(),
-		"workingMode":          data.NodeRegistration().WorkingMode,
-		"organizations":        data.NodeRegistration().Organizations,
-		"yurthubServerAddr":    data.YurtHubServer(),
-	}
-
-	yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx)
-	if err != nil {
+	if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
 		return err
 	}
-
-	if err := os.WriteFile(filepath.Join(podManifestPath, constants.YurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil {
+	if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil {
 		return err
 	}
-	klog.Info("[join-node] Add hub agent static yaml is ok")
 	return nil
 }
diff --git a/pkg/yurtadm/cmd/renew/certificate/certificate.go b/pkg/yurtadm/cmd/renew/certificate/certificate.go
new file mode 100644
index 00000000000..172f7d1face
--- /dev/null
+++ b/pkg/yurtadm/cmd/renew/certificate/certificate.go
@@ -0,0 +1,164 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package certificate
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/spf13/cobra"
+	flag "github.com/spf13/pflag"
+	"k8s.io/klog/v2"
+
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	yurtconstants "github.com/openyurtio/openyurt/pkg/yurtadm/constants"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
+)
+
+type certificateOptions struct {
+	token                    string
+	caCertHashes             []string
+	unsafeSkipCAVerification bool
+	serverAddr               string
+	yurthubServer            string
+}
+
+// newCertificateOptions returns a struct ready for being used for creating cmd renew flags.
+func newCertificateOptions() *certificateOptions {
+	return &certificateOptions{
+		caCertHashes:             make([]string, 0),
+		unsafeSkipCAVerification: true,
+		yurthubServer:            yurtconstants.DefaultYurtHubServerAddr,
+	}
+}
+
+// NewCmdCertificate returns "yurtadm renew certificate" command.
+func NewCmdCertificate() *cobra.Command {
+	o := newCertificateOptions()
+
+	certificateCmd := &cobra.Command{
+		Use:   "certificate",
+		Short: "Create bootstrap file for yurthub to update certificate",
+		RunE: func(certificateCmd *cobra.Command, args []string) error {
+			if err := o.validate(); err != nil {
+				klog.Fatalf("validate options: %v", err)
+			}
+
+			// Check if YurtHub's certificate is ready or not.
+			// No update if ready.
+			if ok := yurthub.CheckYurthubReadyzOnce(o.yurthubServer); ok {
+				klog.Infoln("The certificate is still valid and does not need to be renewed")
+				return nil
+			}
+
+			us, err := parseRemoteServers(o.serverAddr)
+			if err != nil {
+				return err
+			}
+
+			// 1.Create a temporary bootstrap file and set to /var/lib/yurthub directory.
+			if err := yurthub.SetHubBootstrapConfig(us[0].Host, o.token, o.caCertHashes); err != nil {
+				return err
+			}
+
+			// 2.Check if YurtHub's certificates is ready or not.
+			if err := yurthub.CheckYurthubReadyz(o.yurthubServer); err != nil {
+				return err
+			}
+
+			// 3.Delete temporary bootstrap file.
+			if err := yurthub.CleanHubBootstrapConfig(); err != nil {
+				return err
+			}
+
+			klog.Infoln("Certificate renewed successfully")
+
+			return nil
+		},
+	}
+
+	addCertificateConfigFlags(certificateCmd.Flags(), o)
+	return certificateCmd
+}
+
+func (options *certificateOptions) validate() error {
+	if len(options.serverAddr) == 0 {
+		return fmt.Errorf("server-address is empty")
+	}
+
+	if len(options.token) == 0 {
+		return fmt.Errorf("bootstrap token is empty")
+	}
+
+	if len(options.caCertHashes) == 0 && !options.unsafeSkipCAVerification {
+		return fmt.Errorf("set --discovery-token-unsafe-skip-ca-verification flag as true or pass CACertHashes to continue")
+	}
+
+	return nil
+}
+
+func parseRemoteServers(serverAddr string) ([]*url.URL, error) {
+	servers := strings.Split(serverAddr, ",")
+	us := make([]*url.URL, 0, len(servers))
+	remoteServers := make([]string, 0, len(servers))
+	for _, server := range servers {
+		u, err := url.Parse(server)
+		if err != nil {
+			klog.Errorf("failed to parse server address %s, %v", servers, err)
+			return us, err
+		}
+		if u.Scheme == "" {
+			u.Scheme = "https"
+		} else if u.Scheme != "https" {
+			return us, fmt.Errorf("only https scheme is supported for server address(%s)", serverAddr)
+		}
+		us = append(us, u)
+		remoteServers = append(remoteServers, u.String())
+	}
+
+	if len(us) < 1 {
+		return us, fmt.Errorf("no server address is set, can not connect remote server")
+	}
+	klog.Infof("%s would connect remote servers: %s", projectinfo.GetHubName(), strings.Join(remoteServers, ","))
+
+	return us, nil
+}
+
+// addCertificateConfigFlags adds certificate flags bound to the config to the specified flagset
+func addCertificateConfigFlags(flagSet *flag.FlagSet, certificateOptions *certificateOptions) {
+	flagSet.StringVar(
+		&certificateOptions.token, yurtconstants.TokenStr, certificateOptions.token,
+		"Use this token for bootstrapping yurthub.",
+	)
+	flagSet.StringSliceVar(
+		&certificateOptions.caCertHashes, yurtconstants.TokenDiscoveryCAHash, certificateOptions.caCertHashes,
+		"For token-based discovery, validate that the root CA public key matches this hash (format: \"<type>:<value>\").",
+	)
+	flagSet.BoolVar(
+		&certificateOptions.unsafeSkipCAVerification, yurtconstants.TokenDiscoverySkipCAHash, certificateOptions.unsafeSkipCAVerification,
+		"For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.",
+	)
+	flagSet.StringVar(
+		&certificateOptions.serverAddr, yurtconstants.ServerAddr, certificateOptions.serverAddr,
+		"The address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"",
+	)
+	flagSet.StringVar(
+		&certificateOptions.yurthubServer, yurtconstants.YurtHubServerAddr, certificateOptions.yurthubServer,
+		"Sets the address for yurthub server addr",
+	)
+}
diff --git a/pkg/yurtadm/cmd/renew/renew.go b/pkg/yurtadm/cmd/renew/renew.go
new file mode 100644
index 00000000000..8d2cd9ba4c1
--- /dev/null
+++ b/pkg/yurtadm/cmd/renew/renew.go
@@ -0,0 +1,67 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package renew
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/renew/certificate"
+	util "github.com/openyurtio/openyurt/pkg/yurtadm/util/error"
+)
+
+// NewCmdRenew returns "yurtadm renew" command.
+func NewCmdRenew(in io.Reader, out io.Writer, outErr io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "renew",
+		Short: "Renew something like certificate",
+		// Without this callback, if a user runs just the "renew"
+		// command without a subcommand, or with an invalid subcommand,
+		// cobra will print usage information, but still exit cleanly.
+		// We want to return an error code in these cases so that the
+		// user knows that their command was invalid.
+		Run: subCmdRun(),
+	}
+
+	cmd.AddCommand(certificate.NewCmdCertificate())
+	return cmd
+}
+
+// subCmdRun returns a function that handles a case where a subcommand must be specified
+// Without this callback, if a user runs just the command without a subcommand,
+// or with an invalid subcommand, cobra will print usage information, but still exit cleanly.
+func subCmdRun() func(c *cobra.Command, args []string) {
+	return func(c *cobra.Command, args []string) {
+		if len(args) > 0 {
+			util.CheckErr(usageErrorf(c, "invalid subcommand %q", strings.Join(args, " ")))
+		}
+		err := c.Help()
+		if err != nil {
+			return
+		}
+		util.CheckErr(util.ErrExit)
+	}
+}
+
+func usageErrorf(c *cobra.Command, format string, args ...interface{}) error {
+	msg := fmt.Sprintf(format, args...)
+	return errors.Errorf("%s\nSee '%s -h' for help and examples", msg, c.CommandPath())
+}
diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go
index 6aac12e21f4..1b73f798229 100644
--- a/pkg/yurtadm/constants/constants.go
+++ b/pkg/yurtadm/constants/constants.go
@@ -23,10 +23,10 @@ const (
 	KubeletConfigureDir      = "/etc/kubernetes"
 	KubeletWorkdir           = "/var/lib/kubelet"
 	YurtHubWorkdir           = "/var/lib/yurthub"
+	YurtHubBootstrapConfig   = "/var/lib/yurthub/bootstrap-hub.conf"
 	OpenyurtDir              = "/var/lib/openyurt"
 	YurttunnelAgentWorkdir   = "/var/lib/yurttunnel-agent"
 	YurttunnelServerWorkdir  = "/var/lib/yurttunnel-server"
-	KubeConfigPath           = "/etc/kubernetes/kubelet.conf"
 	KubeCniDir               = "/opt/cni/bin"
 	KubeCniVersion           = "v0.8.0"
 	KubeletServiceFilepath   = "/etc/systemd/system/kubelet.service"
@@ -97,11 +97,14 @@ const (
 	YurtHubImage = "yurthub-image"
 	// YurtHubServerAddr flag set the address of yurthub server (not proxy server!)
 	YurtHubServerAddr = "yurthub-server-addr"
+	// ServerAddr flag set the address of kubernetes kube-apiserver
+	ServerAddr = "server-addr"
 	// ReuseCNIBin flag sets whether to reuse local CNI binaries or not.
 	ReuseCNIBin = "reuse-cni-bin"
 
 	ServerHealthzServer          = "127.0.0.1:10267"
 	ServerHealthzURLPath         = "/v1/healthz"
+	ServerReadyzURLPath          = "/v1/readyz"
 	DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt"
 	DefaultOpenYurtVersion       = "latest"
 	Yurthub                      = "yurthub"
@@ -219,7 +222,7 @@ spec:
     - --bind-address={{.yurthubServerAddr}}
     - --server-addr={{.kubernetesServerAddr}}
     - --node-name=$(NODE_NAME)
-    - --join-token={{.joinToken}}
+    - --bootstrap-file={{.bootstrapFile}}
     - --working-mode={{.workingMode}}
       {{if .enableDummyIf }}
     - --enable-dummy-if={{.enableDummyIf}}
diff --git a/pkg/yurtadm/util/error/error.go b/pkg/yurtadm/util/error/error.go
new file mode 100644
index 00000000000..dc0aa7a55f8
--- /dev/null
+++ b/pkg/yurtadm/util/error/error.go
@@ -0,0 +1,114 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	errorsutil "k8s.io/apimachinery/pkg/util/errors"
+)
+
+const (
+	// DefaultErrorExitCode defines exit the code for failed action generally
+	DefaultErrorExitCode = 1
+	// PreFlightExitCode defines exit the code for preflight checks
+	PreFlightExitCode = 2
+	// ValidationExitCode defines the exit code validation checks
+	ValidationExitCode = 3
+)
+
+var (
+	ErrInvalidSubCommandMsg = "invalid subcommand"
+	ErrExit                 = errors.New("exit")
+)
+
+// fatal prints the message if set and then exits.
+func fatal(msg string, code int) {
+	if len(msg) > 0 {
+		// add newline if needed
+		if !strings.HasSuffix(msg, "\n") {
+			msg += "\n"
+		}
+
+		fmt.Fprint(os.Stderr, msg)
+	}
+	os.Exit(code)
+}
+
+// CheckErr prints a user friendly error to STDERR and exits with a non-zero
+// exit code. Unrecognized errors will be printed with an "error: " prefix.
+//
+// This method is generic to the command in use and may be used by non-Kubectl
+// commands.
+func CheckErr(err error) {
+	checkErr(err, fatal)
+}
+
+// preflightError allows us to know if the error is a preflight error or not
+// defining the interface here avoids an import cycle of pulling in preflight into the util package
+type preflightError interface {
+	Preflight() bool
+}
+
+// checkErr formats a given error as a string and calls the passed handleErr
+// func with that string and an exit code.
+func checkErr(err error, handleErr func(string, int)) {
+
+	var msg string
+	if err != nil {
+		msg = fmt.Sprintf("%s\nTo see the stack trace of this error execute with --v=5 or higher", err.Error())
+		// check if the verbosity level in klog is high enough and print a stack trace.
+		f := flag.CommandLine.Lookup("v")
+		if f != nil {
+			// assume that the "v" flag contains a parseable Int32 as per klog's "Level" type alias,
+			// thus no error from ParseInt is handled here.
+			if v, e := strconv.ParseInt(f.Value.String(), 10, 32); e == nil {
+				// https://git.k8s.io/community/contributors/devel/sig-instrumentation/logging.md
+				// klog.V(5) - Trace level verbosity
+				if v > 4 {
+					msg = fmt.Sprintf("%+v", err)
+				}
+			}
+		}
+	}
+
+	if err == nil {
+		return
+	}
+	switch {
+	case err == ErrExit:
+		handleErr("", DefaultErrorExitCode)
+	case strings.Contains(err.Error(), ErrInvalidSubCommandMsg):
+		handleErr(err.Error(), DefaultErrorExitCode)
+	default:
+		switch err.(type) {
+		case preflightError:
+			handleErr(msg, PreFlightExitCode)
+		case errorsutil.Aggregate:
+			handleErr(msg, ValidationExitCode)
+
+		default:
+			handleErr(msg, DefaultErrorExitCode)
+		}
+	}
+}
diff --git a/pkg/yurtadm/util/error/error_test.go b/pkg/yurtadm/util/error/error_test.go
new file mode 100644
index 00000000000..3c53a302c43
--- /dev/null
+++ b/pkg/yurtadm/util/error/error_test.go
@@ -0,0 +1,59 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+type pferror struct{}
+
+func (p *pferror) Preflight() bool { return true }
+func (p *pferror) Error() string   { return "" }
+func TestCheckErr(t *testing.T) {
+	var codeReturned int
+	errHandle := func(err string, code int) {
+		codeReturned = code
+	}
+
+	var tests = []struct {
+		name     string
+		e        error
+		expected int
+	}{
+		{"error is nil", nil, 0},
+		{"empty error", errors.New(""), DefaultErrorExitCode},
+		{"preflight error", &pferror{}, PreFlightExitCode},
+	}
+
+	for _, rt := range tests {
+		t.Run(rt.name, func(t *testing.T) {
+			codeReturned = 0
+			checkErr(rt.e, errHandle)
+			if codeReturned != rt.expected {
+				t.Errorf(
+					"failed checkErr:\n\texpected: %d\n\t  actual: %d",
+					rt.expected,
+					codeReturned,
+				)
+			}
+		})
+	}
+}
diff --git a/pkg/yurtadm/util/kubernetes/kubernetes.go b/pkg/yurtadm/util/kubernetes/kubernetes.go
index d9f26646aec..1ef2ab89f5c 100644
--- a/pkg/yurtadm/util/kubernetes/kubernetes.go
+++ b/pkg/yurtadm/util/kubernetes/kubernetes.go
@@ -47,6 +47,7 @@ import (
 	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/util"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem"
 )
 
 const (
@@ -475,3 +476,15 @@ func RetrieveBootstrapConfig(data joindata.YurtJoinData) (*clientcmdapi.Config,
 		data.JoinToken(),
 	), nil
 }
+
+// CheckKubeletStatus check if kubelet is healthy.
+func CheckKubeletStatus() error {
+	initSystem, err := initsystem.GetInitSystem()
+	if err != nil {
+		return err
+	}
+	if ok := initSystem.ServiceIsActive("kubelet"); !ok {
+		return fmt.Errorf("kubelet is not active. ")
+	}
+	return nil
+}
diff --git a/pkg/yurtadm/util/yurthub/yurthub.go b/pkg/yurtadm/util/yurthub/yurthub.go
new file mode 100644
index 00000000000..27539ce9ed8
--- /dev/null
+++ b/pkg/yurtadm/util/yurthub/yurthub.go
@@ -0,0 +1,169 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package yurthub
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog/v2"
+
+	kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig"
+	"github.com/openyurtio/openyurt/pkg/util/templates"
+	"github.com/openyurtio/openyurt/pkg/util/token"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
+)
+
+// AddYurthubStaticYaml generate YurtHub static yaml for worker node.
+func AddYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) error {
+	klog.Info("[join-node] Adding edge hub static yaml")
+	if _, err := os.Stat(podManifestPath); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(podManifestPath, os.ModePerm)
+			if err != nil {
+				return err
+			}
+		} else {
+			klog.Errorf("Describe dir %s fail: %v", podManifestPath, err)
+			return err
+		}
+	}
+
+	// There can be multiple master IP addresses
+	serverAddrs := strings.Split(data.ServerAddr(), ",")
+	for i := 0; i < len(serverAddrs); i++ {
+		serverAddrs[i] = fmt.Sprintf("https://%s", serverAddrs[i])
+	}
+
+	kubernetesServerAddrs := strings.Join(serverAddrs, ",")
+
+	ctx := map[string]string{
+		"kubernetesServerAddr": kubernetesServerAddrs,
+		"image":                data.YurtHubImage(),
+		"bootstrapFile":        constants.YurtHubBootstrapConfig,
+		"workingMode":          data.NodeRegistration().WorkingMode,
+		"organizations":        data.NodeRegistration().Organizations,
+		"yurthubServerAddr":    data.YurtHubServer(),
+	}
+
+	yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(filepath.Join(podManifestPath, constants.YurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil {
+		return err
+	}
+	klog.Info("[join-node] Add hub agent static yaml is ok")
+	return nil
+}
+
+func SetHubBootstrapConfig(serverAddr string, joinToken string, caCertHashes []string) error {
+	if cfg, err := token.RetrieveValidatedConfigInfo(nil, &token.BootstrapData{
+		ServerAddr:   serverAddr,
+		JoinToken:    joinToken,
+		CaCertHashes: caCertHashes,
+	}); err != nil {
+		return errors.Wrap(err, "couldn't retrieve bootstrap config info")
+	} else {
+		clusterInfo := kubeconfigutil.GetClusterFromKubeConfig(cfg)
+		tlsBootstrapCfg := kubeconfigutil.CreateWithToken(
+			fmt.Sprintf("https://%s", serverAddr),
+			"kubernetes",
+			"token-bootstrap-client",
+			clusterInfo.CertificateAuthorityData,
+			joinToken,
+		)
+		if err = kubeconfigutil.WriteToDisk(constants.YurtHubBootstrapConfig, tlsBootstrapCfg); err != nil {
+			return errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk")
+		}
+	}
+
+	return nil
+}
+
+// CheckYurthubHealthz check if YurtHub is healthy.
+func CheckYurthubHealthz(yurthubServer string) error {
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerHealthzURLPath), nil)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{}
+	return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) {
+		resp, err := client.Do(req)
+		if err != nil {
+			return false, nil
+		}
+		ok, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return false, nil
+		}
+		return string(ok) == "OK", nil
+	})
+}
+
+// CheckYurthubReadyz check if YurtHub's certificates are ready or not
+func CheckYurthubReadyz(yurthubServer string) error {
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerReadyzURLPath), nil)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{}
+	return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) {
+		resp, err := client.Do(req)
+		if err != nil {
+			return false, nil
+		}
+		ok, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return false, nil
+		}
+		return string(ok) == "OK", nil
+	})
+}
+
+func CheckYurthubReadyzOnce(yurthubServer string) bool {
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerReadyzURLPath), nil)
+	if err != nil {
+		return false
+	}
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	ok, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return false
+	}
+	return string(ok) == "OK"
+}
+
+func CleanHubBootstrapConfig() error {
+	if err := os.RemoveAll(constants.YurtHubBootstrapConfig); err != nil {
+		klog.Warningf("Clean file %s fail: %v, please clean it manually.", constants.YurtHubBootstrapConfig, err)
+	}
+	return nil
+}

From 75719ae0926cc5f23ef194a9fa83851974970a1e Mon Sep 17 00:00:00 2001
From: Zhen Zhao <70508195+JameKeal@users.noreply.github.com>
Date: Wed, 29 Mar 2023 20:14:51 +0800
Subject: [PATCH 13/14] fix daemonset pod updater pointer error (#1340)

---
 pkg/controller/daemonpodupdater/util.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/controller/daemonpodupdater/util.go b/pkg/controller/daemonpodupdater/util.go
index 75b08db65a4..0daa6fc58b6 100644
--- a/pkg/controller/daemonpodupdater/util.go
+++ b/pkg/controller/daemonpodupdater/util.go
@@ -73,7 +73,7 @@ func IsDaemonsetPodLatest(ds *appsv1.DaemonSet, pod *corev1.Pod) bool {
 	klog.V(5).Infof("daemonset %v revision hash is %v", ds.Name, hash)
 	klog.V(5).Infof("daemonset %v generation is %v", ds.Name, generation)
 
-	templateMatches := generation != nil && pod.Labels[extensions.DaemonSetTemplateGenerationKey] == fmt.Sprint(generation)
+	templateMatches := generation != nil && pod.Labels[extensions.DaemonSetTemplateGenerationKey] == fmt.Sprint(*generation)
 	hashMatches := len(hash) > 0 && pod.Labels[extensions.DefaultDaemonSetUniqueLabelKey] == hash
 	return hashMatches || templateMatches
 }

From eafe3c067bbd44891bae199372e9150a4bd703c4 Mon Sep 17 00:00:00 2001
From: lu chen <1813927768@qq.com>
Date: Thu, 30 Mar 2023 10:39:51 +0800
Subject: [PATCH 14/14] migrate controller poolcoordinatorcert (#1336)

---
 .../yurt-manager-auto-generated.yaml          |  19 +++
 charts/openyurt/templates/yurt-manager.yaml   |   4 +-
 charts/openyurt/values.yaml                   |  13 +-
 .../app/controllermanager.go                  |   1 -
 cmd/yurt-controller-manager/app/core.go       |  10 --
 .../apps/v1alpha1/zz_generated.deepcopy.go    |   2 +-
 .../apps/v1beta1/zz_generated.deepcopy.go     |   2 +-
 .../raven/v1alpha1/zz_generated.deepcopy.go   |   2 +-
 pkg/controller/add_poolcoordinatorcert.go     |  30 ++++
 .../poolcoordinator/cert/certificate.go       |  16 +-
 .../poolcoordinator/cert/certificate_test.go  |   2 +-
 ...r.go => poolcoordinatorcert_controller.go} | 151 +++++++++---------
 ...=> poolcoordinatorcert_controller_test.go} |   2 +-
 pkg/controller/poolcoordinator/cert/secret.go |   6 +-
 .../poolcoordinator/cert/secret_test.go       |   2 +-
 pkg/controller/poolcoordinator/cert/util.go   |   6 +-
 .../poolcoordinator/cert/util_test.go         |   2 +-
 17 files changed, 152 insertions(+), 118 deletions(-)
 create mode 100644 pkg/controller/add_poolcoordinatorcert.go
 rename pkg/controller/poolcoordinator/cert/{poolcoordinator_cert_manager.go => poolcoordinatorcert_controller.go} (76%)
 rename pkg/controller/poolcoordinator/cert/{poolcoordinator_cert_manager_test.go => poolcoordinatorcert_controller_test.go} (98%)

diff --git a/charts/openyurt/templates/yurt-manager-auto-generated.yaml b/charts/openyurt/templates/yurt-manager-auto-generated.yaml
index a122f941a7e..ffb36fdb063 100644
--- a/charts/openyurt/templates/yurt-manager-auto-generated.yaml
+++ b/charts/openyurt/templates/yurt-manager-auto-generated.yaml
@@ -503,6 +503,24 @@ metadata:
   creationTimestamp: null
   name: yurt-manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secret
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
 - apiGroups:
   - admissionregistration.k8s.io
   resources:
@@ -558,6 +576,7 @@ rules:
   resources:
   - certificatesigningrequests
   verbs:
+  - create
   - get
   - list
   - watch
diff --git a/charts/openyurt/templates/yurt-manager.yaml b/charts/openyurt/templates/yurt-manager.yaml
index 4d5cfee2a1c..a1da9ccd6e7 100644
--- a/charts/openyurt/templates/yurt-manager.yaml
+++ b/charts/openyurt/templates/yurt-manager.yaml
@@ -60,11 +60,11 @@ spec:
           command:
             - /usr/local/bin/yurt-manager
           image: {{ .Values.yurtManager.image.repository }}:{{ .Values.yurtManager.image.tag }}
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
           name: yurt-manager
           env:
             - name: WEBHOOK_PORT
-              value: {{ .Values.yurtManager.webhook.port }}
+              value: {{ .Values.yurtManager.webhook.port | quote }}
           ports:
             - containerPort: {{ .Values.yurtManager.webhook.port }}
               name: webhook-server
diff --git a/charts/openyurt/values.yaml b/charts/openyurt/values.yaml
index 809a7aedc52..c8c097320be 100644
--- a/charts/openyurt/values.yaml
+++ b/charts/openyurt/values.yaml
@@ -1,4 +1,3 @@
-
 imagePullSecrets: []
 imagePullPolicy: IfNotPresent
 
@@ -57,12 +56,12 @@ yurtManager:
     requests:
       cpu: 100m
       memory: 256Mi
-  affinity: 
+  affinity:
     nodeAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
         nodeSelectorTerms:
-        - matchExpressions:
-          - key: openyurt.io/is-edge-worker
-            operator: In
-            values:
-            - "false"
+          - matchExpressions:
+              - key: openyurt.io/is-edge-worker
+                operator: In
+                values:
+                  - "false"
diff --git a/cmd/yurt-controller-manager/app/controllermanager.go b/cmd/yurt-controller-manager/app/controllermanager.go
index 28312a436fd..73ef85fac71 100644
--- a/cmd/yurt-controller-manager/app/controllermanager.go
+++ b/cmd/yurt-controller-manager/app/controllermanager.go
@@ -309,7 +309,6 @@ var ControllersDisabledByDefault = sets.NewString()
 // paired to their InitFunc.  This allows for structured downstream composition and subdivision.
 func NewControllerInitializers() map[string]InitFunc {
 	controllers := map[string]InitFunc{}
-	controllers["poolcoordinatorcertmanager"] = startPoolCoordinatorCertManager
 	controllers["poolcoordinator"] = startPoolCoordinatorController
 	controllers["daemonpodupdater"] = startDaemonPodUpdaterController
 	controllers["servicetopologycontroller"] = startServiceTopologyController
diff --git a/cmd/yurt-controller-manager/app/core.go b/cmd/yurt-controller-manager/app/core.go
index d20b0d4e690..22e54a5fb0c 100644
--- a/cmd/yurt-controller-manager/app/core.go
+++ b/cmd/yurt-controller-manager/app/core.go
@@ -25,21 +25,11 @@ import (
 	"net/http"
 
 	daemonpodupdater "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater"
-	poolcoordinatorcertmanager "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/cert"
 	poolcoordinator "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/delegatelease"
 	"github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/podbinding"
 	"github.com/openyurtio/openyurt/pkg/controller/servicetopology"
 )
 
-func startPoolCoordinatorCertManager(ctx ControllerContext) (http.Handler, bool, error) {
-	poolcoordinatorCertManager := poolcoordinatorcertmanager.NewPoolCoordinatorCertManager(
-		ctx.ClientBuilder.ClientOrDie("poolcoordinator-cert-manager"),
-		ctx.InformerFactory.Core().V1().Pods(),
-	)
-	go poolcoordinatorCertManager.Run(1, ctx.Stop)
-	return nil, true, nil
-}
-
 func startPoolCoordinatorController(ctx ControllerContext) (http.Handler, bool, error) {
 	poolcoordinatorController := poolcoordinator.NewController(
 		ctx.ClientBuilder.ClientOrDie("poolcoordinator-delegate-lease"),
diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go
index 69e3dc08139..00b18209ced 100644
--- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go
@@ -23,7 +23,7 @@ package v1alpha1
 
 import (
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
diff --git a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go
index a62c4266aa2..ebb32758258 100644
--- a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go
+++ b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go
@@ -23,7 +23,7 @@ package v1beta1
 
 import (
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
diff --git a/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go
index d2d42342df3..c736ea956f0 100644
--- a/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go
@@ -22,7 +22,7 @@ limitations under the License.
 package v1alpha1
 
 import (
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
diff --git a/pkg/controller/add_poolcoordinatorcert.go b/pkg/controller/add_poolcoordinatorcert.go
new file mode 100644
index 00000000000..54e3e6b60a1
--- /dev/null
+++ b/pkg/controller/add_poolcoordinatorcert.go
@@ -0,0 +1,30 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the License);
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	poolcoordinatorcert "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/cert"
+)
+
+// Note !!! @kadisi
+// Do not change the name of the file @kadisi
+// Auto generate by make addcontroller command !!!
+// Note !!!
+
+func init() {
+	controllerAddFuncs = append(controllerAddFuncs, poolcoordinatorcert.Add)
+}
diff --git a/pkg/controller/poolcoordinator/cert/certificate.go b/pkg/controller/poolcoordinator/cert/certificate.go
index 0105cf1ff18..5beb1d3aefe 100644
--- a/pkg/controller/poolcoordinator/cert/certificate.go
+++ b/pkg/controller/poolcoordinator/cert/certificate.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
 	"context"
@@ -200,7 +200,7 @@ func IsCertFromCA(cert *x509.Certificate, caCert *x509.Certificate) bool {
 	}
 
 	if _, err := cert.Verify(verifyOptions); err != nil {
-		klog.Infof("cert not authorized by current CA: %v", err)
+		klog.Infof(Format("cert not authorized by current CA: %v", err))
 		return false
 	}
 
@@ -301,10 +301,10 @@ func GetCertAndKeyFromCertMgr(certManager certificate.Manager, stopCh <-chan str
 	err = wait.PollUntil(5*time.Second, func() (bool, error) {
 		// keep polling until the certificate is signed
 		if certManager.Current() != nil {
-			klog.Infof("%s certificate signed successfully", ComponentName)
+			klog.Infof(Format("%s certificate signed successfully", ComponentName))
 			return true, nil
 		}
-		klog.Infof("waiting for the master to sign the %s certificate", ComponentName)
+		klog.Infof(Format("waiting for the master to sign the %s certificate", ComponentName))
 		return false, nil
 	}, stopCh)
 
@@ -349,7 +349,7 @@ func WriteCertIntoSecret(clientSet client.Interface, certName, secretName string
 		return err
 	}
 
-	klog.Infof("successfully write %s cert/key pair into %s", certName, secretName)
+	klog.Infof(Format("successfully write %s cert/key pair into %s", certName, secretName))
 
 	return nil
 }
@@ -385,7 +385,7 @@ func WriteCertAndKeyIntoSecret(clientSet client.Interface, certName, secretName
 		}
 	}
 
-	klog.Infof("successfully write %s cert/key into %s", certName, secretName)
+	klog.Infof(Format("successfully write %s cert/key into %s", certName, secretName))
 
 	return nil
 }
@@ -400,7 +400,7 @@ func WriteKubeConfigIntoSecret(clientSet client.Interface, secretName, kubeConfi
 		return err
 	}
 
-	klog.Infof("successfully write kubeconfig into secret %s", secretName)
+	klog.Infof(Format("successfully write kubeconfig into secret %s", secretName))
 
 	return nil
 }
@@ -430,7 +430,7 @@ func WriteKeyPairIntoSecret(clientSet client.Interface, secretName, keyName stri
 		return errors.Wrapf(err, "fail to write %s.pub into secret %s", keyName, secretName)
 	}
 
-	klog.Infof("successfully write key pair into secret %s", secretName)
+	klog.Infof(Format("successfully write key pair into secret %s", secretName))
 
 	return nil
 }
diff --git a/pkg/controller/poolcoordinator/cert/certificate_test.go b/pkg/controller/poolcoordinator/cert/certificate_test.go
index 7bf3e423b5b..1f61f2f3073 100644
--- a/pkg/controller/poolcoordinator/cert/certificate_test.go
+++ b/pkg/controller/poolcoordinator/cert/certificate_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
 	"crypto/x509"
diff --git a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager.go b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller.go
similarity index 76%
rename from pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager.go
rename to pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller.go
index eb97eff7601..82a6f15e204 100644
--- a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager.go
+++ b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller.go
@@ -1,44 +1,54 @@
 /*
-Copyright 2022 The OpenYurt Authors.
+Copyright 2023 The OpenYurt Authors.
 
-Licensed under the Apache License, Version 2.0 (the "License");
+Licensed under the Apache License, Version 2.0 (the License);
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
+distributed under the License is distributed on an AS IS BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
+	"context"
 	"crypto"
 	"crypto/x509"
+	"flag"
 	"fmt"
 	"net"
-	"time"
 
 	"github.com/pkg/errors"
 	certificatesv1 "k8s.io/api/certificates/v1"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/wait"
-	coreinformers "k8s.io/client-go/informers/core/v1"
 	client "k8s.io/client-go/kubernetes"
-	corelisters "k8s.io/client-go/listers/core/v1"
-	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/util/workqueue"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config"
 	certfactory "github.com/openyurtio/openyurt/pkg/util/certmanager/factory"
-	ip "github.com/openyurtio/openyurt/pkg/util/ip"
+	"github.com/openyurtio/openyurt/pkg/util/ip"
+)
+
+func init() {
+	flag.IntVar(&concurrentReconciles, "poolcoordinatorcert-workers", concurrentReconciles, "Max concurrent workers for Poolcoordinatorcert controller.")
+}
+
+var (
+	concurrentReconciles = 3
 )
 
 const (
+	controllerName = "Poolcoordinatorcert-controller"
+
 	// tmp file directory for certmanager to write cert files
 	certDir = "/tmp"
 
@@ -175,84 +185,71 @@ var allSelfSignedCerts []CertConfig = []CertConfig{
 	},
 }
 
-// PoolCoordinatorCertManager manages certificates releted with poolcoordinator pod
-type PoolCoordinatorCertManager struct {
-	kubeclientset client.Interface
-	podLister     corelisters.PodLister
-	podSynced     cache.InformerSynced
-	podWorkQueue  workqueue.RateLimitingInterface
+func Format(format string, args ...interface{}) string {
+	s := fmt.Sprintf(format, args...)
+	return fmt.Sprintf("%s: %s", controllerName, s)
 }
 
-func NewPoolCoordinatorCertManager(kc client.Interface, podInformer coreinformers.PodInformer) *PoolCoordinatorCertManager {
-
-	certManager := PoolCoordinatorCertManager{
-		kubeclientset: kc,
+// Add creates a new Poolcoordinatorcert Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
+// and Start it when the Manager is Started.
+func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error {
+	r := &ReconcilePoolcoordinatorcert{}
 
-		podLister: podInformer.Lister(),
-		podSynced: podInformer.Informer().HasSynced,
-
-		podWorkQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
+	// Create a new controller
+	_, err := controller.New(controllerName, mgr, controller.Options{
+		Reconciler: r, MaxConcurrentReconciles: concurrentReconciles,
+	})
+	if err != nil {
+		return err
 	}
 
-	// Watch for poolcoordinator pod changes to manage related certs (including kubeconfig)
-	podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc:    func(obj interface{}) {},
-		UpdateFunc: func(oldObj, newObj interface{}) {},
-		DeleteFunc: func(obj interface{}) {},
-	})
+	// init PoolCoordinator
+	// prepare some necessary assets (CA, certs, kubeconfigs) for pool-coordinator
+	err = initPoolCoordinator(r.Client, nil)
+	if err != nil {
+		return err
+	}
 
-	return &certManager
+	return nil
 }
 
-func (c *PoolCoordinatorCertManager) Run(threadiness int, stopCh <-chan struct{}) {
-	defer utilruntime.HandleCrash()
+var _ reconcile.Reconciler = &ReconcilePoolcoordinatorcert{}
 
-	klog.Info("Starting poolcoordinatorCertManager controller")
-	defer klog.Info("Shutting down poolcoordinatorCertManager controller")
-	defer c.podWorkQueue.ShutDown()
+// ReconcilePoolcoordinatorcert reconciles a Poolcoordinatorcert object
+type ReconcilePoolcoordinatorcert struct {
+	Client client.Interface
+}
 
-	// prepare some necessary assets (CA, certs, kubeconfigs) for pool-coordinator
-	err := initPoolCoordinator(c.kubeclientset, stopCh)
+// InjectConfig
+func (r *ReconcilePoolcoordinatorcert) InjectConfig(cfg *rest.Config) error {
+	client, err := client.NewForConfig(cfg)
 	if err != nil {
-		klog.Errorf("fail to init poolcoordinator %v", err)
-	}
-
-	// Synchronize the cache before starting to process events
-	if !cache.WaitForCacheSync(stopCh, c.podSynced) {
-		klog.Error("sync poolcoordinatorCertManager controller timeout")
-	}
-
-	// The main Controller loop
-	for i := 0; i < threadiness; i++ {
-		go wait.Until(c.runWorker, time.Second, stopCh)
+		klog.Errorf("failed to create kube client, %v", err)
+		return err
 	}
-
-	<-stopCh
+	r.Client = client
+	return nil
 }
 
-func (c *PoolCoordinatorCertManager) runWorker() {
-	for {
-		obj, shutdown := c.podWorkQueue.Get()
-		if shutdown {
-			return
-		}
+// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=create
+// +kubebuilder:rbac:groups="",resources=secret,verbs=get;update;patch;create;list
+// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;watch;list
 
-		if err := c.syncHandler(obj.(string)); err != nil {
-			utilruntime.HandleError(err)
-		}
+// todo: make customized certificate for each poolcoordinator pod
+func (r *ReconcilePoolcoordinatorcert) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) {
 
-		c.podWorkQueue.Forget(obj)
-		c.podWorkQueue.Done(obj)
-	}
-}
+	// Note !!!!!!!!!!
+	// We strongly recommend use Format() to  encapsulation because Format() can print logs by module
+	// @kadisi
+	klog.Infof(Format("Reconcile Poolcoordinatorcert %s/%s", request.Namespace, request.Name))
 
-func (c *PoolCoordinatorCertManager) syncHandler(key string) error {
-	// todo: make customized certificate for each poolcoordinator pod
-	return nil
+	return reconcile.Result{}, nil
 }
 
 func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) error {
 
+	klog.Infof(Format("init poolcoordinator started"))
+
 	// Prepare CA certs
 	caCert, caKey, reuseCA, err := initCA(clientSet)
 	if err != nil {
@@ -272,14 +269,14 @@ func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) err
 			// 1.1 check if cert exist
 			cert, _, err := loadCertAndKeyFromSecret(clientSet, certConf)
 			if err != nil {
-				klog.Infof("can not load cert %s from %s secret", certConf.CertName, certConf.SecretName)
+				klog.Infof(Format("can not load cert %s from %s secret", certConf.CertName, certConf.SecretName))
 				selfSignedCerts = append(selfSignedCerts, certConf)
 				continue
 			}
 
 			// 1.2 check if cert is autorized by current CA
 			if !IsCertFromCA(cert, caCert) {
-				klog.Infof("existing cert %s is not authorized by current CA", certConf.CertName)
+				klog.Infof(Format("existing cert %s is not authorized by current CA", certConf.CertName))
 				selfSignedCerts = append(selfSignedCerts, certConf)
 				continue
 			}
@@ -290,20 +287,20 @@ func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) err
 				ips, _, err := certConf.certInit(clientSet, stopCh)
 				if err != nil {
 					// if cert init failed, skip this cert
-					klog.Errorf("fail to init cert %s when checking dynamic attrs: %v", certConf.CertName, err)
+					klog.Errorf(Format("fail to init cert %s when checking dynamic attrs: %v", certConf.CertName, err))
 					continue
 				} else {
 					// check if dynamic IP addresses already exist in cert
 					changed := ip.SearchAllIP(cert.IPAddresses, ips)
 					if changed {
-						klog.Infof("cert %s IP has changed", certConf.CertName)
+						klog.Infof(Format("cert %s IP has changed", certConf.CertName))
 						selfSignedCerts = append(selfSignedCerts, certConf)
 						continue
 					}
 				}
 			}
 
-			klog.Infof("cert %s not change, reuse it", certConf.CertName)
+			klog.Infof(Format("cert %s not change, reuse it", certConf.CertName))
 		}
 	} else {
 		// create all certs with new CA
@@ -313,7 +310,7 @@ func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) err
 	// create self signed certs
 	for _, certConf := range selfSignedCerts {
 		if err := initPoolCoordinatorCert(clientSet, certConf, caCert, caKey, stopCh); err != nil {
-			klog.Errorf("create cert %s fail: %v", certConf.CertName, err)
+			klog.Errorf(Format("create cert %s fail: %v", certConf.CertName, err))
 			return err
 		}
 	}
@@ -358,12 +355,12 @@ func initCA(clientSet client.Interface) (caCert *x509.Certificate, caKey crypto.
 
 	if err == nil {
 		// if CA already exist
-		klog.Info("CA already exist in secret, reuse it")
+		klog.Info(Format("CA already exist in secret, reuse it"))
 		return caCert, caKey, true, nil
 	} else {
 		// if not exist
 		// create new CA certs
-		klog.Infof("fail to get CA from secret: %v, create new CA", err)
+		klog.Infof(Format("fail to get CA from secret: %v, create new CA", err))
 		// write it into the secret
 		caCert, caKey, err = NewSelfSignedCA()
 		if err != nil {
diff --git a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager_test.go b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller_test.go
similarity index 98%
rename from pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager_test.go
rename to pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller_test.go
index 90431373e8f..7bbcb9a0c9d 100644
--- a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager_test.go
+++ b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
 	"testing"
diff --git a/pkg/controller/poolcoordinator/cert/secret.go b/pkg/controller/poolcoordinator/cert/secret.go
index 61a9883fb38..2bb49e66542 100644
--- a/pkg/controller/poolcoordinator/cert/secret.go
+++ b/pkg/controller/poolcoordinator/cert/secret.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
 	"context"
@@ -54,12 +54,12 @@ func NewSecretClient(clientSet client.Interface, ns, name string) (*SecretClient
 		// if this secret already exist, reuse it
 		if kerrors.IsAlreadyExists(err) {
 			secret, _ = clientSet.CoreV1().Secrets(ns).Get(context.TODO(), name, metav1.GetOptions{})
-			klog.V(4).Infof("secret %s already exisit: %v", name, secret)
+			klog.V(4).Infof(Format("secret %s already exisit", secret.Name))
 		} else {
 			return nil, fmt.Errorf("create secret client %s fail: %v", name, err)
 		}
 	} else {
-		klog.V(4).Infof("secret %s not exisit, create one: %v", name, secret)
+		klog.V(4).Infof(Format("secret %s not exisit, create one", secret.Name))
 	}
 
 	return &SecretClient{
diff --git a/pkg/controller/poolcoordinator/cert/secret_test.go b/pkg/controller/poolcoordinator/cert/secret_test.go
index 2d18bb4b09e..601939a7f8a 100644
--- a/pkg/controller/poolcoordinator/cert/secret_test.go
+++ b/pkg/controller/poolcoordinator/cert/secret_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
 	"testing"
diff --git a/pkg/controller/poolcoordinator/cert/util.go b/pkg/controller/poolcoordinator/cert/util.go
index d7c39abc42f..26f52d0a444 100644
--- a/pkg/controller/poolcoordinator/cert/util.go
+++ b/pkg/controller/poolcoordinator/cert/util.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
 	"context"
@@ -59,10 +59,10 @@ func waitUntilSVCReady(clientSet client.Interface, serviceName string, stopCh <-
 	if err = wait.PollUntil(1*time.Second, func() (bool, error) {
 		serverSVC, err = clientSet.CoreV1().Services(PoolcoordinatorNS).Get(context.TODO(), serviceName, metav1.GetOptions{})
 		if err == nil {
-			klog.Infof("%s service is ready for poolcoordinator_cert_manager", serviceName)
+			klog.Infof(Format("%s service is ready for poolcoordinator_cert_manager", serviceName))
 			return true, nil
 		}
-		klog.Infof("waiting for the poolcoordinator %s service", serviceName)
+		klog.Infof(Format("waiting for the poolcoordinator %s service", serviceName))
 		return false, nil
 	}, stopCh); err != nil {
 		return nil, nil, err
diff --git a/pkg/controller/poolcoordinator/cert/util_test.go b/pkg/controller/poolcoordinator/cert/util_test.go
index ea95c1c7078..85544a660de 100644
--- a/pkg/controller/poolcoordinator/cert/util_test.go
+++ b/pkg/controller/poolcoordinator/cert/util_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cert
+package poolcoordinatorcert
 
 import (
 	"testing"