diff --git a/Makefile b/Makefile index 0e38e74563d..105dd2683cf 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ GIT_COMMIT = $(shell git rev-parse HEAD) ENABLE_AUTONOMY_TESTS ?=true CRD_OPTIONS ?= "crd:crdVersions=v1" BUILD_KUSTOMIZE ?= _output/manifest +GOPROXY ?= $(shell go env GOPROXY) ifeq ($(shell git tag --points-at ${GIT_COMMIT}),) GIT_VERSION=$(IMAGE_TAG)-$(shell echo ${GIT_COMMIT} | cut -c 1-7) @@ -38,7 +39,8 @@ endif DOCKER_BUILD_ARGS = --build-arg GIT_VERSION=${GIT_VERSION} ifeq (${REGION}, cn) -DOCKER_BUILD_ARGS += --build-arg GOPROXY=https://goproxy.cn --build-arg MIRROR_REPO=mirrors.aliyun.com +GOPROXY=https://goproxy.cn +DOCKER_BUILD_ARGS += --build-arg GOPROXY=$(GOPROXY) --build-arg MIRROR_REPO=mirrors.aliyun.com endif ifneq (${http_proxy},) @@ -55,7 +57,7 @@ all: test build # Build binaries in the host environment build: - bash hack/make-rules/build.sh $(WHAT) + GOPROXY=$(GOPROXY) bash hack/make-rules/build.sh $(WHAT) # Run test test: diff --git a/charts/openyurt/templates/yurt-controller-manager.yaml b/charts/openyurt/templates/yurt-controller-manager.yaml index df521373d84..0540fc3e36d 100644 --- a/charts/openyurt/templates/yurt-controller-manager.yaml +++ b/charts/openyurt/templates/yurt-controller-manager.yaml @@ -29,6 +29,7 @@ rules: - list - update - watch + - delete - apiGroups: - "" resources: diff --git a/charts/openyurt/templates/yurt-manager-auto-generated.yaml b/charts/openyurt/templates/yurt-manager-auto-generated.yaml index a122f941a7e..ffb36fdb063 100644 --- a/charts/openyurt/templates/yurt-manager-auto-generated.yaml +++ b/charts/openyurt/templates/yurt-manager-auto-generated.yaml @@ -503,6 +503,24 @@ metadata: creationTimestamp: null name: yurt-manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secret + verbs: + - create + - get + - list + - patch + - update - apiGroups: - admissionregistration.k8s.io resources: @@ -558,6 +576,7 @@ rules: resources: - certificatesigningrequests verbs: + - create - get - list - watch diff --git a/charts/openyurt/templates/yurt-manager.yaml b/charts/openyurt/templates/yurt-manager.yaml index 4d5cfee2a1c..a1da9ccd6e7 100644 --- a/charts/openyurt/templates/yurt-manager.yaml +++ b/charts/openyurt/templates/yurt-manager.yaml @@ -60,11 +60,11 @@ spec: command: - /usr/local/bin/yurt-manager image: {{ .Values.yurtManager.image.repository }}:{{ .Values.yurtManager.image.tag }} - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: yurt-manager env: - name: WEBHOOK_PORT - value: {{ .Values.yurtManager.webhook.port }} + value: {{ .Values.yurtManager.webhook.port | quote }} ports: - containerPort: {{ .Values.yurtManager.webhook.port }} name: webhook-server diff --git a/charts/openyurt/values.yaml b/charts/openyurt/values.yaml index 809a7aedc52..c8c097320be 100644 --- a/charts/openyurt/values.yaml +++ b/charts/openyurt/values.yaml @@ -1,4 +1,3 @@ - imagePullSecrets: [] imagePullPolicy: IfNotPresent @@ -57,12 +56,12 @@ yurtManager: requests: cpu: 100m memory: 256Mi - affinity: + affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: openyurt.io/is-edge-worker - operator: In - values: - - "false" + - matchExpressions: + - key: openyurt.io/is-edge-worker + operator: In + values: + - "false" diff --git a/cmd/yurt-controller-manager/app/controllermanager.go b/cmd/yurt-controller-manager/app/controllermanager.go index 28312a436fd..73ef85fac71 100644 --- a/cmd/yurt-controller-manager/app/controllermanager.go +++ b/cmd/yurt-controller-manager/app/controllermanager.go @@ -309,7 +309,6 @@ var ControllersDisabledByDefault = sets.NewString() // paired to their InitFunc. This allows for structured downstream composition and subdivision. func NewControllerInitializers() map[string]InitFunc { controllers := map[string]InitFunc{} - controllers["poolcoordinatorcertmanager"] = startPoolCoordinatorCertManager controllers["poolcoordinator"] = startPoolCoordinatorController controllers["daemonpodupdater"] = startDaemonPodUpdaterController controllers["servicetopologycontroller"] = startServiceTopologyController diff --git a/cmd/yurt-controller-manager/app/core.go b/cmd/yurt-controller-manager/app/core.go index d20b0d4e690..22e54a5fb0c 100644 --- a/cmd/yurt-controller-manager/app/core.go +++ b/cmd/yurt-controller-manager/app/core.go @@ -25,21 +25,11 @@ import ( "net/http" daemonpodupdater "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater" - poolcoordinatorcertmanager "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/cert" poolcoordinator "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/delegatelease" "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/podbinding" "github.com/openyurtio/openyurt/pkg/controller/servicetopology" ) -func startPoolCoordinatorCertManager(ctx ControllerContext) (http.Handler, bool, error) { - poolcoordinatorCertManager := poolcoordinatorcertmanager.NewPoolCoordinatorCertManager( - ctx.ClientBuilder.ClientOrDie("poolcoordinator-cert-manager"), - ctx.InformerFactory.Core().V1().Pods(), - ) - go poolcoordinatorCertManager.Run(1, ctx.Stop) - return nil, true, nil -} - func startPoolCoordinatorController(ctx ControllerContext) (http.Handler, bool, error) { poolcoordinatorController := poolcoordinator.NewController( ctx.ClientBuilder.ClientOrDie("poolcoordinator-delegate-lease"), diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index b95b50303f7..f6a8847b005 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -343,6 +343,7 @@ func createCertManager(options *options.YurtHubOptions, remoteServers []*url.URL RootDir: options.RootDir, NodeName: options.NodeName, JoinToken: options.JoinToken, + BootstrapFile: options.BootstrapFile, CaCertHashes: options.CACertHashes, YurtHubCertOrganizations: options.YurtHubCertOrganizations, CertIPs: certIPs, diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index db34ca60e01..32f90ed85d8 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -61,6 +61,7 @@ type YurtHubOptions struct { HeartbeatIntervalSeconds int MaxRequestInFlight int JoinToken string + BootstrapFile string RootDir string Version bool EnableProfiling bool @@ -143,8 +144,8 @@ func (options *YurtHubOptions) Validate() error { return fmt.Errorf("server-address is empty") } - if len(options.JoinToken) == 0 { - return fmt.Errorf("bootstrap token is empty") + if len(options.JoinToken) == 0 && len(options.BootstrapFile) == 0 { + return fmt.Errorf("bootstrap token and bootstrap file are empty, one of them must be set") } if !util.IsSupportedLBMode(options.LBMode) { @@ -183,7 +184,9 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.HeartbeatTimeoutSeconds, "heartbeat-timeout-seconds", o.HeartbeatTimeoutSeconds, " number of seconds after which the heartbeat times out.") fs.IntVar(&o.HeartbeatIntervalSeconds, "heartbeat-interval-seconds", o.HeartbeatIntervalSeconds, " number of seconds for omitting one time heartbeat to remote server.") fs.IntVar(&o.MaxRequestInFlight, "max-requests-in-flight", o.MaxRequestInFlight, "the maximum number of parallel requests.") - fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent when --cert-mgr-mode=hubself.") + fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent.") + fs.MarkDeprecated("join-token", "It is planned to be removed from OpenYurt in the version v1.5. Please use --bootstrap-file to bootstrap hub agent.") + fs.StringVar(&o.BootstrapFile, "bootstrap-file", o.BootstrapFile, "the bootstrap file for bootstrapping hub agent.") fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).") fs.BoolVar(&o.Version, "version", o.Version, "print the version information.") fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling, "enable profiling via web interface host:port/debug/pprof/") diff --git a/go.mod b/go.mod index ce9a8fc870a..41b7e416f68 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.0 go.etcd.io/etcd/client/v3 v3.5.0 golang.org/x/sys v0.6.0 - google.golang.org/grpc v1.40.0 + google.golang.org/grpc v1.54.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/square/go-jose.v2 v2.6.0 k8s.io/api v0.22.3 @@ -64,7 +64,8 @@ require ( ) require ( - cloud.google.com/go v0.81.0 // indirect + cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/BurntSushi/toml v0.4.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect @@ -72,7 +73,7 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect @@ -89,7 +90,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -114,6 +115,7 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spf13/afero v1.6.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect go.opentelemetry.io/contrib v0.20.0 // indirect @@ -131,15 +133,15 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.5.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect diff --git a/go.sum b/go.sum index 5c28c9732fb..6626920d985 100644 --- a/go.sum +++ b/go.sum @@ -13,18 +13,16 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -105,8 +103,9 @@ github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -114,9 +113,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= @@ -163,10 +160,8 @@ github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQm github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -258,9 +253,8 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -293,18 +287,16 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -312,10 +304,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -377,7 +365,6 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -413,6 +400,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -521,6 +509,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -585,8 +574,9 @@ github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -670,8 +660,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= @@ -720,6 +708,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -749,7 +738,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -760,8 +748,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -802,33 +788,22 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= @@ -895,20 +870,13 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -927,8 +895,8 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -939,8 +907,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -999,12 +967,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1031,11 +994,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1075,20 +1033,11 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1102,17 +1051,12 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/hack/dockerfiles/build/Dockerfile.yurthub b/hack/dockerfiles/build/Dockerfile.yurthub index b233165c2ca..688b5384518 100644 --- a/hack/dockerfiles/build/Dockerfile.yurthub +++ b/hack/dockerfiles/build/Dockerfile.yurthub @@ -5,4 +5,6 @@ ARG TARGETOS TARGETARCH MIRROR_REPO RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \ apk add ca-certificates bash libc6-compat iptables ip6tables && update-ca-certificates && rm /var/cache/apk/* COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurthub /usr/local/bin/yurthub +COPY hack/iptables-wrapper-installer.sh / +RUN chmod +x /iptables-wrapper-installer.sh && /iptables-wrapper-installer.sh ENTRYPOINT ["/usr/local/bin/yurthub"] \ No newline at end of file diff --git a/hack/dockerfiles/release/Dockerfile.yurthub b/hack/dockerfiles/release/Dockerfile.yurthub index 53b83d1e117..d6f8cfbe9d3 100644 --- a/hack/dockerfiles/release/Dockerfile.yurthub +++ b/hack/dockerfiles/release/Dockerfile.yurthub @@ -11,4 +11,6 @@ ARG TARGETOS TARGETARCH MIRROR_REPO RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \ apk add ca-certificates bash libc6-compat iptables ip6tables && update-ca-certificates && rm /var/cache/apk/* COPY --from=builder /build/_output/local/bin/${TARGETOS}/${TARGETARCH}/yurthub /usr/local/bin/yurthub +COPY hack/iptables-wrapper-installer.sh / +RUN chmod +x /iptables-wrapper-installer.sh && /iptables-wrapper-installer.sh ENTRYPOINT ["/usr/local/bin/yurthub"] \ No newline at end of file diff --git a/hack/iptables-wrapper-installer.sh b/hack/iptables-wrapper-installer.sh new file mode 100644 index 00000000000..e6eb7d8210d --- /dev/null +++ b/hack/iptables-wrapper-installer.sh @@ -0,0 +1,208 @@ +#!/bin/sh + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# +# iptables-wrapper-installer.sh [--no-sanity-check] +# +# Installs a wrapper iptables script in a container that will figure out +# whether iptables-legacy or iptables-nft is in use on the host and then +# replaces itself with the correct underlying iptables version. +# +# Unless "--no-sanity-check" is passed, it will first verify that the +# container already contains a suitable version of iptables. + +# NOTE: This can only use POSIX /bin/sh features; the build container +# might not contain bash. + +set -eu + +# Find iptables binary location +if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then + sbin="/usr/sbin" +elif [ -d /sbin -a -e /sbin/iptables ]; then + sbin="/sbin" +else + echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2 + exit 1 +fi + +# Determine how the system selects between iptables-legacy and iptables-nft +if [ -x /usr/sbin/alternatives ]; then + # Fedora/SUSE style alternatives + altstyle="fedora" +elif [ -x /usr/sbin/update-alternatives ]; then + # Debian style alternatives + altstyle="debian" +else + # No alternatives system + altstyle="none" +fi + +if [ "${1:-}" != "--no-sanity-check" ]; then + # Ensure dependencies are installed + if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then + echo "ERROR: iptables-nft is not installed" 1>&2 + exit 1 + fi + if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then + echo "ERROR: iptables-legacy is not installed" 1>&2 + exit 1 + fi + + case "${version}" in + *v1.8.[0123]\ *) + echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2 + echo " Upgrade to 1.8.4 or newer." 1>&2 + exit 1 + ;; + *) + # 1.8.4+ are OK + ;; + esac +fi + +# Start creating the wrapper... +rm -f "${sbin}/iptables-wrapper" +cat > "${sbin}/iptables-wrapper" </dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l) +if [ "\${nft_kubelet_rules}" -ne 0 ]; then + mode=nft +else + # Check for kubernetes 1.17-or-later with iptables-legacy. We + # can't pass "-t mangle" to iptables-legacy-save because it would + # cause the kernel to create that table if it didn't already + # exist, which we don't want. So we have to grab all the rules + legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l) + if [ "\${legacy_kubelet_rules}" -ne 0 ]; then + mode=legacy + else + # With older kubernetes releases there may not be any _specific_ + # rules we can look for, but we assume that some non-containerized process + # (possibly kubelet) will have created _some_ iptables rules. + num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l) + num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l) + if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then + mode=legacy + else + mode=nft + fi + fi +fi + +EOF + +# Write out the appropriate alternatives-selection commands +case "${altstyle}" in + fedora) +cat >> "${sbin}/iptables-wrapper" < /dev/null || failed=1 +EOF + ;; + + debian) +cat >> "${sbin}/iptables-wrapper" < /dev/null || failed=1 +update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1 +EOF + ;; + + *) +cat >> "${sbin}/iptables-wrapper" </dev/null || failed=1 +EOF + ;; +esac + +# Write out the post-alternatives-selection error checking and final wrap-up +cat >> "${sbin}/iptables-wrapper" <&2 + # fake it, though this will probably also fail if they aren't root + exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@" +fi + +# Now re-exec the original command with the newly-selected alternative +exec "\$0" "\$@" +EOF +chmod +x "${sbin}/iptables-wrapper" + +# Now back in the installer script, point the iptables binaries at our +# wrapper +case "${altstyle}" in + fedora) + alternatives \ + --install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \ + --slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper + ;; + + debian) + update-alternatives \ + --install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \ + --slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper + update-alternatives \ + --install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \ + --slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \ + --slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper + ;; + + *) + for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do + rm -f "${sbin}/${cmd}" + ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}" + done + ;; +esac + +# Cleanup +rm -f "$0" diff --git a/hack/make-rules/image_build.sh b/hack/make-rules/image_build.sh index 3c16d5ebc07..b069cfcbc0b 100755 --- a/hack/make-rules/image_build.sh +++ b/hack/make-rules/image_build.sh @@ -34,6 +34,7 @@ REGION=${REGION:-} IMAGE_REPO=${IMAGE_REPO:-"openyurt"} IMAGE_TAG=${IMAGE_TAG:-$(get_image_tag)} DOCKER_BUILD_ARGS="" +DOCKER_EXTRA_ENVS="" BUILD_BASE_IMAGE="golang:1.18" BUILD_GOPROXY=$(go env GOPROXY) GOPROXY_CN="https://goproxy.cn" @@ -46,10 +47,12 @@ fi if [[ ! -z ${http_proxy} ]]; then DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg http_proxy=${http_proxy}" + DOCKER_EXTRA_ENVS="--env http_proxy=${http_proxy}" fi if [[ ! -z ${https_proxy} ]]; then DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg https_proxy=${https_proxy}" + DOCKER_EXTRA_ENVS="${DOCKER_EXTRA_ENVS=} --env https_proxy=${https_proxy}" fi if [[ ! -z ${TARGET_PLATFORMS} ]]; then @@ -74,6 +77,7 @@ docker run \ --env GOOS=${TARGETOS} \ --env GOARCH=${TARGETARCH} \ --env GOCACHE=/tmp/ \ + ${DOCKER_EXTRA_ENVS} \ --user $(id -u ${USER}):$(id -g ${USER}) \ ${BUILD_BASE_IMAGE} \ ./hack/make-rules/build.sh ${targets[@]} diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go index 69e3dc08139..00b18209ced 100644 --- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go index a62c4266aa2..ebb32758258 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1beta1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go index d2d42342df3..c736ea956f0 100644 --- a/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/controller/add_poolcoordinatorcert.go b/pkg/controller/add_poolcoordinatorcert.go new file mode 100644 index 00000000000..54e3e6b60a1 --- /dev/null +++ b/pkg/controller/add_poolcoordinatorcert.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + poolcoordinatorcert "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/cert" +) + +// Note !!! @kadisi +// Do not change the name of the file @kadisi +// Auto generate by make addcontroller command !!! +// Note !!! + +func init() { + controllerAddFuncs = append(controllerAddFuncs, poolcoordinatorcert.Add) +} diff --git a/pkg/controller/csrapprover/csrapprover_controller.go b/pkg/controller/csrapprover/csrapprover_controller.go index f7a9ce121ff..21e051ac2ce 100644 --- a/pkg/controller/csrapprover/csrapprover_controller.go +++ b/pkg/controller/csrapprover/csrapprover_controller.go @@ -28,10 +28,12 @@ import ( certificatesv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -41,11 +43,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - registryClient "github.com/openyurtio/openyurt/pkg/client" poolcoordinatorCert "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/cert" "github.com/openyurtio/openyurt/pkg/projectinfo" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -56,7 +55,7 @@ func init() { var ( concurrentReconciles = 3 - csrV1Kind = certificatesv1.SchemeGroupVersion.WithKind("CertificateSigningRequest") + csrV1Resource = certificatesv1.SchemeGroupVersion.WithResource("certificatesigningrequests") yurtCsr = fmt.Sprintf("%s-csr", strings.TrimRightFunc(projectinfo.GetProjectPrefix(), func(c rune) bool { return c == '-' })) yurtHubNodeCertOrgPrefix = "openyurt:tenant:" @@ -105,8 +104,22 @@ type csrRecognizer struct { // Add creates a new CsrApprover Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - return add(mgr, newReconciler(c, mgr)) +func Add(_ *appconfig.CompletedConfig, mgr manager.Manager) error { + r := &ReconcileCsrApprover{} + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, + }) + if err != nil { + return err + } + + // Watch for csr changes + if r.csrV1Supported { + return c.Watch(&source.Kind{Type: &certificatesv1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}) + } else { + return c.Watch(&source.Kind{Type: &certificatesv1beta1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}) + } } var _ reconcile.Reconciler = &ReconcileCsrApprover{} @@ -118,33 +131,31 @@ type ReconcileCsrApprover struct { csrApproverClient kubernetes.Interface } -// newReconciler returns a new reconcile.Reconciler -func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - genericClient := registryClient.GetGenericClientWithName(controllerName) +func (r *ReconcileCsrApprover) InjectClient(c client.Client) error { + r.Client = c + return nil +} - return &ReconcileCsrApprover{ - Client: utilclient.NewClientFromManager(mgr, controllerName), - csrV1Supported: utildiscovery.DiscoverGVK(csrV1Kind), - csrApproverClient: genericClient.KubeClient, +func (r *ReconcileCsrApprover) InjectMapper(mapper meta.RESTMapper) error { + if gvk, err := mapper.KindFor(csrV1Resource); err != nil { + klog.Errorf("v1.CertificateSigningRequest is not supported, %v", err) + r.csrV1Supported = false + } else { + klog.Infof("%s is supported", gvk.String()) + r.csrV1Supported = true } + + return nil } -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New(controllerName, mgr, controller.Options{ - Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, - }) +func (r *ReconcileCsrApprover) InjectConfig(cfg *rest.Config) error { + client, err := kubernetes.NewForConfig(cfg) if err != nil { + klog.Errorf("failed to create kube client, %v", err) return err } - - // Watch for changes to CsrApprover - if utildiscovery.DiscoverGVK(csrV1Kind) { - return c.Watch(&source.Kind{Type: &certificatesv1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}) - } else { - return c.Watch(&source.Kind{Type: &certificatesv1beta1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}) - } + r.csrApproverClient = client + return nil } // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=get;list;watch diff --git a/pkg/controller/daemonpodupdater/util.go b/pkg/controller/daemonpodupdater/util.go index 75b08db65a4..0daa6fc58b6 100644 --- a/pkg/controller/daemonpodupdater/util.go +++ b/pkg/controller/daemonpodupdater/util.go @@ -73,7 +73,7 @@ func IsDaemonsetPodLatest(ds *appsv1.DaemonSet, pod *corev1.Pod) bool { klog.V(5).Infof("daemonset %v revision hash is %v", ds.Name, hash) klog.V(5).Infof("daemonset %v generation is %v", ds.Name, generation) - templateMatches := generation != nil && pod.Labels[extensions.DaemonSetTemplateGenerationKey] == fmt.Sprint(generation) + templateMatches := generation != nil && pod.Labels[extensions.DaemonSetTemplateGenerationKey] == fmt.Sprint(*generation) hashMatches := len(hash) > 0 && pod.Labels[extensions.DefaultDaemonSetUniqueLabelKey] == hash return hashMatches || templateMatches } diff --git a/pkg/controller/poolcoordinator/cert/certificate.go b/pkg/controller/poolcoordinator/cert/certificate.go index 0105cf1ff18..5beb1d3aefe 100644 --- a/pkg/controller/poolcoordinator/cert/certificate.go +++ b/pkg/controller/poolcoordinator/cert/certificate.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( "context" @@ -200,7 +200,7 @@ func IsCertFromCA(cert *x509.Certificate, caCert *x509.Certificate) bool { } if _, err := cert.Verify(verifyOptions); err != nil { - klog.Infof("cert not authorized by current CA: %v", err) + klog.Infof(Format("cert not authorized by current CA: %v", err)) return false } @@ -301,10 +301,10 @@ func GetCertAndKeyFromCertMgr(certManager certificate.Manager, stopCh <-chan str err = wait.PollUntil(5*time.Second, func() (bool, error) { // keep polling until the certificate is signed if certManager.Current() != nil { - klog.Infof("%s certificate signed successfully", ComponentName) + klog.Infof(Format("%s certificate signed successfully", ComponentName)) return true, nil } - klog.Infof("waiting for the master to sign the %s certificate", ComponentName) + klog.Infof(Format("waiting for the master to sign the %s certificate", ComponentName)) return false, nil }, stopCh) @@ -349,7 +349,7 @@ func WriteCertIntoSecret(clientSet client.Interface, certName, secretName string return err } - klog.Infof("successfully write %s cert/key pair into %s", certName, secretName) + klog.Infof(Format("successfully write %s cert/key pair into %s", certName, secretName)) return nil } @@ -385,7 +385,7 @@ func WriteCertAndKeyIntoSecret(clientSet client.Interface, certName, secretName } } - klog.Infof("successfully write %s cert/key into %s", certName, secretName) + klog.Infof(Format("successfully write %s cert/key into %s", certName, secretName)) return nil } @@ -400,7 +400,7 @@ func WriteKubeConfigIntoSecret(clientSet client.Interface, secretName, kubeConfi return err } - klog.Infof("successfully write kubeconfig into secret %s", secretName) + klog.Infof(Format("successfully write kubeconfig into secret %s", secretName)) return nil } @@ -430,7 +430,7 @@ func WriteKeyPairIntoSecret(clientSet client.Interface, secretName, keyName stri return errors.Wrapf(err, "fail to write %s.pub into secret %s", keyName, secretName) } - klog.Infof("successfully write key pair into secret %s", secretName) + klog.Infof(Format("successfully write key pair into secret %s", secretName)) return nil } diff --git a/pkg/controller/poolcoordinator/cert/certificate_test.go b/pkg/controller/poolcoordinator/cert/certificate_test.go index 7bf3e423b5b..1f61f2f3073 100644 --- a/pkg/controller/poolcoordinator/cert/certificate_test.go +++ b/pkg/controller/poolcoordinator/cert/certificate_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( "crypto/x509" diff --git a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager.go b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller.go similarity index 76% rename from pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager.go rename to pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller.go index eb97eff7601..82a6f15e204 100644 --- a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager.go +++ b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller.go @@ -1,44 +1,54 @@ /* -Copyright 2022 The OpenYurt Authors. +Copyright 2023 The OpenYurt Authors. -Licensed under the Apache License, Version 2.0 (the "License"); +Licensed under the Apache License, Version 2.0 (the License); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, +distributed under the License is distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( + "context" "crypto" "crypto/x509" + "flag" "fmt" "net" - "time" "github.com/pkg/errors" certificatesv1 "k8s.io/api/certificates/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - coreinformers "k8s.io/client-go/informers/core/v1" client "k8s.io/client-go/kubernetes" - corelisters "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" + "k8s.io/client-go/rest" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" certfactory "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" - ip "github.com/openyurtio/openyurt/pkg/util/ip" + "github.com/openyurtio/openyurt/pkg/util/ip" +) + +func init() { + flag.IntVar(&concurrentReconciles, "poolcoordinatorcert-workers", concurrentReconciles, "Max concurrent workers for Poolcoordinatorcert controller.") +} + +var ( + concurrentReconciles = 3 ) const ( + controllerName = "Poolcoordinatorcert-controller" + // tmp file directory for certmanager to write cert files certDir = "/tmp" @@ -175,84 +185,71 @@ var allSelfSignedCerts []CertConfig = []CertConfig{ }, } -// PoolCoordinatorCertManager manages certificates releted with poolcoordinator pod -type PoolCoordinatorCertManager struct { - kubeclientset client.Interface - podLister corelisters.PodLister - podSynced cache.InformerSynced - podWorkQueue workqueue.RateLimitingInterface +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", controllerName, s) } -func NewPoolCoordinatorCertManager(kc client.Interface, podInformer coreinformers.PodInformer) *PoolCoordinatorCertManager { - - certManager := PoolCoordinatorCertManager{ - kubeclientset: kc, +// Add creates a new Poolcoordinatorcert Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { + r := &ReconcilePoolcoordinatorcert{} - podLister: podInformer.Lister(), - podSynced: podInformer.Informer().HasSynced, - - podWorkQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + // Create a new controller + _, err := controller.New(controllerName, mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, + }) + if err != nil { + return err } - // Watch for poolcoordinator pod changes to manage related certs (including kubeconfig) - podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) {}, - UpdateFunc: func(oldObj, newObj interface{}) {}, - DeleteFunc: func(obj interface{}) {}, - }) + // init PoolCoordinator + // prepare some necessary assets (CA, certs, kubeconfigs) for pool-coordinator + err = initPoolCoordinator(r.Client, nil) + if err != nil { + return err + } - return &certManager + return nil } -func (c *PoolCoordinatorCertManager) Run(threadiness int, stopCh <-chan struct{}) { - defer utilruntime.HandleCrash() +var _ reconcile.Reconciler = &ReconcilePoolcoordinatorcert{} - klog.Info("Starting poolcoordinatorCertManager controller") - defer klog.Info("Shutting down poolcoordinatorCertManager controller") - defer c.podWorkQueue.ShutDown() +// ReconcilePoolcoordinatorcert reconciles a Poolcoordinatorcert object +type ReconcilePoolcoordinatorcert struct { + Client client.Interface +} - // prepare some necessary assets (CA, certs, kubeconfigs) for pool-coordinator - err := initPoolCoordinator(c.kubeclientset, stopCh) +// InjectConfig +func (r *ReconcilePoolcoordinatorcert) InjectConfig(cfg *rest.Config) error { + client, err := client.NewForConfig(cfg) if err != nil { - klog.Errorf("fail to init poolcoordinator %v", err) - } - - // Synchronize the cache before starting to process events - if !cache.WaitForCacheSync(stopCh, c.podSynced) { - klog.Error("sync poolcoordinatorCertManager controller timeout") - } - - // The main Controller loop - for i := 0; i < threadiness; i++ { - go wait.Until(c.runWorker, time.Second, stopCh) + klog.Errorf("failed to create kube client, %v", err) + return err } - - <-stopCh + r.Client = client + return nil } -func (c *PoolCoordinatorCertManager) runWorker() { - for { - obj, shutdown := c.podWorkQueue.Get() - if shutdown { - return - } +// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=create +// +kubebuilder:rbac:groups="",resources=secret,verbs=get;update;patch;create;list +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;watch;list - if err := c.syncHandler(obj.(string)); err != nil { - utilruntime.HandleError(err) - } +// todo: make customized certificate for each poolcoordinator pod +func (r *ReconcilePoolcoordinatorcert) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) { - c.podWorkQueue.Forget(obj) - c.podWorkQueue.Done(obj) - } -} + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.Infof(Format("Reconcile Poolcoordinatorcert %s/%s", request.Namespace, request.Name)) -func (c *PoolCoordinatorCertManager) syncHandler(key string) error { - // todo: make customized certificate for each poolcoordinator pod - return nil + return reconcile.Result{}, nil } func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) error { + klog.Infof(Format("init poolcoordinator started")) + // Prepare CA certs caCert, caKey, reuseCA, err := initCA(clientSet) if err != nil { @@ -272,14 +269,14 @@ func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) err // 1.1 check if cert exist cert, _, err := loadCertAndKeyFromSecret(clientSet, certConf) if err != nil { - klog.Infof("can not load cert %s from %s secret", certConf.CertName, certConf.SecretName) + klog.Infof(Format("can not load cert %s from %s secret", certConf.CertName, certConf.SecretName)) selfSignedCerts = append(selfSignedCerts, certConf) continue } // 1.2 check if cert is autorized by current CA if !IsCertFromCA(cert, caCert) { - klog.Infof("existing cert %s is not authorized by current CA", certConf.CertName) + klog.Infof(Format("existing cert %s is not authorized by current CA", certConf.CertName)) selfSignedCerts = append(selfSignedCerts, certConf) continue } @@ -290,20 +287,20 @@ func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) err ips, _, err := certConf.certInit(clientSet, stopCh) if err != nil { // if cert init failed, skip this cert - klog.Errorf("fail to init cert %s when checking dynamic attrs: %v", certConf.CertName, err) + klog.Errorf(Format("fail to init cert %s when checking dynamic attrs: %v", certConf.CertName, err)) continue } else { // check if dynamic IP addresses already exist in cert changed := ip.SearchAllIP(cert.IPAddresses, ips) if changed { - klog.Infof("cert %s IP has changed", certConf.CertName) + klog.Infof(Format("cert %s IP has changed", certConf.CertName)) selfSignedCerts = append(selfSignedCerts, certConf) continue } } } - klog.Infof("cert %s not change, reuse it", certConf.CertName) + klog.Infof(Format("cert %s not change, reuse it", certConf.CertName)) } } else { // create all certs with new CA @@ -313,7 +310,7 @@ func initPoolCoordinator(clientSet client.Interface, stopCh <-chan struct{}) err // create self signed certs for _, certConf := range selfSignedCerts { if err := initPoolCoordinatorCert(clientSet, certConf, caCert, caKey, stopCh); err != nil { - klog.Errorf("create cert %s fail: %v", certConf.CertName, err) + klog.Errorf(Format("create cert %s fail: %v", certConf.CertName, err)) return err } } @@ -358,12 +355,12 @@ func initCA(clientSet client.Interface) (caCert *x509.Certificate, caKey crypto. if err == nil { // if CA already exist - klog.Info("CA already exist in secret, reuse it") + klog.Info(Format("CA already exist in secret, reuse it")) return caCert, caKey, true, nil } else { // if not exist // create new CA certs - klog.Infof("fail to get CA from secret: %v, create new CA", err) + klog.Infof(Format("fail to get CA from secret: %v, create new CA", err)) // write it into the secret caCert, caKey, err = NewSelfSignedCA() if err != nil { diff --git a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager_test.go b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller_test.go similarity index 98% rename from pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager_test.go rename to pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller_test.go index 90431373e8f..7bbcb9a0c9d 100644 --- a/pkg/controller/poolcoordinator/cert/poolcoordinator_cert_manager_test.go +++ b/pkg/controller/poolcoordinator/cert/poolcoordinatorcert_controller_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( "testing" diff --git a/pkg/controller/poolcoordinator/cert/secret.go b/pkg/controller/poolcoordinator/cert/secret.go index 61a9883fb38..2bb49e66542 100644 --- a/pkg/controller/poolcoordinator/cert/secret.go +++ b/pkg/controller/poolcoordinator/cert/secret.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( "context" @@ -54,12 +54,12 @@ func NewSecretClient(clientSet client.Interface, ns, name string) (*SecretClient // if this secret already exist, reuse it if kerrors.IsAlreadyExists(err) { secret, _ = clientSet.CoreV1().Secrets(ns).Get(context.TODO(), name, metav1.GetOptions{}) - klog.V(4).Infof("secret %s already exisit: %v", name, secret) + klog.V(4).Infof(Format("secret %s already exisit", secret.Name)) } else { return nil, fmt.Errorf("create secret client %s fail: %v", name, err) } } else { - klog.V(4).Infof("secret %s not exisit, create one: %v", name, secret) + klog.V(4).Infof(Format("secret %s not exisit, create one", secret.Name)) } return &SecretClient{ diff --git a/pkg/controller/poolcoordinator/cert/secret_test.go b/pkg/controller/poolcoordinator/cert/secret_test.go index 2d18bb4b09e..601939a7f8a 100644 --- a/pkg/controller/poolcoordinator/cert/secret_test.go +++ b/pkg/controller/poolcoordinator/cert/secret_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( "testing" diff --git a/pkg/controller/poolcoordinator/cert/util.go b/pkg/controller/poolcoordinator/cert/util.go index d7c39abc42f..26f52d0a444 100644 --- a/pkg/controller/poolcoordinator/cert/util.go +++ b/pkg/controller/poolcoordinator/cert/util.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( "context" @@ -59,10 +59,10 @@ func waitUntilSVCReady(clientSet client.Interface, serviceName string, stopCh <- if err = wait.PollUntil(1*time.Second, func() (bool, error) { serverSVC, err = clientSet.CoreV1().Services(PoolcoordinatorNS).Get(context.TODO(), serviceName, metav1.GetOptions{}) if err == nil { - klog.Infof("%s service is ready for poolcoordinator_cert_manager", serviceName) + klog.Infof(Format("%s service is ready for poolcoordinator_cert_manager", serviceName)) return true, nil } - klog.Infof("waiting for the poolcoordinator %s service", serviceName) + klog.Infof(Format("waiting for the poolcoordinator %s service", serviceName)) return false, nil }, stopCh); err != nil { return nil, nil, err diff --git a/pkg/controller/poolcoordinator/cert/util_test.go b/pkg/controller/poolcoordinator/cert/util_test.go index ea95c1c7078..85544a660de 100644 --- a/pkg/controller/poolcoordinator/cert/util_test.go +++ b/pkg/controller/poolcoordinator/cert/util_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cert +package poolcoordinatorcert import ( "testing" diff --git a/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go b/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go index 27a3f050025..d6b224f2438 100644 --- a/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go +++ b/pkg/controller/poolcoordinator/delegatelease/poolcoordinator_controller.go @@ -55,7 +55,8 @@ type Controller struct { leaseLister leaselisterv1.LeaseNamespaceLister nodeUpdateQueue workqueue.Interface - ldc *utils.LeaseDelegatedCounter + ldc *utils.LeaseDelegatedCounter + delLdc *utils.LeaseDelegatedCounter } func (c *Controller) onLeaseCreate(n interface{}) { @@ -87,6 +88,7 @@ func NewController(kc client.Interface, informerFactory informers.SharedInformer client: kc, nodeUpdateQueue: workqueue.NewNamed("poolcoordinator_node"), ldc: utils.NewLeaseDelegatedCounter(), + delLdc: utils.NewLeaseDelegatedCounter(), } if informerFactory != nil { @@ -152,6 +154,7 @@ func (c *Controller) doDeTaintNodeNotSchedulable(node *corev1.Node) *corev1.Node taints := node.Spec.Taints taints, deleted := utils.DeleteTaintsByKey(taints, constant.NodeNotSchedulableTaint) if !deleted { + c.delLdc.Inc(node.Name) klog.Infof("detaint %s: no key %s exists, nothing to do\n", node.Name, constant.NodeNotSchedulableTaint) return node } @@ -162,6 +165,8 @@ func (c *Controller) doDeTaintNodeNotSchedulable(node *corev1.Node) *corev1.Node nn, err = c.client.CoreV1().Nodes().Update(context.TODO(), nn, metav1.UpdateOptions{}) if err != nil { klog.Error(err) + } else { + c.delLdc.Inc(node.Name) } } return nn @@ -186,9 +191,10 @@ func (c *Controller) syncHandler(key string) error { c.ldc.Inc(nl.Name) if c.ldc.Counter(nl.Name) >= constant.LeaseDelegationThreshold { c.taintNodeNotSchedulable(nl.Name) + c.delLdc.Reset(nl.Name) } } else { - if c.ldc.Counter(nl.Name) >= constant.LeaseDelegationThreshold { + if c.delLdc.Counter(nl.Name) == 0 { c.deTaintNodeNotSchedulable(nl.Name) } c.ldc.Reset(nl.Name) diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go index f363deda7d1..7114ee87a11 100644 --- a/pkg/node-servant/components/yurthub.go +++ b/pkg/node-servant/components/yurthub.go @@ -27,14 +27,16 @@ import ( "strings" "time" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -42,6 +44,8 @@ import ( const ( hubHealthzCheckFrequency = 10 * time.Second fileMode = 0666 + DefaultRootDir = "/var/lib" + DefaultCaPath = "/etc/kubernetes/pki/ca.crt" ) type yurtHubOperator struct { @@ -79,7 +83,7 @@ func (op *yurtHubOperator) Install() error { "yurthubServerAddr": constants.DefaultYurtHubServerAddr, "kubernetesServerAddr": op.apiServerAddr, "image": op.yurthubImage, - "joinToken": op.joinToken, + "bootstrapFile": constants.YurtHubBootstrapConfig, "workingMode": string(op.workingMode), "enableDummyIf": strconv.FormatBool(op.enableDummyIf), "enableNodePool": strconv.FormatBool(op.enableNodePool), @@ -88,7 +92,16 @@ func (op *yurtHubOperator) Install() error { return err } - // 1-2. create yurthub.yaml + // 1-2. create /var/lib/yurthub/bootstrap-hub.conf + if err := enutil.EnsureDir(constants.YurtHubWorkdir); err != nil { + return err + } + if err := setHubBootstrapConfig(op.apiServerAddr, op.joinToken); err != nil { + return err + } + klog.Infof("create the %s", constants.YurtHubBootstrapConfig) + + // 1-3. create yurthub.yaml podManifestPath := enutil.GetPodManifestPath() if err := enutil.EnsureDir(podManifestPath); err != nil { return err @@ -151,7 +164,7 @@ func getYurthubYaml(podManifestPath string) string { } func getYurthubConf() string { - return filepath.Join(token.DefaultRootDir, projectinfo.GetHubName()) + return filepath.Join(DefaultRootDir, projectinfo.GetHubName()) } func getYurthubCacheDir() string { @@ -221,3 +234,26 @@ func pingClusterHealthz(client *http.Client, addr string) (bool, error) { return true, nil } + +func setHubBootstrapConfig(serverAddr string, joinToken string) error { + caData, err := os.ReadFile(DefaultCaPath) + if err != nil { + return err + } + tlsBootstrapCfg := kubeconfigutil.CreateWithToken( + serverAddr, + "openyurt-e2e-test", + "token-bootstrap-client", + caData, + joinToken, + ) + content, err := clientcmd.Write(*tlsBootstrapCfg) + if err != nil { + return err + } + if err := os.WriteFile(constants.YurtHubBootstrapConfig, content, fileMode); err != nil { + return errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk") + } + + return nil +} diff --git a/pkg/node-servant/preflight-convert/preflight.go b/pkg/node-servant/preflight-convert/preflight.go index 5f33dcf535a..d404e40fc5e 100644 --- a/pkg/node-servant/preflight-convert/preflight.go +++ b/pkg/node-servant/preflight-convert/preflight.go @@ -17,11 +17,9 @@ limitations under the License. package preflight_convert import ( - "fmt" - "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/preflight" + "github.com/openyurtio/openyurt/pkg/node-servant/preflight" ) // ConvertPreflighter do the preflight-convert-convert job @@ -42,8 +40,8 @@ func (n *ConvertPreflighter) Do() error { return err } - fmt.Println("[preflight-convert] Pulling images required for converting a Kubernetes cluster to an OpenYurt cluster") - fmt.Println("[preflight-convert] This might take a minute or two, depending on the speed of your internet connection") + klog.Infof("[preflight-convert] Pulling images required for converting a Kubernetes cluster to an OpenYurt cluster") + klog.Infof("[preflight-convert] This might take a minute or two, depending on the speed of your internet connection") if err := preflight.RunPullImagesCheck(n, n.IgnorePreflightErrors); err != nil { return err } diff --git a/pkg/preflight/checks.go b/pkg/node-servant/preflight/checks.go similarity index 100% rename from pkg/preflight/checks.go rename to pkg/node-servant/preflight/checks.go diff --git a/pkg/preflight/constants.go b/pkg/node-servant/preflight/constants.go similarity index 100% rename from pkg/preflight/constants.go rename to pkg/node-servant/preflight/constants.go diff --git a/pkg/preflight/interface.go b/pkg/node-servant/preflight/interface.go similarity index 100% rename from pkg/preflight/interface.go rename to pkg/node-servant/preflight/interface.go diff --git a/pkg/yurtadm/cmd/cmd.go b/pkg/yurtadm/cmd/cmd.go index 2af8ffc80af..761310725a6 100644 --- a/pkg/yurtadm/cmd/cmd.go +++ b/pkg/yurtadm/cmd/cmd.go @@ -28,6 +28,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/docs" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join" + "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/renew" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/reset" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/token" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/yurtinit" @@ -50,6 +51,7 @@ func NewYurtadmCommand() *cobra.Command { cmds.AddCommand(reset.NewCmdReset(os.Stdin, os.Stdout, os.Stderr)) cmds.AddCommand(token.NewCmdToken(os.Stdin, os.Stdout, os.Stderr)) cmds.AddCommand(docs.NewDocsCmd(cmds)) + cmds.AddCommand(renew.NewCmdRenew(os.Stdin, os.Stdout, os.Stderr)) klog.InitFlags(nil) // goflag.Parse() diff --git a/pkg/yurtadm/cmd/join/phases/postcheck.go b/pkg/yurtadm/cmd/join/phases/postcheck.go index aaa77e17dcc..a70cae1cb52 100644 --- a/pkg/yurtadm/cmd/join/phases/postcheck.go +++ b/pkg/yurtadm/cmd/join/phases/postcheck.go @@ -17,64 +17,31 @@ limitations under the License. package phases import ( - "fmt" - "io" - "net/http" - "time" - - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" - "github.com/openyurtio/openyurt/pkg/yurtadm/constants" - "github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" ) -// RunPostCheck executes the node health check process. +// RunPostCheck executes the node health check and clean process. func RunPostCheck(data joindata.YurtJoinData) error { klog.V(1).Infof("check kubelet status.") - if err := checkKubeletStatus(); err != nil { + if err := kubernetes.CheckKubeletStatus(); err != nil { return err } klog.V(1).Infof("kubelet service is active") klog.V(1).Infof("waiting hub agent ready.") - if err := checkYurthubHealthz(data); err != nil { + if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil { return err } klog.V(1).Infof("hub agent is ready") - return nil -} - -// checkKubeletStatus check if kubelet is healthy. -func checkKubeletStatus() error { - initSystem, err := initsystem.GetInitSystem() - if err != nil { + if err := yurthub.CleanHubBootstrapConfig(); err != nil { return err } - if ok := initSystem.ServiceIsActive("kubelet"); !ok { - return fmt.Errorf("kubelet is not active. ") - } - return nil -} + klog.V(1).Infof("clean yurthub bootstrap config file success") -// checkYurthubHealthz check if YurtHub is healthy. -func checkYurthubHealthz(joinData joindata.YurtJoinData) error { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", joinData.YurtHubServer()), constants.ServerHealthzURLPath), nil) - if err != nil { - return err - } - client := &http.Client{} - return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { - resp, err := client.Do(req) - if err != nil { - return false, nil - } - ok, err := io.ReadAll(resp.Body) - if err != nil { - return false, nil - } - return string(ok) == "OK", nil - }) + return nil } diff --git a/pkg/yurtadm/cmd/join/phases/prepare.go b/pkg/yurtadm/cmd/join/phases/prepare.go index acc46c4f2a1..6d21954b642 100644 --- a/pkg/yurtadm/cmd/join/phases/prepare.go +++ b/pkg/yurtadm/cmd/join/phases/prepare.go @@ -17,18 +17,16 @@ limitations under the License. package phases import ( - "fmt" "os" "path/filepath" - "strings" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" "github.com/openyurtio/openyurt/pkg/yurtadm/util/system" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" ) // RunPrepare executes the node initialization process. @@ -66,58 +64,17 @@ func RunPrepare(data joindata.YurtJoinData) error { if err := yurtadmutil.SetKubeletConfigForNode(); err != nil { return err } - if err := addYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil { + if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil { return err } - if err := yurtadmutil.SetDiscoveryConfig(data); err != nil { + if err := yurthub.AddYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil { return err } - if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil { - return err - } - return nil -} - -// addYurthubStaticYaml generate YurtHub static yaml for worker node. -func addYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) error { - klog.Info("[join-node] Adding edge hub static yaml") - if _, err := os.Stat(podManifestPath); err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(podManifestPath, os.ModePerm) - if err != nil { - return err - } - } else { - klog.Errorf("Describe dir %s fail: %v", podManifestPath, err) - return err - } - } - - // There can be multiple master IP addresses - serverAddrs := strings.Split(data.ServerAddr(), ",") - for i := 0; i < len(serverAddrs); i++ { - serverAddrs[i] = fmt.Sprintf("https://%s", serverAddrs[i]) - } - - kubernetesServerAddrs := strings.Join(serverAddrs, ",") - - ctx := map[string]string{ - "kubernetesServerAddr": kubernetesServerAddrs, - "image": data.YurtHubImage(), - "joinToken": data.JoinToken(), - "workingMode": data.NodeRegistration().WorkingMode, - "organizations": data.NodeRegistration().Organizations, - "yurthubServerAddr": data.YurtHubServer(), - } - - yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx) - if err != nil { + if err := yurtadmutil.SetDiscoveryConfig(data); err != nil { return err } - - if err := os.WriteFile(filepath.Join(podManifestPath, constants.YurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil { + if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil { return err } - klog.Info("[join-node] Add hub agent static yaml is ok") return nil } diff --git a/pkg/yurtadm/cmd/renew/certificate/certificate.go b/pkg/yurtadm/cmd/renew/certificate/certificate.go new file mode 100644 index 00000000000..172f7d1face --- /dev/null +++ b/pkg/yurtadm/cmd/renew/certificate/certificate.go @@ -0,0 +1,164 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "fmt" + "net/url" + "strings" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + yurtconstants "github.com/openyurtio/openyurt/pkg/yurtadm/constants" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" +) + +type certificateOptions struct { + token string + caCertHashes []string + unsafeSkipCAVerification bool + serverAddr string + yurthubServer string +} + +// newCertificateOptions returns a struct ready for being used for creating cmd renew flags. +func newCertificateOptions() *certificateOptions { + return &certificateOptions{ + caCertHashes: make([]string, 0), + unsafeSkipCAVerification: true, + yurthubServer: yurtconstants.DefaultYurtHubServerAddr, + } +} + +// NewCmdCertificate returns "yurtadm renew certificate" command. +func NewCmdCertificate() *cobra.Command { + o := newCertificateOptions() + + certificateCmd := &cobra.Command{ + Use: "certificate", + Short: "Create bootstrap file for yurthub to update certificate", + RunE: func(certificateCmd *cobra.Command, args []string) error { + if err := o.validate(); err != nil { + klog.Fatalf("validate options: %v", err) + } + + // Check if YurtHub's certificate is ready or not. + // No update if ready. + if ok := yurthub.CheckYurthubReadyzOnce(o.yurthubServer); ok { + klog.Infoln("The certificate is still valid and does not need to be renewed") + return nil + } + + us, err := parseRemoteServers(o.serverAddr) + if err != nil { + return err + } + + // 1.Create a temporary bootstrap file and set to /var/lib/yurthub directory. + if err := yurthub.SetHubBootstrapConfig(us[0].Host, o.token, o.caCertHashes); err != nil { + return err + } + + // 2.Check if YurtHub's certificates is ready or not. + if err := yurthub.CheckYurthubReadyz(o.yurthubServer); err != nil { + return err + } + + // 3.Delete temporary bootstrap file. + if err := yurthub.CleanHubBootstrapConfig(); err != nil { + return err + } + + klog.Infoln("Certificate renewed successfully") + + return nil + }, + } + + addCertificateConfigFlags(certificateCmd.Flags(), o) + return certificateCmd +} + +func (options *certificateOptions) validate() error { + if len(options.serverAddr) == 0 { + return fmt.Errorf("server-address is empty") + } + + if len(options.token) == 0 { + return fmt.Errorf("bootstrap token is empty") + } + + if len(options.caCertHashes) == 0 && !options.unsafeSkipCAVerification { + return fmt.Errorf("set --discovery-token-unsafe-skip-ca-verification flag as true or pass CACertHashes to continue") + } + + return nil +} + +func parseRemoteServers(serverAddr string) ([]*url.URL, error) { + servers := strings.Split(serverAddr, ",") + us := make([]*url.URL, 0, len(servers)) + remoteServers := make([]string, 0, len(servers)) + for _, server := range servers { + u, err := url.Parse(server) + if err != nil { + klog.Errorf("failed to parse server address %s, %v", servers, err) + return us, err + } + if u.Scheme == "" { + u.Scheme = "https" + } else if u.Scheme != "https" { + return us, fmt.Errorf("only https scheme is supported for server address(%s)", serverAddr) + } + us = append(us, u) + remoteServers = append(remoteServers, u.String()) + } + + if len(us) < 1 { + return us, fmt.Errorf("no server address is set, can not connect remote server") + } + klog.Infof("%s would connect remote servers: %s", projectinfo.GetHubName(), strings.Join(remoteServers, ",")) + + return us, nil +} + +// addCertificateConfigFlags adds certificate flags bound to the config to the specified flagset +func addCertificateConfigFlags(flagSet *flag.FlagSet, certificateOptions *certificateOptions) { + flagSet.StringVar( + &certificateOptions.token, yurtconstants.TokenStr, certificateOptions.token, + "Use this token for bootstrapping yurthub.", + ) + flagSet.StringSliceVar( + &certificateOptions.caCertHashes, yurtconstants.TokenDiscoveryCAHash, certificateOptions.caCertHashes, + "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").", + ) + flagSet.BoolVar( + &certificateOptions.unsafeSkipCAVerification, yurtconstants.TokenDiscoverySkipCAHash, certificateOptions.unsafeSkipCAVerification, + "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.", + ) + flagSet.StringVar( + &certificateOptions.serverAddr, yurtconstants.ServerAddr, certificateOptions.serverAddr, + "The address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"", + ) + flagSet.StringVar( + &certificateOptions.yurthubServer, yurtconstants.YurtHubServerAddr, certificateOptions.yurthubServer, + "Sets the address for yurthub server addr", + ) +} diff --git a/pkg/yurtadm/cmd/renew/renew.go b/pkg/yurtadm/cmd/renew/renew.go new file mode 100644 index 00000000000..8d2cd9ba4c1 --- /dev/null +++ b/pkg/yurtadm/cmd/renew/renew.go @@ -0,0 +1,67 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package renew + +import ( + "fmt" + "io" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/renew/certificate" + util "github.com/openyurtio/openyurt/pkg/yurtadm/util/error" +) + +// NewCmdRenew returns "yurtadm renew" command. +func NewCmdRenew(in io.Reader, out io.Writer, outErr io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "renew", + Short: "Renew something like certificate", + // Without this callback, if a user runs just the "renew" + // command without a subcommand, or with an invalid subcommand, + // cobra will print usage information, but still exit cleanly. + // We want to return an error code in these cases so that the + // user knows that their command was invalid. + Run: subCmdRun(), + } + + cmd.AddCommand(certificate.NewCmdCertificate()) + return cmd +} + +// subCmdRun returns a function that handles a case where a subcommand must be specified +// Without this callback, if a user runs just the command without a subcommand, +// or with an invalid subcommand, cobra will print usage information, but still exit cleanly. +func subCmdRun() func(c *cobra.Command, args []string) { + return func(c *cobra.Command, args []string) { + if len(args) > 0 { + util.CheckErr(usageErrorf(c, "invalid subcommand %q", strings.Join(args, " "))) + } + err := c.Help() + if err != nil { + return + } + util.CheckErr(util.ErrExit) + } +} + +func usageErrorf(c *cobra.Command, format string, args ...interface{}) error { + msg := fmt.Sprintf(format, args...) + return errors.Errorf("%s\nSee '%s -h' for help and examples", msg, c.CommandPath()) +} diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go index 0c9515b2d8a..151bc5efc41 100644 --- a/pkg/yurtadm/constants/constants.go +++ b/pkg/yurtadm/constants/constants.go @@ -23,10 +23,10 @@ const ( KubeletConfigureDir = "/etc/kubernetes" KubeletWorkdir = "/var/lib/kubelet" YurtHubWorkdir = "/var/lib/yurthub" + YurtHubBootstrapConfig = "/var/lib/yurthub/bootstrap-hub.conf" OpenyurtDir = "/var/lib/openyurt" YurttunnelAgentWorkdir = "/var/lib/yurttunnel-agent" YurttunnelServerWorkdir = "/var/lib/yurttunnel-server" - KubeConfigPath = "/etc/kubernetes/kubelet.conf" KubeCniDir = "/opt/cni/bin" KubeCniVersion = "v0.8.0" KubeletServiceFilepath = "/etc/systemd/system/kubelet.service" @@ -97,11 +97,14 @@ const ( YurtHubImage = "yurthub-image" // YurtHubServerAddr flag set the address of yurthub server (not proxy server!) YurtHubServerAddr = "yurthub-server-addr" + // ServerAddr flag set the address of kubernetes kube-apiserver + ServerAddr = "server-addr" // ReuseCNIBin flag sets whether to reuse local CNI binaries or not. ReuseCNIBin = "reuse-cni-bin" ServerHealthzServer = "127.0.0.1:10267" ServerHealthzURLPath = "/v1/healthz" + ServerReadyzURLPath = "/v1/readyz" DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt" DefaultOpenYurtVersion = "latest" Yurthub = "yurthub" @@ -220,7 +223,7 @@ spec: - --bind-address={{.yurthubServerAddr}} - --server-addr={{.kubernetesServerAddr}} - --node-name=$(NODE_NAME) - - --join-token={{.joinToken}} + - --bootstrap-file={{.bootstrapFile}} - --working-mode={{.workingMode}} {{if .enableDummyIf }} - --enable-dummy-if={{.enableDummyIf}} diff --git a/pkg/yurtadm/util/error/error.go b/pkg/yurtadm/util/error/error.go new file mode 100644 index 00000000000..dc0aa7a55f8 --- /dev/null +++ b/pkg/yurtadm/util/error/error.go @@ -0,0 +1,114 @@ +/* +Copyright 2023 The OpenYurt Authors. +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/pkg/errors" + errorsutil "k8s.io/apimachinery/pkg/util/errors" +) + +const ( + // DefaultErrorExitCode defines exit the code for failed action generally + DefaultErrorExitCode = 1 + // PreFlightExitCode defines exit the code for preflight checks + PreFlightExitCode = 2 + // ValidationExitCode defines the exit code validation checks + ValidationExitCode = 3 +) + +var ( + ErrInvalidSubCommandMsg = "invalid subcommand" + ErrExit = errors.New("exit") +) + +// fatal prints the message if set and then exits. +func fatal(msg string, code int) { + if len(msg) > 0 { + // add newline if needed + if !strings.HasSuffix(msg, "\n") { + msg += "\n" + } + + fmt.Fprint(os.Stderr, msg) + } + os.Exit(code) +} + +// CheckErr prints a user friendly error to STDERR and exits with a non-zero +// exit code. Unrecognized errors will be printed with an "error: " prefix. +// +// This method is generic to the command in use and may be used by non-Kubectl +// commands. +func CheckErr(err error) { + checkErr(err, fatal) +} + +// preflightError allows us to know if the error is a preflight error or not +// defining the interface here avoids an import cycle of pulling in preflight into the util package +type preflightError interface { + Preflight() bool +} + +// checkErr formats a given error as a string and calls the passed handleErr +// func with that string and an exit code. +func checkErr(err error, handleErr func(string, int)) { + + var msg string + if err != nil { + msg = fmt.Sprintf("%s\nTo see the stack trace of this error execute with --v=5 or higher", err.Error()) + // check if the verbosity level in klog is high enough and print a stack trace. + f := flag.CommandLine.Lookup("v") + if f != nil { + // assume that the "v" flag contains a parseable Int32 as per klog's "Level" type alias, + // thus no error from ParseInt is handled here. + if v, e := strconv.ParseInt(f.Value.String(), 10, 32); e == nil { + // https://git.k8s.io/community/contributors/devel/sig-instrumentation/logging.md + // klog.V(5) - Trace level verbosity + if v > 4 { + msg = fmt.Sprintf("%+v", err) + } + } + } + } + + if err == nil { + return + } + switch { + case err == ErrExit: + handleErr("", DefaultErrorExitCode) + case strings.Contains(err.Error(), ErrInvalidSubCommandMsg): + handleErr(err.Error(), DefaultErrorExitCode) + default: + switch err.(type) { + case preflightError: + handleErr(msg, PreFlightExitCode) + case errorsutil.Aggregate: + handleErr(msg, ValidationExitCode) + + default: + handleErr(msg, DefaultErrorExitCode) + } + } +} diff --git a/pkg/yurtadm/util/error/error_test.go b/pkg/yurtadm/util/error/error_test.go new file mode 100644 index 00000000000..3c53a302c43 --- /dev/null +++ b/pkg/yurtadm/util/error/error_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The OpenYurt Authors. +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/pkg/errors" +) + +type pferror struct{} + +func (p *pferror) Preflight() bool { return true } +func (p *pferror) Error() string { return "" } +func TestCheckErr(t *testing.T) { + var codeReturned int + errHandle := func(err string, code int) { + codeReturned = code + } + + var tests = []struct { + name string + e error + expected int + }{ + {"error is nil", nil, 0}, + {"empty error", errors.New(""), DefaultErrorExitCode}, + {"preflight error", &pferror{}, PreFlightExitCode}, + } + + for _, rt := range tests { + t.Run(rt.name, func(t *testing.T) { + codeReturned = 0 + checkErr(rt.e, errHandle) + if codeReturned != rt.expected { + t.Errorf( + "failed checkErr:\n\texpected: %d\n\t actual: %d", + rt.expected, + codeReturned, + ) + } + }) + } +} diff --git a/pkg/yurtadm/util/kubernetes/kubernetes.go b/pkg/yurtadm/util/kubernetes/kubernetes.go index d9f26646aec..1ef2ab89f5c 100644 --- a/pkg/yurtadm/util/kubernetes/kubernetes.go +++ b/pkg/yurtadm/util/kubernetes/kubernetes.go @@ -47,6 +47,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtadm/constants" "github.com/openyurtio/openyurt/pkg/yurtadm/util" "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem" ) const ( @@ -475,3 +476,15 @@ func RetrieveBootstrapConfig(data joindata.YurtJoinData) (*clientcmdapi.Config, data.JoinToken(), ), nil } + +// CheckKubeletStatus check if kubelet is healthy. +func CheckKubeletStatus() error { + initSystem, err := initsystem.GetInitSystem() + if err != nil { + return err + } + if ok := initSystem.ServiceIsActive("kubelet"); !ok { + return fmt.Errorf("kubelet is not active. ") + } + return nil +} diff --git a/pkg/yurtadm/util/yurthub/yurthub.go b/pkg/yurtadm/util/yurthub/yurthub.go new file mode 100644 index 00000000000..27539ce9ed8 --- /dev/null +++ b/pkg/yurtadm/util/yurthub/yurthub.go @@ -0,0 +1,169 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurthub + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" + + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" + "github.com/openyurtio/openyurt/pkg/util/templates" + "github.com/openyurtio/openyurt/pkg/util/token" + "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" + "github.com/openyurtio/openyurt/pkg/yurtadm/constants" +) + +// AddYurthubStaticYaml generate YurtHub static yaml for worker node. +func AddYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) error { + klog.Info("[join-node] Adding edge hub static yaml") + if _, err := os.Stat(podManifestPath); err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(podManifestPath, os.ModePerm) + if err != nil { + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", podManifestPath, err) + return err + } + } + + // There can be multiple master IP addresses + serverAddrs := strings.Split(data.ServerAddr(), ",") + for i := 0; i < len(serverAddrs); i++ { + serverAddrs[i] = fmt.Sprintf("https://%s", serverAddrs[i]) + } + + kubernetesServerAddrs := strings.Join(serverAddrs, ",") + + ctx := map[string]string{ + "kubernetesServerAddr": kubernetesServerAddrs, + "image": data.YurtHubImage(), + "bootstrapFile": constants.YurtHubBootstrapConfig, + "workingMode": data.NodeRegistration().WorkingMode, + "organizations": data.NodeRegistration().Organizations, + "yurthubServerAddr": data.YurtHubServer(), + } + + yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx) + if err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(podManifestPath, constants.YurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil { + return err + } + klog.Info("[join-node] Add hub agent static yaml is ok") + return nil +} + +func SetHubBootstrapConfig(serverAddr string, joinToken string, caCertHashes []string) error { + if cfg, err := token.RetrieveValidatedConfigInfo(nil, &token.BootstrapData{ + ServerAddr: serverAddr, + JoinToken: joinToken, + CaCertHashes: caCertHashes, + }); err != nil { + return errors.Wrap(err, "couldn't retrieve bootstrap config info") + } else { + clusterInfo := kubeconfigutil.GetClusterFromKubeConfig(cfg) + tlsBootstrapCfg := kubeconfigutil.CreateWithToken( + fmt.Sprintf("https://%s", serverAddr), + "kubernetes", + "token-bootstrap-client", + clusterInfo.CertificateAuthorityData, + joinToken, + ) + if err = kubeconfigutil.WriteToDisk(constants.YurtHubBootstrapConfig, tlsBootstrapCfg); err != nil { + return errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk") + } + } + + return nil +} + +// CheckYurthubHealthz check if YurtHub is healthy. +func CheckYurthubHealthz(yurthubServer string) error { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerHealthzURLPath), nil) + if err != nil { + return err + } + client := &http.Client{} + return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { + resp, err := client.Do(req) + if err != nil { + return false, nil + } + ok, err := io.ReadAll(resp.Body) + if err != nil { + return false, nil + } + return string(ok) == "OK", nil + }) +} + +// CheckYurthubReadyz check if YurtHub's certificates are ready or not +func CheckYurthubReadyz(yurthubServer string) error { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerReadyzURLPath), nil) + if err != nil { + return err + } + client := &http.Client{} + return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { + resp, err := client.Do(req) + if err != nil { + return false, nil + } + ok, err := io.ReadAll(resp.Body) + if err != nil { + return false, nil + } + return string(ok) == "OK", nil + }) +} + +func CheckYurthubReadyzOnce(yurthubServer string) bool { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerReadyzURLPath), nil) + if err != nil { + return false + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return false + } + ok, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + return string(ok) == "OK" +} + +func CleanHubBootstrapConfig() error { + if err := os.RemoveAll(constants.YurtHubBootstrapConfig); err != nil { + klog.Warningf("Clean file %s fail: %v, please clean it manually.", constants.YurtHubBootstrapConfig, err) + } + return nil +} diff --git a/pkg/yurthub/certificate/token/token.go b/pkg/yurthub/certificate/token/token.go index ca7bd075478..4920ca14dd3 100644 --- a/pkg/yurthub/certificate/token/token.go +++ b/pkg/yurthub/certificate/token/token.go @@ -68,6 +68,7 @@ type CertificateManagerConfiguration struct { RootDir string NodeName string JoinToken string + BootstrapFile string CaCertHashes []string YurtHubCertOrganizations []string CertIPs []net.IP @@ -86,6 +87,7 @@ type yurtHubCertManager struct { hubRunDir string hubName string joinToken string + bootstrapFile string dialer *util.Dialer } @@ -104,6 +106,7 @@ func NewYurtHubCertManager(cfg *CertificateManagerConfiguration) (hubCert.YurtCe hubRunDir: hubRunDir, hubName: projectinfo.GetHubName(), joinToken: cfg.JoinToken, + bootstrapFile: cfg.BootstrapFile, caCertHashes: cfg.CaCertHashes, dialer: util.NewDialer("hub certificate manager"), } @@ -150,10 +153,10 @@ func removeDirContents(dir string) error { } func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { - if cfg, err := clientcmd.LoadFromFile(ycm.getBootstrapConfFile()); err == nil { + if cfg, err := clientcmd.LoadFromFile(ycm.GetHubConfFile()); err == nil { cluster := kubeconfigutil.GetClusterFromKubeConfig(cfg) if serverURL, err := url.Parse(cluster.Server); err != nil { - klog.Errorf("couldn't get server info from %s, %v", ycm.getBootstrapConfFile(), err) + klog.Errorf("couldn't get server info from %s, %v", ycm.GetHubConfFile(), err) } else { for i := range servers { if servers[i].Host == serverURL.Host { @@ -187,7 +190,58 @@ func (ycm *yurtHubCertManager) Start() { // if these files already exist, just reuse them. func (ycm *yurtHubCertManager) prepareConfigAndCaFile() error { var tlsBootstrapCfg *clientcmdapi.Config + var hubKubeConfig *clientcmdapi.Config + var err error + + // A bootstrap-file is prepared by yurtadm join command and configured as parameter for yurthub, + // yurthub only need to use it. + if len(ycm.bootstrapFile) != 0 { + // 1. load bootstrap config + if tlsBootstrapCfg, err = clientcmd.LoadFromFile(ycm.getBootstrapConfFile()); err != nil { + klog.Errorf("maybe hub agent restarted, failed to load bootstrap config file(%s), %v.", ycm.getBootstrapConfFile()) + } else { + klog.V(2).Infof("%s file is configured, just use it", ycm.getBootstrapConfFile()) + } + + // 2. prepare kubeconfig file(/var/lib/yurthub/yurthub.conf) for yurthub + if exist, err := util.FileExists(ycm.GetHubConfFile()); err != nil { + return errors.Wrap(err, "couldn't stat hub kubeconfig file") + } else if exist { + klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetHubConfFile()) + if hubKubeConfig, err = clientcmd.LoadFromFile(ycm.GetHubConfFile()); err != nil { + return errors.Wrapf(err, "couldn't load hub kubeconfig file(%s)", ycm.GetHubConfFile()) + } + } else if tlsBootstrapCfg == nil { + return errors.Errorf("neither boostrap file(%s) nor kubeconfig file(%s) exist when hub agent started", ycm.bootstrapFile, ycm.GetHubConfFile()) + } else { + // hub kubeconfig file doesn't exist, but bootstrap file is ready, so create hub.conf by bootstrap config + hubKubeConfig = createHubConfig(tlsBootstrapCfg, ycm.apiServerClientCertStore.CurrentPath()) + if err = kubeconfigutil.WriteToDisk(ycm.GetHubConfFile(), hubKubeConfig); err != nil { + return errors.Wrapf(err, "couldn't save %s to disk", hubConfigFileName) + } + } + // 3. prepare ca.crt file(/var/lib/yurthub/pki/ca.crt) for yurthub + if exist, err := util.FileExists(ycm.GetCaFile()); err != nil { + return errors.Wrap(err, "couldn't stat ca.crt file") + } else if !exist { + cluster := kubeconfigutil.GetClusterFromKubeConfig(hubKubeConfig) + if cluster != nil { + if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil { + return errors.Wrap(err, "couldn't save the CA certificate to disk") + } + } else { + return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile()) + } + } else { + klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile()) + } + return nil + } + + // in order to keep consistency with old version(with join token), + // if join token instead of bootstrap-file is set, we will use join token to create boostrap-hub.conf + // use join token to create bootstrap-hub.conf and will be removed in the future version // 1. prepare bootstrap config file(/var/lib/yurthub/bootstrap-hub.conf) for yurthub if exist, err := util.FileExists(ycm.getBootstrapConfFile()); err != nil { return errors.Wrap(err, "couldn't stat bootstrap config file") @@ -274,6 +328,9 @@ func (ycm *yurtHubCertManager) getPkiDir() string { // getBootstrapConfFile returns the path of yurthub bootstrap conf file func (ycm *yurtHubCertManager) getBootstrapConfFile() string { + if len(ycm.bootstrapFile) != 0 { + return ycm.bootstrapFile + } return filepath.Join(ycm.hubRunDir, bootstrapConfigFileName) } diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index d7b9b6e98ed..fead050d390 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -25,6 +25,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/profile" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" ota "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate" "github.com/openyurtio/openyurt/pkg/yurthub/util" @@ -77,6 +78,7 @@ func registerHandlers(c *mux.Router, cfg *config.YurtHubConfiguration, rest *res // register handler for health check c.HandleFunc("/v1/healthz", healthz).Methods("GET") + c.Handle("/v1/readyz", readyz(cfg.CertManager)).Methods("GET") // register handler for profile if cfg.EnableProfiling { @@ -97,3 +99,16 @@ func healthz(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "OK") } + +// readyz is used for checking certificates are ready or not +func readyz(certificateMgr certificate.YurtCertificateManager) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ready := certificateMgr.Ready() + if ready { + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "OK") + } else { + http.Error(w, "certificates are not ready", http.StatusInternalServerError) + } + }) +} diff --git a/test/e2e/cmd/init/constants/constants.go b/test/e2e/cmd/init/constants/constants.go index 6642a122325..fc38c78f9b7 100644 --- a/test/e2e/cmd/init/constants/constants.go +++ b/test/e2e/cmd/init/constants/constants.go @@ -106,6 +106,9 @@ spec: image: {{.image}} imagePullPolicy: IfNotPresent name: yurt-manager + env: + - name: WEBHOOK_PORT + value: "10270" ports: - containerPort: 10270 name: webhook-server diff --git a/test/e2e/cmd/init/converter.go b/test/e2e/cmd/init/converter.go index 028f7177f66..80c00c7e92a 100644 --- a/test/e2e/cmd/init/converter.go +++ b/test/e2e/cmd/init/converter.go @@ -153,18 +153,16 @@ func (c *ClusterConverter) deployYurthub() error { } // print logs of yurt-manager - labelSelector, logErr := metav1.ParseToLabelSelector("app.kubernetes.io/name=yurt-manager") - if logErr != nil { - return err - } podList, logErr := c.ClientSet.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ - LabelSelector: labelSelector.String(), + LabelSelector: labels.SelectorFromSet(map[string]string{"app.kubernetes.io/name": "yurt-manager"}).String(), }) if logErr != nil { + klog.Errorf("failed to get yurt-manager pod, %v", logErr) return err } if len(podList.Items) == 0 { + klog.Errorf("yurt-manager pod doesn't exist") return err } if logErr = kubeutil.PrintPodLog(c.ClientSet, &podList.Items[0], os.Stderr); logErr != nil { diff --git a/test/e2e/cmd/init/util/kubernetes/util.go b/test/e2e/cmd/init/util/kubernetes/util.go index c2b638957fc..2c158c2b110 100644 --- a/test/e2e/cmd/init/util/kubernetes/util.go +++ b/test/e2e/cmd/init/util/kubernetes/util.go @@ -256,7 +256,12 @@ func RunJobAndCleanup(cliSet kubeclientset.Interface, job *batchv1.Job, timeout, func PrintPodLog(client kubeclientset.Interface, pod *corev1.Pod, w io.Writer) error { klog.Infof("start to print logs for pod(%s/%s):", pod.Namespace, pod.Name) req := client.CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.Name, &corev1.PodLogOptions{}) - return kubectllogs.DefaultConsumeRequest(req, w) + if err := kubectllogs.DefaultConsumeRequest(req, w); err != nil { + klog.Errorf("failed to print logs for pod(%s/%s), %v", pod.Namespace, pod.Name, err) + return err + } + + return nil } // RunServantJobs launch servant jobs on specified nodes and wait all jobs to finish.