From 37e64cc9271ff245bda33c7ee5c67cbe93c378fd Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Mon, 5 Feb 2024 13:51:31 +0100 Subject: [PATCH] Use manager in test extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com --- Makefile | 15 +- exp/runtime/server/server.go | 30 +- test/e2e/config/docker.yaml | 3 + .../config/default/kustomization.yaml | 5 +- test/extension/config/default/manager.yaml | 24 +- .../config/default/manager_image_patch.yaml | 3 +- .../config/default/manager_pull_policy.yaml | 3 +- .../config/default/manager_webhook_patch.yaml | 3 +- test/extension/config/default/role.yaml | 18 -- test/extension/config/rbac/kustomization.yaml | 8 + .../config/rbac/leader_election_role.yaml | 25 ++ .../rbac/leader_election_role_binding.yaml | 12 + test/extension/config/rbac/role.yaml | 29 ++ .../role_binding.yaml} | 6 +- .../{default => rbac}/service_account.yaml | 1 + .../handlers/topologymutation/handler.go | 2 + test/extension/main.go | 262 ++++++++++++------ 17 files changed, 324 insertions(+), 125 deletions(-) delete mode 100644 test/extension/config/default/role.yaml create mode 100644 test/extension/config/rbac/kustomization.yaml create mode 100644 test/extension/config/rbac/leader_election_role.yaml create mode 100644 test/extension/config/rbac/leader_election_role_binding.yaml create mode 100644 test/extension/config/rbac/role.yaml rename test/extension/config/{default/rolebinding.yaml => rbac/role_binding.yaml} (84%) rename test/extension/config/{default => rbac}/service_account.yaml (75%) diff --git a/Makefile b/Makefile index c435d035f8c6..75ec8df8531b 100644 --- a/Makefile +++ b/Makefile @@ -268,7 +268,7 @@ help: # Display this help ##@ generate: -ALL_GENERATE_MODULES = core kubeadm-bootstrap kubeadm-control-plane docker-infrastructure in-memory-infrastructure +ALL_GENERATE_MODULES = core kubeadm-bootstrap kubeadm-control-plane docker-infrastructure in-memory-infrastructure test-extension .PHONY: generate generate: ## Run all generate-manifests-*, generate-go-deepcopy-*, generate-go-conversions-* and generate-go-openapi targets @@ -368,6 +368,13 @@ generate-manifests-in-memory-infrastructure: $(CONTROLLER_GEN) ## Generate manif output:webhook:dir=./config/webhook \ webhook +.PHONY: generate-manifests-test-extension +generate-manifests-test-extension: $(CONTROLLER_GEN) ## Generate manifests e.g. RBAC for test-extension provider + cd ./test/extension; $(CONTROLLER_GEN) \ + paths=./... \ + output:rbac:dir=./config/rbac \ + rbac:roleName=manager-role + .PHONY: generate-go-deepcopy generate-go-deepcopy: ## Run all generate-go-deepcopy-* targets $(MAKE) $(addprefix generate-go-deepcopy-,$(ALL_GENERATE_MODULES)) @@ -418,6 +425,9 @@ generate-go-deepcopy-in-memory-infrastructure: $(CONTROLLER_GEN) ## Generate dee paths=./api/... \ paths=./internal/cloud/api/... +.PHONY: generate-go-deepcopy-test-extension +generate-go-deepcopy-test-extension: $(CONTROLLER_GEN) ## Generate deepcopy go code for test-extension + .PHONY: generate-go-conversions generate-go-conversions: ## Run all generate-go-conversions-* targets $(MAKE) $(addprefix generate-go-conversions-,$(ALL_GENERATE_MODULES)) @@ -472,6 +482,9 @@ generate-go-conversions-docker-infrastructure: $(CONVERSION_GEN) ## Generate con .PHONY: generate-go-conversions-in-memory-infrastructure generate-go-conversions-in-memory-infrastructure: $(CONVERSION_GEN) ## Generate conversions go code for in-memory infrastructure provider +.PHONY: generate-go-conversions-test-extension +generate-go-conversions-test-extension: $(CONVERSION_GEN) ## Generate conversions go code for in-memory infrastructure provider + # The tmp/sigs.k8s.io/cluster-api symlink is a workaround to make this target run outside of GOPATH .PHONY: generate-go-openapi generate-go-openapi: $(OPENAPI_GEN) $(CONTROLLER_GEN) ## Generate openapi go code for runtime SDK diff --git a/exp/runtime/server/server.go b/exp/runtime/server/server.go index 73b6a757b779..7572f163953d 100644 --- a/exp/runtime/server/server.go +++ b/exp/runtime/server/server.go @@ -43,8 +43,8 @@ var DefaultPort = 9443 // Server is a runtime webhook server. type Server struct { + webhook.Server catalog *runtimecatalog.Catalog - server webhook.Server handlers map[string]ExtensionHandler } @@ -53,20 +53,26 @@ type Options struct { // Catalog is the catalog used to handle requests. Catalog *runtimecatalog.Catalog - // Port is the port that the webhook server serves at. - // It is used to set webhook.Server.Port. - Port int - - // Host is the hostname that the webhook server binds to. + // Host is the address that the server will listen on. + // Defaults to "" - all addresses. // It is used to set webhook.Server.Host. Host string + // Port is the port number that the server will serve. + // It will be defaulted to 9443 if unspecified. + // It is used to set webhook.Server.Port. + Port int + // CertDir is the directory that contains the server key and certificate. // If not set, webhook server would look up the server key and certificate in // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate // must be named tls.key and tls.crt, respectively. // It is used to set webhook.Server.CertDir. CertDir string + + // TLSOpts is used to allow configuring the TLS config used for the server. + // This also allows providing a certificate via GetCertificate. + TLSOpts []func(*tls.Config) } // New creates a new runtime webhook server based on the given Options. @@ -88,18 +94,14 @@ func New(options Options) (*Server, error) { CertDir: options.CertDir, CertName: "tls.crt", KeyName: "tls.key", + TLSOpts: options.TLSOpts, WebhookMux: http.NewServeMux(), - TLSOpts: []func(*tls.Config){ - func(cfg *tls.Config) { - cfg.MinVersion = tls.VersionTLS13 - }, - }, }, ) return &Server{ + Server: webhookServer, catalog: options.Catalog, - server: webhookServer, handlers: map[string]ExtensionHandler{}, }, nil } @@ -232,10 +234,10 @@ func (s *Server) Start(ctx context.Context) error { handler := h wrappedHandler := s.wrapHandler(handler) - s.server.Register(handlerPath, http.HandlerFunc(wrappedHandler)) + s.Server.Register(handlerPath, http.HandlerFunc(wrappedHandler)) } - return s.server.Start(ctx) + return s.Server.Start(ctx) } // discoveryHandler generates a discovery handler based on a list of handlers. diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index df8d181f853b..8453fe64a89b 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -229,6 +229,9 @@ providers: versions: - name: v1.7.99 # next; use manifest from source files value: ../../../test/extension/config/default + replacements: + - old: "--leader-elect" + new: "--leader-elect\n - --logging-format=json" files: - sourcePath: "../data/shared/main/metadata.yaml" diff --git a/test/extension/config/default/kustomization.yaml b/test/extension/config/default/kustomization.yaml index e56b07ac75bd..c06afd5e4697 100644 --- a/test/extension/config/default/kustomization.yaml +++ b/test/extension/config/default/kustomization.yaml @@ -10,13 +10,10 @@ resources: - namespace.yaml - manager.yaml - service.yaml -- service_account.yaml -# Note: resources specific of the CAPI test-extension, other Runtime extensions provider might want to drop this -- role.yaml -- rolebinding.yaml bases: - ../certmanager +- ../rbac patchesStrategicMerge: # Enable webhook with corresponding certificate mount. diff --git a/test/extension/config/default/manager.yaml b/test/extension/config/default/manager.yaml index 7d7afd36e3d9..4a1e2359f18e 100644 --- a/test/extension/config/default/manager.yaml +++ b/test/extension/config/default/manager.yaml @@ -2,7 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: manager + name: controller-manager + namespace: system + labels: + app: test-extension-manager spec: selector: matchLabels: @@ -16,8 +19,27 @@ spec: containers: - command: - /manager + args: + - "--leader-elect" + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" image: controller:latest name: manager + ports: + - containerPort: 9440 + name: healthz + protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/test/extension/config/default/manager_image_patch.yaml b/test/extension/config/default/manager_image_patch.yaml index eb73af96da95..a33cefd11a33 100644 --- a/test/extension/config/default/manager_image_patch.yaml +++ b/test/extension/config/default/manager_image_patch.yaml @@ -1,7 +1,8 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: manager + name: controller-manager + namespace: system spec: template: spec: diff --git a/test/extension/config/default/manager_pull_policy.yaml b/test/extension/config/default/manager_pull_policy.yaml index 866aaaf1d1d8..74a0879c604a 100644 --- a/test/extension/config/default/manager_pull_policy.yaml +++ b/test/extension/config/default/manager_pull_policy.yaml @@ -1,7 +1,8 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: manager + name: controller-manager + namespace: system spec: template: spec: diff --git a/test/extension/config/default/manager_webhook_patch.yaml b/test/extension/config/default/manager_webhook_patch.yaml index bb0576db44a3..bccef6d70db8 100644 --- a/test/extension/config/default/manager_webhook_patch.yaml +++ b/test/extension/config/default/manager_webhook_patch.yaml @@ -1,7 +1,8 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: manager + name: controller-manager + namespace: system spec: template: spec: diff --git a/test/extension/config/default/role.yaml b/test/extension/config/default/role.yaml deleted file mode 100644 index d920535dd50d..000000000000 --- a/test/extension/config/default/role.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Note: this is specific of the CAPI test-extension, because it uses a ConfigMap to define life-cycle hooks answers. -# other Runtime extensions provider might want to drop this ClusterRole or make it scoped to a namespace changing kind to Role. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: manager-role # NOTE: this value will be prefixed with namePrefix value in kustomization.yaml -rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - patch - - update - - create diff --git a/test/extension/config/rbac/kustomization.yaml b/test/extension/config/rbac/kustomization.yaml new file mode 100644 index 000000000000..74cf1eaeb2d6 --- /dev/null +++ b/test/extension/config/rbac/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- leader_election_role.yaml +- leader_election_role_binding.yaml +- role.yaml +- role_binding.yaml +- service_account.yaml diff --git a/test/extension/config/rbac/leader_election_role.yaml b/test/extension/config/rbac/leader_election_role.yaml new file mode 100644 index 000000000000..197f0681744b --- /dev/null +++ b/test/extension/config/rbac/leader_election_role.yaml @@ -0,0 +1,25 @@ + +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete diff --git a/test/extension/config/rbac/leader_election_role_binding.yaml b/test/extension/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 000000000000..d5e0044679ab --- /dev/null +++ b/test/extension/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: manager + namespace: system diff --git a/test/extension/config/rbac/role.yaml b/test/extension/config/rbac/role.yaml new file mode 100644 index 000000000000..8e90952e79ec --- /dev/null +++ b/test/extension/config/rbac/role.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/test/extension/config/default/rolebinding.yaml b/test/extension/config/rbac/role_binding.yaml similarity index 84% rename from test/extension/config/default/rolebinding.yaml rename to test/extension/config/rbac/role_binding.yaml index 0c9189aaa76d..5dfbe23e48ea 100644 --- a/test/extension/config/default/rolebinding.yaml +++ b/test/extension/config/rbac/role_binding.yaml @@ -3,11 +3,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: + creationTimestamp: null name: manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: manager-role subjects: - - kind: ServiceAccount - name: manager \ No newline at end of file +- kind: ServiceAccount + name: manager + namespace: system diff --git a/test/extension/config/default/service_account.yaml b/test/extension/config/rbac/service_account.yaml similarity index 75% rename from test/extension/config/default/service_account.yaml rename to test/extension/config/rbac/service_account.yaml index 635bc1056b8c..77f747b53c9e 100644 --- a/test/extension/config/default/service_account.yaml +++ b/test/extension/config/rbac/service_account.yaml @@ -2,3 +2,4 @@ apiVersion: v1 kind: ServiceAccount metadata: name: manager + namespace: system diff --git a/test/extension/handlers/topologymutation/handler.go b/test/extension/handlers/topologymutation/handler.go index 22370412e755..b2da6b4e2825 100644 --- a/test/extension/handlers/topologymutation/handler.go +++ b/test/extension/handlers/topologymutation/handler.go @@ -49,6 +49,8 @@ var ( cgroupDriverPatchVersionCeiling = semver.Version{Major: 1, Minor: 24} ) +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;patch;update;create + // ExtensionHandlers provides a common struct shared across the topology mutation hooks handlers; // this is convenient because in Cluster API's E2E tests all of them are using a decoder for working with typed // API objects, which makes code easier to read and less error prone than using unstructured or working with raw json/yaml. diff --git a/test/extension/main.go b/test/extension/main.go index 0246efc601f0..1c24ade7f67e 100644 --- a/test/extension/main.go +++ b/test/extension/main.go @@ -21,22 +21,28 @@ limitations under the License. package main import ( + "context" "flag" - "fmt" - "net/http" "os" + goruntime "runtime" "time" "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/leaderelection/resourcelock" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/logs" logsv1 "k8s.io/component-base/logs/api/v1" _ "k8s.io/component-base/logs/json/register" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/remote" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" @@ -46,6 +52,7 @@ import ( "sigs.k8s.io/cluster-api/test/extension/handlers/lifecycle" "sigs.k8s.io/cluster-api/test/extension/handlers/topologymutation" infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" + "sigs.k8s.io/cluster-api/util/flags" "sigs.k8s.io/cluster-api/version" ) @@ -55,25 +62,44 @@ var ( // scheme is a Kubernetes runtime scheme containing all the information about API types used by the test extension. // NOTE: it is not mandatory to use scheme in custom RuntimeExtension, but working with typed API objects makes code - // easier to read and less error prone than using unstructured or working with raw json/yaml. - scheme = runtime.NewScheme() + // easier to read and less error-prone than using unstructured or working with raw json/yaml. + scheme = runtime.NewScheme() + // Creates a logger to be used during the main func using controller runtime utilities + // NOTE: it is not mandatory to use controller runtime utilities in custom RuntimeExtension, but it is recommended + // because it makes log from those components similar to log from controllers. + setupLog = ctrl.Log.WithName("setup") controllerName = "cluster-api-test-extension-manager" - // Flags. - profilerAddress string - webhookPort int - webhookCertDir string - logOptions = logs.NewOptions() + // flags. + enableLeaderElection bool + leaderElectionLeaseDuration time.Duration + leaderElectionRenewDeadline time.Duration + leaderElectionRetryPeriod time.Duration + profilerAddress string + enableContentionProfiling bool + syncPeriod time.Duration + restConfigQPS float32 + restConfigBurst int + webhookPort int + webhookCertDir string + healthAddr string + tlsOptions = flags.TLSOptions{} + diagnosticsOptions = flags.DiagnosticsOptions{} + logOptions = logs.NewOptions() ) func init() { - // Adds to the catalog all the RuntimeHooks defined in cluster API. - _ = runtimehooksv1.AddToCatalog(catalog) - // Adds to the scheme all the API types we used by the test extension. - _ = infrav1.AddToScheme(scheme) - _ = controlplanev1.AddToScheme(scheme) + _ = clientgoscheme.AddToScheme(scheme) + _ = apiextensionsv1.AddToScheme(scheme) + + _ = clusterv1.AddToScheme(scheme) _ = bootstrapv1.AddToScheme(scheme) + _ = controlplanev1.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + // Register the RuntimeHook types into the catalog. + _ = runtimehooksv1.AddToCatalog(catalog) } // InitFlags initializes the flags. @@ -83,27 +109,56 @@ func InitFlags(fs *pflag.FlagSet) { // recommended because it helps in ensuring consistency across different components in the cluster. logsv1.AddFlags(logOptions, fs) - // Add test-extension specific flags - // NOTE: it is not mandatory to use the same flag names in all RuntimeExtension, but it is recommended when - // addressing common concerns like profiler-address, webhook-port, webhook-cert-dir etc. because it helps in ensuring - // consistency across different components in the cluster. + fs.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + + fs.DurationVar(&leaderElectionLeaseDuration, "leader-elect-lease-duration", 15*time.Second, + "Interval at which non-leader candidates will wait to force acquire leadership (duration string)") + + fs.DurationVar(&leaderElectionRenewDeadline, "leader-elect-renew-deadline", 10*time.Second, + "Duration that the leading controller manager will retry refreshing leadership before giving up (duration string)") + + fs.DurationVar(&leaderElectionRetryPeriod, "leader-elect-retry-period", 2*time.Second, + "Duration the LeaderElector clients should wait between tries of actions (duration string)") fs.StringVar(&profilerAddress, "profiler-address", "", "Bind address to expose the pprof profiler (e.g. localhost:6060)") + fs.BoolVar(&enableContentionProfiling, "contention-profiling", false, + "Enable block profiling") + + fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, + "The minimum interval at which watched resources are reconciled (e.g. 15m)") + + fs.Float32Var(&restConfigQPS, "kube-api-qps", 20, + "Maximum queries per second from the controller client to the Kubernetes API server. Defaults to 20") + + fs.IntVar(&restConfigBurst, "kube-api-burst", 30, + "Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30") + fs.IntVar(&webhookPort, "webhook-port", 9443, "Webhook Server port") fs.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/", "Webhook cert dir, only used when webhook-port is specified.") + + fs.StringVar(&healthAddr, "health-addr", ":9440", + "The address the health endpoint binds to.") + + flags.AddDiagnosticsOptions(fs, &diagnosticsOptions) + flags.AddTLSOptions(fs, &tlsOptions) + + // Add test-extension specific flags + // NOTE: it is not mandatory to use the same flag names in all RuntimeExtension, but it is recommended when + // addressing common concerns like profiler-address, webhook-port, webhook-cert-dir etc. because it helps in ensuring + // consistency across different components in the cluster. } -func main() { - // Creates a logger to be used during the main func using controller runtime utilities - // NOTE: it is not mandatory to use controller runtime utilities in custom RuntimeExtension, but it is recommended - // because it makes log from those components similar to log from controllers. - setupLog := ctrl.Log.WithName("main") +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create +func main() { // Initialize and parse command line flags. InitFlags(pflag.CommandLine) pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) @@ -126,49 +181,98 @@ func main() { // Add the klog logger in the context. // NOTE: it is not mandatory to use contextual logging in custom RuntimeExtension, but it is recommended // because it allows to use a log stored in the context across the entire chain of calls (without - // requiring an addition log parameter in all the functions. + // requiring an addition log parameter in all the functions). ctrl.SetLogger(klog.Background()) - // Initialize the golang profiler server, if required. - // TODO(sbueringer): remove this once we switched to using a manager. - if profilerAddress != "" { - setupLog.Info(fmt.Sprintf("Profiler listening for requests at %s", profilerAddress)) - go func() { - srv := http.Server{Addr: profilerAddress, ReadHeaderTimeout: 2 * time.Second} - if err := srv.ListenAndServe(); err != nil { - setupLog.Error(err, "problem running profiler server") - } - }() + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = restConfigQPS + restConfig.Burst = restConfigBurst + restConfig.UserAgent = remote.DefaultClusterAPIUserAgent(controllerName) + + tlsOptionOverrides, err := flags.GetTLSOptionOverrideFuncs(tlsOptions) + if err != nil { + setupLog.Error(err, "unable to add TLS settings to the webhook server") + os.Exit(1) } - // **************************************************** - // Create a http server for serving runtime extensions - // **************************************************** + diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions) - webhookServer, err := server.New(server.Options{ - Catalog: catalog, + if enableContentionProfiling { + goruntime.SetBlockProfileRate(1) + } + + // Create an HTTP server for serving Runtime Extensions. + runtimeExtensionWebhookServer, err := server.New(server.Options{ Port: webhookPort, CertDir: webhookCertDir, + TLSOpts: tlsOptionOverrides, + Catalog: catalog, }) if err != nil { - setupLog.Error(err, "error creating webhook server") + setupLog.Error(err, "error creating runtime extension webhook server") os.Exit(1) } - // **************************************************** - // Add handlers for the Runtime hooks you are - // interested in, in the test-extension we are registering all - // of them for ensuring proper test coverage - // **************************************************** + ctrlOptions := ctrl.Options{ + Scheme: scheme, + LeaderElection: enableLeaderElection, + LeaderElectionID: "controller-leader-election-capv-test-extension", + LeaseDuration: &leaderElectionLeaseDuration, + RenewDeadline: &leaderElectionRenewDeadline, + RetryPeriod: &leaderElectionRetryPeriod, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + HealthProbeBindAddress: healthAddr, + PprofBindAddress: profilerAddress, + Metrics: diagnosticsOpts, + Cache: cache.Options{ + SyncPeriod: &syncPeriod, + }, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{ + &corev1.ConfigMap{}, + &corev1.Secret{}, + }, + }, + }, + WebhookServer: runtimeExtensionWebhookServer, + } + + // Start the manager + mgr, err := ctrl.NewManager(restConfig, ctrlOptions) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } - // Topology Mutation Hooks (Runtime Patches) + // Set up a context listening for SIGINT. + ctx := ctrl.SetupSignalHandler() + // Setup Runtime Extensions. + setupTopologyMutationHookHandlers(runtimeExtensionWebhookServer) + setupLifecycleHookHandlers(mgr, runtimeExtensionWebhookServer) + + // Setup checks, indexes, reconcilers and webhooks. + setupChecks(mgr) + setupIndexes(ctx, mgr) + setupReconcilers(ctx, mgr) + setupWebhooks(mgr) + + setupLog.Info("Starting manager", "version", version.Get().String()) + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +// setupTopologyMutationHookHandlers sets up Topology Mutation Hooks (Runtime Patches). +func setupTopologyMutationHookHandlers(runtimeExtensionWebhookServer *server.Server) { // Create the ExtensionHandlers for the Topology Mutation Hooks. // NOTE: it is not mandatory to group all the ExtensionHandlers using a struct, what is important // is to have HandlerFunc with the signature defined in sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1. topologyMutationExtensionHandlers := topologymutation.NewExtensionHandlers(scheme) - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.GeneratePatches, Name: "generate-patches", HandlerFunc: topologyMutationExtensionHandlers.GeneratePatches, @@ -177,7 +281,7 @@ func main() { os.Exit(1) } - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.ValidateTopology, Name: "validate-topology", HandlerFunc: topologyMutationExtensionHandlers.ValidateTopology, @@ -186,7 +290,7 @@ func main() { os.Exit(1) } - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.DiscoverVariables, Name: "discover-variables", HandlerFunc: topologyMutationExtensionHandlers.DiscoverVariables, @@ -194,30 +298,16 @@ func main() { setupLog.Error(err, "error adding handler") os.Exit(1) } +} - // Lifecycle Hooks - - // Gets a client to access the Kubernetes cluster where this RuntimeExtension will be deployed to; - // this is a requirement specific of the lifecycle hooks implementation for Cluster APIs E2E tests. - restConfig, err := ctrl.GetConfig() - if err != nil { - setupLog.Error(err, "error getting config for the cluster") - os.Exit(1) - } - restConfig.UserAgent = remote.DefaultClusterAPIUserAgent(controllerName) - - client, err := client.New(restConfig, client.Options{}) - if err != nil { - setupLog.Error(err, "error creating client to the cluster") - os.Exit(1) - } - +// setupLifecycleHookHandlers sets up Lifecycle Hooks. +func setupLifecycleHookHandlers(mgr ctrl.Manager, runtimeExtensionWebhookServer *server.Server) { // Create the ExtensionHandlers for the lifecycle hooks // NOTE: it is not mandatory to group all the ExtensionHandlers using a struct, what is important // is to have HandlerFunc with the signature defined in sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1. - lifecycleExtensionHandlers := lifecycle.NewExtensionHandlers(client) + lifecycleExtensionHandlers := lifecycle.NewExtensionHandlers(mgr.GetClient()) - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.BeforeClusterCreate, Name: "before-cluster-create", HandlerFunc: lifecycleExtensionHandlers.DoBeforeClusterCreate, @@ -226,7 +316,7 @@ func main() { os.Exit(1) } - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.AfterControlPlaneInitialized, Name: "after-control-plane-initialized", HandlerFunc: lifecycleExtensionHandlers.DoAfterControlPlaneInitialized, @@ -235,7 +325,7 @@ func main() { os.Exit(1) } - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.BeforeClusterUpgrade, Name: "before-cluster-upgrade", HandlerFunc: lifecycleExtensionHandlers.DoBeforeClusterUpgrade, @@ -244,7 +334,7 @@ func main() { os.Exit(1) } - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.AfterControlPlaneUpgrade, Name: "after-control-plane-upgrade", HandlerFunc: lifecycleExtensionHandlers.DoAfterControlPlaneUpgrade, @@ -253,7 +343,7 @@ func main() { os.Exit(1) } - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.AfterClusterUpgrade, Name: "after-cluster-upgrade", HandlerFunc: lifecycleExtensionHandlers.DoAfterClusterUpgrade, @@ -262,7 +352,7 @@ func main() { os.Exit(1) } - if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.BeforeClusterDelete, Name: "before-cluster-delete", HandlerFunc: lifecycleExtensionHandlers.DoBeforeClusterDelete, @@ -270,17 +360,25 @@ func main() { setupLog.Error(err, "error adding handler") os.Exit(1) } +} - // **************************************************** - // Start the https server - // **************************************************** - - // Setup a context listening for SIGINT. - ctx := ctrl.SetupSignalHandler() +func setupChecks(mgr ctrl.Manager) { + if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + setupLog.Error(err, "unable to create ready check") + os.Exit(1) + } - setupLog.Info("starting RuntimeExtension", "version", version.Get().String()) - if err := webhookServer.Start(ctx); err != nil { - setupLog.Error(err, "error running webhook server") + if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + setupLog.Error(err, "unable to create health check") os.Exit(1) } } + +func setupIndexes(_ context.Context, _ ctrl.Manager) { +} + +func setupReconcilers(_ context.Context, _ ctrl.Manager) { +} + +func setupWebhooks(_ ctrl.Manager) { +}