Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 Use CAPI's ClusterCacheTracker #1911

Merged
merged 1 commit into from
Sep 16, 2024
Merged

🌱 Use CAPI's ClusterCacheTracker #1911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions controllers/metal3cluster_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/utils/ptr"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
capierrors "sigs.k8s.io/cluster-api/errors"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
Expand All @@ -53,6 +54,7 @@ const (
// Metal3ClusterReconciler reconciles a Metal3Cluster object.
type Metal3ClusterReconciler struct {
Client client.Client
Tracker *remote.ClusterCacheTracker
ManagerFactory baremetal.ManagerFactoryInterface
Log logr.Logger
WatchFilterValue string
Expand Down Expand Up @@ -124,11 +126,25 @@ func (r *Metal3ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques

// Handle deleted clusters
if !metal3Cluster.DeletionTimestamp.IsZero() {
return reconcileDelete(ctx, clusterMgr)
res, err := reconcileDelete(ctx, clusterMgr)
// Requeue if the reconcile failed because the ClusterCacheTracker was locked for
// the current cluster because of concurrent access.
if errors.Is(err, remote.ErrClusterLocked) {
clusterLog.Info("Requeuing because another worker has the lock on the ClusterCacheTracker")
return ctrl.Result{Requeue: true}, nil
}
return res, err
}

// Handle non-deleted clusters
return reconcileNormal(ctx, clusterMgr)
res, err := reconcileNormal(ctx, clusterMgr)
// Requeue if the reconcile failed because the ClusterCacheTracker was locked for
// the current cluster because of concurrent access.
if errors.Is(err, remote.ErrClusterLocked) {
clusterLog.Info("Requeuing because another worker has the lock on the ClusterCacheTracker")
return ctrl.Result{Requeue: true}, nil
}
return res, err
}

func patchMetal3Cluster(ctx context.Context, patchHelper *patch.Helper, metal3Cluster *infrav1.Metal3Cluster, options ...patch.Option) error {
Expand Down
2 changes: 2 additions & 0 deletions controllers/metal3data_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/patch"
Expand All @@ -45,6 +46,7 @@ const (
// Metal3DataReconciler reconciles a Metal3Data object.
type Metal3DataReconciler struct {
Client client.Client
Tracker *remote.ClusterCacheTracker
ManagerFactory baremetal.ManagerFactoryInterface
Log logr.Logger
WatchFilterValue string
Expand Down
2 changes: 2 additions & 0 deletions controllers/metal3datatemplate_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand All @@ -43,6 +44,7 @@ const (
// Metal3DataTemplateReconciler reconciles a Metal3DataTemplate object.
type Metal3DataTemplateReconciler struct {
Client client.Client
Tracker *remote.ClusterCacheTracker
ManagerFactory baremetal.ManagerFactoryInterface
Log logr.Logger
WatchFilterValue string
Expand Down
2 changes: 2 additions & 0 deletions controllers/metal3labelsync_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"k8s.io/client-go/tools/cache"
k8strings "k8s.io/utils/strings"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/patch"
Expand All @@ -60,6 +61,7 @@ const (
// Metal3LabelSyncReconciler reconciles label updates to BareMetalHost objects with the corresponding K Node objects in the workload cluster.
type Metal3LabelSyncReconciler struct {
Client client.Client
Tracker *remote.ClusterCacheTracker
ManagerFactory baremetal.ManagerFactoryInterface
Log logr.Logger
CapiClientGetter baremetal.ClientGetter
Expand Down
2 changes: 2 additions & 0 deletions controllers/metal3machine_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
capierrors "sigs.k8s.io/cluster-api/errors"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
Expand All @@ -50,6 +51,7 @@ const (
type Metal3MachineReconciler struct {
Client client.Client
ManagerFactory baremetal.ManagerFactoryInterface
Tracker *remote.ClusterCacheTracker
Log logr.Logger
CapiClientGetter baremetal.ClientGetter
WatchFilterValue string
Expand Down
2 changes: 2 additions & 0 deletions controllers/metal3machinetemplate_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand All @@ -45,6 +46,7 @@ const (
// Metal3MachineTemplateReconciler reconciles a Metal3MachineTemplate object.
type Metal3MachineTemplateReconciler struct {
Client client.Client
Tracker *remote.ClusterCacheTracker
ManagerFactory baremetal.ManagerFactoryInterface
Log logr.Logger
WatchFilterValue string
Expand Down
2 changes: 2 additions & 0 deletions controllers/metal3remediation_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/patch"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -39,6 +40,7 @@ import (
// Metal3RemediationReconciler reconciles a Metal3Remediation object.
type Metal3RemediationReconciler struct {
client.Client
Tracker *remote.ClusterCacheTracker
ManagerFactory baremetal.ManagerFactoryInterface
Log logr.Logger
IsOutOfServiceTaintEnabled bool
Expand Down
55 changes: 54 additions & 1 deletion main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
_ "k8s.io/component-base/logs/json/register"
"k8s.io/klog/v2/klogr"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
caipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1alpha1"
"sigs.k8s.io/cluster-api/util/flags"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -67,12 +68,15 @@ const (
var (
myscheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
controllerName = "cluster-api-provider-metal3-manager"
waitForMetal3Controller = false
enableLeaderElection bool
leaderElectionLeaseDuration time.Duration
leaderElectionRenewDeadline time.Duration
leaderElectionRetryPeriod time.Duration
syncPeriod time.Duration
clusterCacheTrackerClientQPS float32
clusterCacheTrackerClientBurst int
metal3MachineConcurrency int
metal3ClusterConcurrency int
metal3DataTemplateConcurrency int
Expand Down Expand Up @@ -120,7 +124,7 @@ func main() {
restConfig := ctrl.GetConfigOrDie()
restConfig.QPS = restConfigQPS
restConfig.Burst = restConfigBurst
restConfig.UserAgent = "cluster-api-provider-metal3-manager"
restConfig.UserAgent = "controllerName"

tlsOptions, metricsOptions, err := flags.GetManagerOptions(managerOptions)
if err != nil {
Expand Down Expand Up @@ -267,6 +271,20 @@ func initFlags(fs *pflag.FlagSet) {
"The minimum interval at which watched resources are reconciled (e.g. 15m)",
)

fs.Float32Var(
&clusterCacheTrackerClientQPS,
"clustercachetracker-client-qps",
20,
"Maximum queries per second from the cluster cache tracker clients to the Kubernetes API server of workload clusters.",
)

fs.IntVar(
&clusterCacheTrackerClientBurst,
"clustercachetracker-client-burst",
30,
"Maximum number of queries that should be allowed in one burst from the cluster cache tracker clients to the Kubernetes API server of workload clusters.",
)

fs.IntVar(
&webhookPort,
"webhook-port",
Expand Down Expand Up @@ -356,8 +374,37 @@ func setupChecks(mgr ctrl.Manager) {
}

func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {
secretCachingClient, err := client.New(mgr.GetConfig(), client.Options{
HTTPClient: mgr.GetHTTPClient(),
Cache: &client.CacheOptions{
Reader: mgr.GetCache(),
},
})
if err != nil {
setupLog.Error(err, "unable to create secret caching client")
os.Exit(1)
}

// Set up a ClusterCacheTracker and ClusterCacheReconciler to provide to controllers
// requiring a connection to a remote cluster
tracker, err := remote.NewClusterCacheTracker(
mgr,
remote.ClusterCacheTrackerOptions{
SecretCachingClient: secretCachingClient,
ControllerName: controllerName,
Log: &ctrl.Log,
ClientQPS: clusterCacheTrackerClientQPS,
ClientBurst: clusterCacheTrackerClientBurst,
},
)
if err != nil {
setupLog.Error(err, "Unable to create cluster cache tracker")
os.Exit(1)
}

if err := (&controllers.Metal3MachineReconciler{
Client: mgr.GetClient(),
Tracker: tracker,
ManagerFactory: baremetal.NewManagerFactory(mgr.GetClient()),
Log: ctrl.Log.WithName("controllers").WithName("Metal3Machine"),
CapiClientGetter: infraremote.NewClusterClient,
Expand All @@ -369,6 +416,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {

if err := (&controllers.Metal3ClusterReconciler{
Client: mgr.GetClient(),
Tracker: tracker,
ManagerFactory: baremetal.NewManagerFactory(mgr.GetClient()),
Log: ctrl.Log.WithName("controllers").WithName("Metal3Cluster"),
WatchFilterValue: watchFilterValue,
Expand All @@ -379,6 +427,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {

if err := (&controllers.Metal3DataTemplateReconciler{
Client: mgr.GetClient(),
Tracker: tracker,
ManagerFactory: baremetal.NewManagerFactory(mgr.GetClient()),
Log: ctrl.Log.WithName("controllers").WithName("Metal3DataTemplate"),
WatchFilterValue: watchFilterValue,
Expand All @@ -389,6 +438,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {

if err := (&controllers.Metal3DataReconciler{
Client: mgr.GetClient(),
Tracker: tracker,
ManagerFactory: baremetal.NewManagerFactory(mgr.GetClient()),
Log: ctrl.Log.WithName("controllers").WithName("Metal3Data"),
WatchFilterValue: watchFilterValue,
Expand All @@ -399,6 +449,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {

if err := (&controllers.Metal3LabelSyncReconciler{
Client: mgr.GetClient(),
Tracker: tracker,
ManagerFactory: baremetal.NewManagerFactory(mgr.GetClient()),
Log: ctrl.Log.WithName("controllers").WithName("Metal3LabelSync"),
CapiClientGetter: infraremote.NewClusterClient,
Expand All @@ -409,6 +460,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {

if err := (&controllers.Metal3MachineTemplateReconciler{
Client: mgr.GetClient(),
Tracker: tracker,
ManagerFactory: baremetal.NewManagerFactory(mgr.GetClient()),
Log: ctrl.Log.WithName("controllers").WithName("Metal3MachineTemplate"),
}).SetupWithManager(ctx, mgr, concurrency(metal3MachineTemplateConcurrency)); err != nil {
Expand All @@ -422,6 +474,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {
}
if err := (&controllers.Metal3RemediationReconciler{
Client: mgr.GetClient(),
Tracker: tracker,
ManagerFactory: baremetal.NewManagerFactory(mgr.GetClient()),
Log: ctrl.Log.WithName("controllers").WithName("Metal3Remediation"),
IsOutOfServiceTaintEnabled: isOOSTSupported,
Expand Down
Loading