From c85ce8e0c72e7668ba6da607a27cd9640155974f Mon Sep 17 00:00:00 2001 From: killianmuldoon Date: Wed, 21 Jun 2023 15:57:50 +0100 Subject: [PATCH] CAPIM: Enable update for coreDNS and kube-proxy Signed-off-by: killianmuldoon --- .../main/clusterclass-in-memory.yaml | 5 - .../controllers/inmemorymachine_controller.go | 114 ++++++++++++++++++ .../inmemory/internal/server/api/const.go | 58 +++++++++ .../inmemory/internal/server/api/handler.go | 11 ++ test/infrastructure/inmemory/main.go | 2 + .../clusterclass-in-memory-quick-start.yaml | 5 - 6 files changed, 185 insertions(+), 10 deletions(-) diff --git a/test/e2e/data/infrastructure-inmemory/main/clusterclass-in-memory.yaml b/test/e2e/data/infrastructure-inmemory/main/clusterclass-in-memory.yaml index 9064a6e254b2..e3c7bc2f741b 100644 --- a/test/e2e/data/infrastructure-inmemory/main/clusterclass-in-memory.yaml +++ b/test/e2e/data/infrastructure-inmemory/main/clusterclass-in-memory.yaml @@ -6,11 +6,6 @@ spec: controlPlane: metadata: annotations: - # The in-memory provider currently does not support looking up coredns - # and kube-proxy information and leads to reconcile errors in KCP. - # With these annotations KCP will skip processing those steps. - controlplane.cluster.x-k8s.io/skip-coredns: "" - controlplane.cluster.x-k8s.io/skip-kube-proxy: "" machineInfrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 diff --git a/test/infrastructure/inmemory/internal/controllers/inmemorymachine_controller.go b/test/infrastructure/inmemory/internal/controllers/inmemorymachine_controller.go index 5786c0459bdc..c432e742702b 100644 --- a/test/infrastructure/inmemory/internal/controllers/inmemorymachine_controller.go +++ b/test/infrastructure/inmemory/internal/controllers/inmemorymachine_controller.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -215,6 +216,8 @@ func (r *InMemoryMachineReconciler) reconcileNormal(ctx context.Context, cluster r.reconcileNormalScheduler, r.reconcileNormalControllerManager, r.reconcileNormalKubeadmObjects, + r.reconcileNormalKubeProxy, + r.reconcileNormalCoredns, } res := ctrl.Result{} @@ -753,6 +756,117 @@ func (r *InMemoryMachineReconciler) reconcileNormalKubeadmObjects(ctx context.Co return ctrl.Result{}, nil } +func (r *InMemoryMachineReconciler) reconcileNormalKubeProxy(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, _ *infrav1.InMemoryMachine) (ctrl.Result, error) { + // No-op if the machine is not a control plane machine. + if !util.IsControlPlaneMachine(machine) { + return ctrl.Result{}, nil + } + + // TODO: Add provisioning time for KubeProxy. + + // Compute the resource group unique name. + // NOTE: We are using reconcilerGroup also as a name for the listener for sake of simplicity. + resourceGroup := klog.KObj(cluster).String() + cloudClient := r.CloudManager.GetResourceGroup(resourceGroup).GetClient() + + // Create the kube-proxy-daemonset + kubeProxyDaemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: "kube-proxy", + Labels: map[string]string{ + "component": "kube-proxy", + }, + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-proxy", + Image: fmt.Sprintf("registry.k8s.io/kube-proxy:%s", *machine.Spec.Version), + }, + }, + }, + }, + }, + } + if err := cloudClient.Get(ctx, client.ObjectKeyFromObject(kubeProxyDaemonSet), kubeProxyDaemonSet); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, errors.Wrapf(err, "failed to get kube-proxy DaemonSet") + } + + if err := cloudClient.Create(ctx, kubeProxyDaemonSet); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, errors.Wrapf(err, "failed to create kube-proxy DaemonSet") + } + } + return ctrl.Result{}, nil +} + +func (r *InMemoryMachineReconciler) reconcileNormalCoredns(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, _ *infrav1.InMemoryMachine) (ctrl.Result, error) { + // No-op if the machine is not a control plane machine. + if !util.IsControlPlaneMachine(machine) { + return ctrl.Result{}, nil + } + + // TODO: Add provisioning time for CoreDNS. + + // Compute the resource group unique name. + // NOTE: We are using reconcilerGroup also as a name for the listener for sake of simplicity. + resourceGroup := klog.KObj(cluster).String() + cloudClient := r.CloudManager.GetResourceGroup(resourceGroup).GetClient() + + // Create the coredns configMap. + corednsConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: "coredns", + }, + Data: map[string]string{ + "Corefile": "ANG", + }, + } + if err := cloudClient.Get(ctx, client.ObjectKeyFromObject(corednsConfigMap), corednsConfigMap); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, errors.Wrapf(err, "failed to get coreDNS configMap") + } + + if err := cloudClient.Create(ctx, corednsConfigMap); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, errors.Wrapf(err, "failed to create coreDNS configMap") + } + } + // Create the coredns deployment. + corednsDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: "coredns", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "coredns", + Image: "registry.k8s.io/coredns/coredns:v1.10.1", + }, + }, + }, + }, + }, + } + + if err := cloudClient.Get(ctx, client.ObjectKeyFromObject(corednsDeployment), corednsDeployment); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, errors.Wrapf(err, "failed to get coreDNS deployment") + } + + if err := cloudClient.Create(ctx, corednsDeployment); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, errors.Wrapf(err, "failed to create coreDNS deployment") + } + } + return ctrl.Result{}, nil +} + func (r *InMemoryMachineReconciler) reconcileDelete(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, inMemoryMachine *infrav1.InMemoryMachine) (ctrl.Result, error) { // Call the inner reconciliation methods. phases := []func(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, inMemoryMachine *infrav1.InMemoryMachine) (ctrl.Result, error){ diff --git a/test/infrastructure/inmemory/internal/server/api/const.go b/test/infrastructure/inmemory/internal/server/api/const.go index b9b57bb6672b..32ea331fde95 100644 --- a/test/infrastructure/inmemory/internal/server/api/const.go +++ b/test/infrastructure/inmemory/internal/server/api/const.go @@ -122,6 +122,19 @@ var ( Version: "v1", }, }, + { + Name: "apps", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "apps/v1", + Version: "v1", + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "apps/v1", + Version: "v1", + }, + }, }, } @@ -200,4 +213,49 @@ var ( }, }, } + appsV1ResourceList = &metav1.APIResourceList{ + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "daemonsets", + SingularName: "daemonset", + Namespaced: true, + Kind: "DaemonSet", + Verbs: []string{ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch", + }, + ShortNames: []string{ + "ds", + }, + StorageVersionHash: "", + }, + { + Name: "deployments", + SingularName: "deployment", + Namespaced: true, + Kind: "Deployment", + Verbs: []string{ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch", + }, + ShortNames: []string{ + "deploy", + }, + StorageVersionHash: "", + }, + }, + } ) diff --git a/test/infrastructure/inmemory/internal/server/api/handler.go b/test/infrastructure/inmemory/internal/server/api/handler.go index 0516ba7eb8f7..009be73248b4 100644 --- a/test/infrastructure/inmemory/internal/server/api/handler.go +++ b/test/infrastructure/inmemory/internal/server/api/handler.go @@ -153,6 +153,14 @@ func (h *apiServerHandler) apisDiscovery(req *restful.Request, resp *restful.Res } return } + if req.PathParameter("group") == "apps" && req.PathParameter("version") == "v1" { + if err := resp.WriteEntity(appsV1ResourceList); err != nil { + _ = resp.WriteErrorString(http.StatusInternalServerError, err.Error()) + return + } + return + } + _ = resp.WriteErrorString(http.StatusInternalServerError, fmt.Sprintf("discovery info not defined for %s/%s", req.PathParameter("group"), req.PathParameter("version"))) return } @@ -552,6 +560,9 @@ func getAPIResourceList(req *restful.Request) *metav1.APIResourceList { if req.PathParameter("group") == "rbac.authorization.k8s.io" && req.PathParameter("version") == "v1" { return rbacv1APIResourceList } + if req.PathParameter("group") == "apps" && req.PathParameter("version") == "v1" { + return appsV1ResourceList + } return nil } return corev1APIResourceList diff --git a/test/infrastructure/inmemory/main.go b/test/infrastructure/inmemory/main.go index dea5a98f9033..43332aba4d60 100644 --- a/test/infrastructure/inmemory/main.go +++ b/test/infrastructure/inmemory/main.go @@ -25,6 +25,7 @@ import ( "time" "github.com/spf13/pflag" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" @@ -88,6 +89,7 @@ func init() { // scheme used for operating on the cloud resource. _ = cloudv1.AddToScheme(cloudScheme) _ = corev1.AddToScheme(cloudScheme) + _ = appsv1.AddToScheme(cloudScheme) _ = rbacv1.AddToScheme(cloudScheme) } diff --git a/test/infrastructure/inmemory/templates/clusterclass-in-memory-quick-start.yaml b/test/infrastructure/inmemory/templates/clusterclass-in-memory-quick-start.yaml index 5449ebd14790..c1dfec5292e8 100644 --- a/test/infrastructure/inmemory/templates/clusterclass-in-memory-quick-start.yaml +++ b/test/infrastructure/inmemory/templates/clusterclass-in-memory-quick-start.yaml @@ -6,11 +6,6 @@ spec: controlPlane: metadata: annotations: - # The in-memory provider currently does not support looking up coredns - # and kube-proxy information and leads to reconcile errors in KCP. - # With these annotations KCP will skip processing those steps. - controlplane.cluster.x-k8s.io/skip-coredns: "" - controlplane.cluster.x-k8s.io/skip-kube-proxy: "" machineInfrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1