Skip to content

Commit

Permalink
feat: install api-server-external-service in virtualcluster
Browse files Browse the repository at this point in the history
Signed-off-by: qiuwei <[email protected]>
  • Loading branch information
qiuwei68 committed Jun 11, 2024
1 parent 5d85238 commit 14ff9e6
Show file tree
Hide file tree
Showing 6 changed files with 251 additions and 1 deletion.
4 changes: 4 additions & 0 deletions pkg/kubenest/constants/constant.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const (
KosmosJoinControllerName = "kosmos-join-controller"
KosmosNs = "kosmos-system"
SystemNs = "kube-system"
DefaultNs = "default"
DefaultImageRepositoryEnv = "IMAGE_REPOSITIRY"
DefaultImageVersionEnv = "IMAGE_VERSION"
DefaultCoreDnsImageTagEnv = "COREDNS_IMAGE_TAG"
Expand Down Expand Up @@ -112,6 +113,9 @@ const (
StateLabelKey = "kosmos-io/state"

KonnectivityServerSuffix = "konnectivity-server"

//in virtual cluster
ApiServerExternalService = "api-server-external-service"
)

type Action string
160 changes: 160 additions & 0 deletions pkg/kubenest/controlplane/endpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
package controlplane

import (
"context"
"fmt"

"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/dynamic"
"k8s.io/klog/v2"

"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/virtualcluster"
"github.com/kosmos.io/kosmos/pkg/kubenest/util"
)

func EnsureApiServerExternalEndPoint(dynamicClient dynamic.Interface) error {
err := installApiServerExternalEndpointInVirtualCluster(dynamicClient)
if err != nil {
return err
}

err = installApiServerExternalServiceInVirtualCluster(dynamicClient)
if err != nil {
return err
}
return nil
}

func installApiServerExternalEndpointInVirtualCluster(dynamicClient dynamic.Interface) error {
klog.Info("begin to get kubernetes endpoint")
kubeEndpointUnstructured, err := dynamicClient.Resource(schema.GroupVersionResource{
Group: "",
Version: "v1",
Resource: "endpoints",
}).Namespace(constants.DefaultNs).Get(context.TODO(), "kubernetes", metav1.GetOptions{})
if err != nil {
klog.Error("get Kubernetes endpoint failed", err)
return errors.Wrap(err, "failed to get kubernetes endpoint")
}
klog.V(4).Info("the Kubernetes endpoint is:", kubeEndpointUnstructured)

if kubeEndpointUnstructured != nil {
kubeEndpoint := &corev1.Endpoints{}
err := runtime.DefaultUnstructuredConverter.FromUnstructured(kubeEndpointUnstructured.Object, kubeEndpoint)
if err != nil {
klog.Error("switch Kubernetes endpoint to typed object failed", err)
return errors.Wrap(err, "failed to convert kubernetes endpoint to typed object")
}

newEndpoint := kubeEndpoint.DeepCopy()
newEndpoint.Name = constants.ApiServerExternalService
newEndpoint.Namespace = constants.DefaultNs
newEndpoint.ResourceVersion = ""
newEndpointUnstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(newEndpoint)
if err != nil {
klog.Error("switch new endpoint to unstructured object failed", err)
return errors.Wrap(err, "failed to convert new endpoint to unstructured object")
}
klog.V(4).Info("after switch the Endpoint unstructured is:", newEndpointUnstructuredObj)

newEndpointUnstructured := &unstructured.Unstructured{Object: newEndpointUnstructuredObj}
createResult, err := dynamicClient.Resource(schema.GroupVersionResource{
Group: "",
Version: "v1",
Resource: "endpoints",
}).Namespace(constants.DefaultNs).Create(context.TODO(), newEndpointUnstructured, metav1.CreateOptions{})
if err != nil {
klog.Error("create api-server-external-service endpoint failed", err)
return errors.Wrap(err, "failed to create api-server-external-service endpoint")
} else {
klog.Info("success create api-server-external-service endpoint:", createResult)
}
} else {
return errors.New("kubernetes endpoint does not exist")
}

return nil
}

func installApiServerExternalServiceInVirtualCluster(dynamicClient dynamic.Interface) error {
port, err := getEndPointPort(dynamicClient)
if err != nil {
return fmt.Errorf("error when getEndPointPort: %w", err)
}
apiServerExternalServiceBytes, err := util.ParseTemplate(virtualcluster.ApiServerExternalService, struct {
ServicePort int32
}{
ServicePort: port,
})
if err != nil {
return fmt.Errorf("error when parsing api-server-external-serive template: %w", err)
}

var obj unstructured.Unstructured
if err := yaml.Unmarshal([]byte(apiServerExternalServiceBytes), &obj); err != nil {
return fmt.Errorf("err when decoding api-server-external service in virtual cluster: %w", err)
}

err = util.CreateObject(dynamicClient, "default", "api-server-external-service", &obj)
if err != nil {
return fmt.Errorf("error when creating api-server-external service in virtual cluster err: %w", err)
}
return nil

}

func getEndPointPort(dynamicClient dynamic.Interface) (int32, error) {
klog.Info("begin to get Endpoints ports...")
endpointsRes := dynamicClient.Resource(schema.GroupVersionResource{
Group: "",
Version: "v1",
Resource: "endpoints",
}).Namespace(constants.DefaultNs)

endpointsRaw, err := endpointsRes.Get(context.TODO(), constants.ApiServerExternalService, metav1.GetOptions{})
if err != nil {
klog.Errorf("get Endpoints failed: %v", err)
return 0, err
}

subsets, found, err := unstructured.NestedSlice(endpointsRaw.Object, "subsets")
if !found || err != nil {
klog.Errorf("The subsets field was not found or parsing error occurred: %v", err)
return 0, fmt.Errorf("subsets field not found or error parsing it")
}

if len(subsets) == 0 {
klog.Errorf("subsets is empty")
return 0, fmt.Errorf("No subsets found in the endpoints")
}

subset := subsets[0].(map[string]interface{})
ports, found, err := unstructured.NestedSlice(subset, "ports")
if !found || err != nil {
klog.Errorf("ports field not found or parsing error: %v", err)
return 0, fmt.Errorf("ports field not found or error parsing it")
}

if len(ports) == 0 {
klog.Errorf("Port not found in the endpoint")
return 0, fmt.Errorf("No ports found in the endpoint")
}

port := ports[0].(map[string]interface{})
portNum, found, err := unstructured.NestedInt64(port, "port")
if !found || err != nil {
klog.Errorf("ports field not found or parsing error: %v", err)
return 0, fmt.Errorf("port field not found or error parsing it")
}

klog.Infof("The port number was successfully obtained: %d", portNum)
return int32(portNum), nil

}
1 change: 1 addition & 0 deletions pkg/kubenest/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func NewInitPhase(opts *InitOptions) *workflow.Phase {
initPhase.AppendTask(tasks.NewCoreDNSTask())
// add server
initPhase.AppendTask(tasks.NewComponentsFromManifestsTask())
initPhase.AppendTask(tasks.NewEndPointTask())

initPhase.SetDataInitializer(func() (workflow.RunData, error) {
return newRunData(opts)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package virtualcluster

const (
ApiServerExternalService = `
apiVersion: v1
kind: Service
metadata:
name: api-server-external-service
namespace: default
spec:
type: NodePort
ports:
- protocol: TCP
port: {{ .ServicePort }}
targetPort: {{ .ServicePort }}
nodePort: 30443
`
)
2 changes: 1 addition & 1 deletion pkg/kubenest/tasks/coredns.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ func runCheckCoreDnsTask(r workflow.RunData) error {
func runCoreDnsVirtualTask(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("Virtual cluster manifests-components task invoked with an invalid data struct")
return errors.New("Virtual cluster coreDns task invoked with an invalid data struct")
}

secret, err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Get(context.TODO(),
Expand Down
67 changes: 67 additions & 0 deletions pkg/kubenest/tasks/endpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package tasks

import (
"context"
"fmt"

"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"

"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
"github.com/kosmos.io/kosmos/pkg/kubenest/controlplane"
"github.com/kosmos.io/kosmos/pkg/kubenest/workflow"
)

func NewEndPointTask() workflow.Task {
return workflow.Task{
Name: "endpoint",
Run: runEndpoint,
RunSubTasks: true,
Tasks: []workflow.Task{
{
Name: "deploy-endpoint-in-virtual-cluster",
Run: runEndPointInVirtualClusterTask,
},
},
}
}

func runEndpoint(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("endPoint task invoked with an invalid data struct")
}

klog.V(4).InfoS("[endPoint] Running endPoint task", "virtual cluster", klog.KObj(data))
return nil
}

func runEndPointInVirtualClusterTask(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("Virtual cluster endpoint task invoked with an invalid data struct")
}

secret, err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Get(context.TODO(),
fmt.Sprintf("%s-%s", data.GetName(), constants.AdminConfig), metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "Get virtualcluster kubeconfig secret error")
}
config, err := clientcmd.RESTConfigFromKubeConfig(secret.Data[constants.KubeConfig])
if err != nil {
return err
}
dynamicClient, err := dynamic.NewForConfig(config)
if err != nil {
return err
}

err = controlplane.EnsureApiServerExternalEndPoint(dynamicClient)
if err != nil {
return err
}
return nil
}

0 comments on commit 14ff9e6

Please sign in to comment.