fix: properly forward apiservice

chart/values.schema.json

@@ -2,6 +2,25 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://vcluster.com/schemas/config",
   "$defs": {
+    "APIServiceTarget": {
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Name is the name of the host service of the apiservice."
+        },
+        "namespace": {
+          "type": "string",
+          "description": "Namespace is the name of the host service of the apiservice."
+        },
+        "port": {
+          "type": "integer",
+          "description": "Port is the target port on the host service to connect to."
+        }
+      },
+      "additionalProperties": false,
+      "type": "object",
+      "description": "APIServiceTarget holds the service name and namespace of the host apiservice."
+    },
     "BackingStore": {
       "properties": {
         "etcd": {
@@ -1694,6 +1713,10 @@
           "type": "boolean",
           "description": "Enabled signals if the integration should be enabled"
         },
+        "service": {
+          "$ref": "#/$defs/APIServiceTarget",
+          "description": "Service holds information about where to find the virt-api service. Defaults to virt-api/kubevirt."
+        },
         "webhook": {
           "$ref": "#/$defs/EnableSwitch",
           "description": "Webhook holds configuration for enabling the webhook within the vCluster"
@@ -1796,6 +1819,10 @@
           "type": "boolean",
           "description": "Enabled signals the metrics server integration should be enabled."
         },
+        "service": {
+          "$ref": "#/$defs/APIServiceTarget",
+          "description": "Service holds information about where to find the metrics-server service. Defaults to metrics-server/kube-system."
+        },
         "nodes": {
           "type": "boolean",
           "description": "Nodes defines if metrics-server nodes api should get proxied from host to virtual cluster."

config/config.go

@@ -89,6 +89,8 @@ type Integrations struct {
 type KubeVirt struct {
 	// Enabled signals if the integration should be enabled
 	Enabled bool `json:"enabled,omitempty"`
+	// Service holds information about where to find the virt-api service. Defaults to virt-api/kubevirt.
+	Service APIServiceTarget `json:"service,omitempty"`
 	// Webhook holds configuration for enabling the webhook within the vCluster
 	Webhook EnableSwitch `json:"webhook,omitempty"`
 	// Sync holds configuration on what resources to sync
@@ -116,13 +118,28 @@ type MetricsServer struct {
 	// Enabled signals the metrics server integration should be enabled.
 	Enabled bool `json:"enabled,omitempty"`
 
+	// Service holds information about where to find the metrics-server service. Defaults to metrics-server/kube-system.
+	Service APIServiceTarget `json:"service,omitempty"`
+
 	// Nodes defines if metrics-server nodes api should get proxied from host to virtual cluster.
 	Nodes bool `json:"nodes,omitempty"`
 
 	// Pods defines if metrics-server pods api should get proxied from host to virtual cluster.
 	Pods bool `json:"pods,omitempty"`
 }
 
+// APIServiceTarget holds the service name and namespace of the host apiservice.
+type APIServiceTarget struct {
+	// Name is the name of the host service of the apiservice.
+	Name string `json:"name,omitempty"`
+
+	// Namespace is the name of the host service of the apiservice.
+	Namespace string `json:"namespace,omitempty"`
+
+	// Port is the target port on the host service to connect to.
+	Port int `json:"port,omitempty"`
+}
+
 // ExternalConfig holds external tool configuration
 type ExternalConfig map[string]interface{}
 

pkg/apiservice/generic.go

@@ -30,8 +30,6 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
-const proxyPort = 9000
-
 func checkExistingAPIService(ctx context.Context, client client.Client, groupVersion schema.GroupVersion) bool {
 	var exists bool
 	_ = applyOperation(ctx, func(ctx context.Context) (bool, error) {
@@ -80,7 +78,7 @@ func deleteOperation(ctrlCtx *config.ControllerContext, groupVersion schema.Grou
 	}
 }
 
-func createOperation(ctrlCtx *config.ControllerContext, serviceName string, groupVersion schema.GroupVersion) wait.ConditionWithContextFunc {
+func createOperation(ctrlCtx *config.ControllerContext, serviceName string, hostPort int, groupVersion schema.GroupVersion) wait.ConditionWithContextFunc {
 	return func(ctx context.Context) (bool, error) {
 		service := &corev1.Service{
 			ObjectMeta: metav1.ObjectMeta{
@@ -93,7 +91,7 @@ func createOperation(ctrlCtx *config.ControllerContext, serviceName string, grou
 			service.Spec.ExternalName = "localhost"
 			service.Spec.Ports = []corev1.ServicePort{
 				{
-					Port: int32(proxyPort),
+					Port: int32(hostPort),
 				},
 			}
 			return nil
@@ -111,7 +109,7 @@ func createOperation(ctrlCtx *config.ControllerContext, serviceName string, grou
 			Service: &apiregistrationv1.ServiceReference{
 				Name:      serviceName,
 				Namespace: "kube-system",
-				Port:      ptr.To(int32(proxyPort)),
+				Port:      ptr.To(int32(hostPort)),
 			},
 			InsecureSkipTLSVerify: true,
 			Group:                 groupVersion.Group,
@@ -141,19 +139,36 @@ func createOperation(ctrlCtx *config.ControllerContext, serviceName string, grou
 	}
 }
 
-func StartAPIServiceProxy(ctx context.Context, hostConfig *rest.Config, tlsCertFile, tlsKeyFile string) error {
+func StartAPIServiceProxy(ctx *config.ControllerContext, targetServiceName, targetServiceNamespace string, targetPort, hostPort int) error {
+	tlsCertFile := ctx.Config.VirtualClusterKubeConfig().ServerCACert
+	tlsKeyFile := ctx.Config.VirtualClusterKubeConfig().ServerCAKey
+
+	hostConfig := rest.CopyConfig(ctx.LocalManager.GetConfig())
+	hostConfig.Host = "https://" + targetServiceName + "." + targetServiceNamespace
+	if targetPort > 0 {
+		hostConfig.Host = hostConfig.Host + ":" + strconv.Itoa(targetPort)
+	}
+	hostConfig.APIPath = ""
+	hostConfig.CAFile = ""
+	hostConfig.CAData = nil
+	hostConfig.KeyFile = ""
+	hostConfig.KeyData = nil
+	hostConfig.CertFile = ""
+	hostConfig.CertData = nil
+	hostConfig.Insecure = true
+
 	proxyHandler, err := handler.Handler("", hostConfig, nil)
 	if err != nil {
 		return fmt.Errorf("create host proxy handler: %w", err)
 	}
 
 	s := serializer.NewCodecFactory(scheme.Scheme)
 	server := &http.Server{
-		Addr: "localhost:" + strconv.Itoa(proxyPort),
+		Addr: "localhost:" + strconv.Itoa(hostPort),
 		Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 			// we only allow traffic to discovery paths
 			if !isAPIServiceProxyPathAllowed(request.Method, request.URL.Path) {
-				klog.FromContext(ctx).Info("Denied access to api service proxy at path", "path", request.URL.Path, "method", request.Method)
+				klog.FromContext(ctx.Context).Info("Denied access to api service proxy at path", "path", request.URL.Path, "method", request.Method)
 				responsewriters.ErrorNegotiated(
 					kerrors.NewForbidden(metav1.SchemeGroupVersion.WithResource("proxy").GroupResource(), "proxy", fmt.Errorf("paths other than discovery paths are not allowed")),
 					s,
@@ -169,10 +184,10 @@ func StartAPIServiceProxy(ctx context.Context, hostConfig *rest.Config, tlsCertF
 	}
 
 	go func() {
-		klog.Infof("Listening apiservice proxy on localhost:%d...", proxyPort)
+		klog.Infof("Listening apiservice proxy on localhost:%d...", hostPort)
 		err = server.ListenAndServeTLS(tlsCertFile, tlsKeyFile)
 		if err != nil {
-			klog.FromContext(ctx).Error(err, "error listening for apiservice proxy and serve tls")
+			klog.FromContext(ctx.Context).Error(err, "error listening for apiservice proxy and serve tls")
 			os.Exit(1)
 		}
 	}()
@@ -216,8 +231,8 @@ func isAPIServiceProxyPathAllowed(method, path string) bool {
 	return false
 }
 
-func RegisterAPIService(ctx *config.ControllerContext, serviceName string, groupVersion schema.GroupVersion) error {
-	return applyOperation(ctx.Context, createOperation(ctx, serviceName, groupVersion))
+func RegisterAPIService(ctx *config.ControllerContext, serviceName string, hostPort int, groupVersion schema.GroupVersion) error {
+	return applyOperation(ctx.Context, createOperation(ctx, serviceName, hostPort, groupVersion))
 }
 
 func DeregisterAPIService(ctx *config.ControllerContext, groupVersion schema.GroupVersion) error {

pkg/config/controller_context.go

@@ -32,9 +32,6 @@ type ControllerContext struct {
 
 	// AcquiredLeaderHooks are hooks to start after vCluster acquired leader
 	AcquiredLeaderHooks []Hook
-
-	// StartAPIServiceProxy will start the api service proxy if needed
-	StartAPIServiceProxy bool
 }
 
 type Filter func(http.Handler, Clients) http.Handler

pkg/integrations/metricsserver/metricsserver.go

@@ -1,6 +1,7 @@
 package metricsserver
 
 import (
+	"cmp"
 	"context"
 	"fmt"
 	"net/http"
@@ -28,6 +29,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const hostPort = 9001
+
 const (
 	RequestVerbList = "list"
 	RequestVerbGet  = "get"
@@ -45,7 +48,14 @@ var GroupVersion = schema.GroupVersion{
 func Register(ctx *config.ControllerContext) error {
 	ctx.AcquiredLeaderHooks = append(ctx.AcquiredLeaderHooks, RegisterOrDeregisterAPIService)
 	if ctx.Config.Integrations.MetricsServer.Enabled {
-		ctx.StartAPIServiceProxy = true
+		targetService := cmp.Or(ctx.Config.Integrations.MetricsServer.Service.Name, "metrics-server")
+		targetServiceNamespace := cmp.Or(ctx.Config.Integrations.MetricsServer.Service.Namespace, "kube-system")
+		targetServicePort := cmp.Or(ctx.Config.Integrations.MetricsServer.Service.Port, 443)
+		err := apiservice.StartAPIServiceProxy(ctx, targetService, targetServiceNamespace, targetServicePort, hostPort)
+		if err != nil {
+			return fmt.Errorf("start api service proxy: %w", err)
+		}
+
 		ctx.PostServerHooks = append(ctx.PostServerHooks, func(h http.Handler, clients config.Clients) http.Handler {
 			return WithMetricsServerProxy(
 				h,
@@ -63,7 +73,7 @@ func Register(ctx *config.ControllerContext) error {
 
 func RegisterOrDeregisterAPIService(ctx *config.ControllerContext) error {
 	if ctx.Config.Integrations.MetricsServer.Enabled {
-		return apiservice.RegisterAPIService(ctx, "metrics-server", GroupVersion)
+		return apiservice.RegisterAPIService(ctx, "metrics-server", hostPort, GroupVersion)
 	}
 
 	return apiservice.DeregisterAPIService(ctx, GroupVersion)

pkg/server/server.go

@@ -2,15 +2,13 @@ package server
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"os"
 	"strconv"
 	"time"
 
-	"github.com/loft-sh/vcluster/pkg/apiservice"
 	"github.com/loft-sh/vcluster/pkg/authentication/delegatingauthenticator"
 	"github.com/loft-sh/vcluster/pkg/authorization/allowall"
 	"github.com/loft-sh/vcluster/pkg/authorization/delegatingauthorizer"
@@ -221,12 +219,6 @@ func NewServer(ctx *config.ControllerContext, requestHeaderCaFile, clientCaFile
 	h = filters.WithMetricsProxy(h, localConfig, cachedVirtualClient)
 
 	// inject apis
-	if ctx.StartAPIServiceProxy {
-		err = apiservice.StartAPIServiceProxy(ctx.Context, ctx.LocalManager.GetConfig(), ctx.Config.VirtualClusterKubeConfig().ServerCACert, ctx.Config.VirtualClusterKubeConfig().ServerCAKey)
-		if err != nil {
-			return nil, fmt.Errorf("start api service proxy: %w", err)
-		}
-	}
 	if ctx.Config.Sync.FromHost.Nodes.Enabled && ctx.Config.Sync.FromHost.Nodes.SyncBackChanges {
 		h = filters.WithNodeChanges(ctx.Context, h, uncachedLocalClient, uncachedVirtualClient, virtualConfig)
 	}