feat(kuma-cp): gateway API behind experimental flag (#4014)

Signed-off-by: Jakub Dyszkiewicz <[email protected]>

pkg/api-server/config_ws_test.go

@@ -345,6 +345,7 @@ var _ = Describe("Config WS", func() {
           },
           "experimental": {
             "meshGateway": false,
+            "gatewayAPI": false,
             "kubeOutboundsAsVIPs": false
           }
         }

pkg/config/app/kuma-cp/config.go

@@ -200,6 +200,7 @@ var DefaultConfig = func() Config {
 		Access:      access.DefaultAccessConfig(),
 		Experimental: ExperimentalConfig{
 			MeshGateway:         false,
+			GatewayAPI:          false,
 			KubeOutboundsAsVIPs: false,
 		},
 	}
@@ -277,6 +278,9 @@ func (c *Config) Validate() error {
 	if err := c.Diagnostics.Validate(); err != nil {
 		return errors.Wrap(err, "Diagnostics validation failed")
 	}
+	if err := c.Experimental.Validate(); err != nil {
+		return errors.Wrap(err, "Experimental validation failed")
+	}
 	return nil
 }
 
@@ -316,7 +320,16 @@ func DefaultGeneralConfig() *GeneralConfig {
 type ExperimentalConfig struct {
 	// If true, experimental built-in gateway is enabled.
 	MeshGateway bool `yaml:"meshGateway" envconfig:"KUMA_EXPERIMENTAL_MESHGATEWAY"`
+	// If true, experimental Gateway API is enabled
+	GatewayAPI bool `yaml:"gatewayAPI" envconfig:"KUMA_EXPERIMENTAL_GATEWAY_API"`
 	// If true, instead of embedding kubernetes outbounds into Dataplane object, they are persisted next to VIPs in ConfigMap
 	// This can improve performance, but it should be enabled only after all instances are migrated to version that supports this config
 	KubeOutboundsAsVIPs bool `yaml:"kubeOutboundsAsVIPs" envconfig:"KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS"`
 }
+
+func (e ExperimentalConfig) Validate() error {
+	if e.GatewayAPI && !e.MeshGateway {
+		return errors.New("GatewayAPI cannot be enabled without MeshGateway")
+	}
+	return nil
+}

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -433,6 +433,8 @@ access:
 experimental:
   # If true, experimental built-in gateway is enabled
   meshGateway: false # ENV: KUMA_EXPERIMENTAL_MESHGATEWAY
+  # If true, experimental Gateway API is enabled
+  gatewayAPI: false # ENV: KUMA_EXPERIMENTAL_GATEWAY_API
   # If true, instead of embedding kubernetes outbounds into Dataplane object, they are persisted next to VIPs in ConfigMap
   # This can improve performance, but it should be enabled only after all instances are migrated to version that supports this config
   kubeOutboundsAsVIPs: false # ENV: KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS

pkg/config/loader_test.go

@@ -250,6 +250,7 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Access.Static.ViewConfigDump.Groups).To(Equal([]string{"zt-group1", "zt-group2"}))
 
 			Expect(cfg.Experimental.MeshGateway).To(BeTrue())
+			Expect(cfg.Experimental.GatewayAPI).To(BeTrue())
 			Expect(cfg.Experimental.KubeOutboundsAsVIPs).To(BeTrue())
 		},
 		Entry("from config file", testCase{
@@ -466,6 +467,7 @@ access:
       groups: ["zt-group1", "zt-group2"]
 experimental:
   meshGateway: true
+  gatewayAPI: true
   kubeOutboundsAsVIPs: true
 `,
 		}),
@@ -614,6 +616,7 @@ experimental:
 				"KUMA_ACCESS_STATIC_GET_CONFIG_DUMP_USERS":                                                 "zt-admin1,zt-admin2",
 				"KUMA_ACCESS_STATIC_GET_CONFIG_DUMP_GROUPS":                                                "zt-group1,zt-group2",
 				"KUMA_EXPERIMENTAL_MESHGATEWAY":                                                            "true",
+				"KUMA_EXPERIMENTAL_GATEWAY_API":                                                            "true",
 				"KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS":                                                 "true",
 			},
 			yamlFileConfig: "",

pkg/plugins/runtime/k8s/plugin_gateway.go

@@ -20,7 +20,7 @@ import (
 	k8s_webhooks "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/webhooks"
 )
 
-func crdsPresent(mgr kube_ctrl.Manager) bool {
+func gatewayAPICRDsPresent(mgr kube_ctrl.Manager) bool {
 	gk := schema.GroupKind{
 		Group: gatewayapi.SchemeGroupVersion.Group,
 		Kind:  "Gateway",
@@ -34,7 +34,7 @@ func crdsPresent(mgr kube_ctrl.Manager) bool {
 	return len(mappings) > 0
 }
 
-func gatewayPresent() bool {
+func meshGatewayCRDsPresent() bool {
 	// If we haven't registered our type, we're not reconciling MeshGatewayInstance
 	// or gatewayapi objects.
 	if _, err := k8s_registry.Global().NewObject(&mesh_proto.MeshGateway{}); err != nil {
@@ -48,7 +48,11 @@ func gatewayPresent() bool {
 }
 
 func addGatewayReconcilers(mgr kube_ctrl.Manager, rt core_runtime.Runtime, converter k8s_common.Converter) error {
-	if !gatewayPresent() {
+	if !rt.Config().Experimental.MeshGateway {
+		return nil
+	}
+	if !meshGatewayCRDsPresent() {
+		log.Info("[WARNING] Experimental MeshGateway feature is enabled, but CRDs are not registered. Disabling support")
 		return nil
 	}
 
@@ -81,8 +85,18 @@ func addGatewayReconcilers(mgr kube_ctrl.Manager, rt core_runtime.Runtime, conve
 		return errors.Wrap(err, "could not setup MeshGatewayInstance reconciler")
 	}
 
-	if !crdsPresent(mgr) {
-		log.Info("Gateway API CRDs not registered")
+	if rt.Config().Experimental.GatewayAPI {
+		if err := addGatewayAPIReconcillers(mgr, rt, proxyFactory); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func addGatewayAPIReconcillers(mgr kube_ctrl.Manager, rt core_runtime.Runtime, proxyFactory *containers.DataplaneProxyFactory) error {
+	if !gatewayAPICRDsPresent(mgr) {
+		log.Info("[WARNING] Experimental GatewayAPI feature is enabled, but CRDs are not registered. Disabling support")
 		return nil
 	}
 
@@ -118,14 +132,13 @@ func addGatewayReconcilers(mgr kube_ctrl.Manager, rt core_runtime.Runtime, conve
 	if err := gatewayAPIHTTPRouteReconciler.SetupWithManager(mgr); err != nil {
 		return errors.Wrap(err, "could not setup Gateway API HTTPRoute reconciler")
 	}
-
 	return nil
 }
 
 // gatewayValidators returns all the Gateway-related validators we want to
 // start.
 func gatewayValidators(rt core_runtime.Runtime, converter k8s_common.Converter) []k8s_common.AdmissionValidator {
-	if !gatewayPresent() {
+	if !meshGatewayCRDsPresent() {
 		return nil
 	}