SVR-185 Serverless Image Builder mutating webhook (#244)

* SVR-185 Image Builder Mutating/Validation Webhook

Support a mutating and validating webhook for creating Pods.

Notable changes:

* Supports replacing obscuring URL cr.union.ai with underlying GAR URL
* Introduce concept of WebhookType to distinguish Secrets and
  ImageBuilder
* Update existing adminissionregistrationv1.MutatingWebhookConfiguration with a **new
  additional Webhook** for ImageBuilder. This was necessary because
  Secrets and ImageBuilder uses different Pod Label Selectors.
  Removed PodMutator containing a list of mutators.
* This introduces somewhat strict requirements that **all serverless
  flyte* FlytePropeller originated Pods must conform to a specific set
  of GAR registry paths.

flytepropeller/pkg/controller/nodes/node_exec_context.go

@@ -3,9 +3,8 @@ package nodes
 import (
 	"context"
 	"fmt"
-	"strconv"
-
 	"slices"
+	"strconv"
 
 	_struct "github.com/golang/protobuf/ptypes/struct"
 	"github.com/pkg/errors"

flytepropeller/pkg/controller/nodes/task/k8s/event_watcher_test.go

@@ -1,7 +1,6 @@
 package k8s
 
 import (
-	"k8s.io/client-go/tools/cache"
 	"testing"
 	"time"
 
@@ -10,6 +9,7 @@ import (
 	eventsv1 "k8s.io/api/events/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/cache"
 )
 
 func TestEventWatcher_OnAdd(t *testing.T) {

flytepropeller/pkg/webhook/config/config.go

@@ -5,6 +5,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/flyteorg/flyte/flytestdlib/config"
 )
@@ -111,6 +112,9 @@ type Config struct {
 	GCPSecretManagerConfig      GCPSecretManagerConfig      `json:"gcpSecretManager" pflag:",GCP Secret Manager config."`
 	VaultSecretManagerConfig    VaultSecretManagerConfig    `json:"vaultSecretManager" pflag:",Vault Secret Manager config."`
 	EmbeddedSecretManagerConfig EmbeddedSecretManagerConfig `json:"embeddedSecretManagerConfig" pflag:",Embedded Secret Manager config without sidecar and which calls into the supported providers directly."`
+
+	// Ignore PFlag for Image Builder
+	ImageBuilderConfig *ImageBuilderConfig `json:"imageBuilderConfig,omitempty" pflag:"-,"`
 }
 
 //go:generate enumer --type=EmbeddedSecretManagerType -json -yaml -trimprefix=EmbeddedSecretManagerType
@@ -155,6 +159,17 @@ type VaultSecretManagerConfig struct {
 	Annotations map[string]string `json:"annotations" pflag:"-,Annotation to be added to user task pod. The annotation can also be used to override default annotations added by Flyte. Useful to customize Vault integration (https://developer.hashicorp.com/vault/docs/platform/k8s/injector/annotations)"`
 }
 
+type HostnameReplacement struct {
+	Existing            string `json:"existing" pflag:",The existing hostname to replace"`
+	Replacement         string `json:"replacement" pflag:",The replacement hostname"`
+	DisableVerification bool   `json:"disableVerification" pflag:",Allow disabling URI verification for development environments"`
+}
+
+type ImageBuilderConfig struct {
+	HostnameReplacement HostnameReplacement  `json:"hostnameReplacement"`
+	LabelSelector       metav1.LabelSelector `json:"labelSelector"`
+}
+
 func GetConfig() *Config {
 	return configSection.GetConfig().(*Config)
 }

flytepropeller/pkg/webhook/config/config_test.go

@@ -0,0 +1,103 @@
+package config
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Ensure HostnameReplacements resolves to non-nil empty list.
+func TestConfig_DefaultImageBuilder(t *testing.T) {
+	assert.Nil(t, DefaultConfig.ImageBuilderConfig)
+}
+
+func TestConfig_LoadSimpleJSON(t *testing.T) {
+	expectedJSON := `{
+		"metrics-prefix": "test-prefix",
+		"certDir": "/test/cert/dir",
+		"localCert": true,
+		"listenPort": 8080,
+		"serviceName": "test-service",
+		"servicePort": 8081,
+		"secretName": "test-secret",
+		"secretManagerType": "K8s"
+	}`
+
+	var config Config
+	err := json.Unmarshal([]byte(expectedJSON), &config)
+	assert.Nil(t, err)
+
+	assert.Nil(t, config.ImageBuilderConfig)
+}
+
+func TestConfig_ImageBuilderConfig(t *testing.T) {
+
+	t.Run("With verification enabled", func(t *testing.T) {
+		expectedJSON := `{
+		"metrics-prefix": "test-prefix",
+		"certDir": "/test/cert/dir",
+		"localCert": true,
+		"listenPort": 8080,
+		"serviceName": "test-service",
+		"servicePort": 8081,
+		"secretName": "test-secret",
+		"secretManagerType": "K8s",
+		"imageBuilderConfig": {
+			"hostnameReplacement": {
+				"existing": "test.existing.hostname",
+				"replacement": "test.replacement.hostname"
+			},
+			"labelSelector": {
+				"matchLabels": {
+					"test-key": "test-value"
+				}
+			}
+			}
+		}`
+
+		var config Config
+		err := json.Unmarshal([]byte(expectedJSON), &config)
+		assert.Nil(t, err)
+
+		assert.Equal(t, "test.existing.hostname", config.ImageBuilderConfig.HostnameReplacement.Existing)
+		assert.Equal(t, "test.replacement.hostname", config.ImageBuilderConfig.HostnameReplacement.Replacement)
+		assert.Equal(t, false, config.ImageBuilderConfig.HostnameReplacement.DisableVerification)
+		assert.Equal(t, "test-value", config.ImageBuilderConfig.LabelSelector.MatchLabels["test-key"])
+	})
+
+	t.Run("With verification disabled", func(t *testing.T) {
+		expectedJSON := `{
+		"metrics-prefix": "test-prefix",
+		"certDir": "/test/cert/dir",
+		"localCert": true,
+		"listenPort": 8080,
+		"serviceName": "test-service",
+		"servicePort": 8081,
+		"secretName": "test-secret",
+		"secretManagerType": "K8s",
+		"imageBuilderConfig": {
+			"hostnameReplacement": {
+				"existing": "test.existing.hostname",
+				"replacement": "test.replacement.hostname",
+				"disableVerification": true
+			},
+			"labelSelector": {
+				"matchLabels": {
+					"test-key": "test-value"
+				}
+			}
+			}
+		}`
+
+		var config Config
+		err := json.Unmarshal([]byte(expectedJSON), &config)
+		assert.Nil(t, err)
+
+		assert.Equal(t, "test.existing.hostname", config.ImageBuilderConfig.HostnameReplacement.Existing)
+		assert.Equal(t, "test.replacement.hostname", config.ImageBuilderConfig.HostnameReplacement.Replacement)
+		assert.Equal(t, true, config.ImageBuilderConfig.HostnameReplacement.DisableVerification)
+		assert.Equal(t, "test-value", config.ImageBuilderConfig.LabelSelector.MatchLabels["test-key"])
+	})
+
+}

flytepropeller/pkg/webhook/entrypoint.go

@@ -54,7 +54,7 @@ func RunWebhook(ctx context.Context, propellerCfg *config.Config, cfg *config2.C
 
 	webhookScope := (*scope).NewSubScope("webhook")
 
-	secretsWebhook, err := NewPodMutator(ctx, cfg, mgr.GetScheme(), webhookScope)
+	secretsWebhook, err := NewPodCreationWebhookConfig(ctx, cfg, mgr.GetScheme(), webhookScope)
 	if err != nil {
 		return err
 	}
@@ -65,7 +65,7 @@ func RunWebhook(ctx context.Context, propellerCfg *config.Config, cfg *config2.C
 		return err
 	}
 
-	err = secretsWebhook.Register(ctx, mgr)
+	err = secretsWebhook.Register(ctx, K8sRuntimeHTTPHookRegisterer{mgr: mgr})
 	if err != nil {
 		logger.Fatalf(ctx, "Failed to register webhook with manager. Error: %v", err)
 	}
@@ -76,7 +76,7 @@ func RunWebhook(ctx context.Context, propellerCfg *config.Config, cfg *config2.C
 	return nil
 }
 
-func createMutationConfig(ctx context.Context, kubeClient *kubernetes.Clientset, webhookObj *PodMutator, defaultNamespace string) error {
+func createMutationConfig(ctx context.Context, kubeClient *kubernetes.Clientset, webhookObj *PodCreationWebhookConfig, defaultNamespace string) error {
 	shouldAddOwnerRef := true
 	podName, found := os.LookupEnv(PodNameEnvVar)
 	if !found {

flytepropeller/pkg/webhook/http_hook_registerer.go

@@ -0,0 +1,21 @@
+package webhook
+
+import (
+	"net/http"
+
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+//go:generate mockery --output=./mocks --case=underscore -name=HTTPHookRegistererIface
+
+type HTTPHookRegistererIface interface {
+	Register(path string, hook http.Handler)
+}
+
+type K8sRuntimeHTTPHookRegisterer struct {
+	mgr manager.Manager
+}
+
+func (k K8sRuntimeHTTPHookRegisterer) Register(path string, hook http.Handler) {
+	k.mgr.GetWebhookServer().Register(path, hook)
+}