From 8152f280934a8e7696bbea05e6ca107f0ba42358 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Tue, 20 Jul 2021 10:48:20 -0700 Subject: [PATCH] Customizable user namespace templating (#223) --- flyteadmin_config.yaml | 2 ++ pkg/clusterresource/controller.go | 2 +- pkg/common/namespace.go | 32 +++++++---------- pkg/common/namespace_test.go | 17 +++++----- .../interfaces/namespace_configuration.go | 8 ++--- pkg/runtime/namespace_config_provider.go | 34 ++++++++----------- pkg/workflowengine/impl/propeller_executor.go | 6 ++-- 7 files changed, 44 insertions(+), 57 deletions(-) diff --git a/flyteadmin_config.yaml b/flyteadmin_config.yaml index 3d2f82f803..70accc006d 100644 --- a/flyteadmin_config.yaml +++ b/flyteadmin_config.yaml @@ -200,3 +200,5 @@ qualityOfService: development: LOW staging: MEDIUM # by default production has an UNDEFINED tier when it is omitted from the configuration +namespace_mapping: + template: "{{ project }}-{{ domain }}" # Default namespace mapping template. diff --git a/pkg/clusterresource/controller.go b/pkg/clusterresource/controller.go index 4c1baf7028..5b80c4e292 100644 --- a/pkg/clusterresource/controller.go +++ b/pkg/clusterresource/controller.go @@ -489,7 +489,7 @@ func (c *controller) Sync(ctx context.Context) error { for _, project := range projects { for _, domain := range *domains { - namespace := common.GetNamespaceName(c.config.NamespaceMappingConfiguration().GetNamespaceMappingConfig(), project.Identifier, domain.Name) + namespace := common.GetNamespaceName(c.config.NamespaceMappingConfiguration().GetNamespaceTemplate(), project.Identifier, domain.Name) customTemplateValues, err := c.getCustomTemplateValues( ctx, project.Identifier, domain.ID, domainTemplateValues[domain.ID]) if err != nil { diff --git a/pkg/common/namespace.go b/pkg/common/namespace.go index 6998fcee7c..207c12585a 100644 --- a/pkg/common/namespace.go +++ b/pkg/common/namespace.go @@ -1,27 +1,19 @@ package common -import "fmt" +import ( + "strings" +) -type NamespaceMapping int +const projectTemplate = "{{ project }}" +const domainTemplate = "{{ domain }}" -const namespaceFormat = "%s-%s" +const replaceAllInstancesOfString = -1 -const ( - NamespaceMappingProjectDomain NamespaceMapping = iota - NamespaceMappingDomain NamespaceMapping = iota - NamespaceMappingProject NamespaceMapping = iota -) +// GetNamespaceName returns kubernetes namespace name according to user defined template from config +func GetNamespaceName(template string, project, domain string) string { + var namespace = template + namespace = strings.Replace(namespace, projectTemplate, project, replaceAllInstancesOfString) + namespace = strings.Replace(namespace, domainTemplate, domain, replaceAllInstancesOfString) -// GetNamespaceName returns kubernetes namespace name -func GetNamespaceName(mapping NamespaceMapping, project, domain string) string { - switch mapping { - case NamespaceMappingDomain: - return domain - case NamespaceMappingProject: - return project - case NamespaceMappingProjectDomain: - fallthrough - default: - return fmt.Sprintf(namespaceFormat, project, domain) - } + return namespace } diff --git a/pkg/common/namespace_test.go b/pkg/common/namespace_test.go index a23945839e..1d002972ad 100644 --- a/pkg/common/namespace_test.go +++ b/pkg/common/namespace_test.go @@ -8,19 +8,18 @@ import ( func TestGetNamespaceName(t *testing.T) { testCases := []struct { - mapping NamespaceMapping - project string - domain string - want string + template string + project string + domain string + want string }{ - {NamespaceMappingProjectDomain, "project", "production", "project-production"}, - {20 /*Dummy enum value that is not supported*/, "project", "development", "project-development"}, - {NamespaceMappingDomain, "project", "production", "production"}, - {NamespaceMappingProject, "project", "production", "project"}, + {"prefix-{{ project }}-{{ domain }}", "flytesnacks", "production", "prefix-flytesnacks-production"}, + {"{{ domain }}", "flytesnacks", "production", "production"}, + {"{{ project }}", "flytesnacks", "production", "flytesnacks"}, } for _, tc := range testCases { - got := GetNamespaceName(tc.mapping, tc.project, tc.domain) + got := GetNamespaceName(tc.template, tc.project, tc.domain) assert.Equal(t, got, tc.want) } } diff --git a/pkg/runtime/interfaces/namespace_configuration.go b/pkg/runtime/interfaces/namespace_configuration.go index 0d549c3de3..dc5856e366 100644 --- a/pkg/runtime/interfaces/namespace_configuration.go +++ b/pkg/runtime/interfaces/namespace_configuration.go @@ -1,11 +1,11 @@ package interfaces -import "github.com/flyteorg/flyteadmin/pkg/common" - type NamespaceMappingConfig struct { - Mapping string `json:"mapping"` + Mapping string `json:"mapping"` // Deprecated + Template string `json:"template"` + TemplateData TemplateData `json:"templateData"` } type NamespaceMappingConfiguration interface { - GetNamespaceMappingConfig() common.NamespaceMapping + GetNamespaceTemplate() string } diff --git a/pkg/runtime/namespace_config_provider.go b/pkg/runtime/namespace_config_provider.go index 931b6e51b9..c252345215 100644 --- a/pkg/runtime/namespace_config_provider.go +++ b/pkg/runtime/namespace_config_provider.go @@ -3,42 +3,36 @@ package runtime import ( "context" - "github.com/flyteorg/flyteadmin/pkg/common" "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/config" "github.com/flyteorg/flytestdlib/logger" ) const ( - namespaceMappingKey = "namespace_mapping" - domainVariable = "domain" - projectVariable = "project" - projectDomainVariable = "project-domain" + namespaceMappingKey = "namespace_mapping" + defaultTemplate = "{{ project }}-{{ domain }}" ) var namespaceMappingConfig = config.MustRegisterSection(namespaceMappingKey, &interfaces.NamespaceMappingConfig{ - Mapping: projectDomainVariable, + Template: defaultTemplate, }) type NamespaceMappingConfigurationProvider struct{} -func (p *NamespaceMappingConfigurationProvider) GetNamespaceMappingConfig() common.NamespaceMapping { - var mapping string +func (p *NamespaceMappingConfigurationProvider) GetNamespaceTemplate() string { + var template string if namespaceMappingConfig != nil && namespaceMappingConfig.GetConfig() != nil { - mapping = namespaceMappingConfig.GetConfig().(*interfaces.NamespaceMappingConfig).Mapping + template = namespaceMappingConfig.GetConfig().(*interfaces.NamespaceMappingConfig).Template + if len(namespaceMappingConfig.GetConfig().(*interfaces.NamespaceMappingConfig).Mapping) > 0 { + logger.Errorf(context.TODO(), "Using `mapping` in namespace configs is deprecated. "+ + "Please use a custom string template like `{{ project }}-{{ domain }}` instead") + } } - - switch mapping { - case domainVariable: - return common.NamespaceMappingDomain - case projectVariable: - return common.NamespaceMappingProject - case projectDomainVariable: - return common.NamespaceMappingProjectDomain - default: - logger.Warningf(context.Background(), "Unsupported value for namespace_mapping in config, defaulting to -") - return common.NamespaceMappingProjectDomain + if len(template) == 0 { + logger.Infof(context.TODO(), "No namespace template specified in config. Using [%+s] by default", defaultTemplate) + template = defaultTemplate } + return template } func NewNamespaceMappingConfigurationProvider() interfaces.NamespaceMappingConfiguration { diff --git a/pkg/workflowengine/impl/propeller_executor.go b/pkg/workflowengine/impl/propeller_executor.go index ba99d4f2fe..1dbf639c3d 100644 --- a/pkg/workflowengine/impl/propeller_executor.go +++ b/pkg/workflowengine/impl/propeller_executor.go @@ -108,7 +108,7 @@ func (c *FlytePropeller) ExecuteWorkflow(ctx context.Context, input interfaces.E c.metrics.InvalidExecutionID.Inc() return nil, errors.NewFlyteAdminErrorf(codes.Internal, "invalid execution id") } - namespace := common.GetNamespaceName(c.config.GetNamespaceMappingConfig(), input.ExecutionID.GetProject(), input.ExecutionID.GetDomain()) + namespace := common.GetNamespaceName(c.config.GetNamespaceTemplate(), input.ExecutionID.GetProject(), input.ExecutionID.GetDomain()) flyteWf, err := c.builder.BuildFlyteWorkflow(&input.WfClosure, input.Inputs, input.ExecutionID, namespace) if err != nil { c.metrics.WorkflowBuildFailure.Inc() @@ -185,7 +185,7 @@ func (c *FlytePropeller) ExecuteTask(ctx context.Context, input interfaces.Execu c.metrics.InvalidExecutionID.Inc() return nil, errors.NewFlyteAdminErrorf(codes.Internal, "invalid execution id") } - namespace := common.GetNamespaceName(c.config.GetNamespaceMappingConfig(), input.ExecutionID.GetProject(), input.ExecutionID.GetDomain()) + namespace := common.GetNamespaceName(c.config.GetNamespaceTemplate(), input.ExecutionID.GetProject(), input.ExecutionID.GetDomain()) flyteWf, err := c.builder.BuildFlyteWorkflow(&input.WfClosure, input.Inputs, input.ExecutionID, namespace) if err != nil { c.metrics.WorkflowBuildFailure.Inc() @@ -263,7 +263,7 @@ func (c *FlytePropeller) TerminateWorkflowExecution( c.metrics.InvalidExecutionID.Inc() return errors.NewFlyteAdminErrorf(codes.Internal, "invalid execution id") } - namespace := common.GetNamespaceName(c.config.GetNamespaceMappingConfig(), input.ExecutionID.GetProject(), input.ExecutionID.GetDomain()) + namespace := common.GetNamespaceName(c.config.GetNamespaceTemplate(), input.ExecutionID.GetProject(), input.ExecutionID.GetDomain()) target, err := c.executionCluster.GetTarget(ctx, &executioncluster.ExecutionTargetSpec{ TargetID: input.Cluster, })