Merge pull request #5410 from aledbf/ingressclass

Add support for IngressClass and ingress.class annotation

charts/ingress-nginx/templates/clusterrole.yaml

@@ -65,4 +65,12 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - "networking.k8s.io" # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
 {{- end }}

charts/ingress-nginx/templates/controller-role.yaml

@@ -49,6 +49,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - "networking.k8s.io" # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:

cmd/nginx/main.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -42,6 +41,7 @@ import (
 	"k8s.io/klog"
 
 	"k8s.io/ingress-nginx/internal/file"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/class"
 	"k8s.io/ingress-nginx/internal/ingress/controller"
 	"k8s.io/ingress-nginx/internal/ingress/metric"
 	"k8s.io/ingress-nginx/internal/k8s"
@@ -87,8 +87,10 @@ func main() {
 			if errors.IsUnauthorized(err) || errors.IsForbidden(err) {
 				klog.Fatal("✖ The cluster seems to be running with a restrictive Authorization mode and the Ingress controller does not have the required permissions to operate normally.")
 			}
+
 			klog.Fatalf("No service with name %v found: %v", conf.DefaultService, err)
 		}
+
 		klog.Infof("Validated %v as the default backend.", conf.DefaultService)
 	}
 
@@ -107,8 +109,28 @@ func main() {
 		klog.Warningf("Using deprecated \"k8s.io/api/extensions/v1beta1\" package because Kubernetes version is < v1.14.0")
 	}
 
-	if !k8s.IsIngressV1Ready {
-		klog.Infof("Enabling new Ingress features availables since v1.18.0")
+	if k8s.IsIngressV1Ready {
+		klog.Infof("Enabling new Ingress features available since Kubernetes v1.18")
+		k8s.IngressClass, err = kubeClient.NetworkingV1beta1().IngressClasses().
+			Get(context.TODO(), class.IngressClass, metav1.GetOptions{})
+		if err != nil {
+			if !errors.IsNotFound(err) {
+				if !errors.IsUnauthorized(err) && !errors.IsForbidden(err) {
+					klog.Fatalf("Error searching IngressClass: %v", err)
+				}
+
+				klog.Errorf("Unexpected error searching IngressClass: %v", err)
+			}
+
+			klog.Warningf("No IngressClass resource with name %v found. Only annotation will be used.", class.IngressClass)
+
+			// TODO: remove once this is fixed in client-go
+			k8s.IngressClass = nil
+		}
+
+		if k8s.IngressClass != nil && k8s.IngressClass.Spec.Controller != k8s.IngressNGINXController {
+			klog.Fatalf("IngressClass with name %v is not valid for ingress-nginx (invalid Spec.Controller)", class.IngressClass)
+		}
 	}
 
 	conf.Client = kubeClient

deploy/static/provider/aws/deploy-tls-termination.yaml

@@ -106,6 +106,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
 ---
 # Source: ingress-nginx/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -184,6 +192,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ''
     resources:

deploy/static/provider/aws/deploy.yaml

@@ -99,6 +99,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
 ---
 # Source: ingress-nginx/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -177,6 +185,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ''
     resources:

deploy/static/provider/baremetal/deploy.yaml

@@ -99,6 +99,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
 ---
 # Source: ingress-nginx/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -177,6 +185,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ''
     resources:

deploy/static/provider/cloud/deploy.yaml

@@ -99,6 +99,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
 ---
 # Source: ingress-nginx/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -177,6 +185,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ''
     resources:

deploy/static/provider/kind/deploy.yaml

@@ -99,6 +99,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
 ---
 # Source: ingress-nginx/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -177,6 +185,14 @@ rules:
       - ingresses/status
     verbs:
       - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ''
     resources:

internal/ingress/annotations/class/main.go

@@ -18,6 +18,7 @@ package class
 
 import (
 	networking "k8s.io/api/networking/v1beta1"
+	"k8s.io/ingress-nginx/internal/k8s"
 )
 
 const (
@@ -37,30 +38,30 @@ var (
 	IngressClass = "nginx"
 )
 
-// IsValid returns true if the given Ingress either doesn't specify
-// the ingress.class annotation, or it's set to the configured in the
-// ingress controller.
+// IsValid returns true if the given Ingress specify the ingress.class
+// annotation or IngressClassName resource for Kubernetes >= v1.18
 func IsValid(ing *networking.Ingress) bool {
-	className := ing.Spec.IngressClassName
+	// 1. with annotation
+	ingress, ok := ing.GetAnnotations()[IngressKey]
+	if ok {
+		// empty annotation and same annotation on ingress
+		if ingress == "" && IngressClass == DefaultClass {
+			return true
+		}
 
-	// we have 2 valid combinations
-	// 1 - ingress with default class | blank annotation on ingress
-	// 2 - ingress with specific class | same annotation on ingress
-	//
-	// and 2 invalid combinations
-	// 3 - ingress with default class | fixed annotation on ingress
-	// 4 - ingress with specific class | different annotation on ingress
-	if className != nil {
-		return *className == IngressClass
+		return ingress == IngressClass
 	}
 
-	if IngressClass == DefaultClass {
-		return true
+	// 2. k8s < v1.18. Check default annotation
+	if !k8s.IsIngressV1Ready {
+		return IngressClass == DefaultClass
 	}
 
-	if IngressClass == "" {
-		return true
+	// 3. without annotation and IngressClass. Check default annotation
+	if k8s.IngressClass == nil {
+		return IngressClass == DefaultClass
 	}
 
-	return false
+	// 4. with IngressClass
+	return k8s.IngressClass.Name == *ing.Spec.IngressClassName
 }

internal/ingress/annotations/class/main_test.go

@@ -54,10 +54,10 @@ func TestIsValidClass(t *testing.T) {
 		},
 	}
 
+	data := map[string]string{}
+	ing.SetAnnotations(data)
 	for _, test := range tests {
-		if test.ingress != "" {
-			ing.Spec.IngressClassName = &test.ingress
-		}
+		ing.Annotations[IngressKey] = test.ingress
 
 		IngressClass = test.controller
 		DefaultClass = test.defClass

internal/ingress/controller/controller_test.go

@@ -190,8 +190,7 @@ func TestCheckIngress(t *testing.T) {
 	}
 
 	t.Run("When the ingress class differs from nginx", func(t *testing.T) {
-		class := "different"
-		ing.Spec.IngressClassName = &class
+		ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "different"
 		nginx.command = testNginxTestCommand{
 			t:   t,
 			err: fmt.Errorf("test error"),
@@ -202,8 +201,7 @@ func TestCheckIngress(t *testing.T) {
 	})
 
 	t.Run("when the class is the nginx one", func(t *testing.T) {
-		class := "nginx"
-		ing.Spec.IngressClassName = &class
+		ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "nginx"
 		nginx.command = testNginxTestCommand{
 			t:        t,
 			err:      nil,

internal/ingress/controller/store/store.go

@@ -949,30 +949,18 @@ func toIngress(obj interface{}) (*networkingv1beta1.Ingress, bool) {
 			return nil, false
 		}
 
-		ing.Spec.IngressClassName = extractClassName(ing)
 		setDefaultPathTypeIfEmpty(ing)
-
 		return ing, true
 	}
 
 	if ing, ok := obj.(*networkingv1beta1.Ingress); ok {
-		ing.Spec.IngressClassName = extractClassName(ing)
 		setDefaultPathTypeIfEmpty(ing)
-
 		return ing, true
 	}
 
 	return nil, false
 }
 
-func extractClassName(ing *networkingv1beta1.Ingress) *string {
-	if c, ok := ing.Annotations[class.IngressKey]; ok {
-		return &c
-	}
-
-	return nil
-}
-
 // Default path type is Prefix to not break existing definitions
 var defaultPathType = networkingv1beta1.PathTypePrefix
 

internal/k8s/main.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/klog"
 
 	apiv1 "k8s.io/api/core/v1"
+	networkingv1beta1 "k8s.io/api/networking/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
@@ -121,6 +122,13 @@ var IsNetworkingIngressAvailable bool
 // IsIngressV1Ready indicates if the running Kubernetes version is at least v1.18.0
 var IsIngressV1Ready bool
 
+// IngressClass indicates the class of the Ingress to use as filter
+var IngressClass *networkingv1beta1.IngressClass
+
+// IngressNGINXController defines the valid value of IngressClass
+// Controller field for ingress-nginx
+const IngressNGINXController = "k8s.io/ingress-nginx"
+
 // NetworkingIngressAvailable checks if the package "k8s.io/api/networking/v1beta1"
 // is available or not and if Ingress V1 is supported (k8s >= v1.18.0)
 func NetworkingIngressAvailable(client clientset.Interface) (bool, bool) {

test/e2e/framework/exec.go

@@ -55,7 +55,7 @@ func (f *Framework) GetLbAlgorithm(serviceName string, servicePort int) (string,
 
 // ExecIngressPod executes a command inside the first container in ingress controller running pod
 func (f *Framework) ExecIngressPod(command string) (string, error) {
-	pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet)
+	pod, err := GetIngressNGINXPod(f.Namespace, f.KubeClientSet)
 	if err != nil {
 		return "", err
 	}