Support exposing ClusterIngress to two or more shared Gateways. (#2666)

* Support exposing ClusterIngress to two or more Gateways. * Fix golint. Update deps. * Run update-dep again. * Add documentation for ConfigMap entry. * Update config map layout per PR feedback. * Address PR feedbacks. * Address PR feedback.
knative · Dec 7, 2018 · 241f470 · 241f470
1 parent d60b53a
commit 241f470
Show file tree

Hide file tree

Showing 11 changed files with 405 additions and 89 deletions.
diff --git a/config/202-gateway.yaml b/config/202-gateway.yaml
@@ -1,32 +1,8 @@
-# Copyright 2018 The Knative Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The Gateway resource here is to attach to a gateway service that
-# receive all external traffic for Knative pods. We don't maintain
-# extra Gateway service and deployment in knative, but use that
-# provided in Istio by default.
-
-# If you want to replace the Gateway service and deployment to that
-# of your own, you'll need to update the label selector and ports
-# fields accordingly.
-
 # This is the shared Gateway for all Knative routes to use.
-
 apiVersion: networking.istio.io/v1alpha3
 kind: Gateway
 metadata:
-  name: knative-shared-gateway
+  name: knative-ingress-gateway
   namespace: knative-serving
 spec:
   selector:
@@ -46,3 +22,255 @@ spec:
     - "*"
     tls:
       mode: PASSTHROUGH
+---
+# This is the pre-0.3 shared Gateway.  We leave it here so that existing clients can
+# still access using their own IP address.  However, this is deprecated and will be
+# removed in releases after 0.3.
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: knative-shared-gateway
+  namespace: knative-serving
+spec:
+  selector:
+    knative: ingressgateway
+  servers:
+  - port:
+      number: 80
+      name: http
+      protocol: HTTP
+    hosts:
+    - "*"
+  - port:
+      number: 443
+      name: https
+      protocol: HTTPS
+    hosts:
+    - "*"
+    tls:
+      mode: PASSTHROUGH
+---
+# This is the Service definition for the ingress pods serving the
+# pre-0.3 shared Gateway.  We leave it here so that existing clients
+# can still access using their own IP address.  However, it is
+# deprecated and will be removed in releases after 0.3.
+apiVersion: v1
+kind: Service
+metadata:
+  name: knative-ingressgateway
+  namespace: istio-system
+  annotations:
+  labels:
+    chart: gateways-1.0.1
+    release: RELEASE-NAME
+    heritage: Tiller
+    app: knative-ingressgateway
+    knative: ingressgateway
+spec:
+  type: LoadBalancer
+  selector:
+    app: knative-ingressgateway
+    knative: ingressgateway
+  ports:
+    -
+      name: http2
+      nodePort: 32380
+      port: 80
+      targetPort: 80
+    -
+      name: https
+      nodePort: 32390
+      port: 443
+    -
+      name: tcp
+      nodePort: 32400
+      port: 31400
+    -
+      name: tcp-pilot-grpc-tls
+      port: 15011
+      targetPort: 15011
+    -
+      name: tcp-citadel-grpc-tls
+      port: 8060
+      targetPort: 8060
+    -
+      name: tcp-dns-tls
+      port: 853
+      targetPort: 853
+    -
+      name: http2-prometheus
+      port: 15030
+      targetPort: 15030
+    -
+      name: http2-grafana
+      port: 15031
+      targetPort: 15031
+---
+# This is the corresponding Deployment to backed the aforementioned Service.
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: knative-ingressgateway
+  namespace: istio-system
+  labels:
+    chart: gateways-1.0.1
+    release: RELEASE-NAME
+    heritage: Tiller
+    app: knative-ingressgateway
+    knative: ingressgateway
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: knative-ingressgateway
+      knative: ingressgateway
+  template:
+    metadata:
+      labels:
+        app: knative-ingressgateway
+        knative: ingressgateway
+      annotations:
+        sidecar.istio.io/inject: "false"
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+    spec:
+      serviceAccountName: istio-ingressgateway-service-account
+      containers:
+        - name: istio-proxy
+          image: "docker.io/istio/proxyv2:1.0.2"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 80
+            - containerPort: 443
+            - containerPort: 31400
+            - containerPort: 15011
+            - containerPort: 8060
+            - containerPort: 853
+            - containerPort: 15030
+            - containerPort: 15031
+          args:
+          - proxy
+          - router
+          - -v
+          - "2"
+          - --discoveryRefreshDelay
+          - '1s' #discoveryRefreshDelay
+          - --drainDuration
+          - '45s' #drainDuration
+          - --parentShutdownDuration
+          - '1m0s' #parentShutdownDuration
+          - --connectTimeout
+          - '10s' #connectTimeout
+          - --serviceCluster
+          - knative-ingressgateway
+          - --zipkinAddress
+          - zipkin:9411
+          - --statsdUdpAddress
+          - istio-statsd-prom-bridge:9125
+          - --proxyAdminPort
+          - "15000"
+          - --controlPlaneAuthPolicy
+          - NONE
+          - --discoveryAddress
+          - istio-pilot:8080
+          resources:
+            requests:
+              cpu: 10m
+
+          env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.namespace
+          - name: INSTANCE_IP
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.podIP
+          - name: ISTIO_META_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          volumeMounts:
+          - name: istio-certs
+            mountPath: /etc/certs
+            readOnly: true
+          - name: ingressgateway-certs
+            mountPath: "/etc/istio/ingressgateway-certs"
+            readOnly: true
+          - name: ingressgateway-ca-certs
+            mountPath: "/etc/istio/ingressgateway-ca-certs"
+            readOnly: true
+      volumes:
+      - name: istio-certs
+        secret:
+          secretName: istio.istio-ingressgateway-service-account
+          optional: true
+      - name: ingressgateway-certs
+        secret:
+          secretName: "istio-ingressgateway-certs"
+          optional: true
+      - name: ingressgateway-ca-certs
+        secret:
+          secretName: "istio-ingressgateway-ca-certs"
+          optional: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - amd64
+                - ppc64le
+                - s390x
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 2
+            preference:
+              matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - amd64
+          - weight: 2
+            preference:
+              matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - ppc64le
+          - weight: 2
+            preference:
+              matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - s390x
+---
+# This is the horizontal pod autoscaler to make sure the ingress Pods
+# scale up to meet traffic demand.
+#
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+    name: knative-ingressgateway
+    namespace: istio-system
+spec:
+    # TODO(1411): Document/fix this.  We are choosing an arbitrary 10 here.
+    maxReplicas: 10
+    minReplicas: 1
+    scaleTargetRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: knative-ingressgateway
+    metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        targetAverageUtilization: 60
diff --git a/config/config-istio.yaml b/config/config-istio.yaml
@@ -18,5 +18,9 @@ metadata:
   name: config-istio
   namespace: knative-serving
 data:
-  # Gateway used for knative traffic. Default to istio-ingressgateway.
-  ingress-gateway: "istio-ingressgateway.istio-system.svc.cluster.local"
+  # TODO(nghia): Extract the .svc.cluster.local suffix into its own config.
+  # Default Knative Gateway after v0.3. It points to the Istio standard istio-ingressgateway,
+  # instead of a custom one that we used pre-0.3.
+  gateway.knative-ingress-gateway: "istio-ingressgateway.istio-system.svc.cluster.local"
+  # This is the pre-0.3 shared Gateway.  We leave it here for backward compatibility.
+  gateway.knative-shared-gateway: "knative-ingressgateway.istio-system.svc.cluster.local"
diff --git a/pkg/reconciler/v1alpha1/clusteringress/clusteringress.go b/pkg/reconciler/v1alpha1/clusteringress/clusteringress.go
@@ -181,7 +181,7 @@ func (c *Reconciler) reconcile(ctx context.Context, ci *v1alpha1.ClusterIngress)
 	ci.SetDefaults()
 
 	ci.Status.InitializeConditions()
-	vs := resources.MakeVirtualService(ci)
+	vs := resources.MakeVirtualService(ci, gatewayNamesFromContext(ctx))
 
 	logger.Infof("Reconciling clusterIngress :%v", ci)
 	logger.Info("Creating/Updating VirtualService")
@@ -195,14 +195,22 @@ func (c *Reconciler) reconcile(ctx context.Context, ci *v1alpha1.ClusterIngress)
 	// is successfully synced.
 	ci.Status.MarkNetworkConfigured()
 	ci.Status.MarkLoadBalancerReady([]v1alpha1.LoadBalancerIngressStatus{
-		{DomainInternal: ingressGatewayFromContext(ctx)},
+		{DomainInternal: gatewayServiceURLFromContext(ctx)},
 	})
 	logger.Info("ClusterIngress successfully synced")
 	return nil
 }
 
-func ingressGatewayFromContext(ctx context.Context) string {
-	return config.FromContext(ctx).Istio.IngressGateway
+func gatewayServiceURLFromContext(ctx context.Context) string {
+	return config.FromContext(ctx).Istio.IngressGateways[0].ServiceURL
+}
+
+func gatewayNamesFromContext(ctx context.Context) []string {
+	gateways := []string{}
+	for _, gw := range config.FromContext(ctx).Istio.IngressGateways {
+		gateways = append(gateways, gw.GatewayName)
+	}
+	return gateways
 }
 
 func (c *Reconciler) reconcileVirtualService(ctx context.Context, ci *v1alpha1.ClusterIngress,