From a69bee485672037242c7bf91855370205d3db4a1 Mon Sep 17 00:00:00 2001 From: Luke Kysow <1034429+lkysow@users.noreply.github.com> Date: Tue, 17 Mar 2020 16:35:41 -0700 Subject: [PATCH] Automatically use LB addresses for mesh gateways Add support for using the external address of Kubernetes load balancer for the mesh gateway wan address. This change uses the new consul-k8s service-address command to get the address of the load balancer. It also removes meshGateway.wanAddress.{useNodeIP, useNodeName, host} config values in favour of meshGateway.wanAddress.{source, static}. The new source value allows selecting NodeIP, NodeName, Service or Static. This is more extensible than the previous boolean values. We also change the default containerPort to 8443 from 443 because that port can't be bound to on GKE. These are backwards incompatible changes. --- templates/mesh-gateway-clusterrole.yaml | 11 +- templates/mesh-gateway-deployment.yaml | 43 ++- test/unit/mesh-gateway-clusterrole.bats | 24 +- test/unit/mesh-gateway-deployment.bats | 375 +++++++++++++++++------- test/unit/mesh-gateway-service.bats | 18 +- values.yaml | 61 ++-- 6 files changed, 369 insertions(+), 163 deletions(-) diff --git a/templates/mesh-gateway-clusterrole.yaml b/templates/mesh-gateway-clusterrole.yaml index 33f4dad13..91092a64a 100644 --- a/templates/mesh-gateway-clusterrole.yaml +++ b/templates/mesh-gateway-clusterrole.yaml @@ -9,7 +9,7 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} component: mesh-gateway -{{- if or .Values.global.bootstrapACLs .Values.global.enablePodSecurityPolicies }} +{{- if or .Values.global.bootstrapACLs .Values.global.enablePodSecurityPolicies (eq .Values.meshGateway.wanAddress.source "Service") }} rules: {{- if .Values.global.enablePodSecurityPolicies }} - apiGroups: ["policy"] @@ -28,6 +28,15 @@ rules: verbs: - get {{- end }} +{{- if eq .Values.meshGateway.wanAddress.source "Service" }} + - apiGroups: [""] + resources: + - services + resourceNames: + - {{ template "consul.fullname" . }}-mesh-gateway + verbs: + - get + {{- end }} {{- else }} rules: [] {{- end }} diff --git a/templates/mesh-gateway-deployment.yaml b/templates/mesh-gateway-deployment.yaml index 666f0beb2..9b0bffe80 100644 --- a/templates/mesh-gateway-deployment.yaml +++ b/templates/mesh-gateway-deployment.yaml @@ -116,12 +116,35 @@ spec: -token-sink-file=/consul/service/acl-token {{ end }} - {{- if .Values.meshGateway.wanAddress.host }} - WAN_ADDR="{{ .Values.meshGateway.wanAddress.host }}" - {{- else if .Values.meshGateway.wanAddress.useNodeName }} - WAN_ADDR="${NODE_NAME}" - {{- else if .Values.meshGateway.wanAddress.useNodeIP }} + {{- $source := .Values.meshGateway.wanAddress.source }} + {{- $serviceType := .Values.meshGateway.service.type }} + {{- if and (eq $source "Service") (not .Values.meshGateway.service.enabled) }}{{ fail "if meshGateway.wanAddress.source=Service then meshGateway.service.enabled must be set to true" }}{{ end }} + {{- if or (eq $source "NodeIP") (and (eq $source "Service") (eq $serviceType "NodePort")) }} WAN_ADDR="${HOST_IP}" + {{- else if eq $source "NodeName" }} + WAN_ADDR="${NODE_NAME}" + {{- else if and (eq $source "Service") (or (eq $serviceType "ClusterIP") (eq $serviceType "LoadBalancer")) }} + consul-k8s service-address \ + -k8s-namespace={{ .Release.Namespace }} \ + -name={{ template "consul.fullname" . }}-mesh-gateway \ + -output-file=address.txt + WAN_ADDR="$(cat address.txt)" + {{- else if eq $source "Static" }} + {{- if eq .Values.meshGateway.wanAddress.static "" }}{{ fail "if meshGateway.wanAddress.source=Static then meshGateway.wanAddress.static cannot be empty" }}{{ end }} + WAN_ADDR="{{ .Values.meshGateway.wanAddress.static }}" + {{- else }} + {{- fail "currently set meshGateway values for wanAddress.source and service.type are not supported" }} + {{- end }} + + {{- if eq $source "Service" }} + {{- if eq $serviceType "NodePort" }} + {{- if not .Values.meshGateway.service.nodePort }}{{ fail "if meshGateway.wanAddress.source=Service and meshGateway.service.type=NodePort, meshGateway.service.nodePort must be set" }}{{ end }} + WAN_PORT="{{ .Values.meshGateway.service.nodePort }}" + {{- else }} + WAN_PORT="{{ .Values.meshGateway.service.port }}" + {{- end }} + {{- else }} + WAN_PORT="{{ .Values.meshGateway.wanAddress.port }}" {{- end }} cat > /consul/service/service.hcl << EOF @@ -140,17 +163,9 @@ spec: address = "${POD_IP}" port = {{ .Values.meshGateway.containerPort }} } - lan_ipv4 { - address = "${POD_IP}" - port = {{ .Values.meshGateway.containerPort }} - } wan { address = "${WAN_ADDR}" - port = {{ .Values.meshGateway.wanAddress.port }} - } - wan_ipv4 { - address = "${WAN_ADDR}" - port = {{ .Values.meshGateway.wanAddress.port }} + port = ${WAN_PORT} } } checks = [ diff --git a/test/unit/mesh-gateway-clusterrole.bats b/test/unit/mesh-gateway-clusterrole.bats index 1f3db8298..7e9e4ab86 100644 --- a/test/unit/mesh-gateway-clusterrole.bats +++ b/test/unit/mesh-gateway-clusterrole.bats @@ -49,19 +49,34 @@ load _helpers [ "${actual}" = "secrets" ] } -@test "meshGateway/ClusterRole: rules is empty if no ACLs or PSPs" { +@test "meshGateway/ClusterRole: rules for meshGateway.wanAddress.source=Service" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-clusterrole.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ + --set 'meshGateway.service.enabled=true' \ + --set 'meshGateway.service.type=LoadBalancer' \ + --set 'meshGateway.wanAddress.source=Service' \ + . | tee /dev/stderr | + yq -r '.rules[0].resources[0]' | tee /dev/stderr) + [ "${actual}" = "services" ] +} + +@test "meshGateway/ClusterRole: rules is empty if no ACLs, PSPs and meshGateway.source != Service" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-clusterrole.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'meshGateway.wanAddress.source=NodeIP' \ + --set 'connectInject.enabled=true' \ --set 'client.grpc=true' \ . | tee /dev/stderr | yq -r '.rules' | tee /dev/stderr) [ "${actual}" = "[]" ] } -@test "meshGateway/ClusterRole: rules for both ACLs and PSPs" { +@test "meshGateway/ClusterRole: rules for ACLs, PSPs and mesh gateways" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-clusterrole.yaml \ @@ -70,7 +85,10 @@ load _helpers --set 'client.grpc=true' \ --set 'global.bootstrapACLs=true' \ --set 'global.enablePodSecurityPolicies=true' \ + --set 'meshGateway.service.enabled=true' \ + --set 'meshGateway.service.type=LoadBalancer' \ + --set 'meshGateway.wanAddress.source=Service' \ . | tee /dev/stderr | yq -r '.rules | length' | tee /dev/stderr) - [ "${actual}" = "2" ] + [ "${actual}" = "3" ] } diff --git a/test/unit/mesh-gateway-deployment.bats b/test/unit/mesh-gateway-deployment.bats index fd6efc7b5..38939c05b 100755 --- a/test/unit/mesh-gateway-deployment.bats +++ b/test/unit/mesh-gateway-deployment.bats @@ -306,7 +306,7 @@ key2: value2' \ #-------------------------------------------------------------------- # containerPort -@test "meshGateway/Deployment: containerPort defaults to 443" { +@test "meshGateway/Deployment: containerPort defaults to 8443" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-deployment.yaml \ @@ -316,9 +316,9 @@ key2: value2' \ . | tee /dev/stderr \ | yq '.spec.template.spec.containers[0]' | tee /dev/stderr) - [ $(echo "$actual" | yq -r '.ports[0].containerPort') = "443" ] - [ $(echo "$actual" | yq -r '.livenessProbe.tcpSocket.port') = "443" ] - [ $(echo "$actual" | yq -r '.readinessProbe.tcpSocket.port') = "443" ] + [ $(echo "$actual" | yq -r '.ports[0].containerPort') = "8443" ] + [ $(echo "$actual" | yq -r '.livenessProbe.tcpSocket.port') = "8443" ] + [ $(echo "$actual" | yq -r '.readinessProbe.tcpSocket.port') = "8443" ] } @test "meshGateway/Deployment: containerPort can be overridden" { @@ -328,13 +328,13 @@ key2: value2' \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'client.grpc=true' \ - --set 'meshGateway.containerPort=8443' \ + --set 'meshGateway.containerPort=9443' \ . | tee /dev/stderr \ | yq '.spec.template.spec.containers[0]' | tee /dev/stderr) - [ $(echo "$actual" | yq -r '.ports[0].containerPort') = "8443" ] - [ $(echo "$actual" | yq -r '.livenessProbe.tcpSocket.port') = "8443" ] - [ $(echo "$actual" | yq -r '.readinessProbe.tcpSocket.port') = "8443" ] + [ $(echo "$actual" | yq -r '.ports[0].containerPort') = "9443" ] + [ $(echo "$actual" | yq -r '.livenessProbe.tcpSocket.port') = "9443" ] + [ $(echo "$actual" | yq -r '.readinessProbe.tcpSocket.port') = "9443" ] } #-------------------------------------------------------------------- @@ -571,37 +571,34 @@ key2: value2' \ . | tee /dev/stderr | yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) - exp='WAN_ADDR="${HOST_IP}" + exp='consul-k8s service-address \ + -k8s-namespace=default \ + -name=release-name-consul-mesh-gateway \ + -output-file=address.txt +WAN_ADDR="$(cat address.txt)" +WAN_PORT="443" cat > /consul/service/service.hcl << EOF service { kind = "mesh-gateway" name = "mesh-gateway" - port = 443 + port = 8443 address = "${POD_IP}" tagged_addresses { lan { address = "${POD_IP}" - port = 443 - } - lan_ipv4 { - address = "${POD_IP}" - port = 443 + port = 8443 } wan { address = "${WAN_ADDR}" - port = 443 - } - wan_ipv4 { - address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} } } checks = [ { name = "Mesh Gateway Listening" interval = "10s" - tcp = "${POD_IP}:443" + tcp = "${POD_IP}:8443" deregister_critical_service_after = "6h" } ] @@ -630,37 +627,34 @@ consul services register \ -init-type="sync" \ -token-sink-file=/consul/service/acl-token -WAN_ADDR="${HOST_IP}" +consul-k8s service-address \ + -k8s-namespace=default \ + -name=release-name-consul-mesh-gateway \ + -output-file=address.txt +WAN_ADDR="$(cat address.txt)" +WAN_PORT="443" cat > /consul/service/service.hcl << EOF service { kind = "mesh-gateway" name = "mesh-gateway" - port = 443 + port = 8443 address = "${POD_IP}" tagged_addresses { lan { address = "${POD_IP}" - port = 443 - } - lan_ipv4 { - address = "${POD_IP}" - port = 443 + port = 8443 } wan { address = "${WAN_ADDR}" - port = 443 - } - wan_ipv4 { - address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} } } checks = [ { name = "Mesh Gateway Listening" interval = "10s" - tcp = "${POD_IP}:443" + tcp = "${POD_IP}:8443" deregister_critical_service_after = "6h" } ] @@ -684,7 +678,12 @@ consul services register \ . | tee /dev/stderr | yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) - exp='WAN_ADDR="${HOST_IP}" + exp='consul-k8s service-address \ + -k8s-namespace=default \ + -name=release-name-consul-mesh-gateway \ + -output-file=address.txt +WAN_ADDR="$(cat address.txt)" +WAN_PORT="443" cat > /consul/service/service.hcl << EOF service { @@ -693,31 +692,23 @@ service { meta { consul-wan-federation = "1" } - port = 443 + port = 8443 address = "${POD_IP}" tagged_addresses { lan { address = "${POD_IP}" - port = 443 - } - lan_ipv4 { - address = "${POD_IP}" - port = 443 + port = 8443 } wan { address = "${WAN_ADDR}" - port = 443 - } - wan_ipv4 { - address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} } } checks = [ { name = "Mesh Gateway Listening" interval = "10s" - tcp = "${POD_IP}:443" + tcp = "${POD_IP}:8443" deregister_critical_service_after = "6h" } ] @@ -737,11 +728,13 @@ consul services register \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'meshGateway.containerPort=8888' \ + --set 'meshGateway.wanAddress.source=NodeIP' \ --set 'meshGateway.wanAddress.port=9999' \ . | tee /dev/stderr | yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) exp='WAN_ADDR="${HOST_IP}" +WAN_PORT="9999" cat > /consul/service/service.hcl << EOF service { @@ -754,17 +747,9 @@ service { address = "${POD_IP}" port = 8888 } - lan_ipv4 { - address = "${POD_IP}" - port = 8888 - } wan { address = "${WAN_ADDR}" - port = 9999 - } - wan_ipv4 { - address = "${WAN_ADDR}" - port = 9999 + port = ${WAN_PORT} } } checks = [ @@ -784,47 +769,40 @@ consul services register \ [ "${actual}" = "${exp}" ] } -@test "meshGateway/Deployment: service-init init container wanAddress.useNodeIP" { +@test "meshGateway/Deployment: service-init init container wanAddress.source=NodeIP" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-deployment.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.wanAddress.useNodeIP=true' \ + --set 'meshGateway.wanAddress.source=NodeIP' \ . | tee /dev/stderr | yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) exp='WAN_ADDR="${HOST_IP}" +WAN_PORT="443" cat > /consul/service/service.hcl << EOF service { kind = "mesh-gateway" name = "mesh-gateway" - port = 443 + port = 8443 address = "${POD_IP}" tagged_addresses { lan { address = "${POD_IP}" - port = 443 - } - lan_ipv4 { - address = "${POD_IP}" - port = 443 + port = 8443 } wan { address = "${WAN_ADDR}" - port = 443 - } - wan_ipv4 { - address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} } } checks = [ { name = "Mesh Gateway Listening" interval = "10s" - tcp = "${POD_IP}:443" + tcp = "${POD_IP}:8443" deregister_critical_service_after = "6h" } ] @@ -837,48 +815,40 @@ consul services register \ [ "${actual}" = "${exp}" ] } -@test "meshGateway/Deployment: service-init init container wanAddress.useNodeName" { +@test "meshGateway/Deployment: service-init init container wanAddress.source=NodeName" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-deployment.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.wanAddress.useNodeIP=false' \ - --set 'meshGateway.wanAddress.useNodeName=true' \ + --set 'meshGateway.wanAddress.source=NodeName' \ . | tee /dev/stderr | yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) exp='WAN_ADDR="${NODE_NAME}" +WAN_PORT="443" cat > /consul/service/service.hcl << EOF service { kind = "mesh-gateway" name = "mesh-gateway" - port = 443 + port = 8443 address = "${POD_IP}" tagged_addresses { lan { address = "${POD_IP}" - port = 443 - } - lan_ipv4 { - address = "${POD_IP}" - port = 443 + port = 8443 } wan { address = "${WAN_ADDR}" - port = 443 - } - wan_ipv4 { - address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} } } checks = [ { name = "Mesh Gateway Listening" interval = "10s" - tcp = "${POD_IP}:443" + tcp = "${POD_IP}:8443" deregister_critical_service_after = "6h" } ] @@ -891,48 +861,122 @@ consul services register \ [ "${actual}" = "${exp}" ] } -@test "meshGateway/Deployment: service-init init container wanAddress.host" { +@test "meshGateway/Deployment: service-init init container wanAddress.source=Static fails if wanAddress.static is empty" { + cd `chart_dir` + run helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.source=Static' \ + --set 'meshGateway.wanAddress.static=' \ + . + + [ "$status" -eq 1 ] + [[ "$output" =~ "if meshGateway.wanAddress.source=Static then meshGateway.wanAddress.static cannot be empty" ]] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.source=Static" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-deployment.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.wanAddress.useNodeIP=false' \ - --set 'meshGateway.wanAddress.host=example.com' \ + --set 'meshGateway.wanAddress.source=Static' \ + --set 'meshGateway.wanAddress.static=example.com' \ . | tee /dev/stderr | yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) exp='WAN_ADDR="example.com" +WAN_PORT="443" cat > /consul/service/service.hcl << EOF service { kind = "mesh-gateway" name = "mesh-gateway" - port = 443 + port = 8443 address = "${POD_IP}" tagged_addresses { lan { address = "${POD_IP}" - port = 443 - } - lan_ipv4 { - address = "${POD_IP}" - port = 443 + port = 8443 } wan { address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:8443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.source=Service fails if service.enable is false" { + cd `chart_dir` + run helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.source=Service' \ + --set 'meshGateway.service.enabled=false' \ + . + + [ "$status" -eq 1 ] + [[ "$output" =~ "if meshGateway.wanAddress.source=Service then meshGateway.service.enabled must be set to true" ]] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.source=Service, type=LoadBalancer" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.source=Service' \ + --set 'meshGateway.wanAddress.port=ignored' \ + --set 'meshGateway.service.enabled=true' \ + --set 'meshGateway.service.type=LoadBalancer' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='consul-k8s service-address \ + -k8s-namespace=default \ + -name=release-name-consul-mesh-gateway \ + -output-file=address.txt +WAN_ADDR="$(cat address.txt)" +WAN_PORT="443" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 8443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 8443 } - wan_ipv4 { + wan { address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} } } checks = [ { name = "Mesh Gateway Listening" interval = "10s" - tcp = "${POD_IP}:443" + tcp = "${POD_IP}:8443" deregister_critical_service_after = "6h" } ] @@ -945,47 +989,160 @@ consul services register \ [ "${actual}" = "${exp}" ] } -@test "meshGateway/Deployment: service-init init container consulServiceName can be changed" { +@test "meshGateway/Deployment: service-init init container wanAddress.source=Service, type=NodePort" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-deployment.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.consulServiceName=new-name' \ + --set 'meshGateway.wanAddress.source=Service' \ + --set 'meshGateway.service.enabled=true' \ + --set 'meshGateway.service.nodePort=9999' \ + --set 'meshGateway.service.type=NodePort' \ . | tee /dev/stderr | yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) exp='WAN_ADDR="${HOST_IP}" +WAN_PORT="9999" cat > /consul/service/service.hcl << EOF service { kind = "mesh-gateway" - name = "new-name" - port = 443 + name = "mesh-gateway" + port = 8443 address = "${POD_IP}" tagged_addresses { lan { address = "${POD_IP}" - port = 443 + port = 8443 + } + wan { + address = "${WAN_ADDR}" + port = ${WAN_PORT} } - lan_ipv4 { + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:8443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.source=Service, type=NodePort fails if service.nodePort is null" { + cd `chart_dir` + run helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.source=Service' \ + --set 'meshGateway.service.enabled=true' \ + --set 'meshGateway.service.type=NodePort' \ + . + + [ "$status" -eq 1 ] + [[ "$output" =~ "if meshGateway.wanAddress.source=Service and meshGateway.service.type=NodePort, meshGateway.service.nodePort must be set" ]] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.source=Service, type=ClusterIP" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.source=Service' \ + --set 'meshGateway.wanAddress.port=ignored' \ + --set 'meshGateway.service.enabled=true' \ + --set 'meshGateway.service.type=ClusterIP' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='consul-k8s service-address \ + -k8s-namespace=default \ + -name=release-name-consul-mesh-gateway \ + -output-file=address.txt +WAN_ADDR="$(cat address.txt)" +WAN_PORT="443" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 8443 + address = "${POD_IP}" + tagged_addresses { + lan { address = "${POD_IP}" - port = 443 + port = 8443 } wan { address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:8443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} +@test "meshGateway/Deployment: service-init init container consulServiceName can be changed" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.consulServiceName=new-name' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='consul-k8s service-address \ + -k8s-namespace=default \ + -name=release-name-consul-mesh-gateway \ + -output-file=address.txt +WAN_ADDR="$(cat address.txt)" +WAN_PORT="443" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "new-name" + port = 8443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 8443 } - wan_ipv4 { + wan { address = "${WAN_ADDR}" - port = 443 + port = ${WAN_PORT} } } checks = [ { name = "Mesh Gateway Listening" interval = "10s" - tcp = "${POD_IP}:443" + tcp = "${POD_IP}:8443" deregister_critical_service_after = "6h" } ] diff --git a/test/unit/mesh-gateway-service.bats b/test/unit/mesh-gateway-service.bats index ac8a7c3ed..a5f35eeb3 100755 --- a/test/unit/mesh-gateway-service.bats +++ b/test/unit/mesh-gateway-service.bats @@ -11,7 +11,7 @@ load _helpers [ "${actual}" = "false" ] } -@test "meshGateway/Service: disabled by default with meshGateway, connectInject and client.grpc enabled" { +@test "meshGateway/Service: enabled by default with meshGateway, connectInject and client.grpc enabled" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-service.yaml \ @@ -20,7 +20,7 @@ load _helpers --set 'client.grpc=true' \ . | tee /dev/stderr | yq 'length > 0' | tee /dev/stderr) - [ "${actual}" = "false" ] + [ "${actual}" = "true" ] } @test "meshGateway/Service: enabled with meshGateway.enabled=true meshGateway.service.enabled" { @@ -109,7 +109,7 @@ load _helpers --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.ports[0].targetPort' | tee /dev/stderr) - [ "${actual}" = "443" ] + [ "${actual}" = "8443" ] } @test "meshGateway/Service: uses targetPort from containerPort" { @@ -120,10 +120,10 @@ load _helpers --set 'connectInject.enabled=true' \ --set 'client.grpc=true' \ --set 'meshGateway.service.enabled=true' \ - --set 'meshGateway.containerPort=8443' \ + --set 'meshGateway.containerPort=9443' \ . | tee /dev/stderr | yq -r '.spec.ports[0].targetPort' | tee /dev/stderr) - [ "${actual}" = "8443" ] + [ "${actual}" = "9443" ] } #-------------------------------------------------------------------- @@ -159,7 +159,7 @@ load _helpers #-------------------------------------------------------------------- # Service type -@test "meshGateway/Service: defaults to type ClusterIP" { +@test "meshGateway/Service: defaults to type LoadBalancer" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-service.yaml \ @@ -169,7 +169,7 @@ load _helpers --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.type' | tee /dev/stderr) - [ "${actual}" = "ClusterIP" ] + [ "${actual}" = "LoadBalancer" ] } @test "meshGateway/Service: can set type" { @@ -180,10 +180,10 @@ load _helpers --set 'connectInject.enabled=true' \ --set 'client.grpc=true' \ --set 'meshGateway.service.enabled=true' \ - --set 'meshGateway.service.type=LoadBalancer' \ + --set 'meshGateway.service.type=ClusterIP' \ . | tee /dev/stderr | yq -r '.spec.type' | tee /dev/stderr) - [ "${actual}" = "LoadBalancer" ] + [ "${actual}" = "ClusterIP" ] } #-------------------------------------------------------------------- diff --git a/values.yaml b/values.yaml index 4b45397f7..d754b6cbc 100644 --- a/values.yaml +++ b/values.yaml @@ -766,46 +766,53 @@ meshGateway: # Number of replicas for the Deployment. replicas: 2 - # What gets registered as wan address for the gateway. + # What gets registered as WAN address for the gateway. wanAddress: - # Port that gets registered. + # source configures where to retrieve the WAN address (and possibly port) + # for the mesh gateway from. + # Can be set to either: Service, NodeIP, NodeName or Static. + # + # Service - Determine the address based on the service type. + # If service.type=LoadBalancer use the external IP or hostname of + # the service. Use the port set by service.port. + # If service.type=NodePort use the Node IP. The port will be set to + # service.nodePort so service.nodePort cannot be null. + # If service.type=ClusterIP use the ClusterIP. The port will be set to + # service.port. + # service.type=ExternalName is not supported. + # NodeIP - The node IP as provided by the Kubernetes downward API. + # NodeName - The name of the node as provided by the Kubernetes downward + # API. This is useful if the node names are DNS entries that + # are routable from other datacenters. + # Static - Use the address hardcoded in meshGateway.wanAddress.static. + source: "Service" + + # Port that gets registered for WAN traffic. + # If source is set to "Service" then this setting will have no effect. + # See the documentation for source as to which port will be used in that + # case. port: 443 - # If true, each Gateway Pod will advertise its NodeIP - # (as provided by the Kubernetes downward API) as the wan address. - # This is useful if the node IPs are routable from other DCs. - # useNodeName and host must be false and "" respectively. - useNodeIP: true - - # If true, each Gateway Pod will advertise its NodeName - # (as provided by the Kubernetes downward API) as the wan address. - # This is useful if the node names are DNS entries that are - # routable from other DCs. - # meshGateway.wanAddress.port will be used as the port for the wan address. - # useNodeIP and host must be false and "" respectively. - useNodeName: false - - # If set, each gateway Pod will use this host as its wan address. - # Users must ensure that this address routes to the Gateway pods, - # for example via a DNS entry that routes to the Service fronting the Deployment. - # meshGateway.wanAddress.port will be used as the port for the wan address. - # useNodeIP and useNodeName must be false. - host: "" + # If source is set to "Static" then this value will be used as the WAN + # address of the mesh gateways. This is useful if you've configured a + # DNS entry to point to your mesh gateways. + static: "" # The service option configures the Service that fronts the Gateway Deployment. service: # Whether to create a Service or not. - enabled: false + enabled: true # Type of service, ex. LoadBalancer, ClusterIP. - type: ClusterIP + type: LoadBalancer # Port that the service will be exposed on. # The targetPort will be set to meshGateway.containerPort. port: 443 - # Optional nodePort of the service. Can be used in conjunction with - # type: NodePort. + # Optionally hardcode the nodePort of the service if using type: NodePort. + # If not set and using type: NodePort, Kubernetes will automatically assign + # a port. nodePort: null # Optional YAML string for additional annotations. @@ -829,7 +836,7 @@ meshGateway: consulServiceName: "" # Port that the gateway will run on inside the container. - containerPort: 443 + containerPort: 8443 # Optional hostPort for the gateway to be exposed on. # This can be used with wanAddress.port and wanAddress.useNodeIP