Skip to content
This repository has been archived by the owner on Aug 25, 2021. It is now read-only.

Support terminating gateways #503

Merged
merged 8 commits into from
Jun 18, 2020
Merged

Support terminating gateways #503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8bf9b14
Support terminating gateways
adilyse May 11, 2020
2e4cdf7
Update service registration to remove unnecessary settings
adilyse Jun 10, 2020
4dca8c4
Update based on PR feedback, add unique consul service id field
adilyse Jun 15, 2020
dc99419
Add proxy-id flag to consul connect envoy command
adilyse Jun 16, 2020
15a91f5
Add terminating-gateway-name label to all gateway components
adilyse Jun 16, 2020
12229e1
Update namespace comments to remove inaccurate outcome
adilyse Jun 17, 2020
6028c7c
Update tests based on PR review
adilyse Jun 18, 2020
c00639e
Update values file comments based on PR review
adilyse Jun 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions templates/server-acl-init-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,23 @@ spec:
{{- end }}
{{- end }}

{{- if .Values.terminatingGateways.enabled }}
{{- if .Values.global.enableConsulNamespaces }}
{{- $root := . }}
{{- range .Values.terminatingGateways.gateways }}
{{- if (or $root.Values.terminatingGateways.defaults.consulNamespace .consulNamespace) }}
-terminating-gateway-name="{{ .name }}.{{ (default $root.Values.terminatingGateways.defaults.consulNamespace .consulNamespace) }}" \
{{- else }}
-terminating-gateway-name="{{ .name }}" \
{{- end }}
{{- end }}
{{- else }}
{{- range .Values.terminatingGateways.gateways }}
-terminating-gateway-name="{{ .name }}" \
{{- end }}
{{- end }}
{{- end }}

{{- if .Values.connectInject.aclBindingRuleSelector }}
-acl-binding-rule-selector={{ .Values.connectInject.aclBindingRuleSelector }} \
{{- end }}
Expand Down
357 changes: 357 additions & 0 deletions templates/terminating-gateways-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,357 @@
{{- if .Values.terminatingGateways.enabled }}
{{- if not .Values.connectInject.enabled }}{{ fail "connectInject.enabled must be true" }}{{ end -}}
{{- if not .Values.client.grpc }}{{ fail "client.grpc must be true" }}{{ end -}}
{{- if not (or (and (ne (.Values.client.enabled | toString) "-") .Values.client.enabled) (and (eq (.Values.client.enabled | toString) "-") .Values.global.enabled)) }}{{ fail "clients must be enabled" }}{{ end -}}

{{- $root := . }}
{{- $defaults := .Values.terminatingGateways.defaults }}
{{- $names := dict }}

{{- range .Values.terminatingGateways.gateways }}

{{- if empty .name }}
# Check that name is not empty
{{ fail "Terminating gateway names cannot be empty"}}
{{ end -}}
{{- if hasKey $names .name }}
# Check that the name doesn't already exist
{{ fail "Terminating gateway names must be unique"}}
{{ end -}}
{{- /* Add the gateway name to the $names dict to ensure uniqueness */ -}}
{{- $_ := set $names .name .name }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "consul.fullname" $root }}-{{ .name }}
namespace: {{ $root.Release.Namespace }}
labels:
app: {{ template "consul.name" $root }}
chart: {{ template "consul.chart" $root }}
heritage: {{ $root.Release.Service }}
release: {{ $root.Release.Name }}
component: terminating-gateway
terminating-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }}
spec:
replicas: {{ default $defaults.replicas .replicas }}
selector:
matchLabels:
app: {{ template "consul.name" $root }}
chart: {{ template "consul.chart" $root }}
heritage: {{ $root.Release.Service }}
release: {{ $root.Release.Name }}
component: terminating-gateway
terminating-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }}
template:
metadata:
labels:
app: {{ template "consul.name" $root }}
chart: {{ template "consul.chart" $root }}
heritage: {{ $root.Release.Service }}
release: {{ $root.Release.Name }}
component: terminating-gateway
terminating-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }}
annotations:
"consul.hashicorp.com/connect-inject": "false"
{{- if $defaults.annotations }}
# We allow both default annotations and gateway-specific annotations
{{- tpl $defaults.annotations $root | nindent 8 }}
{{- end }}
{{- if .annotations }}
{{- tpl .annotations $root | nindent 8 }}
{{- end }}
spec:
{{- if (or $defaults.affinity .affinity) }}
affinity:
{{ tpl (default $defaults.affinity .affinity) $root | nindent 8 | trim }}
{{- end }}
{{- if (or $defaults.tolerations .tolerations) }}
tolerations:
{{ tpl (default $defaults.tolerations .tolerations) $root | nindent 8 | trim }}
{{- end }}
terminationGracePeriodSeconds: 10
serviceAccountName: {{ template "consul.fullname" $root }}-{{ .name }}
volumes:
- name: consul-bin
emptyDir: {}
- name: consul-service
emptyDir:
medium: "Memory"
{{- range (default $defaults.extraVolumes .extraVolumes) }}
- name: userconfig-{{ .name }}
{{ .type }}:
{{- if (eq .type "configMap") }}
name: {{ .name }}
{{- else if (eq .type "secret") }}
secretName: {{ .name }}
{{- end }}
{{- with .items }}
items:
{{- range . }}
- key: {{.key}}
path: {{.path}}
{{- end }}
{{- end }}
{{- end }}
{{- if $root.Values.global.tls.enabled }}
{{- if not (and $root.Values.externalServers.enabled $root.Values.externalServers.useSystemRoots) }}
- name: consul-ca-cert
secret:
{{- if $root.Values.global.tls.caCert.secretName }}
secretName: {{ $root.Values.global.tls.caCert.secretName }}
{{- else }}
secretName: {{ template "consul.fullname" $root }}-ca-cert
{{- end }}
items:
- key: {{ default "tls.crt" $root.Values.global.tls.caCert.secretKey }}
path: tls.crt
{{- end }}
{{- if $root.Values.global.tls.enableAutoEncrypt }}
- name: consul-auto-encrypt-ca-cert
emptyDir:
medium: "Memory"
{{- end }}
{{- end }}
initContainers:
# We use the Envoy image as our base image so we use an init container to
# copy the Consul binary to a shared directory that can be used when
# starting Envoy.
- name: copy-consul-bin
image: {{ $root.Values.global.image | quote }}
command:
- cp
- /bin/consul
- /consul-bin/consul
volumeMounts:
- name: consul-bin
mountPath: /consul-bin
{{- if (and $root.Values.global.tls.enabled $root.Values.global.tls.enableAutoEncrypt) }}
{{- include "consul.getAutoEncryptClientCA" $root | nindent 8 }}
{{- end }}
# service-init registers the terminating gateway service.
- name: service-init
image: {{ $root.Values.global.imageK8S }}
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
{{- if $root.Values.global.tls.enabled }}
- name: CONSUL_HTTP_ADDR
value: https://$(HOST_IP):8501
- name: CONSUL_CACERT
value: /consul/tls/ca/tls.crt
{{- else }}
- name: CONSUL_HTTP_ADDR
value: http://$(HOST_IP):8500
{{- end }}
command:
- "/bin/sh"
- "-ec"
- |
{{- if (or $root.Values.global.acls.manageSystemACLs $root.Values.global.bootstrapACLs) }}
consul-k8s acl-init \
-secret-name="{{ template "consul.fullname" $root }}-{{ .name }}-terminating-gateway-acl-token" \
-k8s-namespace={{ $root.Release.Namespace }} \
-token-sink-file=/consul/service/acl-token
{{- end }}

cat > /consul/service/service.hcl << EOF
service {
kind = "terminating-gateway"
name = "{{ .name }}"
id = "${POD_NAME}"
{{- if $root.Values.global.enableConsulNamespaces }}
namespace = "{{ (default $defaults.consulNamespace .consulNamespace) }}"
{{- end }}
address = "${POD_IP}"
port = 8443
checks = [
{
name = "Terminating Gateway Listening"
interval = "10s"
tcp = "${POD_IP}:8443"
deregister_critical_service_after = "6h"
}
]
}
EOF

/consul-bin/consul services register \
{{- if (or $root.Values.global.acls.manageSystemACLs $root.Values.global.bootstrapACLs) }}
-token-file=/consul/service/acl-token \
{{- end }}
/consul/service/service.hcl
volumeMounts:
- name: consul-service
mountPath: /consul/service
- name: consul-bin
mountPath: /consul-bin
{{- if $root.Values.global.tls.enabled }}
{{- if $root.Values.global.tls.enableAutoEncrypt }}
- name: consul-auto-encrypt-ca-cert
{{- else }}
- name: consul-ca-cert
{{- end }}
mountPath: /consul/tls/ca
readOnly: true
{{- end }}
containers:
- name: terminating-gateway
image: {{ $root.Values.global.imageEnvoy | quote }}
{{- if (default $defaults.resources .resources) }}
resources: {{ toYaml (default $defaults.resources .resources) | nindent 12 }}
{{- end }}
volumeMounts:
- name: consul-bin
mountPath: /consul-bin
{{- if $root.Values.global.tls.enabled }}
{{- if $root.Values.global.tls.enableAutoEncrypt }}
- name: consul-auto-encrypt-ca-cert
{{- else }}
- name: consul-ca-cert
{{- end }}
mountPath: /consul/tls/ca
readOnly: true
{{- end }}
{{- range (default $defaults.extraVolumes .extraVolumes) }}
- name: userconfig-{{ .name }}
readOnly: true
mountPath: /consul/userconfig/{{ .name }}
{{- end }}
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
{{- if (or $root.Values.global.acls.manageSystemACLs $root.Values.global.bootstrapACLs) }}
- name: CONSUL_HTTP_TOKEN
valueFrom:
secretKeyRef:
name: "{{ template "consul.fullname" $root }}-{{ .name }}-terminating-gateway-acl-token"
key: "token"
{{- end}}
{{- if $root.Values.global.tls.enabled }}
- name: CONSUL_HTTP_ADDR
value: https://$(HOST_IP):8501
- name: CONSUL_GRPC_ADDR
value: https://$(HOST_IP):8502
- name: CONSUL_CACERT
value: /consul/tls/ca/tls.crt
{{- else }}
- name: CONSUL_HTTP_ADDR
value: http://$(HOST_IP):8500
- name: CONSUL_GRPC_ADDR
value: $(HOST_IP):8502
{{- end }}
command:
- /consul-bin/consul
- connect
- envoy
- -gateway=terminating
- -proxy-id=$(POD_NAME)
{{- if $root.Values.global.enableConsulNamespaces }}
- -namespace={{ default $defaults.consulNamespace .consulNamespace }}
{{- end }}
livenessProbe:
tcpSocket:
port: 8443
failureThreshold: 3
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
tcpSocket:
port: 8443
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
ports:
- name: gateway
containerPort: 8443
lifecycle:
preStop:
exec:
command:
- "/bin/sh"
- "-ec"
- |
/consul-bin/consul services deregister \
{{- if $root.Values.global.enableConsulNamespaces }}
-namespace={{ default $defaults.consulNamespace .consulNamespace }} \
{{- end }}
-id="${POD_NAME}"

# lifecycle-sidecar ensures the terminating gateway is always registered with
# the local Consul agent, even if it loses the initial registration.
- name: lifecycle-sidecar
image: {{ $root.Values.global.imageK8S }}
volumeMounts:
- name: consul-service
mountPath: /consul/service
readOnly: true
- name: consul-bin
mountPath: /consul-bin
{{- if $root.Values.global.tls.enabled }}
{{- if $root.Values.global.tls.enableAutoEncrypt }}
- name: consul-auto-encrypt-ca-cert
{{- else }}
- name: consul-ca-cert
{{- end }}
mountPath: /consul/tls/ca
readOnly: true
{{- end }}
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
{{- if $root.Values.global.tls.enabled }}
- name: CONSUL_HTTP_ADDR
value: https://$(HOST_IP):8501
- name: CONSUL_CACERT
value: /consul/tls/ca/tls.crt
{{- else }}
- name: CONSUL_HTTP_ADDR
value: http://$(HOST_IP):8500
{{- end }}
command:
- consul-k8s
- lifecycle-sidecar
- -service-config=/consul/service/service.hcl
- -consul-binary=/consul-bin/consul
{{- if (or $root.Values.global.acls.manageSystemACLs $root.Values.global.bootstrapACLs) }}
- -token-file=/consul/service/acl-token
{{- end }}
{{- if (default $defaults.priorityClassName .priorityClassName) }}
priorityClassName: {{ (default $defaults.priorityClassName .priorityClassName) | quote }}
{{- end }}
{{- if (default $defaults.nodeSelector .nodeSelector) }}
nodeSelector:
{{ tpl (default $defaults.nodeSelector .nodeSelector) $root | indent 8 | trim }}
{{- end }}
---
{{- end }}
{{- end }}
Loading