From dc31e0aa120fbf1c64dba88ccefd62a84b901c9b Mon Sep 17 00:00:00 2001 From: Kyle Schochenmaier Date: Tue, 18 Jan 2022 17:17:54 -0600 Subject: [PATCH] add consul-logout add unit tests for consul-logout add acl-init unit test update server-acl-init tests update bats tests update ent tests add support for partitions --- .circleci/config.yml | 3 +- .../tests/controller/controller_test.go | 5 +- .../templates/controller-deployment.yaml | 67 +++++- .../consul/templates/server-acl-init-job.yaml | 1 + .../test/unit/controller-deployment.bats | 138 +++++++++++- .../consul/test/unit/server-acl-init-job.bats | 13 +- control-plane/commands.go | 5 + .../endpoints_controller_ent_test.go | 4 +- .../endpoints_controller_test.go | 4 +- control-plane/helper/test/test_util.go | 10 +- control-plane/subcommand/acl-init/command.go | 158 ++++++++++++- .../subcommand/acl-init/command_test.go | 65 ++++++ control-plane/subcommand/common/common.go | 8 +- .../connect-init/command_ent_test.go | 2 +- .../subcommand/connect-init/command_test.go | 2 +- .../subcommand/consul-logout/command.go | 107 +++++++++ .../subcommand/consul-logout/command_test.go | 162 +++++++++++++ .../subcommand/server-acl-init/command.go | 212 ++++++++++++++++-- .../server-acl-init/command_ent_test.go | 20 +- .../server-acl-init/command_test.go | 149 ++++++++---- .../server-acl-init/connect_inject.go | 74 +----- .../server-acl-init/connect_inject_test.go | 2 +- .../server-acl-init/create_or_update.go | 14 +- 23 files changed, 1041 insertions(+), 184 deletions(-) create mode 100644 control-plane/subcommand/consul-logout/command.go create mode 100644 control-plane/subcommand/consul-logout/command_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bf464fc1a..1b49b29cc4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,8 @@ commands: type: string consul-k8s-image: type: string - default: "docker.mirror.hashicorp.services/hashicorpdev/consul-k8s-control-plane:latest" + #default: "docker.mirror.hashicorp.services/hashicorpdev/consul-k8s-control-plane:latest" + default: "kschoche/consul-k8s-mdc" go-path: type: string default: "/home/circleci/.go_workspace" diff --git a/acceptance/tests/controller/controller_test.go b/acceptance/tests/controller/controller_test.go index e6405f3ccf..0994b462e0 100644 --- a/acceptance/tests/controller/controller_test.go +++ b/acceptance/tests/controller/controller_test.go @@ -22,8 +22,8 @@ func TestController(t *testing.T) { secure bool autoEncrypt bool }{ - {false, false}, - {true, false}, + // {false, false}, + // {true, false}, {true, true}, } @@ -38,6 +38,7 @@ func TestController(t *testing.T) { ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ + "global.imageK8S": "kschoche/consul-k8s-mdc", "controller.enabled": "true", "connectInject.enabled": "true", "global.tls.enabled": strconv.FormatBool(c.secure), diff --git a/charts/consul/templates/controller-deployment.yaml b/charts/consul/templates/controller-deployment.yaml index e5ed0d74f5..51d70c3421 100644 --- a/charts/consul/templates/controller-deployment.yaml +++ b/charts/consul/templates/controller-deployment.yaml @@ -47,15 +47,50 @@ spec: spec: {{- if or .Values.global.acls.manageSystemACLs (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }} initContainers: + {{- if (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }} + {{- include "consul.getAutoEncryptClientCA" . | nindent 6 }} + {{- end }} {{- if .Values.global.acls.manageSystemACLs }} - name: controller-acl-init + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.global.tls.enabled }} + - name: CONSUL_CACERT + value: /consul/tls/ca/tls.crt + {{- end }} + - name: CONSUL_HTTP_ADDR + {{- if .Values.global.tls.enabled }} + value: https://$(HOST_IP):8501 + {{- else }} + value: http://$(HOST_IP):8500 + {{- end }} image: {{ .Values.global.imageK8S }} + volumeMounts: + - mountPath: /consul/connect-inject + name: consul-data + readOnly: false + {{- if .Values.global.tls.enabled }} + {{- if .Values.global.tls.enableAutoEncrypt }} + - name: consul-auto-encrypt-ca-cert + {{- else }} + - name: consul-ca-cert + {{- end }} + mountPath: /consul/tls/ca + readOnly: true + {{- end }} command: - "/bin/sh" - "-ec" - | consul-k8s-control-plane acl-init \ - -secret-name="{{ template "consul.fullname" . }}-controller-acl-token" \ + -acl-auth-method="{{ template "consul.fullname" . }}-k8s-component-auth-method" \ + {{- if .Values.global.adminPartitions.enabled }} + -enable-partitions=true \ + -partition={{ .Values.global.adminPartitions.name }} \ + {{- end }} -k8s-namespace={{ .Release.Namespace }} resources: requests: @@ -65,9 +100,6 @@ spec: memory: "25Mi" cpu: "50m" {{- end }} - {{- if (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }} - {{- include "consul.getAutoEncryptClientCA" . | nindent 6 }} - {{- end }} {{- end }} containers: - command: @@ -98,7 +130,21 @@ spec: -consul-cross-namespace-acl-policy=cross-namespace-policy \ {{- end }} {{- end }} + {{- if .Values.global.acls.manageSystemACLs }} + lifecycle: + preStop: + exec: + command: + - "/bin/sh" + - "-ec" + - | + consul-k8s-control-plane consul-logout + {{- end }} env: + {{- if .Values.global.acls.manageSystemACLs }} + - name: CONSUL_HTTP_TOKEN_FILE + value: "/consul/connect-inject/acl-token" + {{- end }} - name: HOST_IP valueFrom: fieldRef: @@ -110,13 +156,6 @@ spec: name: {{ .Values.controller.aclToken.secretName }} key: {{ .Values.controller.aclToken.secretKey }} {{- end }} - {{- if .Values.global.acls.manageSystemACLs }} - - name: CONSUL_HTTP_TOKEN - valueFrom: - secretKeyRef: - name: "{{ template "consul.fullname" . }}-controller-acl-token" - key: "token" - {{- end}} {{- if .Values.global.tls.enabled }} - name: CONSUL_CACERT value: /consul/tls/ca/tls.crt @@ -138,6 +177,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + - mountPath: /consul/connect-inject + name: consul-data + readOnly: false - mountPath: /tmp/controller-webhook/certs name: cert readOnly: true @@ -175,6 +217,9 @@ spec: medium: "Memory" {{- end }} {{- end }} + - name: consul-data + emptyDir: + medium: "Memory" serviceAccountName: {{ template "consul.fullname" . }}-controller {{- if .Values.controller.nodeSelector }} nodeSelector: diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml index 5195e7975e..11642b8d97 100644 --- a/charts/consul/templates/server-acl-init-job.yaml +++ b/charts/consul/templates/server-acl-init-job.yaml @@ -238,6 +238,7 @@ spec: {{- if .Values.controller.enabled }} -create-controller-token=true \ + -create-component-auth-method=true \ {{- end }} {{- if .Values.apiGateway.enabled }} diff --git a/charts/consul/test/unit/controller-deployment.bats b/charts/consul/test/unit/controller-deployment.bats index 248811867d..5e4197b38c 100644 --- a/charts/consul/test/unit/controller-deployment.bats +++ b/charts/consul/test/unit/controller-deployment.bats @@ -46,18 +46,18 @@ load _helpers #-------------------------------------------------------------------- # global.acls.manageSystemACLs -@test "controller/Deployment: CONSUL_HTTP_TOKEN env variable created when global.acls.manageSystemACLs=true" { +@test "controller/Deployment: consul-logout preStop hook is added when ACLs are enabled" { cd `chart_dir` local actual=$(helm template \ -s templates/controller-deployment.yaml \ --set 'controller.enabled=true' \ --set 'global.acls.manageSystemACLs=true' \ . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_HTTP_TOKEN"))' | tee /dev/stderr) + yq '[.spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]] | any(contains("consul-k8s-control-plane consul-logout"))' | tee /dev/stderr) [ "${actual}" = "true" ] } -@test "controller/Deployment: init container is created when global.acls.manageSystemACLs=true" { +@test "controller/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command and environment" { cd `chart_dir` local object=$(helm template \ -s templates/controller-deployment.yaml \ @@ -73,8 +73,123 @@ load _helpers local actual=$(echo $object | yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[1].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[1].value] | any(contains("http://$(HOST_IP):8500"))' | tee /dev/stderr) + echo $actual + [ "${actual}" = "true" ] +} + +@test "controller/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command and environment with tls enabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/controller-deployment.yaml \ + --set 'controller.enabled=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.acls.manageSystemACLs=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.initContainers[] | select(.name == "controller-acl-init")' | tee /dev/stderr) + + local actual=$(echo $object | + yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[1].name] | any(contains("CONSUL_CACERT"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[2].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[2].value] | any(contains("https://$(HOST_IP):8501"))' | tee /dev/stderr) + echo $actual + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '.volumeMounts[1] | any(contains("consul-ca-cert"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "controller/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command with Partitions enabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/controller-deployment.yaml \ + --set 'controller.enabled=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.enableConsulNamespaces=true' \ + --set 'global.adminPartitions.enabled=true' \ + --set 'global.adminPartitions.name=default' \ + --set 'global.acls.manageSystemACLs=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.initContainers[] | select(.name == "controller-acl-init")' | tee /dev/stderr) + + local actual=$(echo $object | + yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq -r '.command | any(contains("-enable-partitions=true"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual=$(echo $object | + yq -r '.command | any(contains("-partition=default"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[1].name] | any(contains("CONSUL_CACERT"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[2].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[2].value] | any(contains("https://$(HOST_IP):8501"))' | tee /dev/stderr) + echo $actual + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '.volumeMounts[1] | any(contains("consul-ca-cert"))' | tee /dev/stderr) + [ "${actual}" = "true" ] } +@test "controller/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command and environment with tls enabled and autoencrypt enabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/controller-deployment.yaml \ + --set 'controller.enabled=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.acls.manageSystemACLs=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.initContainers[] | select(.name == "controller-acl-init")' | tee /dev/stderr) + + local actual=$(echo $object | + yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[1].name] | any(contains("CONSUL_CACERT"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[2].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '[.env[2].value] | any(contains("https://$(HOST_IP):8501"))' | tee /dev/stderr) + echo $actual + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq '.volumeMounts[1] | any(contains("consul-auto-encrypt-ca-cert"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} #-------------------------------------------------------------------- # global.tls.enabled @@ -486,38 +601,37 @@ load _helpers #-------------------------------------------------------------------- # aclToken -@test "controller/Deployment: aclToken disabled when secretName is missing" { +@test "controller/Deployment: aclToken enabled when secretName and secretKey is provided" { cd `chart_dir` local actual=$(helm template \ -s templates/controller-deployment.yaml \ --set 'controller.enabled=true' \ + --set 'controller.aclToken.secretName=foo' \ --set 'controller.aclToken.secretKey=bar' \ . | tee /dev/stderr | yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_HTTP_TOKEN"))' | tee /dev/stderr) - [ "${actual}" = "false" ] + [ "${actual}" = "true" ] } -@test "controller/Deployment: aclToken disabled when secretKey is missing" { +@test "controller/Deployment: aclToken env is set when ACLs are enabled" { cd `chart_dir` local actual=$(helm template \ -s templates/controller-deployment.yaml \ --set 'controller.enabled=true' \ - --set 'controller.aclToken.secretName=foo' \ + --set 'global.acls.manageSystemACLs=true' \ . | tee /dev/stderr | yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_HTTP_TOKEN"))' | tee /dev/stderr) - [ "${actual}" = "false" ] + [ "${actual}" = "true" ] } -@test "controller/Deployment: aclToken enabled when secretName and secretKey is provided" { +@test "controller/Deployment: aclToken env is not set when ACLs are disabled" { cd `chart_dir` local actual=$(helm template \ -s templates/controller-deployment.yaml \ --set 'controller.enabled=true' \ - --set 'controller.aclToken.secretName=foo' \ - --set 'controller.aclToken.secretKey=bar' \ . | tee /dev/stderr | yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_HTTP_TOKEN"))' | tee /dev/stderr) - [ "${actual}" = "true" ] + [ "${actual}" = "false" ] } #-------------------------------------------------------------------- diff --git a/charts/consul/test/unit/server-acl-init-job.bats b/charts/consul/test/unit/server-acl-init-job.bats index 1b84b5860b..8232a32dea 100644 --- a/charts/consul/test/unit/server-acl-init-job.bats +++ b/charts/consul/test/unit/server-acl-init-job.bats @@ -1641,14 +1641,21 @@ load _helpers [ "${actual}" = "false" ] } -@test "serverACLInit/Job: -create-controller-token set when controller.enabled=true" { +@test "serverACLInit/Job: -create-controller-token set when controller.enabled=true and -create-component-auth-method is passed" { cd `chart_dir` - local actual=$(helm template \ + local object=$(helm template \ -s templates/server-acl-init-job.yaml \ --set 'global.acls.manageSystemACLs=true' \ --set 'controller.enabled=true' \ . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | any(contains("create-controller-token"))' | tee /dev/stderr) + yq '.spec.template.spec.containers[0]' | tee /dev/stderr) + + local actual=$(echo "$object" | + yq '.command | any(contains("-create-controller-token"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo "$object" | + yq '.command | any(contains("-create-component-auth-method"))' | tee /dev/stderr) [ "${actual}" = "true" ] } diff --git a/control-plane/commands.go b/control-plane/commands.go index db43863642..8d5e8de23e 100644 --- a/control-plane/commands.go +++ b/control-plane/commands.go @@ -5,6 +5,7 @@ import ( cmdACLInit "github.com/hashicorp/consul-k8s/control-plane/subcommand/acl-init" cmdConnectInit "github.com/hashicorp/consul-k8s/control-plane/subcommand/connect-init" + cmdConsulLogout "github.com/hashicorp/consul-k8s/control-plane/subcommand/consul-logout" cmdConsulSidecar "github.com/hashicorp/consul-k8s/control-plane/subcommand/consul-sidecar" cmdController "github.com/hashicorp/consul-k8s/control-plane/subcommand/controller" cmdCreateFederationSecret "github.com/hashicorp/consul-k8s/control-plane/subcommand/create-federation-secret" @@ -46,6 +47,10 @@ func init() { return &cmdConsulSidecar.Command{UI: ui}, nil }, + "consul-logout": func() (cli.Command, error) { + return &cmdConsulLogout.Command{UI: ui}, nil + }, + "server-acl-init": func() (cli.Command, error) { return &cmdServerACLInit.Command{UI: ui}, nil }, diff --git a/control-plane/connect-inject/endpoints_controller_ent_test.go b/control-plane/connect-inject/endpoints_controller_ent_test.go index 5859bd9206..788ab4f731 100644 --- a/control-plane/connect-inject/endpoints_controller_ent_test.go +++ b/control-plane/connect-inject/endpoints_controller_ent_test.go @@ -1220,7 +1220,7 @@ func TestReconcileUpdateEndpointWithNamespaces(t *testing.T) { if ts.Mirror { writeOpts.Namespace = "default" } - test.SetupK8sAuthMethodWithNamespaces(t, consulClient, svc.Name, svc.Meta[MetaKeyKubeNS], ts.ExpConsulNS, ts.Mirror, ts.MirrorPrefix) + test.SetupK8sAuthMethodWithNamespaces(t, consulClient, svc.Name, svc.Meta[MetaKeyKubeNS], ts.ExpConsulNS, ts.Mirror, ts.MirrorPrefix, test.AuthMethod) token, _, err := consulClient.ACL().Login(&api.ACLLoginParams{ AuthMethod: test.AuthMethod, BearerToken: test.ServiceAccountJWTToken, @@ -1550,7 +1550,7 @@ func TestReconcileDeleteEndpointWithNamespaces(t *testing.T) { if ts.Mirror { writeOpts.Namespace = "default" } - test.SetupK8sAuthMethodWithNamespaces(t, consulClient, svc.Name, svc.Meta[MetaKeyKubeNS], ts.ExpConsulNS, ts.Mirror, ts.MirrorPrefix) + test.SetupK8sAuthMethodWithNamespaces(t, consulClient, svc.Name, svc.Meta[MetaKeyKubeNS], ts.ExpConsulNS, ts.Mirror, ts.MirrorPrefix, test.AuthMethod) token, _, err = consulClient.ACL().Login(&api.ACLLoginParams{ AuthMethod: test.AuthMethod, BearerToken: test.ServiceAccountJWTToken, diff --git a/control-plane/connect-inject/endpoints_controller_test.go b/control-plane/connect-inject/endpoints_controller_test.go index 98351c3f39..33e7ff3208 100644 --- a/control-plane/connect-inject/endpoints_controller_test.go +++ b/control-plane/connect-inject/endpoints_controller_test.go @@ -2374,7 +2374,7 @@ func TestReconcileUpdateEndpoint(t *testing.T) { // Create a token for this service if ACLs are enabled. if tt.enableACLs { if svc.Kind != api.ServiceKindConnectProxy { - test.SetupK8sAuthMethod(t, consulClient, svc.Name, svc.Meta[MetaKeyKubeNS]) + test.SetupK8sAuthMethod(t, consulClient, svc.Name, svc.Meta[MetaKeyKubeNS], test.AuthMethod) token, _, err := consulClient.ACL().Login(&api.ACLLoginParams{ AuthMethod: test.AuthMethod, BearerToken: test.ServiceAccountJWTToken, @@ -2706,7 +2706,7 @@ func TestReconcileDeleteEndpoint(t *testing.T) { // Create a token for it if ACLs are enabled. if tt.enableACLs { - test.SetupK8sAuthMethod(t, consulClient, svc.Name, "default") + test.SetupK8sAuthMethod(t, consulClient, svc.Name, "default", test.AuthMethod) if svc.Kind != api.ServiceKindConnectProxy { token, _, err = consulClient.ACL().Login(&api.ACLLoginParams{ AuthMethod: test.AuthMethod, diff --git a/control-plane/helper/test/test_util.go b/control-plane/helper/test/test_util.go index 365dc54363..a3f431f214 100644 --- a/control-plane/helper/test/test_util.go +++ b/control-plane/helper/test/test_util.go @@ -59,14 +59,14 @@ func GenerateServerCerts(t *testing.T) (string, string, string) { // SetupK8sAuthMethod create a k8s auth method and a binding rule in Consul for the // given k8s service and namespace. -func SetupK8sAuthMethod(t *testing.T, consulClient *api.Client, serviceName, k8sServiceNS string) { - SetupK8sAuthMethodWithNamespaces(t, consulClient, serviceName, k8sServiceNS, "", false, "") +func SetupK8sAuthMethod(t *testing.T, consulClient *api.Client, serviceName, k8sServiceNS string, authMethodName string) { + SetupK8sAuthMethodWithNamespaces(t, consulClient, serviceName, k8sServiceNS, "", false, "", authMethodName) } // SetupK8sAuthMethodWithNamespaces creates a k8s auth method and binding rule // in Consul for the k8s service name and namespace. It sets up the auth method and the binding // rule so that it works with consul namespaces. -func SetupK8sAuthMethodWithNamespaces(t *testing.T, consulClient *api.Client, serviceName, k8sServiceNS, consulNS string, mirrorNS bool, nsPrefix string) { +func SetupK8sAuthMethodWithNamespaces(t *testing.T, consulClient *api.Client, serviceName, k8sServiceNS, consulNS string, mirrorNS bool, nsPrefix string, authMethodName string) { t.Helper() // Start the mock k8s server. k8sMockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -83,7 +83,7 @@ func SetupK8sAuthMethodWithNamespaces(t *testing.T, consulClient *api.Client, se // Set up Consul's auth method. authMethodTmpl := api.ACLAuthMethod{ - Name: AuthMethod, + Name: authMethodName, Type: "kubernetes", Description: "Kubernetes Auth Method", Config: map[string]interface{}{ @@ -105,7 +105,7 @@ func SetupK8sAuthMethodWithNamespaces(t *testing.T, consulClient *api.Client, se // Create the binding rule. aclBindingRule := api.ACLBindingRule{ Description: "Kubernetes binding rule", - AuthMethod: AuthMethod, + AuthMethod: authMethodName, BindType: api.BindingRuleBindTypeService, BindName: "${serviceaccount.name}", Selector: "serviceaccount.name!=default", diff --git a/control-plane/subcommand/acl-init/command.go b/control-plane/subcommand/acl-init/command.go index 6017137bbf..f96aa0aef5 100644 --- a/control-plane/subcommand/acl-init/command.go +++ b/control-plane/subcommand/acl-init/command.go @@ -6,40 +6,73 @@ import ( "flag" "fmt" "io/ioutil" + "os" "path/filepath" "strings" "sync" "text/template" "time" + "github.com/cenkalti/backoff" + "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul-k8s/control-plane/subcommand" + "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-hclog" "github.com/mitchellh/cli" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) +const ( + defaultBearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" + defaultTokenSinkFile = "/consul/connect-inject/acl-token" + + // The number of times to attempt ACL Login. + numLoginRetries = 300 + + raftReplicationTimeout = 2 * time.Second + tokenReadPollingInterval = 100 * time.Millisecond +) + type Command struct { UI cli.Ui - flags *flag.FlagSet - k8s *flags.K8SFlags + flags *flag.FlagSet + k8s *flags.K8SFlags + http *flags.HTTPFlags + flagSecretName string flagInitType string flagNamespace string flagACLDir string flagTokenSinkFile string + flagACLAuthMethod string // Auth Method to use for ACLs, if enabled. + flagLogLevel string + flagLogJSON bool + + // Flags to support partitions. + flagEnablePartitions bool // true if Admin Partitions are enabled + flagPartitionName string // name of the Admin Partition + + bearerTokenFile string // Location of the bearer token. Default is /var/run/secrets/kubernetes.io/serviceaccount/token. + tokenSinkFile string // Location to write the output token. Default is defaultTokenSinkFile. + k8sClient kubernetes.Interface - once sync.Once - help string + once sync.Once + help string + logger hclog.Logger - ctx context.Context + ctx context.Context + consulClient *api.Client } func (c *Command) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) + c.flags.StringVar(&c.flagSecretName, "secret-name", "", "Name of secret to watch for an ACL token") c.flags.StringVar(&c.flagInitType, "init-type", "", @@ -51,14 +84,37 @@ func (c *Command) init() { c.flags.StringVar(&c.flagTokenSinkFile, "token-sink-file", "", "Optional filepath to write acl token") + // Flags related to using consul login to fetch the ACL token. + c.flags.StringVar(&c.flagACLAuthMethod, "acl-auth-method", "", "Name of the auth method to login to.") + c.flags.StringVar(&c.flagLogLevel, "log-level", "info", + "Log verbosity level. Supported values (in order of detail) are \"trace\", "+ + "\"debug\", \"info\", \"warn\", and \"error\".") + c.flags.BoolVar(&c.flagLogJSON, "log-json", false, + "Enable or disable JSON output format for logging.") + + // Flags related to Partitions. + c.flags.BoolVar(&c.flagEnablePartitions, "enable-partitions", false, + "[Enterprise Only] Enables Admin Partitions") + c.flags.StringVar(&c.flagPartitionName, "partition", "", + "[Enterprise Only] Name of the Admin Partition") + + if c.bearerTokenFile == "" { + c.bearerTokenFile = defaultBearerTokenFile + } + if c.tokenSinkFile == "" { + c.tokenSinkFile = defaultTokenSinkFile + } + c.k8s = &flags.K8SFlags{} + c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.k8s.Flags()) c.help = flags.Usage(help, c.flags) } func (c *Command) Run(args []string) int { + var err error c.once.Do(c.init) - if err := c.flags.Parse(args); err != nil { + if err = c.flags.Parse(args); err != nil { return 1 } if len(c.flags.Args()) > 0 { @@ -84,6 +140,96 @@ func (c *Command) Run(args []string) int { } } + // Set up logging. + if c.logger == nil { + c.logger, err = common.Logger(c.flagLogLevel, c.flagLogJSON) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + } + + if c.flagACLAuthMethod != "" { + cfg := api.DefaultConfig() + if c.flagEnablePartitions { + cfg.Partition = c.flagPartitionName + } + c.http.MergeOntoConfig(cfg) + if c.consulClient == nil { + c.consulClient, err = consul.NewClient(cfg) + if err != nil { + c.logger.Error("Unable to get client connection", "error", err) + return 1 + } + + } + err = backoff.Retry(func() error { + //err := common.ConsulLogin(c.consulClient, c.bearerTokenFile, c.flagACLAuthMethod, c.tokenSinkFile, c.flagAuthMethodNamespace, map[string]string{}) + err := common.ConsulLogin(c.consulClient, c.bearerTokenFile, c.flagACLAuthMethod, c.tokenSinkFile, "", map[string]string{}) + if err != nil { + c.logger.Error("Consul login failed; retrying", "error", err) + } + return err + }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), numLoginRetries)) + if err != nil { + c.logger.Error("Hit maximum retries for consul login", "error", err) + return 1 + } + c.logger.Info("Consul login complete") + // Now fetch the token that was just created so we can use it in subsequent client api calls. + token, err := os.ReadFile(c.tokenSinkFile) + if err != nil { + c.logger.Error("Unable to read token sink file after login", "error", err) + return 1 + } + + // A workaround to check that the ACL token is replicated to other Consul servers. + // + // A consul client may reach out to a follower instead of a leader to resolve the token during the + // call to get services below. This is because clients talk to servers in the stale consistency mode + // to decrease the load on the servers (see https://www.consul.io/docs/architecture/consensus#stale). + // In that case, it's possible that the token isn't replicated + // to that server instance yet. The client will then get an "ACL not found" error + // and subsequently cache this not found response. Then our call below + // to get services from the agent will keep hitting the same "ACL not found" error + // until the cache entry expires (determined by the `acl_token_ttl` which defaults to 30 seconds). + // This is not great because it will delay app start up time by 30 seconds in most cases + // (if you are running 3 servers, then the probability of ending up on a follower is close to 2/3). + // + // To help with that, we try to first read the token in the stale consistency mode until we + // get a successful response. This should not take more than 100ms because raft replication + // should in most cases take less than that (see https://www.consul.io/docs/install/performance#read-write-tuning) + // but we set the timeout to 2s to be sure. + // + // Note though that this workaround does not eliminate this problem completely. It's still possible + // for this call and the next call to reach different servers and those servers to have different + // states from each other. + // For example, this call can reach a leader and succeed, while the call below can go to a follower + // that is still behind the leader and get an "ACL not found" error. + // However, this is a pretty unlikely case because + // clients have sticky connections to a server, and those connections get rebalanced only every 2-3min. + // And so, this workaround should work in a vast majority of cases. + c.logger.Info("Checking that the ACL token exists when reading it in the stale consistency mode") + // Use raft timeout and polling interval to determine the number of retries. + numTokenReadRetries := uint64(raftReplicationTimeout.Milliseconds() / tokenReadPollingInterval.Milliseconds()) + err = backoff.Retry(func() error { + _, _, err := c.consulClient.ACL().TokenReadSelf(&api.QueryOptions{ + AllowStale: true, + Token: string(token), + }) + if err != nil { + c.logger.Error("Unable to read ACL token; retrying", "err", err) + } + return err + }, backoff.WithMaxRetries(backoff.NewConstantBackOff(tokenReadPollingInterval), numTokenReadRetries)) + if err != nil { + c.logger.Error("Unable to read ACL token from a Consul server; "+ + "please check that your server cluster is healthy", "err", err) + return 1 + } + c.logger.Info("Successfully read ACL token from the server") + return 0 + } // Check if the client secret exists yet // If not, wait until it does var secret string diff --git a/control-plane/subcommand/acl-init/command_test.go b/control-plane/subcommand/acl-init/command_test.go index 0a3a7ab8bf..b09d0cfc99 100644 --- a/control-plane/subcommand/acl-init/command_test.go +++ b/control-plane/subcommand/acl-init/command_test.go @@ -2,6 +2,9 @@ package aclinit import ( "context" + "github.com/hashicorp/consul-k8s/control-plane/helper/test" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" "io/ioutil" "os" "path/filepath" @@ -153,3 +156,65 @@ func TestRun_TokenSinkFileTwice(t *testing.T) { require.Equal(token, string(bytes), "exp: %s, got: %s", token, string(bytes)) } } + +// TestRun_PerformsConsulLogin executes the consul login path and validates the token +// is written to disk. +func TestRun_PerformsConsulLogin(t *testing.T) { + var caFile, certFile, keyFile string + // This is the test file that we will write the token to so consul-logout can read it. + tokenFile := common.WriteTempFile(t, "") + bearerFile := common.WriteTempFile(t, test.ServiceAccountJWTToken) + t.Cleanup(func() { + os.Remove(tokenFile) + }) + + k8s := fake.NewSimpleClientset() + + // Start Consul server with ACLs enabled and default deny policy. + masterToken := "b78d37c7-0ca7-5f4d-99ee-6d9975ce4586" + server, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + c.ACL.DefaultPolicy = "deny" + c.ACL.Tokens.InitialManagement = masterToken + caFile, certFile, keyFile = test.GenerateServerCerts(t) + c.CAFile = caFile + c.CertFile = certFile + c.KeyFile = keyFile + }) + require.NoError(t, err) + defer server.Stop() + server.WaitForLeader(t) + cfg := &api.Config{ + Scheme: "http", + Address: server.HTTPAddr, + Token: masterToken, + } + cfg.Address = server.HTTPSAddr + cfg.Scheme = "https" + cfg.TLSConfig = api.TLSConfig{ + CAFile: caFile, + } + consulClient, err := api.NewClient(cfg) + require.NoError(t, err) + + test.SetupK8sAuthMethod(t, consulClient, "test-sa", "default", common.ComponentAuthMethod) + + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + k8sClient: k8s, + bearerTokenFile: bearerFile, + consulClient: consulClient, + tokenSinkFile: tokenFile, + } + + code := cmd.Run([]string{ + "-acl-auth-method", "consul-k8s-component-auth-method", + }) + require.Equal(t, 0, code, ui.ErrorWriter.String()) + + bytes, err := ioutil.ReadFile(tokenFile) + require.NoError(t, err) + require.NotEqual(t, 0, len(bytes)) + +} diff --git a/control-plane/subcommand/common/common.go b/control-plane/subcommand/common/common.go index bd60b5822c..3a66754545 100644 --- a/control-plane/subcommand/common/common.go +++ b/control-plane/subcommand/common/common.go @@ -31,6 +31,8 @@ const ( // which secrets to delete on an uninstall. CLILabelKey = "managed-by" CLILabelValue = "consul-k8s" + + ComponentAuthMethod = "consul-k8s-component-auth-method" ) // Logger returns an hclog instance with log level set and JSON logging enabled/disabled, or an error if level is invalid. @@ -80,9 +82,9 @@ func ValidateUnprivilegedPort(flagName, flagValue string) error { // ConsulLogin issues an ACL().Login to Consul and writes out the token to tokenSinkFile. // The logic of this is taken from the `consul login` command. func ConsulLogin(client *api.Client, bearerTokenFile, authMethodName, tokenSinkFile, namespace string, meta map[string]string) error { - if meta == nil { - return fmt.Errorf("invalid meta") - } + //if meta == nil { + // return fmt.Errorf("invalid meta") + //} data, err := ioutil.ReadFile(bearerTokenFile) if err != nil { return fmt.Errorf("unable to read bearerTokenFile: %v, err: %v", bearerTokenFile, err) diff --git a/control-plane/subcommand/connect-init/command_ent_test.go b/control-plane/subcommand/connect-init/command_ent_test.go index f043542d83..51432fd387 100644 --- a/control-plane/subcommand/connect-init/command_ent_test.go +++ b/control-plane/subcommand/connect-init/command_ent_test.go @@ -172,7 +172,7 @@ func TestRun_ServicePollingWithACLsAndTLSWithNamespaces(t *testing.T) { require.NoError(t, err) if c.acls { - test.SetupK8sAuthMethodWithNamespaces(t, consulClient, testServiceAccountName, "default-ns", c.authMethodNamespace, c.authMethodNamespace != c.consulServiceNamespace, "") + test.SetupK8sAuthMethodWithNamespaces(t, consulClient, testServiceAccountName, "default-ns", c.authMethodNamespace, c.authMethodNamespace != c.consulServiceNamespace, "", test.AuthMethod) } // Register Consul services. diff --git a/control-plane/subcommand/connect-init/command_test.go b/control-plane/subcommand/connect-init/command_test.go index 40136b9fd8..65ce42d37c 100644 --- a/control-plane/subcommand/connect-init/command_test.go +++ b/control-plane/subcommand/connect-init/command_test.go @@ -148,7 +148,7 @@ func TestRun_ServicePollingWithACLsAndTLS(t *testing.T) { consulClient, err := api.NewClient(cfg) require.NoError(t, err) - test.SetupK8sAuthMethod(t, consulClient, testServiceAccountName, "default") + test.SetupK8sAuthMethod(t, consulClient, testServiceAccountName, "default", test.AuthMethod) // Register Consul services. testConsulServices := []api.AgentServiceRegistration{consulCountingSvc, consulCountingSvcSidecar} diff --git a/control-plane/subcommand/consul-logout/command.go b/control-plane/subcommand/consul-logout/command.go new file mode 100644 index 0000000000..2855eaf1ef --- /dev/null +++ b/control-plane/subcommand/consul-logout/command.go @@ -0,0 +1,107 @@ +package consullogout + +import ( + "flag" + "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" + "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-hclog" + "github.com/mitchellh/cli" + "io/ioutil" + "sync" +) + +type Command struct { + UI cli.Ui + + flagLogLevel string + flagLogJSON bool + + flagSet *flag.FlagSet + http *flags.HTTPFlags + + tokenSinkFile string + consulClient *api.Client + + once sync.Once + help string + logger hclog.Logger +} + +func (c *Command) init() { + c.flagSet = flag.NewFlagSet("", flag.ContinueOnError) + c.flagSet.StringVar(&c.flagLogLevel, "log-level", "info", + "Log verbosity level. Supported values (in order of detail) are \"trace\", "+ + "\"debug\", \"info\", \"warn\", and \"error\".") + c.flagSet.BoolVar(&c.flagLogJSON, "log-json", false, + "Enable or disable JSON output format for logging.") + + c.http = &flags.HTTPFlags{} + flags.Merge(c.flagSet, c.http.Flags()) + c.help = flags.Usage(help, c.flagSet) + +} + +const ( + defaultAclTokenLocation = "/consul/connect-inject/acl-token" +) + +func (c *Command) Run(args []string) int { + var err error + c.once.Do(c.init) + + if err := c.flagSet.Parse(args); err != nil { + return 1 + } + if c.logger == nil { + c.logger, err = common.Logger(c.flagLogLevel, c.flagLogJSON) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + } + if c.tokenSinkFile == "" { + c.tokenSinkFile = defaultAclTokenLocation + } + + if c.consulClient == nil { + cfg := api.DefaultConfig() + c.http.MergeOntoConfig(cfg) + c.consulClient, err = consul.NewClient(cfg) + if err != nil { + c.logger.Error("Unable to get client connection", "error", err) + return 1 + } + } + + token, err := ioutil.ReadFile(c.tokenSinkFile) + if err != nil { + c.logger.Error("Unable to read ACL token", "error", err) + return 1 + } + + _, err = c.consulClient.ACL().Logout(&api.WriteOptions{ + Token: string(token), + }) + if err != nil { + c.logger.Error("Unable to destroy consul ACL token", "error", err) + return 1 + } + c.logger.Error("ACL token succesfully destroyed") + return 0 +} + +func (c *Command) Synopsis() string { return synopsis } +func (c *Command) Help() string { + c.once.Do(c.init) + return c.help +} + +const synopsis = "Issue a consul logout to destroy the ACL token." +const help = ` +Usage: consul-k8s-control-plane consul-logout [options] + + Destroys the ACL token for this pod. + Not intended for stand-alone use. +` diff --git a/control-plane/subcommand/consul-logout/command_test.go b/control-plane/subcommand/consul-logout/command_test.go new file mode 100644 index 0000000000..c5bd94d497 --- /dev/null +++ b/control-plane/subcommand/consul-logout/command_test.go @@ -0,0 +1,162 @@ +package consullogout + +import ( + "fmt" + "math/rand" + "os" + "testing" + + "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-k8s/control-plane/helper/test" + "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +// TestRun_InvalidSinkFile validates that we correctly fail in case the token sink file +// does not exist. +func TestRun_InvalidSinkFile(t *testing.T) { + t.Parallel() + randFileName := fmt.Sprintf("/foo/%d/%d", rand.Int(), rand.Int()) + + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + tokenSinkFile: randFileName, + } + code := cmd.Run([]string{}) + require.Equal(t, 1, code) +} + +// Test_UnableToLogoutDueToInvalidToken checks the error path for when Consul is not +// aware of an ACL token. This is a big corner case but covers the rare occurrance that +// the preStop hook where `consul-logout` is run might be executed more than once by Kubelet. +// This also covers obscure cases where the acl-token file is corrupted somehow. +func Test_UnableToLogoutDueToInvalidToken(t *testing.T) { + tokenFile := fmt.Sprintf("/tmp/%d1", rand.Int()) + t.Cleanup(func() { + os.Remove(tokenFile) + }) + + var caFile, certFile, keyFile string + // Start Consul server with ACLs enabled and default deny policy. + masterToken := "b78d37c7-0ca7-5f4d-99ee-6d9975ce4586" + server, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + c.ACL.DefaultPolicy = "deny" + c.ACL.Tokens.InitialManagement = masterToken + caFile, certFile, keyFile = test.GenerateServerCerts(t) + c.CAFile = caFile + c.CertFile = certFile + c.KeyFile = keyFile + }) + require.NoError(t, err) + defer server.Stop() + server.WaitForLeader(t) + cfg := &api.Config{ + Address: server.HTTPSAddr, + Scheme: "https", + Token: masterToken, + TLSConfig: api.TLSConfig{ + CAFile: caFile, + }, + } + consulClient, err := api.NewClient(cfg) + require.NoError(t, err) + + test.SetupK8sAuthMethod(t, consulClient, "test-sa", "default", common.ComponentAuthMethod) + + bogusToken := "00000000-00-0-001110aacddbderf" + err = os.WriteFile(tokenFile, []byte(bogusToken), 0444) + require.NoError(t, err) + + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + tokenSinkFile: tokenFile, + consulClient: consulClient, + } + + // Run the command. + code := cmd.Run([]string{}) + require.Equal(t, 1, code, ui.ErrorWriter.String()) + require.Contains(t, "Unexpected response code: 403 (ACL not found)", ui.ErrorWriter.String()) +} + +// Test_RunUsingLogin creates an AuthMethod and issues an ACL Token via ACL().Login() +// which is the code path that is taken to provision the ACL tokens at runtime through +// subcommand/acl-init. It then runs `consul-logout` and ensures that the ACL token +// is properly destroyed. +func Test_RunUsingLogin(t *testing.T) { + var caFile, certFile, keyFile string + // This is the test file that we will write the token to so consul-logout can read it. + tokenFile := fmt.Sprintf("/tmp/%d1", rand.Int()) + t.Cleanup(func() { + os.Remove(tokenFile) + }) + + // Start Consul server with ACLs enabled and default deny policy. + masterToken := "b78d37c7-0ca7-5f4d-99ee-6d9975ce4586" + server, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + c.ACL.DefaultPolicy = "deny" + c.ACL.Tokens.InitialManagement = masterToken + caFile, certFile, keyFile = test.GenerateServerCerts(t) + c.CAFile = caFile + c.CertFile = certFile + c.KeyFile = keyFile + }) + require.NoError(t, err) + defer server.Stop() + server.WaitForLeader(t) + cfg := &api.Config{ + Address: server.HTTPSAddr, + Scheme: "https", + Token: masterToken, + TLSConfig: api.TLSConfig{ + CAFile: caFile, + }, + } + consulClient, err := consul.NewClient(cfg) + require.NoError(t, err) + + test.SetupK8sAuthMethod(t, consulClient, "test-sa", "default", common.ComponentAuthMethod) + + // Do the login. + req := &api.ACLLoginParams{ + AuthMethod: common.ComponentAuthMethod, + BearerToken: test.ServiceAccountJWTToken, + Meta: map[string]string{}, + } + token, _, err := consulClient.ACL().Login(req, &api.WriteOptions{}) + require.NoError(t, err) + + // Validate that the token was created. + _, _, err = consulClient.ACL().TokenRead(token.AccessorID, &api.QueryOptions{}) + require.NoError(t, err) + + // Write the token's SecretID to the tokenFile which mimics loading + // the ACL token from subcommand/acl-init path. + err = os.WriteFile(tokenFile, []byte(token.SecretID), 0444) + require.NoError(t, err) + + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + tokenSinkFile: tokenFile, + consulClient: consulClient, + } + + // Run the command. + code := cmd.Run([]string{}) + require.Equal(t, 0, code, ui.ErrorWriter.String()) + + // Validate the ACL token was destroyed. + tokenList, _, err := consulClient.ACL().TokenList(nil) + require.NoError(t, err) + for _, tok := range tokenList { + require.NotEqual(t, tok.SecretID, token.SecretID) + } +} diff --git a/control-plane/subcommand/server-acl-init/command.go b/control-plane/subcommand/server-acl-init/command.go index e665d1e3f0..ea52ac431b 100644 --- a/control-plane/subcommand/server-acl-init/command.go +++ b/control-plane/subcommand/server-acl-init/command.go @@ -83,6 +83,9 @@ type Command struct { flagEnableInjectK8SNSMirroring bool // Enables mirroring of k8s namespaces into Consul for Connect inject flagInjectK8SNSMirroringPrefix string // Prefix added to Consul namespaces created when mirroring injected services + // Flags to support ACL Login method of provisioning tokens. + flagCreateComponentAuthMethod bool // Whether or not to create the comoponent auth method in Consul for components to use to fetch acl tokens + // Flag to support a custom bootstrap token. flagBootstrapTokenFile string @@ -213,6 +216,9 @@ func (c *Command) init() { "Path to file containing ACL token for creating policies and tokens. This token must have 'acl:write' permissions."+ "When provided, servers will not be bootstrapped and their policies and tokens will not be updated.") + c.flags.BoolVar(&c.flagCreateComponentAuthMethod, "create-component-auth-method", false, + "Toggle for creating an auth method for components to use to fetch their ACL tokens.") + c.flags.DurationVar(&c.flagTimeout, "timeout", 10*time.Minute, "How long we'll try to bootstrap ACLs for before timing out, e.g. 1ms, 2s, 3m") c.flags.StringVar(&c.flagLogLevel, "log-level", "info", @@ -384,7 +390,7 @@ func (c *Command) Run(args []string) int { if c.flagEnablePartitions && c.flagPartitionName == consulDefaultPartition && isPrimary { // Partition token is local because only the Primary datacenter can have Admin Partitions. - err := c.createLocalACL("partitions", partitionRules, consulDC, isPrimary, consulClient) + err := c.createLocalACL("partitions", partitionRules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -440,6 +446,13 @@ func (c *Command) Run(args []string) int { } } + if c.flagCreateComponentAuthMethod { + err := c.configureComponentAuthMethod(consulClient) + if err != nil { + c.log.Error(err.Error()) + return 1 + } + } if c.flagCreateClientToken { agentRules, err := c.agentRules() if err != nil { @@ -447,7 +460,7 @@ func (c *Command) Run(args []string) int { return 1 } - err = c.createLocalACL("client", agentRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("client", agentRules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -486,9 +499,9 @@ func (c *Command) Run(args []string) int { // If namespaces are enabled, the policy and token needs to be global // to be allowed to create namespaces. if c.flagEnableNamespaces { - err = c.createGlobalACL("catalog-sync", syncRules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL("catalog-sync", syncRules, consulDC, isPrimary, true, consulClient) } else { - err = c.createLocalACL("catalog-sync", syncRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("catalog-sync", syncRules, consulDC, isPrimary, true, consulClient) } if err != nil { c.log.Error(err.Error()) @@ -497,7 +510,8 @@ func (c *Command) Run(args []string) int { } if c.flagCreateInjectToken { - err := c.configureConnectInjectAuthMethod(consulClient) + authMethodName := c.withPrefix("k8s-auth-method") + err := c.configureConnectInjectAuthMethod(consulClient, authMethodName, true) if err != nil { c.log.Error(err.Error()) return 1 @@ -513,9 +527,9 @@ func (c *Command) Run(args []string) int { // If namespaces are enabled, the policy and token need to be global // to be allowed to create namespaces. if c.flagEnableNamespaces { - err = c.createGlobalACL("connect-inject", injectRules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL("connect-inject", injectRules, consulDC, isPrimary, true, consulClient) } else { - err = c.createLocalACL("connect-inject", injectRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("connect-inject", injectRules, consulDC, isPrimary, true, consulClient) } if err != nil { @@ -527,9 +541,9 @@ func (c *Command) Run(args []string) int { if c.flagCreateEntLicenseToken { var err error if c.flagEnablePartitions { - err = c.createLocalACL("enterprise-license", entPartitionLicenseRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("enterprise-license", entPartitionLicenseRules, consulDC, isPrimary, true, consulClient) } else { - err = c.createLocalACL("enterprise-license", entLicenseRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("enterprise-license", entLicenseRules, consulDC, isPrimary, true, consulClient) } if err != nil { c.log.Error(err.Error()) @@ -538,7 +552,7 @@ func (c *Command) Run(args []string) int { } if c.flagCreateSnapshotAgentToken { - err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, isPrimary, consulClient) + err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -551,7 +565,7 @@ func (c *Command) Run(args []string) int { c.log.Error("Error templating api gateway rules", "err", err) return 1 } - err = c.createLocalACL("api-gateway-controller", apigwRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("api-gateway-controller", apigwRules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -567,7 +581,7 @@ func (c *Command) Run(args []string) int { // Mesh gateways require a global policy/token because they must // discover services in other datacenters. - err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -622,7 +636,7 @@ func (c *Command) Run(args []string) int { // the words "ingress-gateway". We need to create unique names for tokens // across all gateway types and so must suffix with `-ingress-gateway`. tokenName := fmt.Sprintf("%s-ingress-gateway", name) - err = c.createLocalACL(tokenName, ingressGatewayRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL(tokenName, ingressGatewayRules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -678,7 +692,7 @@ func (c *Command) Run(args []string) int { // the words "ingress-gateway". We need to create unique names for tokens // across all gateway types and so must suffix with `-terminating-gateway`. tokenName := fmt.Sprintf("%s-terminating-gateway", name) - err = c.createLocalACL(tokenName, terminatingGatewayRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL(tokenName, terminatingGatewayRules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -694,7 +708,7 @@ func (c *Command) Run(args []string) int { } // Policy must be global because it replicates from the primary DC // and so the primary DC needs to be able to accept the token. - err = c.createGlobalACL(common.ACLReplicationTokenName, rules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL(common.ACLReplicationTokenName, rules, consulDC, isPrimary, true, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -710,13 +724,32 @@ func (c *Command) Run(args []string) int { // Controller token must be global because config entry writes all // go to the primary datacenter. This means secondary datacenters need // a token that is known by the primary datacenters. - err = c.createGlobalACL("controller", rules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL("controller", rules, consulDC, isPrimary, false, consulClient) if err != nil { c.log.Error(err.Error()) return 1 } - } + policyName := "controller-token" + if c.flagFederation && !isPrimary { + // If performing ACL replication, we must ensure policy names are + // globally unique so we append the datacenter name but only in secondary datacenters.. + policyName += fmt.Sprintf("-%s", consulDC) + } + ap := &api.ACLRolePolicyLink{ + Name: policyName, + } + apl := []*api.ACLRolePolicyLink{} + apl = append(apl, ap) + + authMethodName := c.withPrefix("k8s-component-auth-method") + serviceAccountName := fmt.Sprintf("%s-controller", c.flagResourcePrefix) + err = c.addRoleAndBindingRule(consulClient, serviceAccountName, authMethodName, apl) + if err != nil { + c.log.Error(err.Error()) + return 1 + } + } c.log.Info("server-acl-init completed successfully") return 0 } @@ -852,6 +885,10 @@ func (c *Command) validateFlags() error { return errors.New("-resource-prefix must be set") } + if c.flagCreateControllerToken && !c.flagCreateComponentAuthMethod { + return errors.New("-create-component-auth-method is required with -create-controller-token") + } + // For the Consul node name to be discoverable via DNS, it must contain only // dashes and alphanumeric characters. Length is also constrained. // These restrictions match those defined in Consul's agent definition. @@ -882,6 +919,147 @@ func (c *Command) validateFlags() error { return nil } +// addRoleAndBindingRule adds a Role and Binding Rule which reference the authMethod. +func (c *Command) addRoleAndBindingRule(client *api.Client, serviceAccountName string, authMethodName string, policies []*api.ACLRolePolicyLink) error { + + // This is the ACL Role which will allow the component which uses the service account + // to be able to do a Consul Login. + aclRoleName := fmt.Sprintf("%s-acl-role", serviceAccountName) + role := &api.ACLRole{ + Name: aclRoleName, + Description: fmt.Sprintf("ACL Role for %s", serviceAccountName), + Policies: policies, + } + + err := c.updateOrCreateACLRole(client, role) + if err != nil { + c.log.Error("unable to update or create ACL Role", err) + return err + } + + // Create the binding rule, this ties the Policies defined in the Role to the service-account and authMethod. + abr := api.ACLBindingRule{ + Description: fmt.Sprintf("Binding Rule for %s", serviceAccountName), + AuthMethod: authMethodName, + Selector: fmt.Sprintf("serviceaccount.name==%q", serviceAccountName), + BindType: api.BindingRuleBindTypeRole, + BindName: aclRoleName, + } + + return c.updateOrCreateBindingRule(client, authMethodName, &abr, true) +} + +// updateOrCreateACLRole will query to see if existing role is in place and update them +// or create them if they do not yet exist. +func (c *Command) updateOrCreateACLRole(client *api.Client, role *api.ACLRole) error { + aclRoleList, _, err := client.ACL().RoleList(nil) + if err != nil { + c.log.Error("unable to read ACL Roles", err) + return err + } + for _, y := range aclRoleList { + if y.Name == role.Name { + role.ID = y.ID + _, _, err := client.ACL().RoleUpdate(role, &api.WriteOptions{}) + if err != nil { + c.log.Error("unable to update role", err) + return err + } + return nil + } + } + _, _, err = client.ACL().RoleCreate(role, &api.WriteOptions{}) + if err != nil { + c.log.Error("unable to create role", err) + return err + } + return nil +} + +// updateOrCreateBindingRule will query to see if existing binding rules are in place and update them +// or create them if they do not yet exist. +func (c *Command) updateOrCreateBindingRule(client *api.Client, authMethodName string, abr *api.ACLBindingRule, skipNamespacing bool) error { + // Binding rule list api call query options + queryOptions := api.QueryOptions{} + + // Add a namespace if appropriate + // If namespaces and mirroring are enabled, this is not necessary because + // the binding rule will fall back to being created in the Consul `default` + // namespace automatically, as is necessary for mirroring. + if !skipNamespacing && c.flagEnableNamespaces && !c.flagEnableInjectK8SNSMirroring { + abr.Namespace = c.flagConsulInjectDestinationNamespace + queryOptions.Namespace = c.flagConsulInjectDestinationNamespace + } + + var existingRules []*api.ACLBindingRule + err := c.untilSucceeds(fmt.Sprintf("listing binding rules for auth method %s", authMethodName), + func() error { + var err error + existingRules, _, err = client.ACL().BindingRuleList(authMethodName, &queryOptions) + return err + }) + if err != nil { + return err + } + + // If the binding rule already exists, update it + // This updates the binding rule any time the acl bootstrapping + // command is rerun, which is a bit of extra overhead, but is + // necessary to pick up any potential config changes. + if len(existingRules) > 0 { + // Find the policy that matches our name and description + // and that's the ID we need + for _, existingRule := range existingRules { + if existingRule.BindName == abr.BindName && existingRule.Description == abr.Description { + abr.ID = existingRule.ID + } + } + + // This will only happen if there are existing policies + // for this auth method, but none that match the binding + // rule set up here in the bootstrap method. + if abr.ID == "" { + return errors.New("unable to find a matching ACL binding rule to update") + } + + err = c.untilSucceeds(fmt.Sprintf("updating acl binding rule for %s", authMethodName), + func() error { + _, _, err := client.ACL().BindingRuleUpdate(abr, nil) + return err + }) + } else { + // Otherwise create the binding rule + err = c.untilSucceeds(fmt.Sprintf("creating acl binding rule for %s", authMethodName), + func() error { + _, _, err := client.ACL().BindingRuleCreate(abr, nil) + return err + }) + } + return err +} + +// configureConnectInject sets up auth methods so that connect injection will +// work. +func (c *Command) configureComponentAuthMethod(consulClient *api.Client) error { + // Create the auth method template. This requires calls to the + // kubernetes environment. + authMethodName := c.withPrefix("k8s-component-auth-method") + authMethodTmpl, err := c.createAuthMethodTmpl(authMethodName, false) + if err != nil { + return err + } + err = c.untilSucceeds(fmt.Sprintf("creating auth method %s", authMethodTmpl.Name), + func() error { + var err error + // `AuthMethodCreate` will also be able to update an existing + // AuthMethod based on the name provided. This means that any + // configuration changes will correctly update the AuthMethod. + _, _, err = consulClient.ACL().AuthMethodCreate(&authMethodTmpl, &api.WriteOptions{}) + return err + }) + return err +} + const consulDefaultNamespace = "default" const consulDefaultPartition = "default" const synopsis = "Initialize ACLs on Consul servers and other components." diff --git a/control-plane/subcommand/server-acl-init/command_ent_test.go b/control-plane/subcommand/server-acl-init/command_ent_test.go index 5824d4af9b..c16315ec49 100644 --- a/control-plane/subcommand/server-acl-init/command_ent_test.go +++ b/control-plane/subcommand/server-acl-init/command_ent_test.go @@ -286,6 +286,7 @@ func TestRun_ACLPolicyUpdates(t *testing.T) { "-terminating-gateway-name=gw", "-terminating-gateway-name=anothergw", "-create-controller-token", + "-create-component-auth-method", } // Our second run, we're going to update from partitions and namespaces disabled to // namespaces enabled with a single destination ns and partitions enabled. @@ -748,10 +749,10 @@ func TestRun_TokensWithNamespacesEnabled(t *testing.T) { LocalToken: false, }, "controller token": { - TokenFlags: []string{"-create-controller-token"}, + TokenFlags: []string{"-create-controller-token", "-create-component-auth-method"}, PolicyNames: []string{"controller-token"}, PolicyDCs: nil, - SecretNames: []string{resourcePrefix + "-controller-acl-token"}, + SecretNames: nil, LocalToken: false, }, "partitions token": { @@ -797,21 +798,28 @@ func TestRun_TokensWithNamespacesEnabled(t *testing.T) { }) require.NoError(err) + // Check that the expected policy was created. + policyNames := map[string]bool{} for i := range c.PolicyNames { policy := policyExists(t, c.PolicyNames[i], consul) require.Equal(c.PolicyDCs, policy.Datacenters) - + policyNames[policy.Name] = true + } + tokens := []string{} + for i := range c.SecretNames { // Test that the token was created as a Kubernetes Secret. tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), c.SecretNames[i], metav1.GetOptions{}) require.NoError(err) require.NotNil(tokenSecret) token, ok := tokenSecret.Data["token"] require.True(ok) - + tokens = append(tokens, string(token)) + } + for i := range tokens { // Test that the token has the expected policies in Consul. - tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) + tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: tokens[i]}) require.NoError(err) - require.Equal(c.PolicyNames[i], tokenData.Policies[0].Name) + require.True(policyNames[tokenData.Policies[0].Name]) require.Equal(c.LocalToken, tokenData.Local) } diff --git a/control-plane/subcommand/server-acl-init/command_test.go b/control-plane/subcommand/server-acl-init/command_test.go index da3cfc3de3..6ec0b7e36e 100644 --- a/control-plane/subcommand/server-acl-init/command_test.go +++ b/control-plane/subcommand/server-acl-init/command_test.go @@ -61,6 +61,14 @@ func TestRun_FlagValidation(t *testing.T) { Flags: []string{"-bootstrap-token-file=/notexist", "-server-address=localhost", "-resource-prefix=prefix"}, ExpErr: "Unable to read bootstrap token from file \"/notexist\": open /notexist: no such file or directory", }, + { + Flags: []string{ + "-server-address=localhost", + "-resource-prefix=prefix", + "-create-controller-token", + }, + ExpErr: "-create-component-auth-method is required with -create-controller-token", + }, { Flags: []string{ "-server-address=localhost", @@ -145,6 +153,45 @@ func TestRun_Defaults(t *testing.T) { // endpoint was called. } +// Test that the component auth method gets created. +func TestRun_ComponentAuthMethod(t *testing.T) { + t.Parallel() + + k8s, testSvr := completeSetup(t) + setUpK8sServiceAccount(t, k8s, ns) + defer testSvr.Stop() + require := require.New(t) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + cmdArgs := []string{ + "-timeout=1m", + "-k8s-namespace=" + ns, + "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], + "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], + "-resource-prefix=" + resourcePrefix, + "-create-component-auth-method"} + + responseCode := cmd.Run(cmdArgs) + require.Equal(0, responseCode, ui.ErrorWriter.String()) + + // Check that the expected policy was created. + bootToken := getBootToken(t, k8s, resourcePrefix, ns) + consulClient, err := api.NewClient(&api.Config{ + Address: testSvr.HTTPAddr, + Token: bootToken, + }) + require.NoError(err) + authMethod, _, err := consulClient.ACL().AuthMethodRead(resourcePrefix+"-k8s-component-auth-method", &api.QueryOptions{}) + require.NoError(err) + require.NotNil(authMethod) +} + // Test the different flags that should create tokens and save them as // Kubernetes secrets. func TestRun_TokensPrimaryDC(t *testing.T) { @@ -244,10 +291,10 @@ func TestRun_TokensPrimaryDC(t *testing.T) { }, { TestName: "Controller token", - TokenFlags: []string{"-create-controller-token"}, + TokenFlags: []string{"-create-controller-token", "-create-component-auth-method"}, PolicyNames: []string{"controller-token"}, PolicyDCs: nil, - SecretNames: []string{resourcePrefix + "-controller-acl-token"}, + SecretNames: nil, LocalToken: false, }, { @@ -292,21 +339,27 @@ func TestRun_TokensPrimaryDC(t *testing.T) { }) require.NoError(err) + policyNames := map[string]bool{} for i := range c.PolicyNames { policy := policyExists(t, c.PolicyNames[i], consul) require.Equal(c.PolicyDCs, policy.Datacenters) - + policyNames[policy.Name] = true + } + tokens := []string{} + for i := range c.SecretNames { // Test that the token was created as a Kubernetes Secret. tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), c.SecretNames[i], metav1.GetOptions{}) require.NoError(err) require.NotNil(tokenSecret) token, ok := tokenSecret.Data["token"] require.True(ok) - + tokens = append(tokens, string(token)) + } + for i := range tokens { // Test that the token has the expected policies in Consul. - tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) + tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: tokens[i]}) require.NoError(err) - require.Equal(c.PolicyNames[i], tokenData.Policies[0].Name) + require.True(policyNames[tokenData.Policies[0].Name]) require.Equal(c.LocalToken, tokenData.Local) } @@ -423,10 +476,10 @@ func TestRun_TokensReplicatedDC(t *testing.T) { }, { TestName: "Controller token", - TokenFlags: []string{"-create-controller-token"}, + TokenFlags: []string{"-create-component-auth-method", "-create-controller-token"}, PolicyNames: []string{"controller-token-dc2"}, PolicyDCs: nil, - SecretNames: []string{resourcePrefix + "-controller-acl-token"}, + SecretNames: nil, LocalToken: false, }, } @@ -435,7 +488,7 @@ func TestRun_TokensReplicatedDC(t *testing.T) { bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" tokenFile := common.WriteTempFile(t, bootToken) - k8s, consul, secondaryAddr, cleanup := mockReplicatedSetup(t, bootToken) + k8s, consul, secondaryAddr, _, sec, cleanup := mockReplicatedSetup(t, bootToken) setUpK8sServiceAccount(t, k8s, ns) defer cleanup() @@ -459,24 +512,34 @@ func TestRun_TokensReplicatedDC(t *testing.T) { responseCode := cmd.Run(cmdArgs) require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + // Sometimes it takes a moment for the policies to be settled when there + // is no kube secret being created. + sec.WaitForServiceIntentions(t) // Check that the expected policy was created. retry.Run(t, func(r *retry.R) { + policyNames := map[string]bool{} for i := range c.PolicyNames { - policy := policyExists(r, c.PolicyNames[i], consul) - require.Equal(r, c.PolicyDCs, policy.Datacenters) - + policy := policyExists(t, c.PolicyNames[i], consul) + require.Equal(t, c.PolicyDCs, policy.Datacenters) + policyNames[policy.Name] = true + } + tokens := []string{} + for i := range c.SecretNames { // Test that the token was created as a Kubernetes Secret. tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), c.SecretNames[i], metav1.GetOptions{}) - require.NoError(r, err) - require.NotNil(r, tokenSecret) + require.NoError(t, err) + require.NotNil(t, tokenSecret) token, ok := tokenSecret.Data["token"] - require.True(r, ok) - + require.True(t, ok) + tokens = append(tokens, string(token)) + } + time.Sleep(time.Second * 1) + for i := range tokens { // Test that the token has the expected policies in Consul. - tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) - require.NoError(r, err) - require.Equal(r, c.PolicyNames[i], tokenData.Policies[0].Name) - require.Equal(r, c.LocalToken, tokenData.Local) + tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: tokens[i]}) + require.NoError(t, err) + require.True(t, policyNames[tokenData.Policies[0].Name]) + require.Equal(t, c.LocalToken, tokenData.Local) } }) }) @@ -567,9 +630,9 @@ func TestRun_TokensWithProvidedBootstrapToken(t *testing.T) { }, { TestName: "Controller token", - TokenFlags: []string{"-create-controller-token"}, + TokenFlags: []string{"-create-controller-token", "-create-component-auth-method"}, PolicyNames: []string{"controller-token"}, - SecretNames: []string{resourcePrefix + "-controller-acl-token"}, + SecretNames: nil, }, } for _, c := range cases { @@ -607,20 +670,26 @@ func TestRun_TokensWithProvidedBootstrapToken(t *testing.T) { // Check that the expected policy was created. retry.Run(t, func(r *retry.R) { + policyNames := map[string]bool{} for i := range c.PolicyNames { - policyExists(r, c.PolicyNames[i], consul) - + policy := policyExists(t, c.PolicyNames[i], consul) + policyNames[policy.Name] = true + } + tokens := []string{} + for i := range c.SecretNames { // Test that the token was created as a Kubernetes Secret. tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), c.SecretNames[i], metav1.GetOptions{}) - require.NoError(r, err) - require.NotNil(r, tokenSecret) + require.NoError(t, err) + require.NotNil(t, tokenSecret) token, ok := tokenSecret.Data["token"] - require.True(r, ok) - + require.True(t, ok) + tokens = append(tokens, string(token)) + } + for i := range tokens { // Test that the token has the expected policies in Consul. - tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) - require.NoError(r, err) - require.Equal(r, c.PolicyNames[i], tokenData.Policies[0].Name) + tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: tokens[i]}) + require.NoError(t, err) + require.True(t, policyNames[tokenData.Policies[0].Name]) } }) }) @@ -688,7 +757,7 @@ func TestRun_AnonymousTokenPolicy(t *testing.T) { if c.SecondaryDC { var cleanup func() bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" - k8s, consul, consulHTTPAddr, cleanup = mockReplicatedSetup(t, bootToken) + k8s, consul, consulHTTPAddr, _, _, cleanup = mockReplicatedSetup(t, bootToken) defer cleanup() tmp, err := ioutil.TempFile("", "") @@ -1913,7 +1982,7 @@ func TestRun_HTTPS(t *testing.T) { func TestRun_ACLReplicationTokenValid(t *testing.T) { t.Parallel() - secondaryK8s, secondaryConsulClient, secondaryAddr, aclReplicationToken, clean := completeReplicatedSetup(t) + secondaryK8s, secondaryConsulClient, secondaryAddr, aclReplicationToken, _, _, clean := completeReplicatedSetup(t) defer clean() // completeReplicatedSetup ran the command in our primary dc so now we @@ -1969,7 +2038,7 @@ func TestRun_AnonPolicy_IgnoredWithReplication(t *testing.T) { t.Run(flag, func(t *testing.T) { bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" tokenFile := common.WriteTempFile(t, bootToken) - k8s, consul, serverAddr, cleanup := mockReplicatedSetup(t, bootToken) + k8s, consul, serverAddr, _, _, cleanup := mockReplicatedSetup(t, bootToken) setUpK8sServiceAccount(t, k8s, ns) defer cleanup() @@ -2146,7 +2215,7 @@ func completeBootstrappedSetup(t *testing.T, masterToken string) (*fake.Clientse // the address of the secondary Consul server, // the replication token generated and a cleanup function // that should be called at the end of the test that cleans up resources. -func completeReplicatedSetup(t *testing.T) (*fake.Clientset, *api.Client, string, string, func()) { +func completeReplicatedSetup(t *testing.T) (*fake.Clientset, *api.Client, string, string, *testutil.TestServer, *testutil.TestServer, func()) { return replicatedSetup(t, "") } @@ -2159,9 +2228,9 @@ func completeReplicatedSetup(t *testing.T) (*fake.Clientset, *api.Client, string // the address of the secondary Consul server, and a // cleanup function that should be called at the end of the test that cleans // up resources. -func mockReplicatedSetup(t *testing.T, bootToken string) (*fake.Clientset, *api.Client, string, func()) { - k8sClient, consulClient, serverAddr, _, cleanup := replicatedSetup(t, bootToken) - return k8sClient, consulClient, serverAddr, cleanup +func mockReplicatedSetup(t *testing.T, bootToken string) (*fake.Clientset, *api.Client, string, *testutil.TestServer, *testutil.TestServer, func()) { + k8sClient, consulClient, serverAddr, _, primary, secondary, cleanup := replicatedSetup(t, bootToken) + return k8sClient, consulClient, serverAddr, primary, secondary, cleanup } // replicatedSetup is a helper function for completeReplicatedSetup and @@ -2172,7 +2241,7 @@ func mockReplicatedSetup(t *testing.T, bootToken string) (*fake.Clientset, *api. // the address of the secondary Consul server, ACL replication token, and a // cleanup function that should be called at the end of the test that cleans // up resources. -func replicatedSetup(t *testing.T, bootToken string) (*fake.Clientset, *api.Client, string, string, func()) { +func replicatedSetup(t *testing.T, bootToken string) (*fake.Clientset, *api.Client, string, string, *testutil.TestServer, *testutil.TestServer, func()) { primarySvr, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { c.ACL.Enabled = true if bootToken != "" { @@ -2264,7 +2333,7 @@ func replicatedSetup(t *testing.T, bootToken string) (*fake.Clientset, *api.Clie // Finally, set up our kube cluster. It will use the secondary dc. k8s := fake.NewSimpleClientset() - return k8s, consul, secondarySvr.HTTPAddr, aclReplicationToken, func() { + return k8s, consul, secondarySvr.HTTPAddr, aclReplicationToken, primarySvr, secondarySvr, func() { primarySvr.Stop() secondarySvr.Stop() } diff --git a/control-plane/subcommand/server-acl-init/connect_inject.go b/control-plane/subcommand/server-acl-init/connect_inject.go index abd10f9f7f..8ae57508ef 100644 --- a/control-plane/subcommand/server-acl-init/connect_inject.go +++ b/control-plane/subcommand/server-acl-init/connect_inject.go @@ -1,10 +1,9 @@ package serveraclinit import ( - "errors" "fmt" - "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/hashicorp/consul/api" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,13 +17,11 @@ const defaultKubernetesHost = "https://kubernetes.default.svc" // configureConnectInject sets up auth methods so that connect injection will // work. -func (c *Command) configureConnectInjectAuthMethod(consulClient *api.Client) error { - - authMethodName := c.withPrefix("k8s-auth-method") +func (c *Command) configureConnectInjectAuthMethod(consulClient *api.Client, authMethodName string, createInjectBindingRule bool) error { // Create the auth method template. This requires calls to the // kubernetes environment. - authMethodTmpl, err := c.createAuthMethodTmpl(authMethodName) + authMethodTmpl, err := c.createAuthMethodTmpl(authMethodName, true) if err != nil { return err } @@ -68,6 +65,7 @@ func (c *Command) configureConnectInjectAuthMethod(consulClient *api.Client) err return err } + c.log.Info("creating inject binding rule") // Create the binding rule. abr := api.ACLBindingRule{ Description: "Kubernetes binding rule", @@ -76,67 +74,10 @@ func (c *Command) configureConnectInjectAuthMethod(consulClient *api.Client) err BindName: "${serviceaccount.name}", Selector: c.flagBindingRuleSelector, } - - // Binding rule list api call query options - queryOptions := api.QueryOptions{} - - // Add a namespace if appropriate - // If namespaces and mirroring are enabled, this is not necessary because - // the binding rule will fall back to being created in the Consul `default` - // namespace automatically, as is necessary for mirroring. - if c.flagEnableNamespaces && !c.flagEnableInjectK8SNSMirroring { - abr.Namespace = c.flagConsulInjectDestinationNamespace - queryOptions.Namespace = c.flagConsulInjectDestinationNamespace - } - - var existingRules []*api.ACLBindingRule - err = c.untilSucceeds(fmt.Sprintf("listing binding rules for auth method %s", authMethodName), - func() error { - var err error - existingRules, _, err = consulClient.ACL().BindingRuleList(authMethodName, &queryOptions) - return err - }) - if err != nil { - return err - } - - // If the binding rule already exists, update it - // This updates the binding rule any time the acl bootstrapping - // command is rerun, which is a bit of extra overhead, but is - // necessary to pick up any potential config changes. - if len(existingRules) > 0 { - // Find the policy that matches our name and description - // and that's the ID we need - for _, existingRule := range existingRules { - if existingRule.BindName == abr.BindName && existingRule.Description == abr.Description { - abr.ID = existingRule.ID - } - } - - // This will only happen if there are existing policies - // for this auth method, but none that match the binding - // rule set up here in the bootstrap method. - if abr.ID == "" { - return errors.New("unable to find a matching ACL binding rule to update") - } - - err = c.untilSucceeds(fmt.Sprintf("updating acl binding rule for %s", authMethodName), - func() error { - _, _, err := consulClient.ACL().BindingRuleUpdate(&abr, nil) - return err - }) - } else { - // Otherwise create the binding rule - err = c.untilSucceeds(fmt.Sprintf("creating acl binding rule for %s", authMethodName), - func() error { - _, _, err := consulClient.ACL().BindingRuleCreate(&abr, nil) - return err - }) - } - return err + return c.updateOrCreateBindingRule(consulClient, authMethodName, &abr, false) } -func (c *Command) createAuthMethodTmpl(authMethodName string) (api.ACLAuthMethod, error) { +func (c *Command) createAuthMethodTmpl(authMethodName string, useNS bool) (api.ACLAuthMethod, error) { // Get the Secret name for the auth method ServiceAccount. var authMethodServiceAccount *apiv1.ServiceAccount saName := c.withPrefix("connect-injector") @@ -197,8 +138,9 @@ func (c *Command) createAuthMethodTmpl(authMethodName string) (api.ACLAuthMethod }, } + // TODO: find out what MapNS/etc do here. // Add options for mirroring namespaces - if c.flagEnableNamespaces && c.flagEnableInjectK8SNSMirroring { + if useNS && c.flagEnableNamespaces && c.flagEnableInjectK8SNSMirroring { authMethodTmpl.Config["MapNamespaces"] = true authMethodTmpl.Config["ConsulNamespacePrefix"] = c.flagInjectK8SNSMirroringPrefix } diff --git a/control-plane/subcommand/server-acl-init/connect_inject_test.go b/control-plane/subcommand/server-acl-init/connect_inject_test.go index a17d635bc1..e9c38595a0 100644 --- a/control-plane/subcommand/server-acl-init/connect_inject_test.go +++ b/control-plane/subcommand/server-acl-init/connect_inject_test.go @@ -64,6 +64,6 @@ func TestCommand_createAuthMethodTmpl_SecretNotFound(t *testing.T) { _, err := k8s.CoreV1().Secrets(ns).Create(ctx, secret, metav1.CreateOptions{}) require.NoError(t, err) - _, err = cmd.createAuthMethodTmpl("test") + _, err = cmd.createAuthMethodTmpl("test", true) require.EqualError(t, err, "found no secret of type 'kubernetes.io/service-account-token' associated with the release-name-consul-connect-injector service account") } diff --git a/control-plane/subcommand/server-acl-init/create_or_update.go b/control-plane/subcommand/server-acl-init/create_or_update.go index a17d78c6e3..fa0f063ee0 100644 --- a/control-plane/subcommand/server-acl-init/create_or_update.go +++ b/control-plane/subcommand/server-acl-init/create_or_update.go @@ -12,22 +12,22 @@ import ( // createLocalACL creates a policy and acl token for this dc (datacenter), i.e. // the policy is only valid for this datacenter and the token is a local token. -func (c *Command) createLocalACL(name, rules, dc string, isPrimary bool, consulClient *api.Client) error { - return c.createACL(name, rules, true, dc, isPrimary, consulClient) +func (c *Command) createLocalACL(name, rules, dc string, isPrimary bool, createKubeSecret bool, consulClient *api.Client) error { + return c.createACL(name, rules, true, dc, isPrimary, createKubeSecret, consulClient) } // createGlobalACL creates a global policy and acl token. The policy is valid // for all datacenters and the token is global. dc must be passed because the // policy name may have the datacenter name appended. -func (c *Command) createGlobalACL(name, rules, dc string, isPrimary bool, consulClient *api.Client) error { - return c.createACL(name, rules, false, dc, isPrimary, consulClient) +func (c *Command) createGlobalACL(name, rules, dc string, isPrimary bool, createKubeSecret bool, consulClient *api.Client) error { + return c.createACL(name, rules, false, dc, isPrimary, createKubeSecret, consulClient) } // createACL creates a policy with rules and name. If localToken is true then // the token will be a local token and the policy will be scoped to only dc. // If localToken is false, the policy will be global. // The token will be written to a Kubernetes secret. -func (c *Command) createACL(name, rules string, localToken bool, dc string, isPrimary bool, consulClient *api.Client) error { +func (c *Command) createACL(name, rules string, localToken bool, dc string, isPrimary bool, createKubeSecret bool, consulClient *api.Client) error { // Create policy with the given rules. policyName := fmt.Sprintf("%s-token", name) if c.flagFederation && !isPrimary { @@ -53,6 +53,10 @@ func (c *Command) createACL(name, rules string, localToken bool, dc string, isPr return err } + if !createKubeSecret { + c.log.Info(fmt.Sprintf("skipping creating kube secret for %s", policyName)) + return err + } // Check if the secret already exists, if so, we assume the ACL has already been // created and return. secretName := c.withPrefix(name + "-acl-token")