Skip to content

vm broker ‐ k8s ‐ tenants ‐ kes vault (TLS) ‐ helm only

Jump to bottom
Allan Roger Reid edited this page Feb 7, 2024 · 1 revision

Install Hashicorp Vault with k8s generated TLS and CA for k8s bases MinIO tenant and KES

loginctl enable-linger ubuntu

Install k3s

sudo touch /dev/kmsg
curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" sh -s - --snapshotter=fuse-overlayfs
systemctl is-active k3s
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml

Install Helm

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh

helm repo add hashicorp https://helm.releases.hashicorp.com
helm search repo vault

Setup standalone Hashicorp Vault with TLS

https://developer.hashicorp.com/vault/docs/platform/k8s/helm/examples/standalone-tls

# SERVICE is the name of the Vault service in kubernetes.
# It does not have to match the actual running service, though it may help for consistency.
export SERVICE=vault-internal

# NAMESPACE where the Vault service is running.
export NAMESPACE=hashicorp-vault

# SECRET_NAME to create in the kubernetes secrets store.
export SECRET_NAME=vault-server-tls

# TMPDIR is a temporary working directory.
export TMPDIR=/tmp

# CSR_NAME will be the name of our certificate signing request as seen by kubernetes.
export CSR_NAME=vault-csr

Create the namespace.

kubectl create namespace ${NAMESPACE}

Create a key for Kubernetes to sign.

openssl genrsa -out ${TMPDIR}/vault.key 2048

Create a Certificate Signing Request (CSR).

cat <<EOF >${TMPDIR}/csr.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = *.${SERVICE}
DNS.2 = *.${SERVICE}.${NAMESPACE}
DNS.3 = *.${SERVICE}.${NAMESPACE}.svc
DNS.4 = *.${SERVICE}.${NAMESPACE}.svc.cluster.local
IP.1 = 127.0.0.1
EOF

openssl req -new \
            -key ${TMPDIR}/vault.key \
            -subj "/CN=system:node:${SERVICE}.${NAMESPACE}.svc;/O=system:nodes" \
            -out ${TMPDIR}/server.csr \
            -config ${TMPDIR}/csr.conf

Create the certificate

kubectl get serviceaccount
kubectl delete secret/service-account-vault
cat <<EOF >${TMPDIR}/service-account-token.yaml
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
  name: service-account-vault
  annotations:
    kubernetes.io/service-account.name: default
EOF
kubectl create -f ${TMPDIR}/service-account-token.yaml

cat <<EOF >${TMPDIR}/csr.yaml
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: ${CSR_NAME}
spec:
  signerName: kubernetes.io/kubelet-serving
  groups:
  - system:authenticated
  request: $(base64 ${TMPDIR}/server.csr | tr -d '\n')
  signerName: kubernetes.io/kubelet-serving
  usages:
  - digital signature
  - key encipherment
  - server auth
EOF

Send the CSR to Kubernetes. Approve the CSR in Kubernetes. Verify that the certificate was approved and issued.

kubectl create -f ${TMPDIR}/csr.yaml
kubectl certificate approve ${CSR_NAME}
kubectl get csr ${CSR_NAME}

Retrieve the certificate. Write the certificate out to a file.

serverCert=$(kubectl get csr ${CSR_NAME} -o jsonpath='{.status.certificate}')
echo "${serverCert}" | openssl base64 -d -A -out ${TMPDIR}/vault.crt
kubectl get secret \
  -o jsonpath="{.items[?(@.type==\"kubernetes.io/service-account-token\")].data['ca\.crt']}" \
  | base64 --decode > ${TMPDIR}/vault.ca

Retrieve Kubernetes CA. Store the key, cert, and Kubernetes CA into Kubernetes secrets.

kubectl create secret generic ${SECRET_NAME} \
    --namespace ${NAMESPACE} \
    --from-file=vault.key=${TMPDIR}/vault.key \
    --from-file=vault.crt=${TMPDIR}/vault.crt \
    --from-file=vault.ca=${TMPDIR}/vault.ca

Create a custom helm configuration

vi custom-values.yaml
#
global:
  enabled: true
  tlsDisable: false

server:
  extraEnvironmentVars:
    VAULT_CACERT: /vault/userconfig/vault-server-tls/vault.ca

  volumes:
    - name: userconfig-vault-server-tls
      secret:
        defaultMode: 420
        secretName: vault-server-tls # Matches the ${SECRET_NAME} from above

  volumeMounts:
    - mountPath: /vault/userconfig/vault-server-tls
      name: userconfig-vault-server-tls
      readOnly: true

  standalone:
    enabled: true
    config: |
      listener "tcp" {
        address = "[::]:8200"
        cluster_address = "[::]:8201"
        tls_cert_file = "/vault/userconfig/vault-server-tls/vault.crt"
        tls_key_file  = "/vault/userconfig/vault-server-tls/vault.key"
        tls_client_ca_file = "/vault/userconfig/vault-server-tls/vault.ca"
      }

      storage "file" {
        path = "/vault/data"
      }

Install vault with helm

helm install vault hashicorp/vault --namespace ${NAMESPACE} -f custom-values.yaml

Observe pods up

kubectl --namespace ${NAMESPACE} get pods
kubectl --namespace ${NAMESPACE} describe pods/vault-0
kubectl --namespace ${NAMESPACE} logs pods/vault-0

Unseal and init vault (within vault-0 pod)

kubectl -n ${NAMESPACE} exec -i -t pod/vault-0 -- /bin/sh

export VAULT_ADDR='https://127.0.0.1:8200'
vault operator init
###
Unseal Key 1: XVTOZVDa+oJ/CV2j38IARFdUnabvzg1ADLgd9HuZh6BA
Unseal Key 2: KmfA8ms3lDRTvsCYIM7lgxokGy8XVm0owWh/0fNHUA2T
Unseal Key 3: Kogd33QkEArNJC3GTHrJGN+zHPbv9cSojYNteiPCccUn
Unseal Key 4: DXXNE3m4fdxCD0TCaiTXCwYiR9mulwzHUSjj0nhbkNRx
Unseal Key 5: b5b6FhX+Y3c+8AR2GETCpMwzJMDCbbF8PNEnqFlm4nyv

Initial Root Token: hvs.dw0PrJlF4Ikh5BpweHRVHyUj
###

Set VAULT_TOKEN (within vault-0 pod)

export VAULT_TOKEN=hvs.dw0PrJlF4Ikh5BpweHRVHyUj

Unseal Vault Server (within vault-0 pod)

vault status
vault operator unseal XVTOZVDa+oJ/CV2j38IARFdUnabvzg1ADLgd9HuZh6BA
vault operator unseal KmfA8ms3lDRTvsCYIM7lgxokGy8XVm0owWh/0fNHUA2T
vault operator unseal Kogd33QkEArNJC3GTHrJGN+zHPbv9cSojYNteiPCccUn
vault status

Configure vault (within vault-0 pod)

export VAULT_FORMAT="json"
vault auth disable approle
vault auth enable approle
vault secrets disable kv
vault secrets enable -version=1 kv
cat << EOF > /tmp/kes-policy.hcl
path "kv/*" {
     capabilities = [ "create", "read", "update", "patch", "delete", "list" ]
}
EOF
vault policy write kes-policy /tmp/kes-policy.hcl
vault write auth/approle/role/kes-role token_num_uses=0 secret_id_num_uses=0 period=5m policies=kes-policy

export VAULT_ROLE_ID=$(vault read auth/approle/role/kes-role/role-id | grep -o '"role_id": "[^"]*' | grep -o '[^"]*$')
echo "${VAULT_ROLE_ID}" > /tmp/VAULT_ROLE_ID.var

export VAULT_SECRET_ID=$(vault write -f auth/approle/role/kes-role/secret-id | grep -o '"secret_id": "[^"]*' | grep -o '[^"]*$')
echo "${VAULT_SECRET_ID}" > /tmp/VAULT_SECRET_ID.var
exit

Obtain vault credentials

export VAULT_ROLE_ID=$(kubectl --namespace=hashicorp-vault exec -it vault-0 --container vault -- /bin/sh -c "cat /tmp/VAULT_ROLE_ID.var | tr -d '\n'")
export VAULT_SECRET_ID=$(kubectl --namespace=hashicorp-vault exec -it vault-0 --container vault -- /bin/sh -c "cat /tmp/VAULT_SECRET_ID.var | tr -d '\n'")
echo $VAULT_ROLE_ID
echo $VAULT_SECRET_ID

Install operator using helm

helm repo add minio-operator https://operator.min.io
helm search repo minio-operator
helm install \
  --namespace minio-operator \
  --create-namespace \
  operator minio-operator/operator

Install tenant using helm

helm install \
  --namespace tenant-kms-encrypted \
  --create-namespace \
  myminio minio-operator/tenant

Tweak tenant size and storeclass/pvcs (due to restrictions on my end)

kubectl patch tenant -n tenant-kms-encrypted myminio --type='merge' -p '{"spec":{"pools":[{"name": "pool-0", "servers": '4', "volumesPerServer": 1, "volumeClaimTemplate": {"apiVersion": "v1", "metadata": {"name": "data"}, "spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}}, "storageClassName": "local-path"}}}]}}'

Check pod health

kubectl -n tenant-kms-encrypted get tenant/myminio
kubectl -n tenant-kms-encrypted get pods
kubectl -n tenant-kms-encrypted logs pod/myminio-pool-0-0

Create kes configuration secret

cat << EOF > ~/kes-configuration-secret.yml apiVersion: v1 kind: Secret metadata: name: kes-configuration namespace: tenant-kms-encrypted type: Opaque stringData: server-config.yaml: |- version: v1 address: :7373 admin: identity: ${MINIO_KES_IDENTITY} tls: key: /tmp/kes/server.key cert: /tmp/kes/server.crt proxy: identities: [] header: cert: X-Tls-Client-Cert policy: my-policy: allow: - /v1/api - /v1/key/create/* - /v1/key/generate/* - /v1/key/decrypt/* - /v1/key/bulk/decrypt/* - /v1/key/list/* - /v1/status identities: cache: expiry: any: 5m0s unused: 20s log: error: on audit: off keystore: vault: version: "v1" endpoint: "https://vault-0.vault-internal.hashicorp-vault.svc.cluster.local:8200" namespace: "" prefix: "my-minio" approle: id: "${VAULT_ROLE_ID}" secret: "${VAULT_SECRET_ID}" retry: 15s status: ping: 10s tls: ca: "/tmp/kes/vault-ca.crt" EOF

kubectl --namespace tenant-kms-encrypted apply -f ./kes-configuration-secret.yml


#### Create secret for /tmp/vault-ca.crt

cat ${TMPDIR}/vault.ca | base64 -w0 cat < vault-ca-secret.yml apiVersion: v1 kind: Secret type: Opaque metadata: name: vault-ca namespace: tenant-kms-encrypted data: vault-ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTURjek16QTRNamt3SGhjTk1qUXdNakEzTVRnek16UTVXaGNOTXpRd01qQTBNVGd6TXpRNQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTURjek16QTRNamt3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFSTFI3QW9LMy9uSGJYWnFhbHhCMDdQcisramlFVzI0QmcwandkZUEwdHkKSmdRR3lod01aMUl2dFErQ0E3T21lNWFUMmdTVnpKNFJQc1k5Sk9uSVhodXZvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXdNbUw3OVlUajliSGRhYjFERDRiClQvSXZ1bTh3Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQUpqWStPWjZLN01jSHY1aEp3cytBUUM3MlNPRktqT1EKaVNwTk5RRWFmSjM0QWlFQXdhOUgxbUd5eDE5WGY4SDJ4bGNFc2RVaUNqOTNXY2xIc3g5OWVhOHA1dTQ9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K EOF

kubectl --namespace tenant-kms-encrypted apply -f ./vault-ca-secret.yml


#### Edit tenant add kes section

kubectl -n tenant-kms-encrypted edit tenant/myminio

apiVersion: minio.min.io/v2 kind: Tenant metadata: name: myminio namespace: tenant-kms-encrypted spec: ... kes: replicas: 2 kesSecret: name: kes-configuration imagePullPolicy: "IfNotPresent" externalCertSecret: null clientCertSecret: name: vault-ca type: Opaque securityContext: runAsUser: 1000 runAsGroup: 1000 runAsNonRoot: true fsGroup: 1000

Check pod health

kubectl -n tenant-kms-encrypted get tenant/myminio
kubectl -n tenant-kms-encrypted get pods
kubectl -n tenant-kms-encrypted logs pod/myminio-kes-0

Access minio

export NODEPORT_HTTP=31092
export NODEPORT_HTTPS=30045

Create a NodePort and access the operator/tenant

kubectl patch service -n tenant-kms-encrypted myminio-console -p '{"spec":{"ports":[{"name": "http-console","port": 9090,"protocol": "TCP","nodePort":'${NODEPORT_HTTP}'},{"name": "https-console","port": 9443,"protocol": "TCP","nodePort":'${NODEPORT_HTTPS}'}],"type": "NodePort"}}'

Login to tenant and test encryption

Extra validation

Install mc and validate

mkdir -p ~/mc && cd ~/mc && rm -rf mc* && wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x mc && cd ~

Add a validating port forward

kubectl port-forward svc/myminio-hl 9000:9000 -n tenant-kms-encrypted &
mc/mc alias set kes-demo https://127.0.0.1:9000 minio minio123 --insecure

Test KMS

mc/mc admin kms key status kes-demo --insecure

Get jwt/Access tenant

SA_TOKEN=$(kubectl -n minio-operator get secret console-sa-secret -o jsonpath="{.data.token}" | base64 --decode) && echo $SA_TOKEN
echo "Access the minio tenant console from https://$(hostname).lab.min.dev:${NODEPORT_HTTPS} using username and password"
Clone this wiki locally