Skip to content

Install stateful KES in k8s

Jump to bottom
Cesar Celis Hernandez edited this page Oct 5, 2022 · 36 revisions

Objective:

Install stateful KES in k8s

Inspired from:

Steps:

  • Delete previous cluster
kind delete clusters kind
  • Create new cluster
kind create cluster --config ~/operator/testing/kind-config.yaml
  • Deploy Operator:
kubectl apply -k github.com/minio/operator/
  • Deploy Tenant
kubectl apply -k ~/operator/examples/kustomization/tenant-lite

  • Create and configure a pod for kes:

    • Create an Ubuntu Pod for KES inside the tenant namespace:
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: ubuntu
  namespace: tenant-lite
  labels:
    app: ubuntu
spec:
  containers:
  - image: ubuntu
    command:
      - "sleep"
      - "604800"
    imagePullPolicy: IfNotPresent
    name: ubuntu
  restartPolicy: Always
EOF
  • Install KES and mc in the Ubuntu Pod:
apt update
apt install wget
apt install vim
wget https://github.com/minio/kes/releases/latest/download/kes-linux-amd64
mv kes-linux-amd64 kes
chmod +x kes
mv kes /usr/local/bin/kes
wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x mc
mv mc /usr/local/bin/mc
  • Create a persistent directory for KES and its configuration file inside the ubuntu pod where kes is located:
rm -rf ~/kes
mkdir ~/kes
cd ~/kes
touch init.yml
  • Create identities:
kes identity new --key sys-admin.key --cert sys-admin.crt kes-sys-admin
kes identity new --key minio-admin.key --cert minio-admin.crt minio-admin
kes identity new --key minio.key --cert minio.crt minio
kes identity new --ip "IP-ADDRESS-OF-THE-POD" localhost # Example: 10.244.2.7
kes identity new --ip "10.244.2.7" localhost # Example: 10.244.2.7
  • Create KES unseal key:
cat /dev/urandom | head -c 32 | base64 # put the result in the .bashrc
vi ~/.bashrc
export KES_UNSEAL_KEY=<VALUE-FROM-ABOVE-COMMAND>
source ~/.bashrc
echo $KES_UNSEAL_KEY # it should print the value
  • Edit/Create KES config file:
cd ~/kes
echo "address: 0.0.0.0:7373" > ~/kes/init.yml
echo "" >> ~/kes/init.yml
echo "tls:" >> ~/kes/init.yml
echo "  key: private.key" >> ~/kes/init.yml
echo "  cert: public.crt" >> ~/kes/init.yml
echo "  client:" >> ~/kes/init.yml
echo "    verify_cert: false" >> ~/kes/init.yml
echo "" >> ~/kes/init.yml
echo "system:" >> ~/kes/init.yml
echo "  admin:" >> ~/kes/init.yml
echo "    identity: $(kes identity of sys-admin.crt)" >> ~/kes/init.yml
echo "" >> ~/kes/init.yml
echo "unseal:" >> ~/kes/init.yml
echo "  environment:" >> ~/kes/init.yml
echo "    name: KES_UNSEAL_KEY" >> ~/kes/init.yml
echo "" >> ~/kes/init.yml
echo "enclave:" >> ~/kes/init.yml
echo "  default:" >> ~/kes/init.yml
echo "    admin:" >> ~/kes/init.yml
echo "      identity: $(kes identity of minio-admin.crt)" >> ~/kes/init.yml
echo "    policy:" >> ~/kes/init.yml
echo "      minio:" >> ~/kes/init.yml
echo "        allow:" >> ~/kes/init.yml
echo "        - /v1/api" >> ~/kes/init.yml
echo "        - /v1/log/audit" >> ~/kes/init.yml
echo "        - /v1/log/error" >> ~/kes/init.yml
echo "        - /v1/key/create/*" >> ~/kes/init.yml
echo "        - /v1/key/generate/*" >> ~/kes/init.yml
echo "        - /v1/key/decrypt/*" >> ~/kes/init.yml
echo "        - /v1/key/bulk/decrypt/*" >> ~/kes/init.yml
  • Initialize KES deployment
cd ~/kes # where init.yml is saved
kes init --config init.yml ~/kes/data
  • Start KES server
kes server ~/kes/data
  • In Ubuntu Pod Terminal where KES is located: Assign MinIO identity to MinIO policy:
cd ~/kes
export KES_SERVER=https://127.0.0.1:7373
export KES_CLIENT_KEY=minio-admin.key
export KES_CLIENT_CERT=minio-admin.crt
kes policy assign -k minio $(kes identity of minio.crt)

  • MinIO Server Setup:

  • Created the secret:

# /Users/cniackz/minio/private.key is ~/kes/minio.key
# /Users/cniackz/minio/public.crt is ~/kes/minio.crt
kubectl create secret generic kes-minio -n tenant-lite --from-file=/Users/cniackz/minio/private.key --from-file=/Users/cniackz/minio/public.crt

# /Users/cniackz/minio/private.key is ~/kes/minio.key
# /Users/cniackz/minio/public.crt is ~/kes/public.crt
kubectl create secret generic kes-minio-public -n tenant-lite --from-file=/Users/cniackz/minio/private.key --from-file=/Users/cniackz/minio/public.crt

k edit tenant -n tenant-lite

apiVersion: minio.min.io/v2
kind: Tenant
metadata:
  name: storage
  namespace: minio-tenant
spec:
  # externalClientCertSecrets is to share the secret with the MinIO Pods:
  # Under: /tmp/certs/client-0 You will find:
  # client.crt and client.key
  # And we can use these files to setup KES in k8s
  externalClientCertSecrets:
  - name: kes-minio
    type: Opaque
  - name: kes-minio-public
    type: Opaque
  env:
    # Set MINIO_KMS_KES_ENDPOINT
    # It is the IP of the Ubuntu Pod.
    - name: MINIO_KMS_KES_ENDPOINT
      value: "https://10.244.2.7:7373"
    # Set MinIO Client Credentials, it comes from kes-minio secret
    - name: MINIO_KMS_KES_CERT_FILE
      value: "/tmp/certs/client-0/client.crt"
    # Set MinIO Client Credentials, it comes from kes-minio secret
    - name: MINIO_KMS_KES_KEY_FILE
      value: "/tmp/certs/client-0/client.key"
    # Set MinIO Default Key
    - name: MINIO_KMS_KES_KEY_NAME
      value: "minio-default-key"
    # Trust the KES Server Certificate, it comes from kes-minio-public secret
    - name: MINIO_KMS_KES_CAPATH
      value: "/tmp/certs/client-1/client.crt"
    # Root User
    - name: MINIO_ROOT_USER
      value: minio
    # ROOT Password:
    - name: MINIO_ROOT_PASSWORD
      value: minio123

mc alias set myminio https://minio.tenant-lite.svc.cluster.local:443 minio minio123
mc rb myminio/my-bucket --force # remove previous bucket to start fresh
mc mb myminio/my-bucket # create new bucket
mc admin kms key create myminio minio-my-bucket # create key
mc encrypt set sse-kms minio-my-bucket myminio/my-bucket # encrypt bucket

Result:

root@ubuntu:/# mc alias set myminio https://minio.tenant-lite.svc.cluster.local:443 minio minio123
Added `myminio` successfully.
root@ubuntu:/# mc rb myminio/my-bucket --force # remove previous bucket to start fresh
mc: <ERROR> Unable to validate target `myminio/my-bucket`. Bucket `my-bucket` does not exist.
root@ubuntu:/# mc mb myminio/my-bucket # create new bucket
Bucket created successfully `myminio/my-bucket`.
root@ubuntu:/# mc admin kms key create myminio minio-my-bucket # create key
Created master key `minio-my-bucket` successfully
root@ubuntu:/# mc encrypt set sse-kms minio-my-bucket myminio/my-bucket # encrypt bucket
Auto encryption configuration has been set successfully for myminio/my-bucket
root@ubuntu:/#
Clone this wiki locally