API-1701: [PoC] Encryption with KMS

.gitleaks.toml

@@ -0,0 +1,7 @@
+[allowlist]
+  description = "Global Allowlist"
+
+  paths = [
+    '''hack-kms\/''',
+  ]
+

bindata/assets/kms/job-pod.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: aws-kms-setup
+  namespace: kube-system
+  labels:
+    name: aws-kms-setup
+spec:
+  containers:
+  - name: aws-kms-setup
+    env:
+    - name: AWS_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: aws-creds
+          key: aws_access_key_id
+    - name: AWS_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: aws-creds
+          key: aws_secret_access_key
+    - name: AWS_DEFAULT_REGION
+      value: "{{.AWSRegion}}"
+    - name: OPENSHIFT_INFRA_NAME
+      value: "{{.OpenShiftInfraId}}"
+    image: public.ecr.aws/aws-cli/aws-cli
+    command:
+    - "bash"
+    args:
+    - "/var/src/kms/setup-kms.sh"
+    # command:
+    # - "sleep"
+    # args:
+    # - "600"
+    resources:
+      limits:
+        memory: "64Mi"
+        cpu: "300m"
+    volumeMounts:
+    - name: kms-script
+      mountPath: /var/src/kms
+  volumes:
+  - name: kms-script
+    configMap:
+      name: kms-script
+  restartPolicy: OnFailure
+  serviceAccountName: kms-setup-sa

bindata/assets/kms/job-sa-role.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kms-setup-role
+  namespace: kube-system
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["create", "update"]

bindata/assets/kms/job-sa-rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kms-setup-rbac
+  namespace: kube-system
+subjects:
+- kind: ServiceAccount
+  name: kms-setup-sa
+  namespace: kube-system
+roleRef:
+  kind: Role
+  name: kms-setup-role
+  apiGroup: rbac.authorization.k8s.io

bindata/assets/kms/job-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kms-setup-sa
+  namespace: kube-system

bindata/assets/kms/job-sh-cm.yaml

@@ -0,0 +1,123 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kms-script
+  namespace: kube-system
+data:
+  policy.json: |-
+    {
+        "Id": "key-policy-01",
+        "Statement": [
+            {
+                "Sid": "Enable IAM User Permissions",
+                "Effect": "Allow",
+                "Principal": {
+                    "AWS": "arn:aws:iam::<AWS_ACCOUNT_ID>:root"
+                },
+                "Action": "kms:*",
+                "Resource": "*"
+            },
+            {
+                "Sid": "Allow use of the key",
+                "Effect": "Allow",
+                "Principal": {
+                    "AWS": "arn:aws:iam::<AWS_ACCOUNT_ID>:role/<OPENSHIFT_INFRA_NAME>-master-role"
+                },
+                "Action": [
+                    "kms:Encrypt",
+                    "kms:Decrypt",
+                    "kms:ReEncrypt*",
+                    "kms:GenerateDataKey*",
+                    "kms:DescribeKey"
+                ],
+                "Resource": "*"
+            },
+            {
+                "Sid": "Allow attachment of persistent resources",
+                "Effect": "Allow",
+                "Principal": {
+                    "AWS": "arn:aws:iam::<AWS_ACCOUNT_ID>:role/<OPENSHIFT_INFRA_NAME>-master-role"
+                },
+                "Action": [
+                    "kms:CreateGrant",
+                    "kms:ListGrants",
+                    "kms:RevokeGrant"
+                ],
+                "Resource": "*",
+                "Condition": {
+                    "Bool": {
+                        "kms:GrantIsForAWSResource": "true"
+                    }
+                }
+            }
+        ]
+    }
+  setup-kms.sh: |-
+    #!/bin/bash
+    AWS_REGION=${AWS_DEFAULT_REGION}
+    KMS_KEY_NAME=${OPENSHIFT_INFRA_NAME}-master-kek
+
+    # create an AWS KMS key
+    KMS_KEY_ID=$(aws kms create-key \
+        --region ${AWS_REGION} \
+        --query KeyMetadata.KeyId \
+        --output text \
+        --description "used with k8s encryption provider" \
+        --key-usage ENCRYPT_DECRYPT)
+
+    aws kms describe-key \
+        --region ${AWS_REGION} \
+        --key-id "${KMS_KEY_ID}"
+
+    # shows the KMS key name on the AWS console, and so 
+    # key is search-able by name specified
+    aws kms create-alias \
+        --region ${AWS_REGION} \
+        --alias-name "alias/${KMS_KEY_NAME}" \
+        --target-key-id "${KMS_KEY_ID}"
+
+    aws iam list-role-policies \
+      --role-name "${OPENSHIFT_INFRA_NAME}"-master-role \
+      --region ${AWS_REGION}
+
+    AWS_ACCOUNT_ID=$(aws sts get-caller-identity \
+      --region ${AWS_REGION} \
+      --query "Account" \
+      --output text)
+    BASE_DIR=$(dirname $0)
+
+    cat "${BASE_DIR}"/policy.json | sed "s/<AWS_ACCOUNT_ID>/${AWS_ACCOUNT_ID}/g" | sed "s/<OPENSHIFT_INFRA_NAME>/${OPENSHIFT_INFRA_NAME}/g" > /tmp/policy-rendered.json
+
+    aws kms put-key-policy \
+      --region ${AWS_REGION} \
+      --key-id "${KMS_KEY_ID}" \
+      --policy-name default \
+      --policy file:///tmp/policy-rendered.json \
+
+    # prints ARN of the KMS key so it can be 
+    # later used with kms plugin
+    KMS_KEY_ARN=$(aws kms describe-key \
+        --region ${AWS_REGION} \
+        --key-id "${KMS_KEY_ID}" \
+        --query KeyMetadata.Arn \
+        --output text)
+    echo ${KMS_KEY_ARN}
+
+    echo '{"kind": "ConfigMap", "metadata": {"name": "kms-key"}, "data": {"aws_kms_arn": "'"${KMS_KEY_ARN}"'"}}' > /tmp/kms-cm.json
+
+    SA=/var/run/secrets/kubernetes.io/serviceaccount
+    for i in {1..10}; 
+    do
+        echo "writing configmap, attempt ${i}"
+        if curl -sS \
+            --cacert ${SA}/ca.crt \
+            -H "Authorization: Bearer $(cat ${SA}/token)" \
+            -H "Content-Type: application/json" \
+            -X POST \
+            --data-binary "@/tmp/kms-cm.json" \
+            https://$KUBERNETES_SERVICE_HOST/api/v1/namespaces/kube-system/configmaps | grep '"kind": "ConfigMap"';
+        then
+            break
+        fi
+        sleep 5;
+    done

bindata/assets/kube-apiserver/pod.yaml

@@ -98,6 +98,8 @@ spec:
       name: cert-dir
     - mountPath: /var/log/kube-apiserver
       name: audit-dir
+    - mountPath: /var/kms-plugin
+      name: var-kms-plugin
     livenessProbe:
       httpGet:
         scheme: HTTPS
@@ -262,6 +264,40 @@ spec:
       requests:
         memory: 50Mi
         cpu: 10m
+  - image: {{.KMSPluginImage}}
+    name: cloud-kms-plugin
+    args:
+    - --debug
+    - --health-port=:18081
+    - --key={{.AWSKMSKeyARN}}
+    - --listen=/var/kms-plugin/socket.sock
+    - --region={{.AWSRegion}}
+    ports:
+    - containerPort: 18081
+      protocol: TCP
+      name: check-kms
+    livenessProbe:
+      httpGet:
+        scheme: HTTP
+        port: 18081
+        path: livez
+      initialDelaySeconds: 15
+      timeoutSeconds: 10
+      periodSeconds: 60
+    readinessProbe:
+      httpGet:
+        scheme: HTTP
+        port: 18081
+        path: healthz
+      initialDelaySeconds: 10
+      timeoutSeconds: 10
+    volumeMounts:
+    - mountPath: /var/kms-plugin
+      name: var-kms-plugin
+    resources:
+      requests:
+        memory: 256Mi
+        cpu: 200m
   terminationGracePeriodSeconds: {{.GracefulTerminationDuration}}
   hostNetwork: true
   priorityClassName: system-node-critical
@@ -277,3 +313,7 @@ spec:
   - hostPath:
       path: /var/log/kube-apiserver
     name: audit-dir
+  - hostPath:
+      path: /var/kms-plugin
+      type: DirectoryOrCreate
+    name: var-kms-plugin

hack-kms/desired_config.yaml

@@ -0,0 +1,21 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: EncryptionConfiguration
+resources:
+  - resources:
+    - configmaps
+    providers:
+    - kms:
+        name: gcp-kms-encryption
+        endpoint: unix:///var/kms-plugin/socket.sock
+        cachesize: 1000
+        timeout: 5s
+    - identity: {}
+  - resources:
+    - secrets
+    providers:
+    - kms:
+        name: gcp-kms-encryption
+        endpoint: unix:///var/kms-plugin/socket.sock
+        cachesize: 1000
+        timeout: 5s
+    - identity: {}

hack-kms/example_decrypted_config.json

@@ -0,0 +1,46 @@
+{
+    "kind": "EncryptionConfiguration",
+    "apiVersion": "apiserver.config.k8s.io/v1",
+    "resources": [
+        {
+            "resources": [
+                "configmaps"
+            ],
+            "providers": [
+                {
+                    "identity": {}
+                },
+                {
+                    "aescbc": {
+                        "keys": [
+                            {
+                                "name": "1",
+                                "secret": "fttcMWlNpNa7BIP6/P2kMEPJoTW72A5HdCEyOBX92lc="
+                            }
+                        ]
+                    }
+                }
+            ]
+        },
+        {
+            "resources": [
+                "secrets"
+            ],
+            "providers": [
+                {
+                    "identity": {}
+                },
+                {
+                    "aescbc": {
+                        "keys": [
+                            {
+                                "name": "1",
+                                "secret": "fttcMWlNpNa7BIP6/P2kMEPJoTW72A5HdCEyOBX92lc="
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    ]
+}

hack-kms/example_encrypted_config.json

@@ -0,0 +1,46 @@
+{
+    "kind": "EncryptionConfiguration",
+    "apiVersion": "apiserver.config.k8s.io/v1",
+    "resources": [
+        {
+            "resources": [
+                "configmaps"
+            ],
+            "providers": [
+                {
+                    "aescbc": {
+                        "keys": [
+                            {
+                                "name": "1",
+                                "secret": "fttcMWlNpNa7BIP6/P2kMEPJoTW72A5HdCEyOBX92lc="
+                            }
+                        ]
+                    }
+                },
+                {
+                    "identity": {}
+                }
+            ]
+        },
+        {
+            "resources": [
+                "secrets"
+            ],
+            "providers": [
+                {
+                    "aescbc": {
+                        "keys": [
+                            {
+                                "name": "1",
+                                "secret": "fttcMWlNpNa7BIP6/P2kMEPJoTW72A5HdCEyOBX92lc="
+                            }
+                        ]
+                    }
+                },
+                {
+                    "identity": {}
+                }
+            ]
+        }
+    ]
+}