From ae8db251d880695d6594cb99cd364fcc186767ef Mon Sep 17 00:00:00 2001 From: telliere Date: Thu, 4 Apr 2024 14:47:45 +0200 Subject: [PATCH] first k8s plan for hpcs --- k8s/hpcs-namespace.yaml | 4 + k8s/hpcs-server-account.yaml | 5 ++ k8s/hpcs-server-configmap.yaml | 61 +++++++++++++ k8s/hpcs-server-service.yaml | 14 +++ k8s/hpcs-server-statefulset.yaml | 62 ++++++++++++++ k8s/hpcs-spire-account.yaml | 5 ++ k8s/spire-oidc-configmap.yaml | 16 ++++ k8s/spire-oidc-service.yaml | 14 +++ k8s/spire-server-account.yaml | 5 ++ k8s/spire-server-cluster-role.yaml | 28 ++++++ k8s/spire-server-configmap.yaml | 73 ++++++++++++++++ k8s/spire-server-nginx-configmap.yaml | 27 ++++++ k8s/spire-server-service.yaml | 14 +++ k8s/spire-server-statefulset.yaml | 119 ++++++++++++++++++++++++++ server/app.py | 5 ++ server/lib/spire_interactions.py | 9 +- 16 files changed, 457 insertions(+), 4 deletions(-) create mode 100644 k8s/hpcs-namespace.yaml create mode 100644 k8s/hpcs-server-account.yaml create mode 100644 k8s/hpcs-server-configmap.yaml create mode 100644 k8s/hpcs-server-service.yaml create mode 100644 k8s/hpcs-server-statefulset.yaml create mode 100644 k8s/hpcs-spire-account.yaml create mode 100644 k8s/spire-oidc-configmap.yaml create mode 100644 k8s/spire-oidc-service.yaml create mode 100644 k8s/spire-server-account.yaml create mode 100644 k8s/spire-server-cluster-role.yaml create mode 100644 k8s/spire-server-configmap.yaml create mode 100644 k8s/spire-server-nginx-configmap.yaml create mode 100644 k8s/spire-server-service.yaml create mode 100644 k8s/spire-server-statefulset.yaml diff --git a/k8s/hpcs-namespace.yaml b/k8s/hpcs-namespace.yaml new file mode 100644 index 0000000..8280228 --- /dev/null +++ b/k8s/hpcs-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: hpcs diff --git a/k8s/hpcs-server-account.yaml b/k8s/hpcs-server-account.yaml new file mode 100644 index 0000000..e51e313 --- /dev/null +++ b/k8s/hpcs-server-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hpcs-server + namespace: hpcs diff --git a/k8s/hpcs-server-configmap.yaml b/k8s/hpcs-server-configmap.yaml new file mode 100644 index 0000000..f58e239 --- /dev/null +++ b/k8s/hpcs-server-configmap.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hpcs-server + namespace: hpcs +data: + hpcs-server.conf: | + [spire-server] + address = localhost + port = 8081 + trust-domain = hpcs + pre-command = "" + spire-server-bin = spire-server + socket-path = /var/run/sockets/server/api.sock + + [spire-agent] + spire-agent-socket = /run/sockets/agent/agent.sock + + [vault] + url = http://vault:8200 + server-role = hpcs-server + + agent.conf: | + agent { + data_dir = "./data/agent" + log_level = "DEBUG" + trust_domain = "hpcs" + server_address = "spire-server" + server_port = 8081 + socket_path = "/var/run/sockets/agent/agent.sock" + admin_socket_path = "/var/run/sockets/admin/admin.sock" + + # Insecure bootstrap is NOT appropriate for production use but is ok for + # simple testing/evaluation purposes. + insecure_bootstrap = true + } + + plugins { + KeyManager "disk" { + plugin_data { + directory = "./data/agent" + } + } + + NodeAttestor "k8s_psat" { + plugin_data { + cluster = "docker-desktop" + } + } + + WorkloadAttestor "k8s" { + plugin_data { + } + } + + WorkloadAttestor "unix" { + plugin_data { + discover_workload_path = true + } + } + } diff --git a/k8s/hpcs-server-service.yaml b/k8s/hpcs-server-service.yaml new file mode 100644 index 0000000..59d45fc --- /dev/null +++ b/k8s/hpcs-server-service.yaml @@ -0,0 +1,14 @@ +# Service definition for spire-oidc (expose the OIDC socket) +apiVersion: v1 +kind: Service +metadata: + name: hpcs-server + namespace: hpcs +spec: + clusterIP: None + selector: + app: hpcs-server + ports: + - name: https + port: 10080 + targetPort: hpcs-server diff --git a/k8s/hpcs-server-statefulset.yaml b/k8s/hpcs-server-statefulset.yaml new file mode 100644 index 0000000..73c6bef --- /dev/null +++ b/k8s/hpcs-server-statefulset.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hpcs-server + namespace: hpcs + labels: + app: hpcs-server +spec: + replicas: 1 + selector: + matchLabels: + app: hpcs-server + serviceName: hpcs-server + template: + metadata: + namespace: hpcs + labels: + app: hpcs-server + spec: + serviceAccountName: hpcs-server + shareProcessNamespace: true + containers: + - name: hpcs-server + image: ghcr.io/cscfi/hpcs/server:k8s_plan + ports: + - containerPort: 10080 + name: hpcs-server + volumeMounts: + - name: hpcs-server-configs + mountPath: /tmp/ + readOnly: false + - name: hpcs-spire-sockets + mountPath: /var/run/sockets + readOnly: false + - name: hpcs-spire-agent-token + mountPath: /var/run/secrets/tokens + readOnly: true + volumes: + - name: hpcs-server-configs + configMap: + name: hpcs-server + - name: hpcs-spire-sockets + hostPath: + path: /run/spire/sockets + type: DirectoryOrCreate + - name: hpcs-spire-agent-token + projected: + sources: + - serviceAccountToken: + path: spire-agent + expirationSeconds: 7200 + audience: spire-server + volumeClaimTemplates: + - metadata: + name: spire-agent-data + namespace: hpcs + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/k8s/hpcs-spire-account.yaml b/k8s/hpcs-spire-account.yaml new file mode 100644 index 0000000..690a8a6 --- /dev/null +++ b/k8s/hpcs-spire-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hpcs-spire + namespace: hpcs diff --git a/k8s/spire-oidc-configmap.yaml b/k8s/spire-oidc-configmap.yaml new file mode 100644 index 0000000..9de2be7 --- /dev/null +++ b/k8s/spire-oidc-configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-oidc + namespace: hpcs +data: + oidc-discovery-provider.conf: | + log_level = "debug" + domains = ["spire-oidc"] + listen_socket_path = "/tmp/spire-server/private/oidc-api.sock" + + server_api { + address = "unix:///tmp/spire-server/private/api.sock" + } + + health_checks {} diff --git a/k8s/spire-oidc-service.yaml b/k8s/spire-oidc-service.yaml new file mode 100644 index 0000000..c425c1e --- /dev/null +++ b/k8s/spire-oidc-service.yaml @@ -0,0 +1,14 @@ +# Service definition for spire-oidc (expose the OIDC socket) +apiVersion: v1 +kind: Service +metadata: + name: spire-oidc + namespace: hpcs +spec: + type: LoadBalancer + selector: + app: spire-server + ports: + - name: https + port: 443 + targetPort: hpcs-nginx diff --git a/k8s/spire-server-account.yaml b/k8s/spire-server-account.yaml new file mode 100644 index 0000000..2135836 --- /dev/null +++ b/k8s/spire-server-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spire-server + namespace: hpcs diff --git a/k8s/spire-server-cluster-role.yaml b/k8s/spire-server-cluster-role.yaml new file mode 100644 index 0000000..41defa1 --- /dev/null +++ b/k8s/spire-server-cluster-role.yaml @@ -0,0 +1,28 @@ +# ClusterRole to allow spire-server node attestor to query Token Review API +# and to be able to push certificate bundles to a configmap +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-server-trust-role +rules: +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: [""] + resources: ["configmaps","pods","nodes"] + verbs: ["patch", "get", "list"] + +--- +# Binds above cluster role to spire-server service account +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-server-trust-role-binding +subjects: +- kind: ServiceAccount + name: spire-server + namespace: hpcs +roleRef: + kind: ClusterRole + name: spire-server-trust-role + apiGroup: rbac.authorization.k8s.io diff --git a/k8s/spire-server-configmap.yaml b/k8s/spire-server-configmap.yaml new file mode 100644 index 0000000..6a9a079 --- /dev/null +++ b/k8s/spire-server-configmap.yaml @@ -0,0 +1,73 @@ +apiVersion: v1 + +kind: ConfigMap +metadata: + name: spire-bundle + namespace: hpcs + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-server + namespace: hpcs +data: + server.conf: | + server { + bind_address = "0.0.0.0" + bind_port = "8081" + socket_path = "/tmp/spire-server/private/api.sock" + trust_domain = "hpcs" + data_dir = "/run/spire/data" + log_level = "DEBUG" + ca_key_type = "rsa-2048" + + jwt_issuer = "spire-server" + default_jwt_svid_ttl = "1h" + + ca_subject = { + country = ["US"], + organization = ["SPIFFE"], + common_name = "", + } + } + + plugins { + DataStore "sql" { + plugin_data { + database_type = "sqlite3" + connection_string = "/run/spire/data/datastore.sqlite3" + } + } + + NodeAttestor "k8s_psat" { + plugin_data { + clusters = { + "docker-desktop" = { + use_token_review_api_validation = true + service_account_allow_list = ["hpcs:hpcs-server"] + } + } + } + } + + KeyManager "disk" { + plugin_data { + keys_path = "/run/spire/data/keys.json" + } + } + + Notifier "k8sbundle" { + plugin_data { + namespace = "hpcs" + } + } + } + + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "8080" + live_path = "/live" + ready_path = "/ready" + } diff --git a/k8s/spire-server-nginx-configmap.yaml b/k8s/spire-server-nginx-configmap.yaml new file mode 100644 index 0000000..0eba7ec --- /dev/null +++ b/k8s/spire-server-nginx-configmap.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hpcs-nginx + namespace: hpcs +data: + nginx.conf: | + events {} + http { + access_log /tmp/access.log; + error_log /tmp/error.log; + + upstream spire-oidc { + server unix:/tmp/spire-server/private/oidc-api.sock; + } + + server{ + listen 443 ssl; + ssl_certificate /certs/selfsigned.crt; + ssl_certificate_key /certs/selfsigned.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + location / { + proxy_pass http://spire-oidc; + } + } + } diff --git a/k8s/spire-server-service.yaml b/k8s/spire-server-service.yaml new file mode 100644 index 0000000..3e2baf2 --- /dev/null +++ b/k8s/spire-server-service.yaml @@ -0,0 +1,14 @@ +# Service definition for spire server +apiVersion: v1 +kind: Service +metadata: + name: spire-server + namespace: hpcs +spec: + type: LoadBalancer + selector: + app: spire-server + ports: + - name: tcp-spire + port: 8081 + targetPort: spire-server diff --git a/k8s/spire-server-statefulset.yaml b/k8s/spire-server-statefulset.yaml new file mode 100644 index 0000000..f1baa0d --- /dev/null +++ b/k8s/spire-server-statefulset.yaml @@ -0,0 +1,119 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: spire-server + namespace: hpcs + labels: + app: spire-server +spec: + replicas: 1 + selector: + matchLabels: + app: spire-server + serviceName: spire-server + template: + metadata: + namespace: hpcs + labels: + app: spire-server + spec: + serviceAccountName: spire-server + shareProcessNamespace: true + containers: + - name: hpcs-nginx + image: nginx + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/ + readOnly: true + - name: spire-server-socket + mountPath: /tmp/spire-server/private + readOnly: false + - name: nginx-certs + mountPath: /certs + readOnly: true + ports: + - containerPort: 443 + name: hpcs-nginx + - name: spire-server + image: ghcr.io/spiffe/spire-server:1.9.0 + args: + - -config + - /run/spire/config/server.conf + ports: + - containerPort: 8081 + name: spire-server + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + - name: spire-data + mountPath: /run/spire/data + readOnly: false + - name: spire-server-socket + mountPath: /tmp/spire-server/private + readOnly: false + livenessProbe: + httpGet: + path: /live + port: 8080 + failureThreshold: 2 + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + - name: spire-oidc + image: ghcr.io/spiffe/oidc-discovery-provider:1.9.0 + args: + - -config + - /run/spire/oidc/config/oidc-discovery-provider.conf + volumeMounts: + - name: spire-server-socket + mountPath: /tmp/spire-server/private + readOnly: false + - name: spire-oidc-config + mountPath: /run/spire/oidc/config/ + readOnly: true + - name: spire-data + mountPath: /run/spire/data + readOnly: false + readinessProbe: + httpGet: + path: /ready + port: 8008 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + volumes: + - name: nginx-config + configMap: + name: hpcs-nginx + - name: spire-config + configMap: + name: spire-server + - name: spire-server-socket + hostPath: + path: /run/spire/sockets/server + type: DirectoryOrCreate + - name: spire-oidc-config + configMap: + name: spire-oidc + - name: nginx-certs + hostPath: + path: /etc/certs/hpcs-spire-oidc + type: DirectoryOrCreate + volumeClaimTemplates: + - metadata: + name: spire-data + namespace: hpcs + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/server/app.py b/server/app.py index fb96bb4..dd98b69 100644 --- a/server/app.py +++ b/server/app.py @@ -53,6 +53,11 @@ "hpcs-server-spiffeid" ) +if configuration["spire-server"].get("socket-path"): + spire_interactions.spire_server_socketpath = configuration["spire-server"].get( + "socket-path" + ) + if configuration["spire-server"].get("pre-command"): spire_interactions.pre_command = configuration["spire-server"]["pre-command"] if configuration["spire-server"]["pre-command"] == '""': diff --git a/server/lib/spire_interactions.py b/server/lib/spire_interactions.py index 7b524c6..6416f64 100644 --- a/server/lib/spire_interactions.py +++ b/server/lib/spire_interactions.py @@ -10,6 +10,7 @@ jwt_workload_api = None hpcs_server_spiffeid = "spiffe://hpcs/hpcs-server/workload" +spire_server_socketpath = "/tmp/spire-server/private/api.sock:" def token_generate(spiffeID: SpiffeId) -> subprocess.CompletedProcess: @@ -23,11 +24,11 @@ def token_generate(spiffeID: SpiffeId) -> subprocess.CompletedProcess: """ if pre_command != "": - command = f"{pre_command} {spire_server_bin} token generate -spiffeID {str(spiffeID)}".split( + command = f"{pre_command} {spire_server_bin} token generate -socketPath {spire_server_socketpath} -spiffeID {str(spiffeID)}".split( " " ) else: - command = f"{spire_server_bin} token generate -spiffeID {str(spiffeID)}".split( + command = f"{spire_server_bin} token generate -socketPath {spire_server_socketpath} -spiffeID {str(spiffeID)}".split( " " ) @@ -48,11 +49,11 @@ def entry_create( subprocess.CompletedProcess: result of the cli command to create the entry """ if pre_command != "": - command = f"{pre_command} {spire_server_bin} entry create -parentID {str(parentID)} -spiffeID {str(spiffeID)}".split( + command = f"{pre_command} {spire_server_bin} entry create -socketPath {spire_server_socketpath} -parentID {str(parentID)} -spiffeID {str(spiffeID)}".split( " " ) else: - command = f"{spire_server_bin} entry create -parentID {str(parentID)} -spiffeID {str(spiffeID)}".split( + command = f"{spire_server_bin} entry create -socketPath {spire_server_socketpath} -parentID {str(parentID)} -spiffeID {str(spiffeID)}".split( " " )