diff --git a/cloud/kubernetes/bring-your-own-certs/client.yaml b/cloud/kubernetes/bring-your-own-certs/client.yaml new file mode 100644 index 000000000000..49c69be5600c --- /dev/null +++ b/cloud/kubernetes/bring-your-own-certs/client.yaml @@ -0,0 +1,35 @@ +# This config file demonstrates how to connect to the CockroachDB StatefulSet +# defined in bring-your-own-certs-statefulset.yaml that uses certificates +# created outside of Kubernetes. See that file for why you may want to use it. +# You should be able to adapt the core ideas to deploy your own custom +# applications and connect them to the database similarly. +# +# The pod that this file defines will sleep in the cluster not using any +# resources. After creating the pod, you can use it to open up a SQL shell to +# the database by running: +# +# kubectl exec -it cockroachdb-client-secure -- ./cockroach sql --url="postgres://root@cockroachdb-public:26257/?sslmode=verify-full&sslcert=/cockroach-certs/client.root.crt&sslkey=/cockroach-certs/client.root.key&sslrootcert=/cockroach-certs/ca.crt" +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-client-secure + labels: + app: cockroachdb-client +spec: + serviceAccountName: cockroachdb + containers: + - name: cockroachdb-client + image: cockroachdb/cockroach:v2.0.5 + # Keep a pod open indefinitely so kubectl exec can be used to get a shell to it + # and run cockroach client commands, such as cockroach sql, cockroach node status, etc. + command: + - sleep + - "2147483648" # 2^31 + volumeMounts: + - name: client-certs + mountPath: /cockroach-certs + volumes: + - name: client-certs + secret: + secretName: cockroachdb.client.root + defaultMode: 256 diff --git a/cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml b/cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml new file mode 100644 index 000000000000..ea6e67fcf36c --- /dev/null +++ b/cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml @@ -0,0 +1,210 @@ +# This config file defines a CockroachDB StatefulSet that uses certificates +# created outside of Kubernetes. You may want to use it if you want to use a +# different certificate authority from the one being used by Kubernetes or if +# your Kubernetes cluster doesn't fully support certificate-signing requests +# (e.g. as of July 2018, EKS doesn't work properly). +# +# To use this config file, first set up your certificates and load them into +# your Kubernetes cluster as Secrets using the commands below: +# +# mkdir certs +# mkdir my-safe-directory +# cockroach cert create-ca --certs-dir=certs --ca-key=my-safe-directory/ca.key +# cockroach cert create-client root --certs-dir=certs --ca-key=my-safe-directory/ca.key +# kubectl create secret generic cockroachdb.client.root --from-file=certs +# cockroach cert create-node --certs-dir=certs --ca-key=my-safe-directory/ca.key localhost 127.0.0.1 cockroachdb-public cockroachdb-public.default cockroachdb-public.default.svc.cluster.local *.cockroachdb *.cockroachdb.default *.cockroachdb.default.svc.cluster.local +# kubectl create secret generic cockroachdb.node --from-file=certs +# kubectl create -f bring-your-own-certs-statefulset.yaml +# kubectl exec -it cockroachdb-0 -- /cockroach/cockroach init --certs-dir=/cockroach/cockroach-certs +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cockroachdb + labels: + app: cockroachdb +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: cockroachdb + labels: + app: cockroachdb +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: cockroachdb + labels: + app: cockroachdb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cockroachdb +subjects: +- kind: ServiceAccount + name: cockroachdb + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + # This service is meant to be used by clients of the database. It exposes a ClusterIP that will + # automatically load balance connections to the different database pods. + name: cockroachdb-public + labels: + app: cockroachdb +spec: + ports: + # The main port, served by gRPC, serves Postgres-flavor SQL, internode + # traffic and the cli. + - port: 26257 + targetPort: 26257 + name: grpc + # The secondary port serves the UI as well as health and debug endpoints. + - port: 8080 + targetPort: 8080 + name: http + selector: + app: cockroachdb +--- +apiVersion: v1 +kind: Service +metadata: + # This service only exists to create DNS entries for each pod in the stateful + # set such that they can resolve each other's IP addresses. It does not + # create a load-balanced ClusterIP and should not be used directly by clients + # in most circumstances. + name: cockroachdb + labels: + app: cockroachdb + annotations: + # Use this annotation in addition to the actual publishNotReadyAddresses + # field below because the annotation will stop being respected soon but the + # field is broken in some versions of Kubernetes: + # https://github.com/kubernetes/kubernetes/issues/58662 + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + # Enable automatic monitoring of all instances when Prometheus is running in the cluster. + prometheus.io/scrape: "true" + prometheus.io/path: "_status/vars" + prometheus.io/port: "8080" +spec: + ports: + - port: 26257 + targetPort: 26257 + name: grpc + - port: 8080 + targetPort: 8080 + name: http + # We want all pods in the StatefulSet to have their addresses published for + # the sake of the other CockroachDB pods even before they're ready, since they + # have to be able to talk to each other in order to become ready. + publishNotReadyAddresses: true + clusterIP: None + selector: + app: cockroachdb +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: cockroachdb-budget + labels: + app: cockroachdb +spec: + selector: + matchLabels: + app: cockroachdb + maxUnavailable: 1 +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: cockroachdb +spec: + serviceName: "cockroachdb" + replicas: 3 + template: + metadata: + labels: + app: cockroachdb + spec: + serviceAccountName: cockroachdb + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - cockroachdb + topologyKey: kubernetes.io/hostname + containers: + - name: cockroachdb + image: cockroachdb/cockroach:v2.0.5 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 26257 + name: grpc + - containerPort: 8080 + name: http + livenessProbe: + httpGet: + path: "/health" + port: http + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: "/health?ready=1" + port: http + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 2 + volumeMounts: + - name: datadir + mountPath: /cockroach/cockroach-data + - name: certs + mountPath: /cockroach/cockroach-certs + env: + - name: COCKROACH_CHANNEL + value: kubernetes-secure + command: + - "/bin/bash" + - "-ecx" + # The use of qualified `hostname -f` is crucial: + # Other nodes aren't able to look up the unqualified hostname. + - "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-host 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%" + # No pre-stop hook is required, a SIGTERM plus some time is all that's + # needed for graceful shutdown of a node. + terminationGracePeriodSeconds: 60 + volumes: + - name: datadir + persistentVolumeClaim: + claimName: datadir + - name: certs + secret: + secretName: cockroachdb.node + defaultMode: 256 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - metadata: + name: datadir + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi