-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
27921: kubernetes: Add a secure config that can use externally created certs r=a-robinson a=a-robinson For Kubernetes clusters that don't sign certificates properly, such as EKS as discovered in a recent forum post: https://forum.cockroachlabs.com/t/secure-cockroachdb-cluster-on-aws-eks/1824 Release note: None -------------------- It's arguable that this isn't worth checking in. I'm not too attached to it if folks don't think we should, I just needed to publish it somewhere for the original poster of https://forum.cockroachlabs.com/t/secure-cockroachdb-cluster-on-aws-eks/1824 to be able to use it. Touches #24527 30256: storage: Transfer lease to least-loaded store when rebalancing replicas r=a-robinson a=a-robinson This logic was totally busted before due to indexing into the wrong replicas slice, meaning we would often transfer the lease to the wrong store. Release note: None Co-authored-by: Alex Robinson <[email protected]>
- Loading branch information
Showing
5 changed files
with
390 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| # This config file demonstrates how to connect to the CockroachDB StatefulSet | ||
| # defined in bring-your-own-certs-statefulset.yaml that uses certificates | ||
| # created outside of Kubernetes. See that file for why you may want to use it. | ||
| # You should be able to adapt the core ideas to deploy your own custom | ||
| # applications and connect them to the database similarly. | ||
| # | ||
| # The pod that this file defines will sleep in the cluster not using any | ||
| # resources. After creating the pod, you can use it to open up a SQL shell to | ||
| # the database by running: | ||
| # | ||
| # kubectl exec -it cockroachdb-client-secure -- ./cockroach sql --url="postgres://root@cockroachdb-public:26257/?sslmode=verify-full&sslcert=/cockroach-certs/client.root.crt&sslkey=/cockroach-certs/client.root.key&sslrootcert=/cockroach-certs/ca.crt" | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: cockroachdb-client-secure | ||
| labels: | ||
| app: cockroachdb-client | ||
| spec: | ||
| serviceAccountName: cockroachdb | ||
| containers: | ||
| - name: cockroachdb-client | ||
| image: cockroachdb/cockroach:v2.0.5 | ||
| # Keep a pod open indefinitely so kubectl exec can be used to get a shell to it | ||
| # and run cockroach client commands, such as cockroach sql, cockroach node status, etc. | ||
| command: | ||
| - sleep | ||
| - "2147483648" # 2^31 | ||
| volumeMounts: | ||
| - name: client-certs | ||
| mountPath: /cockroach-certs | ||
| volumes: | ||
| - name: client-certs | ||
| secret: | ||
| secretName: cockroachdb.client.root | ||
| defaultMode: 256 |
210 changes: 210 additions & 0 deletions
210
cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,210 @@ | ||
| # This config file defines a CockroachDB StatefulSet that uses certificates | ||
| # created outside of Kubernetes. You may want to use it if you want to use a | ||
| # different certificate authority from the one being used by Kubernetes or if | ||
| # your Kubernetes cluster doesn't fully support certificate-signing requests | ||
| # (e.g. as of July 2018, EKS doesn't work properly). | ||
| # | ||
| # To use this config file, first set up your certificates and load them into | ||
| # your Kubernetes cluster as Secrets using the commands below: | ||
| # | ||
| # mkdir certs | ||
| # mkdir my-safe-directory | ||
| # cockroach cert create-ca --certs-dir=certs --ca-key=my-safe-directory/ca.key | ||
| # cockroach cert create-client root --certs-dir=certs --ca-key=my-safe-directory/ca.key | ||
| # kubectl create secret generic cockroachdb.client.root --from-file=certs | ||
| # cockroach cert create-node --certs-dir=certs --ca-key=my-safe-directory/ca.key localhost 127.0.0.1 cockroachdb-public cockroachdb-public.default cockroachdb-public.default.svc.cluster.local *.cockroachdb *.cockroachdb.default *.cockroachdb.default.svc.cluster.local | ||
| # kubectl create secret generic cockroachdb.node --from-file=certs | ||
| # kubectl create -f bring-your-own-certs-statefulset.yaml | ||
| # kubectl exec -it cockroachdb-0 -- /cockroach/cockroach init --certs-dir=/cockroach/cockroach-certs | ||
| apiVersion: v1 | ||
| kind: ServiceAccount | ||
| metadata: | ||
| name: cockroachdb | ||
| labels: | ||
| app: cockroachdb | ||
| --- | ||
| apiVersion: rbac.authorization.k8s.io/v1beta1 | ||
| kind: Role | ||
| metadata: | ||
| name: cockroachdb | ||
| labels: | ||
| app: cockroachdb | ||
| rules: | ||
| - apiGroups: | ||
| - "" | ||
| resources: | ||
| - secrets | ||
| verbs: | ||
| - get | ||
| --- | ||
| apiVersion: rbac.authorization.k8s.io/v1beta1 | ||
| kind: RoleBinding | ||
| metadata: | ||
| name: cockroachdb | ||
| labels: | ||
| app: cockroachdb | ||
| roleRef: | ||
| apiGroup: rbac.authorization.k8s.io | ||
| kind: Role | ||
| name: cockroachdb | ||
| subjects: | ||
| - kind: ServiceAccount | ||
| name: cockroachdb | ||
| namespace: default | ||
| --- | ||
| apiVersion: v1 | ||
| kind: Service | ||
| metadata: | ||
| # This service is meant to be used by clients of the database. It exposes a ClusterIP that will | ||
| # automatically load balance connections to the different database pods. | ||
| name: cockroachdb-public | ||
| labels: | ||
| app: cockroachdb | ||
| spec: | ||
| ports: | ||
| # The main port, served by gRPC, serves Postgres-flavor SQL, internode | ||
| # traffic and the cli. | ||
| - port: 26257 | ||
| targetPort: 26257 | ||
| name: grpc | ||
| # The secondary port serves the UI as well as health and debug endpoints. | ||
| - port: 8080 | ||
| targetPort: 8080 | ||
| name: http | ||
| selector: | ||
| app: cockroachdb | ||
| --- | ||
| apiVersion: v1 | ||
| kind: Service | ||
| metadata: | ||
| # This service only exists to create DNS entries for each pod in the stateful | ||
| # set such that they can resolve each other's IP addresses. It does not | ||
| # create a load-balanced ClusterIP and should not be used directly by clients | ||
| # in most circumstances. | ||
| name: cockroachdb | ||
| labels: | ||
| app: cockroachdb | ||
| annotations: | ||
| # Use this annotation in addition to the actual publishNotReadyAddresses | ||
| # field below because the annotation will stop being respected soon but the | ||
| # field is broken in some versions of Kubernetes: | ||
| # https://github.com/kubernetes/kubernetes/issues/58662 | ||
| service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" | ||
| # Enable automatic monitoring of all instances when Prometheus is running in the cluster. | ||
| prometheus.io/scrape: "true" | ||
| prometheus.io/path: "_status/vars" | ||
| prometheus.io/port: "8080" | ||
| spec: | ||
| ports: | ||
| - port: 26257 | ||
| targetPort: 26257 | ||
| name: grpc | ||
| - port: 8080 | ||
| targetPort: 8080 | ||
| name: http | ||
| # We want all pods in the StatefulSet to have their addresses published for | ||
| # the sake of the other CockroachDB pods even before they're ready, since they | ||
| # have to be able to talk to each other in order to become ready. | ||
| publishNotReadyAddresses: true | ||
| clusterIP: None | ||
| selector: | ||
| app: cockroachdb | ||
| --- | ||
| apiVersion: policy/v1beta1 | ||
| kind: PodDisruptionBudget | ||
| metadata: | ||
| name: cockroachdb-budget | ||
| labels: | ||
| app: cockroachdb | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| app: cockroachdb | ||
| maxUnavailable: 1 | ||
| --- | ||
| apiVersion: apps/v1beta1 | ||
| kind: StatefulSet | ||
| metadata: | ||
| name: cockroachdb | ||
| spec: | ||
| serviceName: "cockroachdb" | ||
| replicas: 3 | ||
| template: | ||
| metadata: | ||
| labels: | ||
| app: cockroachdb | ||
| spec: | ||
| serviceAccountName: cockroachdb | ||
| affinity: | ||
| podAntiAffinity: | ||
| preferredDuringSchedulingIgnoredDuringExecution: | ||
| - weight: 100 | ||
| podAffinityTerm: | ||
| labelSelector: | ||
| matchExpressions: | ||
| - key: app | ||
| operator: In | ||
| values: | ||
| - cockroachdb | ||
| topologyKey: kubernetes.io/hostname | ||
| containers: | ||
| - name: cockroachdb | ||
| image: cockroachdb/cockroach:v2.0.5 | ||
| imagePullPolicy: IfNotPresent | ||
| ports: | ||
| - containerPort: 26257 | ||
| name: grpc | ||
| - containerPort: 8080 | ||
| name: http | ||
| livenessProbe: | ||
| httpGet: | ||
| path: "/health" | ||
| port: http | ||
| scheme: HTTPS | ||
| initialDelaySeconds: 30 | ||
| periodSeconds: 5 | ||
| readinessProbe: | ||
| httpGet: | ||
| path: "/health?ready=1" | ||
| port: http | ||
| scheme: HTTPS | ||
| initialDelaySeconds: 10 | ||
| periodSeconds: 5 | ||
| failureThreshold: 2 | ||
| volumeMounts: | ||
| - name: datadir | ||
| mountPath: /cockroach/cockroach-data | ||
| - name: certs | ||
| mountPath: /cockroach/cockroach-certs | ||
| env: | ||
| - name: COCKROACH_CHANNEL | ||
| value: kubernetes-secure | ||
| command: | ||
| - "/bin/bash" | ||
| - "-ecx" | ||
| # The use of qualified `hostname -f` is crucial: | ||
| # Other nodes aren't able to look up the unqualified hostname. | ||
| - "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-host 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%" | ||
| # No pre-stop hook is required, a SIGTERM plus some time is all that's | ||
| # needed for graceful shutdown of a node. | ||
| terminationGracePeriodSeconds: 60 | ||
| volumes: | ||
| - name: datadir | ||
| persistentVolumeClaim: | ||
| claimName: datadir | ||
| - name: certs | ||
| secret: | ||
| secretName: cockroachdb.node | ||
| defaultMode: 256 | ||
| podManagementPolicy: Parallel | ||
| updateStrategy: | ||
| type: RollingUpdate | ||
| volumeClaimTemplates: | ||
| - metadata: | ||
| name: datadir | ||
| spec: | ||
| accessModes: | ||
| - "ReadWriteOnce" | ||
| resources: | ||
| requests: | ||
| storage: 1Gi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.