From 367c1336afa4f6e1610d6a419f4a784ce631378c Mon Sep 17 00:00:00 2001 From: "kevin.qiao" Date: Mon, 3 Apr 2023 16:28:42 +0800 Subject: [PATCH] support ssl encryption (#191) * support ssl encryption * simplify conditions --- Makefile | 1 - apis/apps/v1alpha1/exporter.go | 2 +- apis/apps/v1alpha1/nebulacluster.go | 24 +++++ apis/apps/v1alpha1/nebulacluster_common.go | 3 - .../v1alpha1/nebulacluster_componentter.go | 1 + apis/apps/v1alpha1/nebulacluster_graphd.go | 87 ++++++++++++++- apis/apps/v1alpha1/nebulacluster_metad.go | 78 ++++++++++++++ apis/apps/v1alpha1/nebulacluster_storaged.go | 78 ++++++++++++++ apis/apps/v1alpha1/nebulacluster_types.go | 41 ++++++- apis/apps/v1alpha1/template.go | 61 +++++++++++ apis/apps/v1alpha1/zz_generated.deepcopy.go | 25 +++++ .../nebula-operator/crds/nebulacluster.yaml | 26 +++++ .../templates/controller-manager-rbac.yaml | 3 + charts/nebula-operator/values.yaml | 16 +-- .../apps.nebula-graph.io_nebulaclusters.yaml | 26 +++++ doc/user/ssl_guide.md | 101 ++++++++++++++++++ pkg/controller/component/metad_cluster.go | 6 +- pkg/controller/component/storaged_cluster.go | 6 +- pkg/controller/component/storaged_scaler.go | 12 ++- pkg/controller/component/storaged_updater.go | 12 ++- .../nebularestore/nebula_restore_manager.go | 22 ++-- pkg/nebula/cert.go | 34 ++++++ pkg/nebula/client.go | 12 ++- pkg/nebula/options.go | 73 ++++++++++++- pkg/util/cert/cert.go | 54 ++++++++++ pkg/util/randstr/randstr.go | 19 ---- 26 files changed, 767 insertions(+), 56 deletions(-) create mode 100644 doc/user/ssl_guide.md create mode 100644 pkg/nebula/cert.go create mode 100644 pkg/util/cert/cert.go delete mode 100644 pkg/util/randstr/randstr.go diff --git a/Makefile b/Makefile index 9818a942..d9ea3b54 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,6 @@ build: generate check ## Build binary. $(GO_BUILD) -ldflags '$(LDFLAGS)' -o images/nebula-operator/bin/scheduler cmd/scheduler/main.go helm-charts: - cp config/crd/bases/*.yaml charts/nebula-operator/crds/ helm package charts/nebula-operator --version $(CHARTS_VERSION) --app-version $(CHARTS_VERSION) helm package charts/nebula-cluster --version $(CHARTS_VERSION) --app-version $(CHARTS_VERSION) mv nebula-operator-*.tgz nebula-cluster-*.tgz charts/ diff --git a/apis/apps/v1alpha1/exporter.go b/apis/apps/v1alpha1/exporter.go index acbac662..d1f407cd 100644 --- a/apis/apps/v1alpha1/exporter.go +++ b/apis/apps/v1alpha1/exporter.go @@ -58,7 +58,7 @@ func (nc *NebulaCluster) GetExporterNodeSelector() map[string]string { } func (nc *NebulaCluster) GetExporterAffinity() *corev1.Affinity { - affinity := nc.Spec.Graphd.PodSpec.Affinity + affinity := nc.Spec.Exporter.PodSpec.Affinity if affinity == nil { affinity = nc.Spec.Affinity } diff --git a/apis/apps/v1alpha1/nebulacluster.go b/apis/apps/v1alpha1/nebulacluster.go index 190af4e1..55f3791b 100644 --- a/apis/apps/v1alpha1/nebulacluster.go +++ b/apis/apps/v1alpha1/nebulacluster.go @@ -116,3 +116,27 @@ func (nc *NebulaCluster) IsBREnabled() bool { func (nc *NebulaCluster) IsLogRotateEnabled() bool { return nc.Spec.LogRotate != nil } + +func (nc *NebulaCluster) InsecureSkipVerify() bool { + skip := nc.Spec.SSLCerts.InsecureSkipVerify + if skip == nil { + return false + } + return *skip +} + +func (nc *NebulaCluster) IsGraphdSSLEnabled() bool { + return nc.Spec.Graphd.Config["enable_graph_ssl"] == "true" +} + +func (nc *NebulaCluster) IsMetadSSLEnabled() bool { + return nc.Spec.Graphd.Config["enable_meta_ssl"] == "true" && + nc.Spec.Metad.Config["enable_meta_ssl"] == "true" && + nc.Spec.Storaged.Config["enable_meta_ssl"] == "true" +} + +func (nc *NebulaCluster) IsClusterEnabled() bool { + return nc.Spec.Graphd.Config["enable_ssl"] == "true" && + nc.Spec.Metad.Config["enable_ssl"] == "true" && + nc.Spec.Storaged.Config["enable_ssl"] == "true" +} diff --git a/apis/apps/v1alpha1/nebulacluster_common.go b/apis/apps/v1alpha1/nebulacluster_common.go index 012a48f7..4ce67775 100644 --- a/apis/apps/v1alpha1/nebulacluster_common.go +++ b/apis/apps/v1alpha1/nebulacluster_common.go @@ -109,9 +109,6 @@ func getKubernetesClusterDomain() string { } func joinHostPort(host string, port int32) string { - if strings.IndexByte(host, ':') >= 0 { - return fmt.Sprintf("[%s]:%d", host, port) - } return fmt.Sprintf("%s:%d", host, port) } diff --git a/apis/apps/v1alpha1/nebulacluster_componentter.go b/apis/apps/v1alpha1/nebulacluster_componentter.go index 1605a207..ce4c7fce 100644 --- a/apis/apps/v1alpha1/nebulacluster_componentter.go +++ b/apis/apps/v1alpha1/nebulacluster_componentter.go @@ -46,6 +46,7 @@ type NebulaClusterComponentter interface { SidecarContainers() []corev1.Container SidecarVolumes() []corev1.Volume ReadinessProbe() *corev1.Probe + IsSSLEnabled() bool IsHeadlessService() bool GetServiceSpec() *ServiceSpec GetServiceName() string diff --git a/apis/apps/v1alpha1/nebulacluster_graphd.go b/apis/apps/v1alpha1/nebulacluster_graphd.go index fdf4e71d..c09afa17 100644 --- a/apis/apps/v1alpha1/nebulacluster_graphd.go +++ b/apis/apps/v1alpha1/nebulacluster_graphd.go @@ -155,6 +155,13 @@ func (c *graphdComponent) ReadinessProbe() *corev1.Probe { return c.nc.Spec.Graphd.PodSpec.ReadinessProbe } +func (c *graphdComponent) IsSSLEnabled() bool { + return (c.nc.Spec.Graphd.Config["enable_graph_ssl"] == "true" || + c.nc.Spec.Graphd.Config["enable_meta_ssl"] == "true" || + c.nc.Spec.Graphd.Config["enable_ssl"] == "true") && + c.nc.Spec.SSLCerts != nil +} + func (c *graphdComponent) IsHeadlessService() bool { return false } @@ -224,13 +231,39 @@ func (c *graphdComponent) GenerateVolumeMounts() []corev1.VolumeMount { } componentType := c.Type().String() - return []corev1.VolumeMount{ + mounts := []corev1.VolumeMount{ { Name: logVolume(componentType), MountPath: "/usr/local/nebula/logs", SubPath: "logs", }, } + + if c.IsSSLEnabled() { + certMounts := []corev1.VolumeMount{ + { + Name: "server-crt", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/server.crt", + SubPath: "server.crt", + }, + { + Name: "server-key", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/server.key", + SubPath: "server.key", + }, + { + Name: "ca-crt", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/ca.crt", + SubPath: "ca.crt", + }, + } + mounts = append(mounts, certMounts...) + } + + return mounts } func (c *graphdComponent) GenerateVolumes() []corev1.Volume { @@ -239,7 +272,7 @@ func (c *graphdComponent) GenerateVolumes() []corev1.Volume { } componentType := c.Type().String() - return []corev1.Volume{ + volumes := []corev1.Volume{ { Name: logVolume(componentType), VolumeSource: corev1.VolumeSource{ @@ -249,6 +282,56 @@ func (c *graphdComponent) GenerateVolumes() []corev1.Volume { }, }, } + + if c.IsSSLEnabled() { + certVolumes := []corev1.Volume{ + { + Name: "server-crt", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.ServerSecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.ServerPublicKey, + Path: "server.crt", + }, + }, + }, + }, + }, + { + Name: "server-key", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.ServerSecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.ServerPrivateKey, + Path: "server.key", + }, + }, + }, + }, + }, + { + Name: "ca-crt", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.CASecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.CAPublicKey, + Path: "ca.crt", + }, + }, + }, + }, + }, + } + volumes = append(volumes, certVolumes...) + } + + return volumes } func (c *graphdComponent) GenerateVolumeClaim() ([]corev1.PersistentVolumeClaim, error) { diff --git a/apis/apps/v1alpha1/nebulacluster_metad.go b/apis/apps/v1alpha1/nebulacluster_metad.go index a811feeb..dab20b25 100644 --- a/apis/apps/v1alpha1/nebulacluster_metad.go +++ b/apis/apps/v1alpha1/nebulacluster_metad.go @@ -169,6 +169,12 @@ func (c *metadComponent) ReadinessProbe() *corev1.Probe { return c.nc.Spec.Metad.PodSpec.ReadinessProbe } +func (c *metadComponent) IsSSLEnabled() bool { + return (c.nc.Spec.Metad.Config["enable_meta_ssl"] == "true" || + c.nc.Spec.Metad.Config["enable_ssl"] == "true") && + c.nc.Spec.SSLCerts != nil +} + func (c *metadComponent) IsHeadlessService() bool { return true } @@ -259,6 +265,30 @@ func (c *metadComponent) GenerateVolumeMounts() []corev1.VolumeMount { }) } + if c.IsSSLEnabled() { + certMounts := []corev1.VolumeMount{ + { + Name: "server-crt", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/server.crt", + SubPath: "server.crt", + }, + { + Name: "server-key", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/server.key", + SubPath: "server.key", + }, + { + Name: "ca-crt", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/ca.crt", + SubPath: "ca.crt", + }, + } + mounts = append(mounts, certMounts...) + } + return mounts } @@ -303,6 +333,54 @@ func (c *metadComponent) GenerateVolumes() []corev1.Volume { }) } + if c.IsSSLEnabled() { + certVolumes := []corev1.Volume{ + { + Name: "server-crt", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.ServerSecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.ServerPublicKey, + Path: "server.crt", + }, + }, + }, + }, + }, + { + Name: "server-key", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.ServerSecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.ServerPrivateKey, + Path: "server.key", + }, + }, + }, + }, + }, + { + Name: "ca-crt", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.CASecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.CAPublicKey, + Path: "ca.crt", + }, + }, + }, + }, + }, + } + volumes = append(volumes, certVolumes...) + } + return volumes } diff --git a/apis/apps/v1alpha1/nebulacluster_storaged.go b/apis/apps/v1alpha1/nebulacluster_storaged.go index b1410478..66813a26 100644 --- a/apis/apps/v1alpha1/nebulacluster_storaged.go +++ b/apis/apps/v1alpha1/nebulacluster_storaged.go @@ -174,6 +174,12 @@ func (c *storagedComponent) ReadinessProbe() *corev1.Probe { return c.nc.Spec.Storaged.PodSpec.ReadinessProbe } +func (c *storagedComponent) IsSSLEnabled() bool { + return (c.nc.Spec.Storaged.Config["enable_meta_ssl"] == "true" || + c.nc.Spec.Storaged.Config["enable_ssl"] == "true") && + c.nc.Spec.SSLCerts != nil +} + func (c *storagedComponent) IsHeadlessService() bool { return true } @@ -271,6 +277,30 @@ func (c *storagedComponent) GenerateVolumeMounts() []corev1.VolumeMount { }) } + if c.IsSSLEnabled() { + certMounts := []corev1.VolumeMount{ + { + Name: "server-crt", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/server.crt", + SubPath: "server.crt", + }, + { + Name: "server-key", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/server.key", + SubPath: "server.key", + }, + { + Name: "ca-crt", + ReadOnly: true, + MountPath: "/usr/local/nebula/certs/ca.crt", + SubPath: "ca.crt", + }, + } + mounts = append(mounts, certMounts...) + } + return mounts } @@ -302,6 +332,54 @@ func (c *storagedComponent) GenerateVolumes() []corev1.Volume { }) } + if c.IsSSLEnabled() { + certVolumes := []corev1.Volume{ + { + Name: "server-crt", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.ServerSecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.ServerPublicKey, + Path: "server.crt", + }, + }, + }, + }, + }, + { + Name: "server-key", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.ServerSecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.ServerPrivateKey, + Path: "server.key", + }, + }, + }, + }, + }, + { + Name: "ca-crt", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.nc.Spec.SSLCerts.CASecret, + Items: []corev1.KeyToPath{ + { + Key: c.nc.Spec.SSLCerts.CAPublicKey, + Path: "ca.crt", + }, + }, + }, + }, + }, + } + volumes = append(volumes, certVolumes...) + } + return volumes } diff --git a/apis/apps/v1alpha1/nebulacluster_types.go b/apis/apps/v1alpha1/nebulacluster_types.go index 5b311eae..6fa5cad8 100644 --- a/apis/apps/v1alpha1/nebulacluster_types.go +++ b/apis/apps/v1alpha1/nebulacluster_types.go @@ -102,6 +102,9 @@ type NebulaClusterSpec struct { // +optional Exporter *ExporterSpec `json:"exporter,omitempty"` + + // SSLCerts defines SSL certs load into secret + SSLCerts *SSLCertsSpec `json:"sslCerts,omitempty"` } // NebulaClusterStatus defines the observed state of NebulaCluster @@ -217,12 +220,48 @@ type ExporterSpec struct { } type LicenseSpec struct { - // Name of the license secret name. + // Name of the license secret. SecretName string `json:"secretName,omitempty"` // The key to nebula license file. LicenseKey string `json:"licenseKey,omitempty"` } +type SSLCertsSpec struct { + // Name of the server cert secret + ServerSecret string `json:"serverSecret,omitempty"` + // The key to server PEM encoded public key certificate + // +kubebuilder:default=tls.crt + // +optional + ServerPublicKey string `json:"serverPublicKey,omitempty"` + // The key to server private key associated with given certificate + // +kubebuilder:default=tls.key + // +optional + ServerPrivateKey string `json:"serverPrivateKey,omitempty"` + + // Name of the client cert secret + ClientSecret string `json:"clientSecret,omitempty"` + // The key to client PEM encoded public key certificate + // +kubebuilder:default=tls.crt + // +optional + ClientPublicKey string `json:"clientPublicKey,omitempty"` + // The key to client private key associated with given certificate + // +kubebuilder:default=tls.key + // +optional + ClientPrivateKey string `json:"clientPrivateKey,omitempty"` + + // Name of the CA cert secret + CASecret string `json:"caSecret,omitempty"` + // The key to CA PEM encoded public key certificate + // +kubebuilder:default=ca.crt + // +optional + CAPublicKey string `json:"caPublicKey,omitempty"` + + // InsecureSkipVerify controls whether a client verifies the server's + // certificate chain and host name. + // +optional + InsecureSkipVerify *bool `json:"insecureSkipVerify,omitempty"` +} + // GraphdSpec defines the desired state of Graphd type GraphdSpec struct { PodSpec `json:",inline"` diff --git a/apis/apps/v1alpha1/template.go b/apis/apps/v1alpha1/template.go index 83f329d2..614ca4fe 100644 --- a/apis/apps/v1alpha1/template.go +++ b/apis/apps/v1alpha1/template.go @@ -222,6 +222,31 @@ const ( --memory_purge_enabled=true # memory background purge interval in seconds --memory_purge_interval_seconds=10 + +# Enable HTTP2 handler for RPC +--enable_http2_routing=false +# HTTP stream timeout in milliseconds +--stream_timeout_ms=30000 + +########## SSL ########## +# whether to enable ssl +--enable_ssl=false +# whether to enable ssl of graph server +--enable_graph_ssl=false +# whether to enable ssl of meta server +--enable_meta_ssl=false +# path to cert pem +--cert_path=certs/server.crt +# path to cert key +--key_path=certs/server.key +# path to trusted CA file +--ca_path=certs/ca.crt +# path to trusted client CA file +--ca_client_path=certs/ca.crt +# path to SSL req challenge password +--password_path=certs/password +# path to SSL config watch path of file or directory +--ssl_watch_path=certs ` // nolint: revive MetadhConfigTemplate = ` @@ -291,6 +316,24 @@ const ( --ng_black_box_dump_period_seconds=5 # Black box log files expire time --ng_black_box_file_lifetime_seconds=1800 + +########## SSL ########## +# whether to enable ssl +--enable_ssl=false +# whether to enable ssl of meta server +--enable_meta_ssl=false +# path to cert pem +--cert_path=certs/server.crt +# path to cert key +--key_path=certs/server.key +# path to trusted CA file +--ca_path=certs/ca.crt +# path to trusted client CA file +--ca_client_path=certs/ca.crt +# path to SSL req challenge password +--password_path=certs/password +# path to SSL config watch path of file or directory +--ssl_watch_path=certs ` // nolint: revive StoragedConfigTemplate = ` @@ -484,5 +527,23 @@ const ( --memory_purge_enabled=true # memory background purge interval in seconds --memory_purge_interval_seconds=10 + +########## SSL ########## +# whether to enable ssl +--enable_ssl=false +# whether to enable ssl of meta server +--enable_meta_ssl=false +# path to cert pem +--cert_path=certs/server.crt +# path to cert key +--key_path=certs/server.key +# path to trusted CA file +--ca_path=certs/ca.crt +# path to trusted client CA file +--ca_client_path=certs/ca.crt +# path to SSL req challenge password +--password_path=certs/password +# path to SSL config watch path of file or directory +--ssl_watch_path=certs ` ) diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index d9da61c6..133f39c4 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -371,6 +371,11 @@ func (in *NebulaClusterSpec) DeepCopyInto(out *NebulaClusterSpec) { *out = new(ExporterSpec) (*in).DeepCopyInto(*out) } + if in.SSLCerts != nil { + in, out := &in.SSLCerts, &out.SSLCerts + *out = new(SSLCertsSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NebulaClusterSpec. @@ -684,6 +689,26 @@ func (in *S3StorageProvider) DeepCopy() *S3StorageProvider { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SSLCertsSpec) DeepCopyInto(out *SSLCertsSpec) { + *out = *in + if in.InsecureSkipVerify != nil { + in, out := &in.InsecureSkipVerify, &out.InsecureSkipVerify + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SSLCertsSpec. +func (in *SSLCertsSpec) DeepCopy() *SSLCertsSpec { + if in == nil { + return nil + } + out := new(SSLCertsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = *in diff --git a/charts/nebula-operator/crds/nebulacluster.yaml b/charts/nebula-operator/crds/nebulacluster.yaml index b0d299f2..ca44c0db 100644 --- a/charts/nebula-operator/crds/nebulacluster.yaml +++ b/charts/nebula-operator/crds/nebulacluster.yaml @@ -7935,6 +7935,32 @@ spec: schedulerName: default: default-scheduler type: string + sslCerts: + properties: + caPublicKey: + default: ca.crt + type: string + caSecret: + type: string + clientPrivateKey: + default: tls.key + type: string + clientPublicKey: + default: tls.crt + type: string + clientSecret: + type: string + insecureSkipVerify: + type: boolean + serverPrivateKey: + default: tls.key + type: string + serverPublicKey: + default: tls.crt + type: string + serverSecret: + type: string + type: object storaged: properties: affinity: diff --git a/charts/nebula-operator/templates/controller-manager-rbac.yaml b/charts/nebula-operator/templates/controller-manager-rbac.yaml index 8c45e05b..b83c4ae1 100644 --- a/charts/nebula-operator/templates/controller-manager-rbac.yaml +++ b/charts/nebula-operator/templates/controller-manager-rbac.yaml @@ -69,6 +69,9 @@ rules: verbs: - get - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/charts/nebula-operator/values.yaml b/charts/nebula-operator/values.yaml index aef61abd..a2450ad5 100644 --- a/charts/nebula-operator/values.yaml +++ b/charts/nebula-operator/values.yaml @@ -3,19 +3,19 @@ image: image: vesoft/nebula-operator:v1.4.2 imagePullPolicy: Always kubeRBACProxy: - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 imagePullPolicy: Always kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.22.12 + image: registry.k8s.io/kube-scheduler:v1.24.11 imagePullPolicy: Always -imagePullSecrets: [] +imagePullSecrets: [ ] kubernetesClusterDomain: "" controllerManager: create: true replicas: 2 - env: [] + env: [ ] resources: limits: cpu: 200m @@ -31,7 +31,7 @@ scheduler: create: false schedulerName: nebula-scheduler replicas: 2 - env: [] + env: [ ] resources: limits: cpu: 200m @@ -61,8 +61,8 @@ webhookBindPort: 9443 # Maximum number of concurrently running reconcile loops for nebula cluster (default 3) maxConcurrentReconciles: 3 -nodeSelector: {} +nodeSelector: { } -tolerations: [] +tolerations: [ ] -affinity: {} \ No newline at end of file +affinity: { } \ No newline at end of file diff --git a/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml b/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml index 39e7b402..e8aba428 100644 --- a/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml +++ b/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml @@ -7935,6 +7935,32 @@ spec: schedulerName: default: default-scheduler type: string + sslCerts: + properties: + caPublicKey: + default: ca.crt + type: string + caSecret: + type: string + clientPrivateKey: + default: tls.key + type: string + clientPublicKey: + default: tls.crt + type: string + clientSecret: + type: string + insecureSkipVerify: + type: boolean + serverPrivateKey: + default: tls.key + type: string + serverPublicKey: + default: tls.crt + type: string + serverSecret: + type: string + type: object storaged: properties: affinity: diff --git a/doc/user/ssl_guide.md b/doc/user/ssl_guide.md new file mode 100644 index 00000000..22e74dee --- /dev/null +++ b/doc/user/ssl_guide.md @@ -0,0 +1,101 @@ +### SSL encryption + +NebulaGraph supports data transmission with SSL encryption between clients, the Graph service, +the Meta service, and the Storage service. This topic describes how to enable SSL encryption. + +NebulaGraph supports three encryption policies: + +- Encrypt the data transmission between clients, the Graph service, the Meta service, and the Storage service. + To enable this policy, add `enable_ssl = true` to each component. + + ```yaml + apiVersion: apps.nebula-graph.io/v1alpha1 + kind: NebulaCluster + metadata: + name: nebula + namespace: default + spec: + sslCerts: + serverSecret: "server-cert" + clientSecret: "client-cert" + caSecret: "ca-cert" + graphd: + config: + enable_ssl: "true" + metad: + config: + enable_ssl: "true" + storaged: + config: + enable_ssl: "true" + ``` + +- Encrypt the data transmission between clients and the Graph service. + This policy is applicable when the clusters are set up in the same server room. Only the port of the Graph service is + open to the outside, + as other services can communicate over the internal network without encryption. + To enable this policy, add `enable_graph_ssl = true` to the graphd component. + + ```yaml + apiVersion: apps.nebula-graph.io/v1alpha1 + kind: NebulaCluster + metadata: + name: nebula + namespace: default + spec: + sslCerts: + serverSecret: "server-cert" + caSecret: "ca-cert" + graphd: + config: + enable_graph_ssl: "true" + ``` + +- Encrypt the data transmission related to the Meta service in the cluster. + This policy applies to transporting classified information to the Meta service. + To enable this policy, add `enable_meta_ssl = true` to each component. + + ```yaml + apiVersion: apps.nebula-graph.io/v1alpha1 + kind: NebulaCluster + metadata: + name: nebula + namespace: default + spec: + sslCerts: + serverSecret: "server-cert" + clientSecret: "client-cert" + caSecret: "ca-cert" + graphd: + config: + enable_meta_ssl: "true" + metad: + config: + enable_meta_ssl: "true" + storaged: + config: + enable_meta_ssl: "true" + ``` + +The full configuration of sslCerts: +```yaml +sslCerts: + # Name of the server cert secret + serverSecret: "server-cert" + # The key to server PEM encoded public key certificate, default name is tls.crt + serverPublicKey: "" + # The key to server private key associated with given certificate, default name is tls.key + serverPrivateKey: "" + # Name of the client cert secret + clientSecret: "client-cert" + # The key to server PEM encoded public key certificate, default name is tls.crt + clientPublicKey: "" + # The key to client private key associated with given certificate, default name is tls.key + clientPrivateKey: "" + # Name of the CA cert secret + caSecret: "ca-cert" + # The key to CA PEM encoded public key certificate, default name is ca.crt + caPublicKey: "" + # InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name + insecureSkipVerify: false +``` \ No newline at end of file diff --git a/pkg/controller/component/metad_cluster.go b/pkg/controller/component/metad_cluster.go index 1553bbae..f69ce3ba 100644 --- a/pkg/controller/component/metad_cluster.go +++ b/pkg/controller/component/metad_cluster.go @@ -176,8 +176,12 @@ func (c *metadCluster) syncMetadConfigMap(nc *v1alpha1.NebulaCluster) (*corev1.C } func (c *metadCluster) setVersion(nc *v1alpha1.NebulaCluster) error { + options, err := nebula.ClientOptions(nc) + if err != nil { + return err + } endpoints := []string{nc.GetMetadThriftConnAddress()} - metaClient, err := nebula.NewMetaClient(endpoints) + metaClient, err := nebula.NewMetaClient(endpoints, options...) if err != nil { return err } diff --git a/pkg/controller/component/storaged_cluster.go b/pkg/controller/component/storaged_cluster.go index 32a0c6dc..d02f3e92 100644 --- a/pkg/controller/component/storaged_cluster.go +++ b/pkg/controller/component/storaged_cluster.go @@ -204,8 +204,12 @@ func (c *storagedCluster) syncStoragedConfigMap(nc *v1alpha1.NebulaCluster) (*co } func (c *storagedCluster) addStorageHosts(nc *v1alpha1.NebulaCluster, oldReplicas, newReplicas int32) error { + options, err := nebula.ClientOptions(nc) + if err != nil { + return err + } endpoints := []string{nc.GetMetadThriftConnAddress()} - metaClient, err := nebula.NewMetaClient(endpoints) + metaClient, err := nebula.NewMetaClient(endpoints, options...) if err != nil { return err } diff --git a/pkg/controller/component/storaged_scaler.go b/pkg/controller/component/storaged_scaler.go index 96edb936..488c39c3 100644 --- a/pkg/controller/component/storaged_scaler.go +++ b/pkg/controller/component/storaged_scaler.go @@ -77,8 +77,12 @@ func (ss *storageScaler) ScaleOut(nc *v1alpha1.NebulaCluster) error { return nil } + options, err := nebula.ClientOptions(nc) + if err != nil { + return err + } endpoints := []string{nc.GetMetadThriftConnAddress()} - metaClient, err := nebula.NewMetaClient(endpoints) + metaClient, err := nebula.NewMetaClient(endpoints, options...) if err != nil { klog.Errorf("create meta client failed: %v", err) return err @@ -124,8 +128,12 @@ func (ss *storageScaler) ScaleIn(nc *v1alpha1.NebulaCluster, oldReplicas, newRep return err } + options, err := nebula.ClientOptions(nc) + if err != nil { + return err + } endpoints := []string{nc.GetMetadThriftConnAddress()} - metaClient, err := nebula.NewMetaClient(endpoints) + metaClient, err := nebula.NewMetaClient(endpoints, options...) if err != nil { return err } diff --git a/pkg/controller/component/storaged_updater.go b/pkg/controller/component/storaged_updater.go index c86f8d46..73943416 100644 --- a/pkg/controller/component/storaged_updater.go +++ b/pkg/controller/component/storaged_updater.go @@ -84,8 +84,12 @@ func (s *storagedUpdater) Update( return err } + options, err := nebula.ClientOptions(nc) + if err != nil { + return err + } endpoints := []string{nc.GetMetadThriftConnAddress()} - mc, err := nebula.NewMetaClient(endpoints) + mc, err := nebula.NewMetaClient(endpoints, options...) if err != nil { return err } @@ -193,9 +197,13 @@ func (s *storagedUpdater) transLeaderIfNecessary( } klog.Infof("set pod %s annotation %v successfully", updatePod.Name, TransLeaderBeginTime) + options, err := nebula.ClientOptions(nc, nebula.SetIsStorage(true)) + if err != nil { + return err + } podFQDN := nc.StoragedComponent().GetPodFQDN(ordinal) endpoint := fmt.Sprintf("%s:%d", podFQDN, nc.StoragedComponent().GetPort(v1alpha1.StoragedPortNameAdmin)) - sc, err := nebula.NewStorageClient([]string{endpoint}) + sc, err := nebula.NewStorageClient([]string{endpoint}, options...) if err != nil { return err } diff --git a/pkg/controller/nebularestore/nebula_restore_manager.go b/pkg/controller/nebularestore/nebula_restore_manager.go index 0f7355d3..df6c90e7 100644 --- a/pkg/controller/nebularestore/nebula_restore_manager.go +++ b/pkg/controller/nebularestore/nebula_restore_manager.go @@ -30,6 +30,7 @@ import ( "github.com/vesoft-inc/nebula-go/v3/nebula/meta" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/klog/v2" "k8s.io/utils/pointer" @@ -40,7 +41,6 @@ import ( "github.com/vesoft-inc/nebula-operator/pkg/util/async" "github.com/vesoft-inc/nebula-operator/pkg/util/condition" utilerrors "github.com/vesoft-inc/nebula-operator/pkg/util/errors" - "github.com/vesoft-inc/nebula-operator/pkg/util/randstr" rtutil "github.com/vesoft-inc/nebula-operator/pkg/util/restore" ) @@ -146,6 +146,11 @@ func (rm *restoreManager) syncRestoreProcess(rt *v1alpha1.NebulaRestore) error { return err } + options, err := nebula.ClientOptions(original) + if err != nil { + return err + } + restored := rm.genNebulaCluster(restoredName, rt, original) if err := rm.clientSet.NebulaCluster().CreateNebulaCluster(restored); err != nil { return err @@ -155,7 +160,7 @@ func (rm *restoreManager) syncRestoreProcess(rt *v1alpha1.NebulaRestore) error { return err } - if err := rm.loadCluster(original, restored); err != nil { + if err := rm.loadCluster(original, restored, options); err != nil { return err } @@ -193,7 +198,8 @@ func (rm *restoreManager) syncRestoreProcess(rt *v1alpha1.NebulaRestore) error { } hostPairs := rm.restore.genHostPairs(rm.restore.bakMetas[0], restored.GetStoragedEndpoints(v1alpha1.StoragedPortNameThrift)) - restoreResp, err := rm.restore.restoreMeta(rm.restore.bakMetas[0], hostPairs, restored.GetMetadEndpoints(v1alpha1.MetadPortNameThrift)) + metaEndpoints := restored.GetMetadEndpoints(v1alpha1.MetadPortNameThrift) + restoreResp, err := rm.restore.restoreMeta(rm.restore.bakMetas[0], hostPairs, metaEndpoints, options) if err != nil { klog.Errorf("restore metad data [%s/%s] failed, error: %v", ns, restoredName, err) return err @@ -285,8 +291,8 @@ func (rm *restoreManager) syncRestoreProcess(rt *v1alpha1.NebulaRestore) error { return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage2, waiting for cluster ready", ns, restoredName) } -func (rm *restoreManager) loadCluster(original, restored *v1alpha1.NebulaCluster) error { - mc, err := nebula.NewMetaClient([]string{original.GetMetadThriftConnAddress()}) +func (rm *restoreManager) loadCluster(original, restored *v1alpha1.NebulaCluster, options []nebula.Option) error { + mc, err := nebula.NewMetaClient([]string{original.GetMetadThriftConnAddress()}, options...) if err != nil { return err } @@ -534,7 +540,7 @@ func (r *Restore) downloadStorageData(parts map[string][]*ng.HostAddr, storageHo return checkpoints, group.Wait() } -func (r *Restore) restoreMeta(backup *meta.BackupMeta, storageMap map[string]string, metaEndpoints []string) (*meta.RestoreMetaResp, error) { +func (r *Restore) restoreMeta(backup *meta.BackupMeta, storageMap map[string]string, metaEndpoints []string, options []nebula.Option) (*meta.RestoreMetaResp, error) { addrMap := make([]*meta.HostPair, 0, len(storageMap)) for from, to := range storageMap { fromAddr, err := rtutil.ParseAddr(from) @@ -556,7 +562,7 @@ func (r *Restore) restoreMeta(backup *meta.BackupMeta, storageMap map[string]str files = append(files, filePath) } - mc, err := nebula.NewMetaClient([]string{metaEndpoint}) + mc, err := nebula.NewMetaClient([]string{metaEndpoint}, options...) if err != nil { if utilerrors.IsDNSError(err) { return nil, utilerrors.ReconcileErrorf("waiting for %s dns lookup is ok", metaEndpoint) @@ -774,7 +780,7 @@ func (rm *restoreManager) clusterReady(namespace, ncName string) (bool, error) { func (rm *restoreManager) getRestoredName(rt *v1alpha1.NebulaRestore) (string, error) { if rt.Status.ClusterName == "" { - genName := "ng" + randstr.Hex(4) + genName := "ng" + rand.String(4) newStatus := &kube.RestoreUpdateStatus{ TimeStarted: &metav1.Time{Time: time.Now()}, ClusterName: pointer.String(genName), diff --git a/pkg/nebula/cert.go b/pkg/nebula/cert.go new file mode 100644 index 00000000..60a670f5 --- /dev/null +++ b/pkg/nebula/cert.go @@ -0,0 +1,34 @@ +package nebula + +import ( + "sigs.k8s.io/controller-runtime/pkg/client/config" + + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + "github.com/vesoft-inc/nebula-operator/pkg/kube" +) + +func getCerts(namespace string, cert *v1alpha1.SSLCertsSpec) ([]byte, []byte, []byte, error) { + cfg, err := config.GetConfig() + if err != nil { + return nil, nil, nil, err + } + + client, err := kube.NewClientSet(cfg) + if err != nil { + return nil, nil, nil, err + } + + caSecret, err := client.Secret().GetSecret(namespace, cert.CASecret) + if err != nil { + return nil, nil, nil, err + } + caCert := caSecret.Data[cert.CAPublicKey] + + clientSecret, err := client.Secret().GetSecret(namespace, cert.ClientSecret) + if err != nil { + return nil, nil, nil, err + } + clientCert := clientSecret.Data[cert.ClientPublicKey] + clientKey := clientSecret.Data[cert.ClientPrivateKey] + return caCert, clientCert, clientKey, nil +} diff --git a/pkg/nebula/client.go b/pkg/nebula/client.go index 38407f7e..e0ad30cc 100644 --- a/pkg/nebula/client.go +++ b/pkg/nebula/client.go @@ -31,13 +31,21 @@ func buildClientTransport(endpoint string, options ...Option) (thrift.Transport, opts := loadOptions(options...) timeoutOption := thrift.SocketTimeout(opts.Timeout) addressOption := thrift.SocketAddr(endpoint) - sock, err := thrift.NewSocket(timeoutOption, addressOption) + + var err error + var sock thrift.Transport + tlsEnabled := opts.EnableClusterTLS || (opts.EnableMetaTLS && !opts.IsStorage) + if tlsEnabled { + sock, err = thrift.NewSSLSocketTimeout(endpoint, opts.TLSConfig, opts.Timeout) + } else { + sock, err = thrift.NewSocket(timeoutOption, addressOption) + } if err != nil { return nil, nil, err } + bufferedTranFactory := thrift.NewBufferedTransportFactory(defaultBufferSize) transport := thrift.NewFramedTransportMaxLength(bufferedTranFactory.GetTransport(sock), frameMaxLength) pf := thrift.NewBinaryProtocolFactoryDefault() - return transport, pf, nil } diff --git a/pkg/nebula/options.go b/pkg/nebula/options.go index 02e04807..71bdacf3 100644 --- a/pkg/nebula/options.go +++ b/pkg/nebula/options.go @@ -16,14 +16,53 @@ limitations under the License. package nebula -import "time" +import ( + "crypto/tls" + "k8s.io/klog/v2" + "time" -const DefaultTimeout = 30 * time.Second + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + "github.com/vesoft-inc/nebula-operator/pkg/util/cert" +) + +const DefaultTimeout = 10 * time.Second type Option func(ops *Options) type Options struct { - Timeout time.Duration + EnableMetaTLS bool + EnableClusterTLS bool + IsStorage bool + Timeout time.Duration + TLSConfig *tls.Config +} + +func ClientOptions(nc *v1alpha1.NebulaCluster, opts ...Option) ([]Option, error) { + options := []Option{SetTimeout(DefaultTimeout)} + if nc.Spec.SSLCerts == nil || (nc.IsGraphdSSLEnabled() && !nc.IsMetadSSLEnabled() && !nc.IsClusterEnabled()) { + return options, nil + } + + if nc.IsMetadSSLEnabled() && !nc.IsClusterEnabled() { + options = append(options, SetMetaTLS(true)) + klog.Infof("cluster [%s/%s] metad SSL enabled", nc.Namespace, nc.Name) + } + if nc.IsClusterEnabled() { + options = append(options, SetClusterTLS(true)) + klog.Infof("cluster [%s/%s] SSL enabled", nc.Namespace, nc.Name) + } + caCert, clientCert, clientKey, err := getCerts(nc.Namespace, nc.Spec.SSLCerts) + if err != nil { + return nil, err + } + tlsConfig, err := cert.LoadTLSConfig(caCert, clientCert, clientKey) + if err != nil { + return nil, err + } + tlsConfig.InsecureSkipVerify = nc.InsecureSkipVerify() + options = append(options, SetTLSConfig(tlsConfig)) + options = append(options, opts...) + return options, nil } func loadOptions(options ...Option) *Options { @@ -34,14 +73,38 @@ func loadOptions(options ...Option) *Options { return opts } -func WithOptions(options Options) Option { +func SetOptions(options Options) Option { return func(opts *Options) { *opts = options } } -func WithTimeout(duration time.Duration) Option { +func SetTimeout(duration time.Duration) Option { return func(options *Options) { options.Timeout = duration } } + +func SetTLSConfig(config *tls.Config) Option { + return func(options *Options) { + options.TLSConfig = config + } +} + +func SetMetaTLS(e bool) Option { + return func(options *Options) { + options.EnableMetaTLS = e + } +} + +func SetClusterTLS(e bool) Option { + return func(options *Options) { + options.EnableClusterTLS = e + } +} + +func SetIsStorage(e bool) Option { + return func(options *Options) { + options.IsStorage = e + } +} diff --git a/pkg/util/cert/cert.go b/pkg/util/cert/cert.go new file mode 100644 index 00000000..409a5328 --- /dev/null +++ b/pkg/util/cert/cert.go @@ -0,0 +1,54 @@ +package cert + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "time" +) + +func LoadTLSConfig(caCert, cert, key []byte) (*tls.Config, error) { + c, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, err + } + rootCAPool := x509.NewCertPool() + ok := rootCAPool.AppendCertsFromPEM(caCert) + if !ok { + return nil, fmt.Errorf("failed to load cert files") + } + return &tls.Config{ + Certificates: []tls.Certificate{c}, + RootCAs: rootCAPool, + }, nil +} + +func ValidCACert(key, cert, caCert []byte, dnsName string, time time.Time) bool { + if len(key) == 0 || len(cert) == 0 || len(caCert) == 0 { + return false + } + _, err := tls.X509KeyPair(cert, key) + if err != nil { + return false + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caCert) { + return false + } + block, _ := pem.Decode(cert) + if block == nil { + return false + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return false + } + ops := x509.VerifyOptions{ + DNSName: dnsName, + Roots: pool, + CurrentTime: time, + } + _, err = c.Verify(ops) + return err == nil +} diff --git a/pkg/util/randstr/randstr.go b/pkg/util/randstr/randstr.go deleted file mode 100644 index ba735f74..00000000 --- a/pkg/util/randstr/randstr.go +++ /dev/null @@ -1,19 +0,0 @@ -package randstr - -import ( - "crypto/rand" - "encoding/hex" -) - -// Bytes generates n random bytes -func Bytes(n int) []byte { - b := make([]byte, n) - _, err := rand.Read(b) - if err != nil { - panic(err) - } - return b -} - -// Hex generates a random hex string with length of n -func Hex(n int) string { return hex.EncodeToString(Bytes(n)) }