Ko 461 nodeport service v2 (#177)

* Initial NodePortConfig added * added configgen port setting * Passes HOST_IP to configbuilder * replaced accidentally removed function * WIP - create nodeport service * WIP - created nodeport test * WIP - test fixes * fixed a port and introduced a couple of test helper methods * do not set a native port in the normal service if the nodeport service is enabled * use cql port instead of cqlSsl port in the test * preserve ClusterIP of existing services * the test runs * updated the user doc for nodeport * Port names now use SSL instead of Ssl * Introduced DefaultCqlPort and DefaultBroadcastPort * Do not remove native port from normal service when nodeport service is configured
datastax · Jul 28, 2020 · b2cf2ec · b2cf2ec
1 parent 5f72962
commit b2cf2ec
Show file tree

Hide file tree

Showing 13 changed files with 513 additions and 57 deletions.
diff --git a/charts/cass-operator-chart/templates/customresourcedefinition.yaml b/charts/cass-operator-chart/templates/customresourcedefinition.yaml
@@ -92,6 +92,20 @@ spec:
                   - serverSecretName
                   type: object
               type: object
+            networking:
+              properties:
+                nodePort:
+                  properties:
+                    broadcast:
+                      type: integer
+                    broadcastSSL:
+                      type: integer
+                    cql:
+                      type: integer
+                    cqlSSL:
+                      type: integer
+                  type: object
+              type: object
             nodeSelector:
               additionalProperties:
                 type: string

diff --git a/docs/user/README.md b/docs/user/README.md
@@ -363,6 +363,24 @@ spec:
   serverImage: private-docker-registry.example.com/dse-img/dse:5f6e7d8c
 ```
 
+## Configuring a NodePort service
+
+A NodePort service may be requested by setting the following fields:
+
+  networking:
+    nodePort:
+      cql: 30001
+      broadcast: 30002
+ 
+The SSL versions of the ports may be requested:
+
+  networking:
+    nodePort:
+      cqlSSL: 30010
+      broadcastSSL: 30020
+      
+If any of the nodePort fields have been configured then a NodePort service will be created that routes from the specified external port to the identically numbered internal port.  Cassandra will be configured to listen on the specified ports.
+
 # Using Your Cluster
 
 ## Connecting from inside the Kubernetes cluster

diff --git a/mage/ginkgo/lib.go b/mage/ginkgo/lib.go
@@ -214,7 +214,6 @@ func (ns *NsWrapper) WaitForOutputContainsAndLog(description string, kcmd kubect
 	Expect(execErr).ToNot(HaveOccurred())
 }
 
-
 func (ns *NsWrapper) WaitForDatacenterCondition(dcName string, conditionType string, value string) {
 	step := fmt.Sprintf("checking that dc condition %s has value %s", conditionType, value)
 	json := fmt.Sprintf("jsonpath={.status.conditions[?(.type=='%s')].status}", conditionType)
@@ -223,7 +222,6 @@ func (ns *NsWrapper) WaitForDatacenterCondition(dcName string, conditionType str
 	ns.WaitForOutputAndLog(step, k, value, 600)
 }
 
-
 func (ns *NsWrapper) WaitForDatacenterToHaveNoPods(dcName string) {
 	step := "checking that no dc pods remain"
 	json := "jsonpath={.items}"
@@ -369,3 +367,27 @@ func (ns NsWrapper) HelmInstall(chartPath string) {
 	err := helm_util.Install(chartPath, "cass-operator", ns.Namespace, overrides)
 	mageutil.PanicOnError(err)
 }
+
+// Note that the actual value will be cast to a string before the comparison with the expectedValue
+func (ns NsWrapper) ExpectKeyValue(m map[string]interface{}, key string, expectedValue string) {
+	actualValue, ok := m[key].(string)
+	if !ok {
+		// Note: floats will end up as strings with six decimal points
+		// example: "12.000000"
+		tryFloat64, ok := m[key].(float64)
+		if !ok {
+			msg := fmt.Sprintf("Actual value for key %s is not expected type", key)
+			err := fmt.Errorf(msg)
+			Expect(err).ToNot(HaveOccurred())
+		}
+		actualValue = fmt.Sprintf("%f", tryFloat64)
+	}
+	Expect(actualValue).To(Equal(expectedValue), "Expected %s %s to be %s", key, m[key], expectedValue)
+}
+
+// Compare all key/values from an expected map to an actual map
+func (ns NsWrapper) ExpectKeyValues(actual map[string]interface{}, expected map[string]string) {
+	for key := range expected {
+		ns.ExpectKeyValue(actual, key, expected[key])
+	}
+}
diff --git a/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml b/operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml
@@ -92,6 +92,20 @@ spec:
                   - serverSecretName
                   type: object
               type: object
+            networking:
+              properties:
+                nodePort:
+                  properties:
+                    broadcast:
+                      type: integer
+                    broadcastSSL:
+                      type: integer
+                    cql:
+                      type: integer
+                    cqlSSL:
+                      type: integer
+                  type: object
+              type: object
             nodeSelector:
               additionalProperties:
                 type: string

diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go
@@ -40,6 +40,10 @@ const (
 	// Progress states for status
 	ProgressUpdating ProgressState = "Updating"
 	ProgressReady    ProgressState = "Ready"
+
+	// Default port numbers
+	DefaultCqlPort       = 9042
+	DefaultBroadcastPort = 7000
 )
 
 // This type exists so there's no chance of pushing random strings to our progress status
@@ -221,11 +225,29 @@ type CassandraDatacenterSpec struct {
 	// Cassandra users to bootstrap
 	Users []CassandraUser `json:"users,omitempty"`
 
+	Networking *NetworkingConfig `json:"networking,omitempty"`
+
 	AdditionalSeeds []string `json:"additionalSeeds,omitempty"`
 
 	Reaper *ReaperConfig `json:"reaper,omitempty"`
 }
 
+type NetworkingConfig struct {
+	NodePort *NodePortConfig `json:"nodePort,omitempty"`
+}
+
+type NodePortConfig struct {
+	Cql          int `json:"cql,omitempty"`
+	CqlSSL       int `json:"cqlSSL,omitempty"`
+	Broadcast    int `json:"broadcast,omitempty"`
+	BroadcastSSL int `json:"broadcastSSL,omitempty"`
+}
+
+// Is the NodePort service enabled?
+func (dc *CassandraDatacenter) IsNodePortEnabled() bool {
+	return dc.Spec.Networking != nil && dc.Spec.Networking.NodePort != nil
+}
+
 type DseWorkloads struct {
 	AnalyticsEnabled bool `json:"analyticsEnabled,omitempty"`
 	GraphEnabled     bool `json:"graphEnabled,omitempty"`
@@ -491,6 +513,10 @@ func (dc *CassandraDatacenter) GetDatacenterServiceName() string {
 	return dc.Spec.ClusterName + "-" + dc.Name + "-service"
 }
 
+func (dc *CassandraDatacenter) GetNodePortServiceName() string {
+	return dc.Spec.ClusterName + "-" + dc.Name + "-node-port-service"
+}
+
 func (dc *CassandraDatacenter) ShouldGenerateSuperuserSecret() bool {
 	return len(dc.Spec.SuperuserSecretName) == 0
 }
@@ -533,13 +559,28 @@ func (dc *CassandraDatacenter) GetConfigAsJSON() (string, error) {
 		}
 	}
 
+	cql := 0
+	cqlSSL := 0
+	broadcast := 0
+	broadcastSSL := 0
+	if dc.IsNodePortEnabled() {
+		cql = dc.Spec.Networking.NodePort.Cql
+		cqlSSL = dc.Spec.Networking.NodePort.CqlSSL
+		broadcast = dc.Spec.Networking.NodePort.Broadcast
+		broadcastSSL = dc.Spec.Networking.NodePort.BroadcastSSL
+	}
+
 	modelValues := serverconfig.GetModelValues(
 		seeds,
 		dc.Spec.ClusterName,
 		dc.Name,
 		graphEnabled,
 		solrEnabled,
-		sparkEnabled)
+		sparkEnabled,
+		cql,
+		cqlSSL,
+		broadcast,
+		broadcastSSL)
 
 	var modelBytes []byte
 
@@ -569,21 +610,61 @@ func (dc *CassandraDatacenter) GetConfigAsJSON() (string, error) {
 	return modelParsed.String(), nil
 }
 
+// Gets the defined CQL port for NodePort.
+// 0 will be returned if NodePort is not configured.
+// The SSL port will be returned if it is defined,
+// otherwise the normal CQL port will be used.
+func (dc *CassandraDatacenter) GetNodePortCqlPort() int {
+	if !dc.IsNodePortEnabled() {
+		return 0
+	}
+
+	if dc.Spec.Networking.NodePort.CqlSSL != 0 {
+		return dc.Spec.Networking.NodePort.CqlSSL
+	} else if dc.Spec.Networking.NodePort.Cql != 0 {
+		return dc.Spec.Networking.NodePort.Cql
+	} else {
+		return DefaultCqlPort
+	}
+}
+
+// Gets the defined broadcast/intranode port for NodePort.
+// 0 will be returned if NodePort is not configured.
+// The SSL port will be returned if it is defined,
+// otherwise the normal broadcast port will be used.
+func (dc *CassandraDatacenter) GetNodePortBroadcastPort() int {
+	if !dc.IsNodePortEnabled() {
+		return 0
+	}
+
+	if dc.Spec.Networking.NodePort.BroadcastSSL != 0 {
+		return dc.Spec.Networking.NodePort.BroadcastSSL
+	} else if dc.Spec.Networking.NodePort.Broadcast != 0 {
+		return dc.Spec.Networking.NodePort.Broadcast
+	} else {
+		return DefaultBroadcastPort
+	}
+}
+
 // GetContainerPorts will return the container ports for the pods in a statefulset based on the provided config
 func (dc *CassandraDatacenter) GetContainerPorts() ([]corev1.ContainerPort, error) {
+
+	cqlPort := DefaultCqlPort
+	broadcastPort := DefaultBroadcastPort
+
 	ports := []corev1.ContainerPort{
 		{
 			// Note: Port Names cannot be more than 15 characters
 			Name:          "native",
-			ContainerPort: 9042,
+			ContainerPort: int32(cqlPort),
 		},
 		{
 			Name:          "inter-node-msg",
 			ContainerPort: 8609,
 		},
 		{
 			Name:          "intra-node",
-			ContainerPort: 7000,
+			ContainerPort: int32(broadcastPort),
 		},
 		{
 			Name:          "tls-intra-node",

diff --git a/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types_test.go b/operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types_test.go
@@ -236,13 +236,13 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) {
 			want: []corev1.ContainerPort{
 				{
 					Name:          "native",
-					ContainerPort: 9042,
+					ContainerPort: DefaultCqlPort,
 				}, {
 					Name:          "inter-node-msg",
 					ContainerPort: 8609,
 				}, {
 					Name:          "intra-node",
-					ContainerPort: 7000,
+					ContainerPort: DefaultBroadcastPort,
 				}, {
 					Name:          "tls-intra-node",
 					ContainerPort: 7001,
@@ -263,13 +263,13 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) {
 			want: []corev1.ContainerPort{
 				{
 					Name:          "native",
-					ContainerPort: 9042,
+					ContainerPort: DefaultCqlPort,
 				}, {
 					Name:          "inter-node-msg",
 					ContainerPort: 8609,
 				}, {
 					Name:          "intra-node",
-					ContainerPort: 7000,
+					ContainerPort: DefaultBroadcastPort,
 				}, {
 					Name:          "tls-intra-node",
 					ContainerPort: 7001,
@@ -292,13 +292,13 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) {
 			want: []corev1.ContainerPort{
 				{
 					Name:          "native",
-					ContainerPort: 9042,
+					ContainerPort: DefaultCqlPort,
 				}, {
 					Name:          "inter-node-msg",
 					ContainerPort: 8609,
 				}, {
 					Name:          "intra-node",
-					ContainerPort: 7000,
+					ContainerPort: DefaultBroadcastPort,
 				}, {
 					Name:          "tls-intra-node",
 					ContainerPort: 7001,

diff --git a/operator/pkg/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/operator/pkg/apis/cassandra/v1beta1/zz_generated.deepcopy.go