Merge pull request #35 from njucjc/lb

Feature: support SLB to expose raven service

charts/raven-controller-manager/values.yaml

@@ -44,8 +44,8 @@ resources:
      memory: 20Mi
 
 nodeSelector:
-  beta.kubernetes.io/arch: amd64
-  beta.kubernetes.io/os: linux
+  kubernetes.io/arch: amd64
+  kubernetes.io/os: linux
 
 affinity:
   nodeAffinity:

config/raven-controller-manager/crd/bases/raven.openyurt.io_gateways.yaml

@@ -63,6 +63,9 @@ spec:
                   - nodeName
                   type: object
                 type: array
+              exposeType:
+                description: ExposeType determines how the Gateway is exposed.
+                type: string
               nodeSelector:
                 description: NodeSelector is a label query over nodes that managed
                   by the gateway. The nodes in the same gateway should share same

config/raven-controller-manager/manager/patches.yaml

@@ -28,5 +28,5 @@ spec:
           operator: "Exists"
           effect: "NoSchedule"
       nodeSelector:
-        beta.kubernetes.io/arch: amd64
-        beta.kubernetes.io/os: linux
+        kubernetes.io/arch: amd64
+        kubernetes.io/os: linux

config/raven-controller-manager/rbac/role.yaml

@@ -30,6 +30,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -64,6 +76,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - crd.projectcalico.org
   resources:
@@ -77,12 +101,8 @@ rules:
   resources:
   - gateways
   verbs:
-  - create
-  - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - raven.openyurt.io

config/setup/all_in_one.yaml

@@ -475,8 +475,8 @@ spec:
               name: cert
               readOnly: true
       nodeSelector:
-        beta.kubernetes.io/arch: amd64
-        beta.kubernetes.io/os: linux
+        kubernetes.io/arch: amd64
+        kubernetes.io/os: linux
       serviceAccountName: raven-controller-manager
       terminationGracePeriodSeconds: 10
       tolerations:

main.go

@@ -32,7 +32,8 @@ import (
 
 	calicov3 "github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/apis/calico/v3"
 	ravenv1alpha1 "github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/apis/raven/v1alpha1"
-	"github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/controllers"
+	"github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/controllers/gateway"
+	"github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/controllers/service"
 	ravenwebhook "github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/webhook"
 	//+kubebuilder:scaffold:imports
 )
@@ -77,14 +78,24 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err = (&controllers.GatewayReconciler{
+	if err = (&gateway.GatewayReconciler{
 		Client: mgr.GetClient(),
 		Log:    ctrl.Log.WithName("controllers").WithName("Gateway"),
 		Scheme: mgr.GetScheme(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Gateway")
 		os.Exit(1)
 	}
+
+	if err = (&service.ServiceReconciler{
+		Client: mgr.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("Service"),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Service")
+		os.Exit(1)
+	}
+
 	if err = (&ravenv1alpha1.Gateway{}).SetupWebhookWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create webhook", "webhook", "Gateway")
 		os.Exit(1)

pkg/ravencontroller/apis/raven/v1alpha1/gateway_types.go

@@ -18,6 +18,7 @@ package v1alpha1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // Event reason.
@@ -28,9 +29,20 @@ const (
 	EventActiveEndpointLost = "ActiveEndpointLost"
 )
 
+var ServiceNamespacedName = types.NamespacedName{
+	Namespace: "kube-system",
+	Name:      "raven-agent-service",
+}
+
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
+type ExposeType string
+
+const (
+	ExposeTypeLoadBalancer = "LoadBalancer"
+)
+
 // GatewaySpec defines the desired state of Gateway
 type GatewaySpec struct {
 	// NodeSelector is a label query over nodes that managed by the gateway.
@@ -39,6 +51,8 @@ type GatewaySpec struct {
 	// TODO add a field to configure using vxlan or host-gw for inner gateway communication?
 	// Endpoints is a list of available Endpoint.
 	Endpoints []Endpoint `json:"endpoints"`
+	// ExposeType determines how the Gateway is exposed.
+	ExposeType ExposeType `json:"exposeType,omitempty"`
 }
 
 // Endpoint stores all essential data for establishing the VPN tunnel.

pkg/ravencontroller/apis/raven/v1alpha1/gateway_webhook.go

@@ -80,16 +80,32 @@ func (g *Gateway) ValidateDelete() error {
 
 func (g *Gateway) validateGateway() error {
 	var errList field.ErrorList
+	if len(g.Spec.Endpoints) == 0 {
+		fldPath := field.NewPath("spec").Child("endpoints")
+		errList = append(errList, field.Invalid(fldPath, g.Spec.Endpoints, "missing required field 'endpoints'"))
+	}
+
+	underNAT := g.Spec.Endpoints[0].UnderNAT
 	for i, ep := range g.Spec.Endpoints {
+		if ep.UnderNAT != underNAT {
+			fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("underNAT")
+			errList = append(errList, field.Invalid(fldPath, ep.UnderNAT, "the 'underNAT' field in endpoints must be the same"))
+		}
+
 		if ep.PublicIP != "" {
+			if g.Spec.ExposeType == ExposeTypeLoadBalancer {
+				fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("publicIP")
+				errList = append(errList, field.Invalid(fldPath, ep.PublicIP, fmt.Sprintf("the 'publicIP' field must not be set when spec.exposeType = %s", ExposeTypeLoadBalancer)))
+			}
+
 			if err := validateIP(ep.PublicIP); err != nil {
 				fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("publicIP")
-				errList = append(errList, field.Invalid(fldPath, ep.PublicIP, fmt.Sprintf("endpoints[%d].publicIP must be a validate IP address", i)))
+				errList = append(errList, field.Invalid(fldPath, ep.PublicIP, "the 'publicIP' field must be a validate IP address"))
 			}
 		}
 		if len(ep.NodeName) == 0 {
 			fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("nodeName")
-			errList = append(errList, field.Invalid(fldPath, ep.NodeName, fmt.Sprintf("endpoints[%d].nodeName cannot be empty", i)))
+			errList = append(errList, field.Invalid(fldPath, ep.NodeName, "the 'nodeName' field must not be empty"))
 		}
 	}
 	if errList != nil {

pkg/ravencontroller/controllers/gateway_controller.go → pkg/ravencontroller/controllers/gateway/gateway_controller.go

@@ -14,14 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package controllers
+package gateway
 
 import (
 	"context"
 	"fmt"
-	"net"
 	"reflect"
 	"strings"
+	"time"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
@@ -40,6 +40,7 @@ import (
 
 	calicov3 "github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/apis/calico/v3"
 	ravenv1alpha1 "github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/apis/raven/v1alpha1"
+	"github.com/openyurtio/raven-controller-manager/pkg/ravencontroller/controllers/util"
 )
 
 // GatewayReconciler reconciles a Gateway object
@@ -50,11 +51,13 @@ type GatewayReconciler struct {
 	recorder record.EventRecorder
 }
 
-//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways,verbs=get;list;watch;
 //+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways/finalizers,verbs=update
 //+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;update;patch
+//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=crd.projectcalico.org,resources=blockaffinities,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
@@ -89,6 +92,18 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	// 1. try to elect an active endpoint if possible
 	activeEp := r.electActiveEndpoint(nodeList, &gw)
 	r.recordEndpointEvent(ctx, &gw, gw.Status.ActiveEndpoint, activeEp)
+	if util.IsGatewayExposeByLB(&gw) {
+		var svc corev1.Service
+		if err := r.Get(ctx, ravenv1alpha1.ServiceNamespacedName, &svc); err != nil {
+			log.V(2).Info("waiting for service sync", "error", err)
+			return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil
+		}
+		if len(svc.Status.LoadBalancer.Ingress) == 0 {
+			log.V(2).Info("waiting for LB ingress sync")
+			return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil
+		}
+		activeEp.PublicIP = svc.Status.LoadBalancer.Ingress[0].IP
+	}
 	gw.Status.ActiveEndpoint = activeEp
 
 	// 2. get nodeInfo list of nodes managed by the Gateway
@@ -101,7 +116,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		}
 		nodes = append(nodes, ravenv1alpha1.NodeInfo{
 			NodeName:  v.Name,
-			PrivateIP: getNodeInternalIP(v),
+			PrivateIP: util.GetNodeInternalIP(v),
 			Subnets:   podCIDRs,
 		})
 	}
@@ -234,18 +249,6 @@ func getNodeCondition(status *corev1.NodeStatus, conditionType corev1.NodeCondit
 	return -1, nil
 }
 
-// getNodeInternalIP returns internal ip of the given `node`.
-func getNodeInternalIP(node corev1.Node) string {
-	var ip string
-	for _, addr := range node.Status.Addresses {
-		if addr.Type == corev1.NodeInternalIP && net.ParseIP(addr.Address) != nil {
-			ip = addr.Address
-			break
-		}
-	}
-	return ip
-}
-
 // getPodCIDRs returns the pod IP ranges assigned to the node.
 func (r *GatewayReconciler) getPodCIDRs(ctx context.Context, node corev1.Node) ([]string, error) {
 	podCIDRs := make([]string, 0)

pkg/ravencontroller/controllers/gateway_controller_test.go → pkg/ravencontroller/controllers/gateway/gateway_controller_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package controllers
+package gateway
 
 import (
 	"testing"

pkg/ravencontroller/controllers/predicates.go → pkg/ravencontroller/controllers/gateway/predicates.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package controllers
+package gateway
 
 import (
 	"github.com/go-logr/logr"