Merge pull request #304 from l1b0k/feature/default-config

improve controlplane default behave

daemon/daemon.go

@@ -25,7 +25,6 @@ import (
 	"github.com/AliyunContainerService/terway/pkg/tracing"
 	"github.com/AliyunContainerService/terway/rpc"
 	"github.com/AliyunContainerService/terway/types"
-	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/containernetworking/cni/libcni"
 	containertypes "github.com/containernetworking/cni/pkg/types"
@@ -1287,7 +1286,6 @@ func validateConfig(cfg *types.Configure) error {
 }
 
 func getPoolConfig(cfg *types.Configure) (*types.PoolConfig, error) {
-	sgIDs := sets.NewString(cfg.SecurityGroup).Insert(cfg.SecurityGroups...)
 	poolConfig := &types.PoolConfig{
 		MaxPoolSize:            cfg.MaxPoolSize,
 		MinPoolSize:            cfg.MinPoolSize,
@@ -1297,10 +1295,13 @@ func getPoolConfig(cfg *types.Configure) (*types.PoolConfig, error) {
 		AccessSecret:           cfg.AccessSecret,
 		EniCapRatio:            cfg.EniCapRatio,
 		EniCapShift:            cfg.EniCapShift,
-		SecurityGroups:         sgIDs.List(),
+		SecurityGroups:         cfg.GetSecurityGroups(),
 		VSwitchSelectionPolicy: cfg.VSwitchSelectionPolicy,
 		EnableENITrunking:      cfg.EnableENITrunking,
 	}
+	if len(poolConfig.SecurityGroups) > 5 {
+		return nil, fmt.Errorf("security groups should not be more than 5, current %d", len(poolConfig.SecurityGroups))
+	}
 	ins := aliyun.GetInstanceMeta()
 	zone := ins.ZoneID
 	if cfg.VSwitches != nil {

pkg/controller/pod/pod_controller.go

@@ -261,7 +261,7 @@ func (m *ReconcilePod) podCreate(ctx context.Context, pod *corev1.Pod) (reconcil
 		if err != nil {
 			return reconcile.Result{}, fmt.Errorf("error get podNetworking %s, %w", podNetwokingName, err)
 		}
-		vsw, err := m.swPool.GetOne(ctx, m.aliyun, nodeInfo.Zone, podNetworking.Spec.VSwitchIDs)
+		vsw, err := m.swPool.GetOne(ctx, m.aliyun, nodeInfo.Zone, podNetworking.Spec.VSwitchIDs, false)
 		if err != nil {
 			return reconcile.Result{}, fmt.Errorf("can not found available vSwitch for zone %s, %w", nodeInfo.Zone, err)
 		}

pkg/controller/vswitch/vswitch.go

@@ -41,8 +41,6 @@ type Switch struct {
 type SwitchPool struct {
 	cache *cache.LRUExpireCache
 	ttl   time.Duration
-
-	ignoreZone bool
 }
 
 // NewSwitchPool create pool and set vSwitches to pool
@@ -56,7 +54,7 @@ func NewSwitchPool(size int, ttl string) (*SwitchPool, error) {
 }
 
 // GetOne get one vSwitch by zone and limit in ids
-func (s *SwitchPool) GetOne(ctx context.Context, client aliyun.VPCOps, zone string, ids []string) (*Switch, error) {
+func (s *SwitchPool) GetOne(ctx context.Context, client aliyun.VPCOps, zone string, ids []string, ignoreZone bool) (*Switch, error) {
 	var fallBackSwitches []*Switch
 	// lookup all vsw in cache and get one matched
 	for _, id := range ids {
@@ -67,7 +65,7 @@ func (s *SwitchPool) GetOne(ctx context.Context, client aliyun.VPCOps, zone stri
 		}
 
 		if vsw.Zone != zone {
-			if s.ignoreZone {
+			if ignoreZone {
 				fallBackSwitches = append(fallBackSwitches, vsw)
 			}
 			continue

pkg/controller/webhook/mutating.go

@@ -112,11 +112,40 @@ func podWebhook(ctx context.Context, req *webhook.AdmissionRequest, client clien
 			return webhook.Denied(fmt.Sprintf("unable parse annotation field %s", types.PodNetworks))
 		}
 		if len(networks.PodNetworks) == 0 {
-			return webhook.Denied("unable pod have no valid network config")
-		}
+			vsws, sgs, err := configFromConfigMap(ctx, client)
+			if err != nil {
+				return webhook.Errored(1, err)
+			}
+			pna := &controlplane.PodNetworksAnnotation{
+				PodNetworks: []controlplane.PodNetworks{
+					{
+						VSwitchIDs:       vsws,
+						SecurityGroupIDs: sgs,
+					},
+				},
+			}
+			pnaBytes, err := json.Marshal(pna)
+			if err != nil {
+				return webhook.Errored(1, err)
+			}
+			pod.Annotations[types.PodNetworks] = string(pnaBytes)
+			memberCount = 1
+		} else {
+			for _, n := range networks.PodNetworks {
+				if len(n.VSwitchIDs) == 0 {
+					return admission.Denied("vSwitchID is not set")
+				}
+				if len(n.SecurityGroupIDs) == 0 {
+					return admission.Denied("security group is not set")
+				}
+				if len(n.SecurityGroupIDs) > 5 {
+					return admission.Denied("security group can not more than 5")
+				}
+			}
 
-		// for now use trunk only
-		memberCount = len(networks.PodNetworks)
+			// for now use trunk only
+			memberCount = len(networks.PodNetworks)
+		}
 	} else {
 		if pod.Annotations[types.PodNetworks] != "" {
 			return webhook.Denied("can not use pod annotation and podNetworking at same time, pod-eni is missing")
@@ -222,30 +251,16 @@ func podNetworkingWebhook(ctx context.Context, req webhook.AdmissionRequest, cli
 	if len(podNetworking.Spec.SecurityGroupIDs) > 0 && len(podNetworking.Spec.VSwitchIDs) > 0 {
 		return webhook.Allowed("podNetworking all set")
 	}
-	cm := &corev1.ConfigMap{}
-	err = client.Get(ctx, k8stypes.NamespacedName{
-		Namespace: "kube-system",
-		Name:      "eni-config",
-	}, cm)
-	if err != nil {
-		return webhook.Errored(1, fmt.Errorf("error get terway configmap eni-config, %w", err))
-	}
-	eniConfStr, ok := cm.Data["eni_conf"]
-	if !ok {
-		return webhook.Errored(1, fmt.Errorf("error parse terway configmap eni-config, %w", err))
-	}
 
-	eniConf, err := types.MergeConfigAndUnmarshal(nil, []byte(eniConfStr))
+	vsws, sgs, err := configFromConfigMap(ctx, client)
 	if err != nil {
-		return webhook.Errored(1, fmt.Errorf("error parse terway configmap eni-config, %w", err))
+		return webhook.Errored(1, err)
 	}
 	if len(podNetworking.Spec.SecurityGroupIDs) == 0 {
-		podNetworking.Spec.SecurityGroupIDs = []string{eniConf.SecurityGroup}
+		podNetworking.Spec.SecurityGroupIDs = sgs
 	}
 	if len(podNetworking.Spec.VSwitchIDs) == 0 {
-		for _, ids := range eniConf.VSwitches {
-			podNetworking.Spec.VSwitchIDs = append(podNetworking.Spec.VSwitchIDs, ids...)
-		}
+		podNetworking.Spec.VSwitchIDs = vsws
 	}
 	podNetworkingPatched, err := json.Marshal(podNetworking)
 	if err != nil {
@@ -282,3 +297,33 @@ func getPreviousZone(client client.Client, pod *corev1.Pod) (string, error) {
 	}
 	return podENI.Spec.Zone, nil
 }
+
+func configFromConfigMap(ctx context.Context, client client.Client) ([]string, []string, error) {
+	cm := &corev1.ConfigMap{}
+	err := client.Get(ctx, k8stypes.NamespacedName{
+		Namespace: "kube-system",
+		Name:      "eni-config",
+	}, cm)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error get terway configmap eni-config, %w", err)
+	}
+	eniConfStr, ok := cm.Data["eni_conf"]
+	if !ok {
+		return nil, nil, fmt.Errorf("error parse terway configmap eni-config, %w", err)
+	}
+
+	eniConf, err := types.MergeConfigAndUnmarshal(nil, []byte(eniConfStr))
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parse terway configmap eni-config, %w", err)
+	}
+
+	sgs := eniConf.GetSecurityGroups()
+	if len(sgs) > 5 {
+		return nil, nil, fmt.Errorf("security groups should not be more than 5, current %d", len(sgs))
+	}
+	var vsws []string
+	for _, ids := range eniConf.VSwitches {
+		vsws = append(vsws, ids...)
+	}
+	return vsws, sgs, nil
+}

types/config.go

@@ -5,6 +5,7 @@ import (
 
 	jsonpatch "github.com/evanphx/json-patch"
 	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // Configure configuration of terway daemon
@@ -34,6 +35,11 @@ type Configure struct {
 	CustomStatefulWorkloadKinds []string `yaml:"custom_stateful_workload_kinds" json:"custom_stateful_workload_kinds"`
 }
 
+func (c *Configure) GetSecurityGroups() []string {
+	sgIDs := sets.NewString(c.SecurityGroup).Insert(c.SecurityGroups...)
+	return sgIDs.List()
+}
+
 // PoolConfig configuration of pool and resource factory
 type PoolConfig struct {
 	MaxPoolSize            int

types/controlplane/annotations_default.go

@@ -19,4 +19,7 @@ limitations under the License.
 
 package controlplane
 
-type PodNetworks struct{}
+type PodNetworks struct {
+	VSwitchIDs       []string `json:"vSwitchIDs"`
+	SecurityGroupIDs []string `json:"securityGroupIDs"`
+}