Skip to content

Commit

Permalink
Merge pull request #686 from l1b0k/deprecate_old
Browse files Browse the repository at this point in the history 
deprecated vpc and enionly mode
  • Loading branch information
@BSWANG
BSWANG authored Sep 12, 2024
2 parents ed64799 + 0eb1f4a commit 4ad2ae9
Show file tree
Hide file tree
Showing 15 changed files with 31 additions and 1,022 deletions.
98 changes: 25 additions & 73 deletions daemon/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func (b *NetworkServiceBuilder) InitService() *NetworkServiceBuilder {
pendingPods: sync.Map{},
}
switch b.daemonMode {
case daemon.ModeENIMultiIP, daemon.ModeVPC, daemon.ModeENIOnly:
case daemon.ModeENIMultiIP:
b.service.daemonMode = b.daemonMode
default:
b.err = fmt.Errorf("unsupported daemon mode")
Expand Down Expand Up @@ -268,9 +268,7 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
}
}

if b.daemonMode != daemon.ModeVPC {
nodeAnnotations[string(types.NormalIPTypeIPs)] = strconv.Itoa(poolConfig.Capacity)
}
nodeAnnotations[string(types.NormalIPTypeIPs)] = strconv.Itoa(poolConfig.Capacity)

attached, err := factory.GetAttachedNetworkInterface(trunkENIID)
if err != nil {
Expand Down Expand Up @@ -327,79 +325,33 @@ func (b *NetworkServiceBuilder) setupENIManager() error {

var eniList []eni.NetworkInterface

if b.daemonMode == daemon.ModeVPC {
eniList = append(eniList, &eni.Veth{})
}

if b.daemonMode == daemon.ModeENIOnly {
if b.config.IPAMType == types.IPAMTypeCRD {
if !b.config.EnableENITrunking {
eniList = append(eniList, eni.NewRemote(b.service.k8s.GetClient(), nil))
} else {
for _, ni := range attached {
if !ni.Trunk {
continue
}
lo := eni.NewLocal(ni, "trunk", factory, poolConfig)
eniList = append(eniList, eni.NewTrunk(b.service.k8s.GetClient(), lo))
}
}
var (
normalENICount int
erdmaENICount int
)
for _, ni := range attached {
serviceLog.V(5).Info("found attached eni", "eni", ni)
if b.config.EnableENITrunking && ni.Trunk && trunkENIID == ni.ID {
lo := eni.NewLocal(ni, "trunk", factory, poolConfig)
eniList = append(eniList, eni.NewTrunk(b.service.k8s.GetClient(), lo))
} else if b.config.EnableERDMA && ni.ERdma {
erdmaENICount++
eniList = append(eniList, eni.NewLocal(ni, "erdma", factory, poolConfig))
} else {
var (
normalENICount int
erdmaENICount int
)
// the legacy mode
for _, ni := range attached {
if b.config.EnableERDMA && ni.ERdma {
erdmaENICount++
eniList = append(eniList, eni.NewLocal(ni, "erdma", factory, poolConfig))
} else {
normalENICount++
eniList = append(eniList, eni.NewLocal(ni, "secondary", factory, poolConfig))
}
}
normalENINeeded := poolConfig.MaxENI - normalENICount
if b.config.EnableERDMA {
normalENINeeded = poolConfig.MaxENI - b.limit.ERdmaAdapters - normalENICount
for i := 0; i < b.limit.ERdmaAdapters-erdmaENICount; i++ {
eniList = append(eniList, eni.NewLocal(nil, "erdma", factory, poolConfig))
}
}

for i := 0; i < normalENINeeded; i++ {
eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig))
}
normalENICount++
eniList = append(eniList, eni.NewLocal(ni, "secondary", factory, poolConfig))
}
} else {
var (
normalENICount int
erdmaENICount int
)
for _, ni := range attached {
serviceLog.V(5).Info("found attached eni", "eni", ni)
if b.config.EnableENITrunking && ni.Trunk && trunkENIID == ni.ID {
lo := eni.NewLocal(ni, "trunk", factory, poolConfig)
eniList = append(eniList, eni.NewTrunk(b.service.k8s.GetClient(), lo))
} else if b.config.EnableERDMA && ni.ERdma {
erdmaENICount++
eniList = append(eniList, eni.NewLocal(ni, "erdma", factory, poolConfig))
} else {
normalENICount++
eniList = append(eniList, eni.NewLocal(ni, "secondary", factory, poolConfig))
}
}
normalENINeeded := poolConfig.MaxENI - normalENICount
if b.config.EnableERDMA {
normalENINeeded = poolConfig.MaxENI - b.limit.ERdmaAdapters - normalENICount
for i := 0; i < b.limit.ERdmaAdapters-erdmaENICount; i++ {
eniList = append(eniList, eni.NewLocal(nil, "erdma", factory, poolConfig))
}
}
normalENINeeded := poolConfig.MaxENI - normalENICount
if b.config.EnableERDMA {
normalENINeeded = poolConfig.MaxENI - b.limit.ERdmaAdapters - normalENICount
for i := 0; i < b.limit.ERdmaAdapters-erdmaENICount; i++ {
eniList = append(eniList, eni.NewLocal(nil, "erdma", factory, poolConfig))
}
}

for i := 0; i < normalENINeeded; i++ {
eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig))
}
for i := 0; i < normalENINeeded; i++ {
eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig))
}

eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, types.EniSelectionPolicy(b.config.EniSelectionPolicy), b.service.k8s)
Expand Down
10 changes: 0 additions & 10 deletions daemon/builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,6 @@ func TestInitService(t *testing.T) {
daemonMode: daemon.ModeENIMultiIP,
expectedError: false,
},
{
name: "Valid daemon mode VPC",
daemonMode: daemon.ModeVPC,
expectedError: false,
},
{
name: "Valid daemon mode ENIOnly",
daemonMode: daemon.ModeENIOnly,
expectedError: false,
},
{
name: "Unsupported daemon mode",
daemonMode: "unsupported",
Expand Down
36 changes: 0 additions & 36 deletions daemon/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,42 +89,6 @@ func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits)
maxMemberENI := 0

switch daemonMode {
case daemon.ModeVPC, daemon.ModeENIOnly:
maxENI = limit.Adapters
maxENI = int(float64(maxENI)*cfg.EniCapRatio) + cfg.EniCapShift - 1

// set max eni node can use
if cfg.MaxENI > 0 && cfg.MaxENI < maxENI {
maxENI = cfg.MaxENI
}

capacity = maxENI

if cfg.MaxPoolSize > maxENI {
poolConfig.MaxPoolSize = maxENI
} else {
poolConfig.MaxPoolSize = cfg.MaxPoolSize
}

poolConfig.MinPoolSize = cfg.MinPoolSize

if cfg.MinENI > 0 {
poolConfig.MinPoolSize = cfg.MinENI
}

if poolConfig.MinPoolSize > poolConfig.MaxPoolSize {
poolConfig.MinPoolSize = poolConfig.MaxPoolSize
}

maxMemberENI = limit.MemberAdapterLimit
if cfg.ENICapPolicy == daemon.ENICapPolicyPreferTrunk {
maxMemberENI = limit.MaxMemberAdapterLimit
}

poolConfig.MaxIPPerENI = 1
if cfg.EnableERDMA {
poolConfig.ERdmaCapacity = limit.ERdmaAdapters
}
case daemon.ModeENIMultiIP:
maxENI = limit.Adapters
maxENI = int(float64(maxENI)*cfg.EniCapRatio) + cfg.EniCapShift - 1
Expand Down
34 changes: 0 additions & 34 deletions daemon/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,40 +26,6 @@ func init() {
})
}

func TestGetPoolConfigWithVPCMode(t *testing.T) {
cfg := &daemon.Config{
MaxPoolSize: 5,
MinPoolSize: 1,
EniCapRatio: 1,
RegionID: "foo",
}
limit := &client.Limits{
Adapters: 10,
MemberAdapterLimit: 5,
}
poolConfig, err := getPoolConfig(cfg, "VPC", limit)
assert.NoError(t, err)
assert.Equal(t, 5, poolConfig.MaxPoolSize)
assert.Equal(t, 1, poolConfig.MinPoolSize)
}

func TestGetPoolConfigWithENIOnlyMode(t *testing.T) {
cfg := &daemon.Config{
MaxPoolSize: 5,
MinPoolSize: 1,
EniCapRatio: 1,
RegionID: "foo",
}
limit := &client.Limits{
Adapters: 10,
MemberAdapterLimit: 5,
}
poolConfig, err := getPoolConfig(cfg, "ENIOnly", limit)
assert.NoError(t, err)
assert.Equal(t, 5, poolConfig.MaxPoolSize)
assert.Equal(t, 1, poolConfig.MinPoolSize)
}

func TestGetPoolConfigWithENIMultiIPMode(t *testing.T) {
cfg := &daemon.Config{
MaxPoolSize: 5,
Expand Down
40 changes: 4 additions & 36 deletions daemon/daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,24 +195,7 @@ func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*r

resourceRequests = append(resourceRequests, req)
}
case daemon.PodNetworkTypeVPCENI:
reply.IPType = rpc.IPType_TypeVPCENI

if pod.PodENI || n.ipamType == types.IPAMTypeCRD {
resourceRequests = append(resourceRequests, &eni.RemoteIPRequest{})
} else {
req := &eni.LocalIPRequest{}

if len(oldRes.GetResourceItemByType(daemon.ResourceTypeENI)) == 1 {
old := oldRes.GetResourceItemByType(daemon.ResourceTypeENI)[0]

setRequest(req, old)
}
resourceRequests = append(resourceRequests, req)
}
case daemon.PodNetworkTypeVPCIP:
reply.IPType = rpc.IPType_TypeVPCIP
resourceRequests = append(resourceRequests, &eni.VethRequest{})
default:
return nil, &types.Error{
Code: types.ErrInternalError,
Expand Down Expand Up @@ -242,9 +225,7 @@ func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*r
c.BasicInfo = &rpc.BasicInfo{}
}
c.BasicInfo.ServiceCIDR = n.k8s.GetServiceCIDR().ToRPC()
if pod.PodNetworkType == daemon.PodNetworkTypeVPCIP {
c.BasicInfo.PodCIDR = n.k8s.GetNodeCidr().ToRPC()
}

c.Pod = &rpc.Pod{
Ingress: pod.TcIngress,
Egress: pod.TcEgress,
Expand Down Expand Up @@ -421,10 +402,6 @@ func (n *networkService) GetIPInfo(ctx context.Context, r *rpc.GetInfoRequest) (
switch pod.PodNetworkType {
case daemon.PodNetworkTypeENIMultiIP:
reply.IPType = rpc.IPType_TypeENIMultiIP
case daemon.PodNetworkTypeVPCIP:
reply.IPType = rpc.IPType_TypeVPCIP
case daemon.PodNetworkTypeVPCENI:
reply.IPType = rpc.IPType_TypeVPCENI

default:
return nil, &types.Error{
Expand Down Expand Up @@ -499,12 +476,7 @@ func (n *networkService) RecordEvent(_ context.Context, r *rpc.EventRequest) (*r
}

func (n *networkService) verifyPodNetworkType(podNetworkMode string) bool {
return (n.daemonMode == daemon.ModeVPC && //vpc
(podNetworkMode == daemon.PodNetworkTypeVPCENI || podNetworkMode == daemon.PodNetworkTypeVPCIP)) ||
// eni-multi-ip
(n.daemonMode == daemon.ModeENIMultiIP && podNetworkMode == daemon.PodNetworkTypeENIMultiIP) ||
// eni-only
(n.daemonMode == daemon.ModeENIOnly && podNetworkMode == daemon.PodNetworkTypeVPCENI)
return n.daemonMode == daemon.ModeENIMultiIP && podNetworkMode == daemon.PodNetworkTypeENIMultiIP // eni-multi-ip
}

func (n *networkService) startGarbageCollectionLoop(ctx context.Context) {
Expand Down Expand Up @@ -801,9 +773,6 @@ func initTrunk(config *daemon.Config, poolConfig *types.PoolConfig, k8sClient k8

func runDevicePlugin(daemonMode string, config *daemon.Config, poolConfig *types.PoolConfig) {
switch daemonMode {
case daemon.ModeVPC, daemon.ModeENIOnly:
dp := deviceplugin.NewENIDevicePlugin(poolConfig.MaxENI, deviceplugin.ENITypeENI)
go dp.Serve()
case daemon.ModeENIMultiIP:
if config.EnableENITrunking {
dp := deviceplugin.NewENIDevicePlugin(poolConfig.MaxMemberENI, deviceplugin.ENITypeMember)
Expand Down Expand Up @@ -833,7 +802,7 @@ func getPodResources(list []interface{}) []daemon.PodResources {

func parseNetworkResource(item daemon.ResourceItem) eni.NetworkResource {
switch item.Type {
case daemon.ResourceTypeENIIP, daemon.ResourceTypeENI:
case daemon.ResourceTypeENIIP:
var v4, v6 netip.Addr
if item.IPv4 != "" {
v4, _ = netip.ParseAddr(item.IPv4)
Expand Down Expand Up @@ -950,8 +919,7 @@ func getPodIPs(netConfs []*rpc.NetConf) []string {
func filterENINotFound(podResources []daemon.PodResources, attachedENIID map[string]*daemon.ENI) []daemon.PodResources {
for i := range podResources {
for j := 0; j < len(podResources[i].Resources); j++ {
if podResources[i].Resources[j].Type == daemon.ResourceTypeENI ||
podResources[i].Resources[j].Type == daemon.ResourceTypeENIIP {
if podResources[i].Resources[j].Type == daemon.ResourceTypeENIIP {

eniID := podResources[i].Resources[j].ENIID
if eniID == "" {
Expand Down
Loading

0 comments on commit 4ad2ae9

Please sign in to comment.