Skip to content

Commit

Permalink
ipam enhance
Browse files Browse the repository at this point in the history 
ipam: will wait until node has confirmed cni del is called
cni: will report cni del

Signed-off-by: l1b0k <[email protected]>
  • Loading branch information
@l1b0k
l1b0k committed Oct 22, 2024
1 parent d637d7f commit 642b960
Show file tree
Hide file tree
Showing 10 changed files with 647 additions and 221 deletions.
2 changes: 1 addition & 1 deletion cmd/terway-controlplane/terway-controlplane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func main() {
os.Exit(1)
}

lo.ForEach([]string{crds.CRDPodENI, crds.CRDPodNetworking, crds.CRDNode}, func(item string, index int) {
lo.ForEach([]string{crds.CRDPodENI, crds.CRDPodNetworking, crds.CRDNode, crds.CRDNodeRuntime}, func(item string, index int) {

Check warning on line 126 in cmd/terway-controlplane/terway-controlplane.go

View check run for this annotation

Codecov / codecov/patch

cmd/terway-controlplane/terway-controlplane.go#L126 

Added line #L126 was not covered by tests
err = crds.CreateOrUpdateCRD(ctx, directClient, item)
if err != nil {
log.Error(err, "unable sync crd")
Expand Down
58 changes: 58 additions & 0 deletions daemon/daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,15 @@ import (

"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/util/sets"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
logf "sigs.k8s.io/controller-runtime/pkg/log"

ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"

"github.com/AliyunContainerService/terway/deviceplugin"
"github.com/AliyunContainerService/terway/pkg/aliyun/client"

networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
"github.com/AliyunContainerService/terway/pkg/eni"
"github.com/AliyunContainerService/terway/pkg/factory"
"github.com/AliyunContainerService/terway/pkg/k8s"
Expand Down Expand Up @@ -540,8 +545,12 @@ func (n *networkService) gcPods(ctx context.Context) error {
}
podResources := getPodResources(objList)

uidInLocal := sets.New[string]()

Check warning on line 548 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L548 

Added line #L548 was not covered by tests
for _, podRes := range podResources {
if podRes.PodInfo != nil {
if podRes.PodInfo.PodUID != "" {
uidInLocal.Insert(podRes.PodInfo.PodUID)
}

Check warning on line 553 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L551-L553 

Added lines #L551 - L553 were not covered by tests
if podRes.PodInfo.PodIPs.IPv4 != nil {
existIPs.Insert(podRes.PodInfo.PodIPs.IPv4.String())
}
Expand Down Expand Up @@ -632,6 +641,55 @@ func (n *networkService) gcPods(ctx context.Context) error {
})
}

// clean runtime node records
err = n.cleanRuntimeNode(ctx, uidInLocal)
if err != nil {
serviceLog.Error(err, "error cleaning runtime node")
}
return nil

Check warning on line 649 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L645-L649 

Added lines #L645 - L649 were not covered by tests
}

func (n *networkService) cleanRuntimeNode(ctx context.Context, localUIDs sets.Set[string]) error {
if n.ipamType != types.IPAMTypeCRD || n.daemonMode != daemon.ModeENIMultiIP {
return nil
}
nodeRuntime := &networkv1beta1.NodeRuntime{}

err := n.k8s.GetClient().Get(ctx, ctrlclient.ObjectKey{Name: n.k8s.NodeName()}, nodeRuntime)
if err != nil {
return err
}

Check warning on line 661 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L652-L661 

Added lines #L652 - L661 were not covered by tests

// compare with local pods
// 1. local pod not found in remote, ignore may be slow
// 2. we found record at initial status , but local don't have this record, update it to delete
for uid, status := range nodeRuntime.Status.Pods {
if !localUIDs.Has(uid) {
continue

Check warning on line 668 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L666-L668 

Added lines #L666 - L668 were not covered by tests
}
if status.Status == networkv1beta1.CNIStatusInitial {
list := strings.Split(status.PodID, ",")
if len(list) != 2 {
continue

Check warning on line 673 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L670-L673 

Added lines #L670 - L673 were not covered by tests
}
ok, err := n.k8s.PodExist(list[0], list[1])
if err != nil || ok {
continue

Check warning on line 677 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L675-L677 

Added lines #L675 - L677 were not covered by tests
}
delete(nodeRuntime.Status.Pods, uid)

Check warning on line 679 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L679 

Added line #L679 was not covered by tests
}
}
update := nodeRuntime.DeepCopy()
_, err = controllerutil.CreateOrPatch(ctx, n.k8s.GetClient(), update, func() error {
update.Status = nodeRuntime.Status
update.Spec = nodeRuntime.Spec
update.Labels = nodeRuntime.Labels
update.Name = nodeRuntime.Name
return nil
})
if err != nil {
return fmt.Errorf("failed to save node runtime status %w", err)
}

Check warning on line 692 in daemon/daemon.go

View check run for this annotation

Codecov / codecov/patch

daemon/daemon.go#L682-L692 

Added lines #L682 - L692 were not covered by tests
return nil
}

Expand Down
71 changes: 36 additions & 35 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,21 @@ require (
github.com/go-playground/mold/v4 v4.2.0
github.com/go-playground/validator/v10 v10.11.1
github.com/google/uuid v1.6.0
github.com/onsi/ginkgo/v2 v2.9.5
github.com/onsi/gomega v1.27.7
github.com/onsi/ginkgo/v2 v2.13.0
github.com/onsi/gomega v1.29.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.15.1
github.com/prometheus/client_golang v1.16.0
github.com/pterm/pterm v0.12.62
github.com/samber/lo v1.39.0
github.com/spf13/cobra v1.6.1
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.9.0
github.com/vishvananda/netlink v1.2.1-beta.2
go.opentelemetry.io/otel v1.10.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0
go.opentelemetry.io/otel/sdk v1.10.0
go.opentelemetry.io/otel/trace v1.10.0
go.uber.org/atomic v1.9.0
go.opentelemetry.io/otel v1.19.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0
go.opentelemetry.io/otel/sdk v1.19.0
go.opentelemetry.io/otel/trace v1.19.0
go.uber.org/atomic v1.10.0
golang.org/x/mod v0.17.0
golang.org/x/net v0.26.0
golang.org/x/sync v0.7.0
Expand All @@ -43,16 +43,16 @@ require (
google.golang.org/protobuf v1.33.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.27.9
k8s.io/apiextensions-apiserver v0.27.9
k8s.io/apimachinery v0.27.9
k8s.io/apiserver v0.27.9
k8s.io/client-go v0.27.9
k8s.io/code-generator v0.27.9
k8s.io/component-base v0.27.9
k8s.io/klog/v2 v2.100.1
k8s.io/kubelet v0.27.9
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
k8s.io/api v0.29.9
k8s.io/apiextensions-apiserver v0.29.9
k8s.io/apimachinery v0.29.9
k8s.io/apiserver v0.29.9
k8s.io/client-go v0.29.9
k8s.io/code-generator v0.29.9
k8s.io/component-base v0.29.9
k8s.io/klog/v2 v2.110.1
k8s.io/kubelet v0.29.9
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
sigs.k8s.io/controller-runtime v0.15.3
sigs.k8s.io/e2e-framework v0.3.0
sigs.k8s.io/yaml v1.3.0
Expand All @@ -64,15 +64,15 @@ require (
atomicgo.dev/schedule v0.0.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/console v1.0.3 // indirect
github.com/coreos/go-iptables v0.6.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-logr/zapr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
Expand All @@ -84,14 +84,15 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
github.com/gookit/color v1.5.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/imdario/mergo v0.3.15 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand All @@ -105,11 +106,12 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/safchain/ethtool v0.3.0 // indirect
github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 // indirect
Expand All @@ -121,9 +123,9 @@ require (
github.com/vladimirvivien/gexe v0.2.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 // indirect
Expand All @@ -132,15 +134,14 @@ require (
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
)

replace github.com/vishvananda/netlink => github.com/BSWANG/netlink v1.0.1-0.20220803105814-1f63f9d61229
Loading

0 comments on commit 642b960

Please sign in to comment.