Skip to content

Commit

Permalink
preheating the informer cache
Browse files Browse the repository at this point in the history
Signed-off-by: l1b0k <[email protected]>
  • Loading branch information
l1b0k committed Aug 2, 2024
1 parent baf6aa1 commit 41b4e0c
Show file tree
Hide file tree
Showing 17 changed files with 354 additions and 55 deletions.
21 changes: 16 additions & 5 deletions cmd/terway-controlplane/terway-controlplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ import (
"github.com/AliyunContainerService/terway/pkg/cert"
register "github.com/AliyunContainerService/terway/pkg/controller"
_ "github.com/AliyunContainerService/terway/pkg/controller/all"
multiipnode "github.com/AliyunContainerService/terway/pkg/controller/multi-ip/node"
multiippod "github.com/AliyunContainerService/terway/pkg/controller/multi-ip/pod"
"github.com/AliyunContainerService/terway/pkg/controller/node"
"github.com/AliyunContainerService/terway/pkg/controller/preheating"
"github.com/AliyunContainerService/terway/pkg/controller/webhook"
"github.com/AliyunContainerService/terway/pkg/metric"
"github.com/AliyunContainerService/terway/pkg/utils"
Expand Down Expand Up @@ -264,6 +268,13 @@ func main() {
}
}

if err = (&preheating.DummyReconcile{
RegisterResource: ctrlCtx.RegisterResource,
}).SetupWithManager(mgr); err != nil {
log.Error(err, "unable to create controller", "controller", "preheating")
os.Exit(1)

Check warning on line 275 in cmd/terway-controlplane/terway-controlplane.go

View check run for this annotation

Codecov / codecov/patch

cmd/terway-controlplane/terway-controlplane.go#L271-L275

Added lines #L271 - L275 were not covered by tests
}

log.Info("controller started")
err = mgr.Start(ctx)
if err != nil {
Expand Down Expand Up @@ -311,15 +322,15 @@ func initOpenTelemetry(ctx context.Context, serviceName, serviceVersion string,
}

func detectMultiIP(ctx context.Context, directClient client.Client, cfg *controlplane.Config) error {
if !lo.Contains(cfg.Controllers, "multi-ip-pod") {
if !lo.Contains(cfg.Controllers, multiipnode.ControllerName) {
return nil
}

var daemonConfig *daemon.Config
var innerErr error
err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 10*time.Second, true, func(ctx context.Context) (done bool, err error) {
var innerErr error
daemonConfig, innerErr = daemon.ConfigFromConfigMap(ctx, directClient, "")
if err != nil {
if innerErr != nil {
if k8sErr.IsNotFound(innerErr) {
return false, nil
}
Expand All @@ -329,7 +340,7 @@ func detectMultiIP(ctx context.Context, directClient client.Client, cfg *control
return true, nil
})
if err != nil {
return err
return fmt.Errorf("error waiting for daemon to be configured: %w, innerErr %s", err, innerErr)
}
switch daemonConfig.IPAMType {
case types.IPAMTypeCRD:
Expand All @@ -338,7 +349,7 @@ func detectMultiIP(ctx context.Context, directClient client.Client, cfg *control

cfg.Controllers = lo.Reject(cfg.Controllers, func(item string, index int) bool {
switch item {
case "node", "multi-ip-pod", "multi-ip-node":
case node.ControllerName, multiipnode.ControllerName, multiippod.ControllerName:
return true
}
return false
Expand Down
124 changes: 124 additions & 0 deletions cmd/terway-controlplane/terway-controlplane_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package main

import (
"context"
"testing"

"github.com/samber/lo"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

"github.com/AliyunContainerService/terway/types/controlplane"
)

func Test_detectMultiIP(t *testing.T) {

type args struct {
ctx context.Context
directClient client.Client
cfg *controlplane.Config
}
tests := []struct {
name string
args args
wantErr bool
checkFunc func(t *testing.T, cfg *controlplane.Config)
}{
{
name: "test empty config",
args: args{
ctx: context.Background(),
directClient: func() client.Client {
nodeReader := fake.NewClientBuilder()
return nodeReader.Build()
}(),
cfg: &controlplane.Config{},
},
wantErr: false,
checkFunc: func(t *testing.T, cfg *controlplane.Config) {},
},
{
name: "test config not found",
args: args{
ctx: context.Background(),
directClient: func() client.Client {
nodeReader := fake.NewClientBuilder()
return nodeReader.Build()
}(),
cfg: &controlplane.Config{
Controllers: []string{"multi-ip-node", "node", "multi-ip-pod"},
},
},
wantErr: true,
checkFunc: func(t *testing.T, cfg *controlplane.Config) {},
},
{
name: "test config ipamv1",
args: args{
ctx: context.Background(),
directClient: func() client.Client {
nodeReader := fake.NewClientBuilder()
nodeReader.WithObjects(
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "eni-config", Namespace: "kube-system"},
Data: map[string]string{
"eni_conf": "{\"ipam_type\":\"\"}",
},
},
)
return nodeReader.Build()
}(),
cfg: &controlplane.Config{
Controllers: []string{"multi-ip-node", "node", "multi-ip-pod", "foo"},
},
},
wantErr: false,
checkFunc: func(t *testing.T, cfg *controlplane.Config) {
assert.False(t, lo.Contains(cfg.Controllers, "multi-ip-node"))
assert.False(t, lo.Contains(cfg.Controllers, "node"))
assert.False(t, lo.Contains(cfg.Controllers, "multi-ip-pod"))
assert.True(t, lo.Contains(cfg.Controllers, "foo"))
},
},
{
name: "test config ipamv2",
args: args{
ctx: context.Background(),
directClient: func() client.Client {
nodeReader := fake.NewClientBuilder()
nodeReader.WithObjects(
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "eni-config", Namespace: "kube-system"},
Data: map[string]string{
"eni_conf": "{\"ipam_type\":\"crd\"}",
},
},
)
return nodeReader.Build()
}(),
cfg: &controlplane.Config{
Controllers: []string{"multi-ip-node", "node", "multi-ip-pod", "foo"},
},
},
wantErr: false,
checkFunc: func(t *testing.T, cfg *controlplane.Config) {
assert.True(t, lo.Contains(cfg.Controllers, "multi-ip-node"))
assert.True(t, lo.Contains(cfg.Controllers, "node"))
assert.True(t, lo.Contains(cfg.Controllers, "multi-ip-pod"))
assert.True(t, lo.Contains(cfg.Controllers, "foo"))
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := detectMultiIP(tt.args.ctx, tt.args.directClient, tt.args.cfg); (err != nil) != tt.wantErr {
t.Errorf("detectMultiIP() error = %v, wantErr %v", err, tt.wantErr)
} else {
tt.checkFunc(t, tt.args.cfg)
}
})
}
}
6 changes: 3 additions & 3 deletions pkg/aliyun/credential/aliyun_client_mgr.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ func (c *ClientMgr) refreshToken() (bool, error) {
if err != nil {
return false, err
}
c.ecs.SetEndpointRules(c.ecs.EndpointMap, "regional", "public")
c.ecs.SetEndpointRules(c.ecs.EndpointMap, "regional", "vpc")

if c.ecsDomainOverride != "" {
c.ecs.Domain = c.ecsDomainOverride
Expand All @@ -191,7 +191,7 @@ func (c *ClientMgr) refreshToken() (bool, error) {
if err != nil {
return false, err
}
c.vpc.SetEndpointRules(c.vpc.EndpointMap, "regional", "public")
c.vpc.SetEndpointRules(c.vpc.EndpointMap, "regional", "vpc")

if c.vpcDomainOverride != "" {
c.vpc.Domain = c.vpcDomainOverride
Expand All @@ -201,7 +201,7 @@ func (c *ClientMgr) refreshToken() (bool, error) {
if err != nil {
return false, err
}
c.eflo.SetEndpointRules(c.eflo.EndpointMap, "regional", "public")
c.eflo.SetEndpointRules(c.eflo.EndpointMap, "regional", "vpc")

if c.efloDomainOverride != "" {
c.eflo.Domain = c.efloDomainOverride
Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/endpoint/endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ import (

var log = ctrl.Log.WithName("endpoint")

const controllerName = "endpoint"
const ControllerName = "endpoint"

func init() {
register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
register.Add(ControllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
ipStr := os.Getenv("MY_POD_IP")
if ipStr == "" {
return fmt.Errorf("podIP is not found")
Expand Down
38 changes: 8 additions & 30 deletions pkg/controller/multi-ip/node/pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ import (
)

const (
ControllerName = "multi-ip-node"

finalizer = "network.alibabacloud.com/node-controller"

batchSize = 10
Expand All @@ -55,8 +57,7 @@ const (
var EventCh = make(chan event.GenericEvent, 1)

func init() {
controllerName := "multi-ip-node"
register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
register.Add(ControllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
fullSyncPeriod, err := time.ParseDuration(ctrlCtx.Config.MultiIPNodeSyncPeriod)
if err != nil {
return err
Expand All @@ -77,17 +78,17 @@ func init() {
return err
}

// metirc and tracer
// metric and tracer

metrics.Registry.MustRegister(ResourcePoolTotal)
tracer := ctrlCtx.TracerProvider.Tracer(controllerName)
tracer := ctrlCtx.TracerProvider.Tracer(ControllerName)

Check warning on line 84 in pkg/controller/multi-ip/node/pool.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/multi-ip/node/pool.go#L84

Added line #L84 was not covered by tests

ctrl, err := controller.New(controllerName, mgr, controller.Options{
MaxConcurrentReconciles: ctrlCtx.Config.MultiIPMaxConcurrent,
ctrl, err := controller.New(ControllerName, mgr, controller.Options{
MaxConcurrentReconciles: ctrlCtx.Config.MultiIPNodeMaxConcurrent,

Check warning on line 87 in pkg/controller/multi-ip/node/pool.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/multi-ip/node/pool.go#L86-L87

Added lines #L86 - L87 were not covered by tests
Reconciler: &ReconcileNode{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
record: mgr.GetEventRecorderFor(controllerName),
record: mgr.GetEventRecorderFor(ControllerName),

Check warning on line 91 in pkg/controller/multi-ip/node/pool.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/multi-ip/node/pool.go#L91

Added line #L91 was not covered by tests
aliyun: ctrlCtx.AliyunClient,
vswpool: ctrlCtx.VSwitchPool,
fullSyncNodePeriod: fullSyncPeriod,
Expand Down Expand Up @@ -1199,26 +1200,3 @@ func addIPToMap(in map[string]*networkv1beta1.IP, ip *networkv1beta1.IP) {
in[ip.IP] = ip
}
}

//
//func (n *ReconcileNode) loadToMetadata(ctx context.Context, name string) {
// prev, ok := n.cache.Load(name)
// if !ok {
// return
// }
// status := prev.(*NodeStatus)
//
// ctxMeta := MetaCtx(ctx)
// ctxMeta.needSyncOpenAPI.Store(status.NeedSyncOpenAPI)
// ctxMeta.lastGCTime = status.LastGCTime
// ctxMeta.lastReconcileTime = status.LastReconcileTime
//}
//
//func (n *ReconcileNode) fromMetadata(ctx context.Context, name string) {
// ctxMeta := MetaCtx(ctx)
//
// n.cache.Store(name, &NodeStatus{
// NeedSyncOpenAPI: ctxMeta.needSyncOpenAPI.Load(),
// LastGCTime: ctxMeta.lastGCTime,
// })
//}
9 changes: 5 additions & 4 deletions pkg/controller/multi-ip/pod/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,20 @@ import (
"github.com/AliyunContainerService/terway/pkg/controller/multi-ip/node"
)

const controllerName = "multi-ip-pod"
const ControllerName = "multi-ip-pod"

func init() {
register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
register.Add(ControllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
ctrlCtx.RegisterResource = append(ctrlCtx.RegisterResource, &corev1.Pod{})
return ctrl.NewControllerManagedBy(mgr).
WithOptions(controller.Options{
MaxConcurrentReconciles: 10,
MaxConcurrentReconciles: ctrlCtx.Config.MultiIPPodMaxConcurrent,
}).
For(&corev1.Pod{}, builder.WithPredicates(&predicateForPodEvent{})).
Complete(&ReconcilePod{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
record: mgr.GetEventRecorderFor(controllerName),
record: mgr.GetEventRecorderFor(ControllerName),
})
}, false)
}
Expand Down
8 changes: 5 additions & 3 deletions pkg/controller/node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,15 @@ import (
)

const (
controllerName = "node"
ControllerName = "node"

finalizer = "network.alibabacloud.com/node-controller"
)

func init() {
register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
register.Add(ControllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
ctrlCtx.RegisterResource = append(ctrlCtx.RegisterResource, &corev1.Node{}, &networkv1beta1.Node{})

Check warning on line 41 in pkg/controller/node/node.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/node/node.go#L41

Added line #L41 was not covered by tests

err := mgr.GetFieldIndexer().IndexField(ctrlCtx.Context, &corev1.Pod{}, "spec.nodeName", func(object client.Object) []string {
pod := object.(*corev1.Pod)
return []string{pod.Spec.NodeName}
Expand All @@ -54,7 +56,7 @@ func init() {
Watches(&networkv1beta1.Node{}, &handler.EnqueueRequestForObject{}).Complete(&ReconcileNode{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
record: mgr.GetEventRecorderFor(controllerName),
record: mgr.GetEventRecorderFor(ControllerName),

Check warning on line 59 in pkg/controller/node/node.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/node/node.go#L59

Added line #L59 was not covered by tests
aliyun: ctrlCtx.AliyunClient,
})
}, false)
Expand Down
2 changes: 0 additions & 2 deletions pkg/controller/node/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,6 @@ var k8sClient client.Client
var testEnv *envtest.Environment
var aliyun register.Interface

var limit aliyunClient.LimitProvider

func TestControllers(t *testing.T) {
ac := mocks.NewInterface(t)
ac.On("DescribeInstanceTypes", mock.Anything, []string{"xxx"}).Return([]ecs.InstanceType{
Expand Down
2 changes: 2 additions & 0 deletions pkg/controller/pod-eni/eni_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ const layout = "2006-01-02T15:04:05Z"

func init() {
register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
ctrlCtx.RegisterResource = append(ctrlCtx.RegisterResource, &v1beta1.PodENI{})

r := NewReconcilePod(mgr, ctrlCtx.AliyunClient)
c, err := controller.NewUnmanaged(controllerName, mgr, controller.Options{
Reconciler: r,
Expand Down
2 changes: 2 additions & 0 deletions pkg/controller/pod-networking/networking.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ const controllerName = "pod-networking"

func init() {
register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
ctrlCtx.RegisterResource = append(ctrlCtx.RegisterResource, &v1beta1.PodNetworking{})

Check warning on line 47 in pkg/controller/pod-networking/networking.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/pod-networking/networking.go#L47

Added line #L47 was not covered by tests

c, err := controller.New(controllerName, mgr, controller.Options{
Reconciler: NewReconcilePodNetworking(mgr, ctrlCtx.AliyunClient, ctrlCtx.VSwitchPool),
MaxConcurrentReconciles: 1,
Expand Down
4 changes: 3 additions & 1 deletion pkg/controller/pod/pod_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ const defaultInterface = "eth0"

func init() {
register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error {
ctrlCtx.RegisterResource = append(ctrlCtx.RegisterResource, &corev1.Pod{})

crdMode := controlplane.GetConfig().IPAMType == types.IPAMTypeCRD

c, err := controller.NewUnmanaged(controllerName, mgr, controller.Options{
Expand Down Expand Up @@ -674,7 +676,7 @@ func (m *ReconcilePod) createENI(ctx context.Context, allocs *[]*v1beta1.Allocat
alloc.AllocationType = *allocType

ch <- alloc
return m.PostENICreate(ctx, alloc)
return nil
})
}
err = g.Wait()
Expand Down
Loading

0 comments on commit 41b4e0c

Please sign in to comment.