diff --git a/ipamd/datastore/data_store.go b/ipamd/datastore/data_store.go index f8b0db9180..52192a593c 100644 --- a/ipamd/datastore/data_store.go +++ b/ipamd/datastore/data_store.go @@ -403,7 +403,6 @@ func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget int) string { deletableENI := ds.getDeletableENI(warmIPTarget) if deletableENI == nil { - log.Debugf("No ENI can be deleted at this time") return "" } diff --git a/ipamd/ipamd.go b/ipamd/ipamd.go index 29fe496df2..b39762f72e 100644 --- a/ipamd/ipamd.go +++ b/ipamd/ipamd.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" log "github.com/cihub/seelog" @@ -161,10 +162,10 @@ type IPAMContext struct { primaryIP map[string]string lastNodeIPPoolAction time.Time lastDecreaseIPPool time.Time - // reconcileCooldownCache keeps timestamps of the last time an IP address was unassigned from an ENI, // so that we don't reconcile and add it back too quickly if IMDS lags behind reality. reconcileCooldownCache ReconcileCooldownCache + terminating int32 // Flag to warn that the pod is about to shut down. } // Keep track of recently freed IPs to avoid reading stale EC2 metadata @@ -231,10 +232,13 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController) log.Errorf("Failed to initialize awsutil interface %v", err) return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface") } - c.awsClient = client + + c.primaryIP = make(map[string]string) + c.reconcileCooldownCache.cache = make(map[string]time.Time) c.warmENITarget = getWarmENITarget() c.warmIPTarget = getWarmIPTarget() + c.useCustomNetworking = UseCustomNetworkCfg() err = c.nodeInit() if err != nil { @@ -265,10 +269,6 @@ func (c *IPAMContext) nodeInit() error { } ipMax.Set(float64(c.maxIPsPerENI * c.maxENI)) - c.useCustomNetworking = UseCustomNetworkCfg() - c.primaryIP = make(map[string]string) - c.reconcileCooldownCache.cache = make(map[string]time.Time) - enis, err := c.awsClient.GetAttachedENIs() if err != nil { log.Error("Failed to retrieve ENI info") @@ -453,11 +453,16 @@ func (c *IPAMContext) decreaseIPPool(interval time.Duration) { // tryFreeENI always tries to free one ENI func (c *IPAMContext) tryFreeENI() { + if c.isTerminating() { + log.Debug("AWS CNI is terminating, not detaching any ENIs") + return + } + eni := c.dataStore.RemoveUnusedENIFromStore(c.warmIPTarget) if eni == "" { - log.Info("No ENI to remove, all ENIs have IPs in use") return } + log.Debugf("Start freeing ENI %s", eni) err := c.awsClient.FreeENI(eni) if err != nil { @@ -560,6 +565,11 @@ func (c *IPAMContext) increaseIPPool() { return } + if c.isTerminating() { + log.Debug("AWS CNI is terminating, will not try to attach any new IPs or ENIs right now") + return + } + // Try to add more IPs to existing ENIs first. increasedPool, err := c.tryAssignIPs() if err != nil { @@ -1050,6 +1060,15 @@ func (c *IPAMContext) ipTargetState() (short int, over int, enabled bool) { return short, over, true } +// setTerminating atomically sets the terminating flag. +func (c *IPAMContext) setTerminating() { + atomic.StoreInt32(&c.terminating, 1) +} + +func (c *IPAMContext) isTerminating() bool { + return atomic.LoadInt32(&c.terminating) > 0 +} + // GetConfigForDebug returns the active values of the configuration env vars (for debugging purposes). func GetConfigForDebug() map[string]interface{} { return map[string]interface{}{ diff --git a/ipamd/ipamd_test.go b/ipamd/ipamd_test.go index c98aa20974..7cd487a49d 100644 --- a/ipamd/ipamd_test.go +++ b/ipamd/ipamd_test.go @@ -78,6 +78,8 @@ func TestNodeInit(t *testing.T) { maxENI: 4, warmENITarget: 1, warmIPTarget: 3, + primaryIP: make(map[string]string), + terminating: int32(0), networkClient: mockNetwork} eni1 := awsutils.ENIMetadata{ @@ -175,6 +177,7 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { useCustomNetworking: UseCustomNetworkCfg(), eniConfig: mockENIConfig, primaryIP: make(map[string]string), + terminating: int32(0), } mockContext.dataStore = datastore.NewDataStore() @@ -252,6 +255,7 @@ func TestTryAddIPToENI(t *testing.T) { networkClient: mockNetwork, eniConfig: mockENIConfig, primaryIP: make(map[string]string), + terminating: int32(0), } mockContext.dataStore = datastore.NewDataStore() @@ -310,6 +314,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { k8sClient: mockK8S, networkClient: mockNetwork, primaryIP: make(map[string]string), + terminating: int32(0), } mockContext.dataStore = datastore.NewDataStore() @@ -397,6 +402,7 @@ func TestGetWarmIPTargetState(t *testing.T) { k8sClient: mockK8S, networkClient: mockNetwork, primaryIP: make(map[string]string), + terminating: int32(0), } mockContext.dataStore = datastore.NewDataStore() diff --git a/ipamd/rpc_handler.go b/ipamd/rpc_handler.go index a63081d7e4..78acd91c80 100644 --- a/ipamd/rpc_handler.go +++ b/ipamd/rpc_handler.go @@ -15,6 +15,9 @@ package ipamd import ( "net" + "os" + "os/signal" + "syscall" "github.com/pkg/errors" @@ -124,9 +127,27 @@ func (c *IPAMContext) RunRPCHandler() error { healthpb.RegisterHealthServer(s, hs) // Register reflection service on gRPC server. reflection.Register(s) + // Add shutdown hook + go c.shutdownListener(s) if err := s.Serve(lis); err != nil { log.Errorf("Failed to start server on gRPC port: %v", err) return errors.Wrap(err, "ipamd: failed to start server on gPRC port") } return nil } + +// shutdownListener - Listen to signals and set ipamd to be in status "terminating" +func (c *IPAMContext) shutdownListener(s *grpc.Server) { + log.Info("Setting up shutdown hook.") + sig := make(chan os.Signal, 1) + + // Interrupt signal sent from terminal + signal.Notify(sig, syscall.SIGINT) + // Terminate signal sent from Kubernetes + signal.Notify(sig, syscall.SIGTERM) + + <-sig + log.Info("Received shutdown signal, setting 'terminating' to true") + // We received an interrupt signal, shut down. + c.setTerminating() +} diff --git a/scripts/install-aws.sh b/scripts/install-aws.sh index c103999281..670e137921 100755 --- a/scripts/install-aws.sh +++ b/scripts/install-aws.sh @@ -9,4 +9,4 @@ if [[ -f /host/etc/cni/net.d/aws.conf ]]; then fi echo "====== Starting amazon-k8s-agent ======" -/app/aws-k8s-agent +exec /app/aws-k8s-agent