diff --git a/pkg/providers/v1/aws.go b/pkg/providers/v1/aws.go index dcdf7a2bb3..a988dd594f 100644 --- a/pkg/providers/v1/aws.go +++ b/pkg/providers/v1/aws.go @@ -400,7 +400,8 @@ type Cloud struct { nodeInformer informercorev1.NodeInformer // Extract the function out to make it easier to test - nodeInformerHasSynced cache.InformerSynced + nodeInformerHasSynced cache.InformerSynced + nodeEventualConsistencyGracePeriod time.Duration eventBroadcaster record.EventBroadcaster eventRecorder record.EventRecorder @@ -615,14 +616,15 @@ func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config. } awsCloud := &Cloud{ - ec2: ec2, - elb: elb, - elbv2: elbv2, - asg: asg, - metadata: metadata, - kms: kms, - cfg: &cfg, - region: regionName, + ec2: ec2, + elb: elb, + elbv2: elbv2, + asg: asg, + metadata: metadata, + kms: kms, + cfg: &cfg, + region: regionName, + nodeEventualConsistencyGracePeriod: cfg.Global.NodeEventualConsistencyGracePeriod, } awsCloud.instanceCache.cloud = awsCloud awsCloud.zoneCache.cloud = awsCloud diff --git a/pkg/providers/v1/config/config.go b/pkg/providers/v1/config/config.go index ef6e371115..7a3764c0c4 100644 --- a/pkg/providers/v1/config/config.go +++ b/pkg/providers/v1/config/config.go @@ -2,8 +2,10 @@ package config import ( "fmt" - "github.com/aws/aws-sdk-go/aws/request" "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/endpoints" @@ -62,6 +64,11 @@ type CloudConfig struct { // NodeIPFamilies determines which IP addresses are added to node objects and their ordering. NodeIPFamilies []string + + // NodeEventualConsistencyGracePeriod is used to account for propogation delays in the EC2 API. + // An instance may not appear in `ec2:DescribeInstances` output for a period of time after launch. + // The cloud-node-lifecycle-controller must not delete the Node prematurely in this case. + NodeEventualConsistencyGracePeriod time.Duration } // [ServiceOverride "1"] // Service = s3 diff --git a/pkg/providers/v1/instances_v2.go b/pkg/providers/v1/instances_v2.go index f7a08a2d16..1a78699d8f 100644 --- a/pkg/providers/v1/instances_v2.go +++ b/pkg/providers/v1/instances_v2.go @@ -22,7 +22,9 @@ package aws import ( "context" - "strconv" + "fmt" + "strconv" + "time" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -45,6 +47,12 @@ func (c *Cloud) getProviderID(ctx context.Context, node *v1.Node) (string, error // InstanceExists returns true if the instance for the given node exists according to the cloud provider. // Use the node.name or node.spec.providerID field to find the node in the cloud provider. func (c *Cloud) InstanceExists(ctx context.Context, node *v1.Node) (bool, error) { + if time.Since(node.CreationTimestamp.Time) < c.nodeEventualConsistencyGracePeriod { + // recently-launched EC2 instances may not appear in `ec2:DescribeInstances` + // we return an error to cause the cloud-node-lifecycle-controller to ignore this node + // until the eventual-consistency grace period elapses + return false, fmt.Errorf("node is within eventual-consistency grace period (%v)", c.nodeEventualConsistencyGracePeriod) + } providerID, err := c.getProviderID(ctx, node) if err != nil { return false, err