Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Completely ignore Instance Metadata when in SQS Queue mode. #735

Merged
merged 5 commits into from
Dec 6, 2022
Merged

Completely ignore Instance Metadata when in SQS Queue mode. #735

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8dfec48
Completely ignore Instance Metadata when in SQS Queue mode.
peteroruba Nov 30, 2022
de95f47
Function parameter typo fix.
peteroruba Dec 4, 2022
3850aa6
Fix imds.GetNodeMetadata() test.
peteroruba Dec 4, 2022
7b81a6b
Merge branch 'aws:main' into main
peteroruba Dec 6, 2022
34ead36
Add unit test when IMDS mode is disabled
snay2 Dec 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 15 additions & 12 deletions cmd/node-termination-handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,12 @@ func main() {
nthConfig.Print()
log.Fatal().Err(err).Msg("Unable to instantiate probes service,")
}
imdsDisabled := nthConfig.EnableSQSTerminationDraining

imds := ec2metadata.New(nthConfig.MetadataURL, nthConfig.MetadataTries)

interruptionEventStore := interruptioneventstore.New(nthConfig)
nodeMetadata := imds.GetNodeMetadata()
nodeMetadata := imds.GetNodeMetadata(imdsDisabled)
// Populate the aws region if available from node metadata and not already explicitly configured
if nthConfig.AWSRegion == "" && nodeMetadata.Region != "" {
nthConfig.AWSRegion = nodeMetadata.Region
Expand Down Expand Up @@ -163,17 +164,19 @@ func main() {
defer close(cancelChan)

monitoringFns := map[string]monitor.Monitor{}
if nthConfig.EnableSpotInterruptionDraining {
imdsSpotMonitor := spotitn.NewSpotInterruptionMonitor(imds, interruptionChan, cancelChan, nthConfig.NodeName)
monitoringFns[spotITN] = imdsSpotMonitor
}
if nthConfig.EnableScheduledEventDraining {
imdsScheduledEventMonitor := scheduledevent.NewScheduledEventMonitor(imds, interruptionChan, cancelChan, nthConfig.NodeName)
monitoringFns[scheduledMaintenance] = imdsScheduledEventMonitor
}
if nthConfig.EnableRebalanceMonitoring || nthConfig.EnableRebalanceDraining {
imdsRebalanceMonitor := rebalancerecommendation.NewRebalanceRecommendationMonitor(imds, interruptionChan, nthConfig.NodeName)
monitoringFns[rebalanceRecommendation] = imdsRebalanceMonitor
if !imdsDisabled {
if nthConfig.EnableSpotInterruptionDraining {
imdsSpotMonitor := spotitn.NewSpotInterruptionMonitor(imds, interruptionChan, cancelChan, nthConfig.NodeName)
monitoringFns[spotITN] = imdsSpotMonitor
}
if nthConfig.EnableScheduledEventDraining {
imdsScheduledEventMonitor := scheduledevent.NewScheduledEventMonitor(imds, interruptionChan, cancelChan, nthConfig.NodeName)
monitoringFns[scheduledMaintenance] = imdsScheduledEventMonitor
}
if nthConfig.EnableRebalanceMonitoring || nthConfig.EnableRebalanceDraining {
imdsRebalanceMonitor := rebalancerecommendation.NewRebalanceRecommendationMonitor(imds, interruptionChan, nthConfig.NodeName)
monitoringFns[rebalanceRecommendation] = imdsRebalanceMonitor
}
}
if nthConfig.EnableSQSTerminationDraining {
cfg := aws.NewConfig().WithRegion(nthConfig.AWSRegion).WithEndpoint(nthConfig.AWSEndpoint).WithSTSRegionalEndpoint(endpoints.RegionalSTSEndpoint)
Expand Down
53 changes: 33 additions & 20 deletions pkg/ec2metadata/ec2metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,30 +325,43 @@ func retry(attempts int, sleep time.Duration, httpReq func() (*http.Response, er
}

// GetNodeMetadata attempts to gather additional ec2 instance information from the metadata service
func (e *Service) GetNodeMetadata() NodeMetadata {
func (e *Service) GetNodeMetadata(imdsDisabled bool) NodeMetadata {
var metadata NodeMetadata
identityDoc, err := e.GetMetadataInfo(IdentityDocPath)
if err != nil {
log.Err(err).Msg("Unable to fetch metadata from IMDS")
return metadata
}
err = json.NewDecoder(strings.NewReader(identityDoc)).Decode(&metadata)
if err != nil {
log.Warn().Msg("Unable to fetch instance identity document from ec2 metadata")
metadata.InstanceID, _ = e.GetMetadataInfo(InstanceIDPath)
metadata.InstanceType, _ = e.GetMetadataInfo(InstanceTypePath)
metadata.LocalIP, _ = e.GetMetadataInfo(LocalIPPath)
metadata.AvailabilityZone, _ = e.GetMetadataInfo(AZPlacementPath)
if (!imdsDisabled) {
identityDoc, err := e.GetMetadataInfo(IdentityDocPath)
if err != nil {
log.Err(err).Msg("Unable to fetch metadata from IMDS")
return metadata
}
err = json.NewDecoder(strings.NewReader(identityDoc)).Decode(&metadata)
if err != nil {
log.Warn().Msg("Unable to fetch instance identity document from ec2 metadata")
metadata.InstanceID, _ = e.GetMetadataInfo(InstanceIDPath)
metadata.InstanceType, _ = e.GetMetadataInfo(InstanceTypePath)
metadata.LocalIP, _ = e.GetMetadataInfo(LocalIPPath)
metadata.AvailabilityZone, _ = e.GetMetadataInfo(AZPlacementPath)
if len(metadata.AvailabilityZone) > 1 {
metadata.Region = metadata.AvailabilityZone[0 : len(metadata.AvailabilityZone)-1]
metadata.Region = metadata.AvailabilityZone[0 : len(metadata.AvailabilityZone)-1]
}
}
metadata.InstanceLifeCycle, _ = e.GetMetadataInfo(InstanceLifeCycle)
metadata.LocalHostname, _ = e.GetMetadataInfo(LocalHostnamePath)
metadata.PublicHostname, _ = e.GetMetadataInfo(PublicHostnamePath)
metadata.PublicIP, _ = e.GetMetadataInfo(PublicIPPath)
}
metadata.InstanceLifeCycle, _ = e.GetMetadataInfo(InstanceLifeCycle)
metadata.LocalHostname, _ = e.GetMetadataInfo(LocalHostnamePath)
metadata.PublicHostname, _ = e.GetMetadataInfo(PublicHostnamePath)
metadata.PublicIP, _ = e.GetMetadataInfo(PublicIPPath)

log.Info().Interface("metadata", metadata).Msg("Startup Metadata Retrieved")
log.Info().Interface("metadata", metadata).Msg("Startup Metadata Retrieved")
} else {
metadata.AccountId = ""
metadata.InstanceID = ""
metadata.InstanceLifeCycle = ""
metadata.InstanceType = ""
metadata.PublicHostname = ""
metadata.PublicIP = ""
metadata.LocalHostname = ""
metadata.LocalIP = ""
metadata.AvailabilityZone = ""
metadata.Region = ""
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this else branch necessary? When imdsDisabled=false, metadata should still be empty, since no properties were set. When you were testing, did you discover this was required?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

He complained about metadata being uninitialized so I've set it to at least empty string values.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Who is "he"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The aws-node-termination-handler process. I ran it both on AWS EKS and on my workstation's local k8s cluster.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, right. If you replace line 329 with the following, it will initialize the whole struct and you won't need to set each property inidividually:

metadata := NodeMetadata{}


return metadata
}
2 changes: 1 addition & 1 deletion pkg/ec2metadata/ec2metadata_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -580,7 +580,7 @@ func TestGetNodeMetadata(t *testing.T) {

// Use URL from our local test server
imds := ec2metadata.New(server.URL, 1)
nodeMetadata := imds.GetNodeMetadata()
nodeMetadata := imds.GetNodeMetadata(false)

h.Assert(t, nodeMetadata.InstanceID == `metadata`, `Missing required NodeMetadata field InstanceID`)
h.Assert(t, nodeMetadata.InstanceType == `metadata`, `Missing required NodeMetadata field InstanceType`)
Expand Down