Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Only metrics #2557

Merged
merged 2 commits into from
Sep 13, 2023
Merged

Only metrics #2557

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
981cfa8
Add metrics, dashboard, and test suite
Aug 29, 2023
c1db805
just metrics
Sep 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions pkg/ipamd/datastore/data_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,19 @@ var (
},
[]string{"cidr"},
)
noAvailableIPAddrs = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "awscni_err_no_avail_addrs",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe change to awscni_no_available_ip_addresses, which

  1. aligns with other metrics's use of "ip_addresses".
  2. and we don't need to have this "err" in metrics name, since it's a expected behavior instead of err.
  3. remove nonstandard use of abbreviation(avail is not a standard abbreviation for available iirc)

Help: "The number of pod IP assignments that fail due to no available IP addresses",
},
)
eniIPsInUse = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "awscni_eni_util",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think this metric should be named awscni_assigned_ip_per_eni, since it serves similar purpose of awscni_assigned_ip_per_cidr metric(expect the label value is eni instead of cidr).
Also, util is not a good abbreviation for "utilization"( i assume you mean utilization here :D)

Help: "The number of allocated ips partitioned by eni",
},
[]string{"eni"},
)
prometheusRegistered = false
)

Expand Down Expand Up @@ -344,6 +357,8 @@ func prometheusRegister() {
prometheus.MustRegister(forceRemovedIPs)
prometheus.MustRegister(totalPrefixes)
prometheus.MustRegister(ipsPerCidr)
prometheus.MustRegister(noAvailableIPAddrs)
prometheus.MustRegister(eniIPsInUse)
prometheusRegistered = true
}
}
Expand Down Expand Up @@ -437,6 +452,8 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error {
cidr.IPAddresses[ipAddr.String()] = addr
ds.assignPodIPAddressUnsafe(addr, allocation.IPAMKey, allocation.Metadata, time.Unix(0, allocation.AllocationTimestamp))
ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address)
// Increment ENI IP usage upon finding assigned ips
eniIPsInUse.WithLabelValues(eni.ID).Inc()
// Update prometheus for ips per cidr
// Secondary IP mode will have /32:1 and Prefix mode will have /28:<number of /32s>
ipsPerCidr.With(prometheus.Labels{"cidr": cidr.Cidr.String()}).Inc()
Expand Down Expand Up @@ -522,6 +539,8 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk,
AvailableIPv4Cidrs: make(map[string]*CidrInfo)}

enis.Set(float64(len(ds.eniPool)))
// Initialize ENI IPs In Use to 0 when an ENI is created
eniIPsInUse.WithLabelValues(eniID).Set(0)
return nil
}

Expand Down Expand Up @@ -711,9 +730,12 @@ func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey, ipamMetadata IPAMMeta
delete(V6Cidr.IPAddresses, addr.Address)
return "", -1, err
}
// Increment ENI IP usage on pod IPv6 allocation
eniIPsInUse.WithLabelValues(eni.ID).Inc()
return addr.Address, eni.DeviceNumber, nil
}
}
noAvailableIPAddrs.Inc()
return "", -1, errors.New("assignPodIPv6AddressUnsafe: no available IP addresses")
}

Expand Down Expand Up @@ -776,11 +798,14 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta
ipsPerCidr.With(prometheus.Labels{"cidr": availableCidr.Cidr.String()}).Dec()
return "", -1, err
}
// Increment ENI IP usage on pod IPv4 allocation
eniIPsInUse.WithLabelValues(eni.ID).Inc()
return addr.Address, eni.DeviceNumber, nil
}
ds.log.Debugf("AssignPodIPv4Address: ENI %s does not have available addresses", eni.ID)
}

noAvailableIPAddrs.Inc()
ds.log.Errorf("DataStore has no available IP/Prefix addresses")
return "", -1, errors.New("assignPodIPv4AddressUnsafe: no available IP/Prefix addresses")
}
Expand Down Expand Up @@ -1072,6 +1097,8 @@ func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget, minimumIPTarget, war

// Prometheus update
enis.Set(float64(len(ds.eniPool)))
// Delete ENI IPs In Use when ENI is removed
eniIPsInUse.DeleteLabelValues(removableENI)
totalIPs.Set(float64(ds.total))
return removableENI
}
Expand Down Expand Up @@ -1126,6 +1153,8 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error {

// Prometheus gauge
enis.Set(float64(len(ds.eniPool)))
// Delete ENI IPs In Use when ENI is removed
eniIPsInUse.DeleteLabelValues(eniID)
return nil
}

Expand Down Expand Up @@ -1166,6 +1195,8 @@ func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, d
ipsPerCidr.With(prometheus.Labels{"cidr": availableCidr.Cidr.String()}).Dec()
ds.log.Infof("UnassignPodIPAddress: sandbox %s's ipAddr %s, DeviceNumber %d",
ipamKey, addr.Address, eni.DeviceNumber)
// Decrement ENI IP usage when a pod is deallocated
eniIPsInUse.WithLabelValues(eni.ID).Dec()
return eni, addr.Address, eni.DeviceNumber, nil
}

Expand Down