Skip to content

Commit

Permalink
More debug logging.
Browse files Browse the repository at this point in the history
  • Loading branch information
igorpeshansky committed Jul 26, 2024
1 parent 6057341 commit d7b2470
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 0 deletions.
2 changes: 2 additions & 0 deletions receiver/dcgmreceiver/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -384,9 +384,11 @@ func (client *dcgmClient) collectDeviceMetrics() (map[uint][]dcgmMetric, error)
var err scrapererror.ScrapeErrors
gpuMetrics := make(map[uint][]dcgmMetric)
for _, gpuIndex := range client.deviceIndices {
client.logger.Debugf("Polling DCGM daemon for GPU %d", gpuIndex)
retry := true
for i := 0; retry && i < client.maxRetries; i++ {
fieldValues, pollErr := dcgmGetLatestValuesForFields(gpuIndex, client.enabledFieldIDs)
client.logger.Debugf("Got %d field values", len(fieldValues))
if pollErr == nil {
gpuMetrics[gpuIndex], retry = client.appendMetrics(gpuMetrics[gpuIndex], gpuIndex, fieldValues)
if retry {
Expand Down
7 changes: 7 additions & 0 deletions receiver/dcgmreceiver/scraper.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,14 +182,21 @@ func (s *dcgmScraper) scrape(_ context.Context) (pmetric.Metrics, error) {
return s.mb.Emit(), err
}

s.settings.Logger.Sugar().Debug("Client created, collecting metrics")
deviceMetrics, err := s.client.collectDeviceMetrics()
if err != nil {
s.settings.Logger.Sugar().Warnf("Metrics not collected; err=%v", err)
return s.mb.Emit(), err
}
s.settings.Logger.Sugar().Debugf("Metrics collected: %d", len(deviceMetrics))

now := pcommon.NewTimestampFromTime(time.Now())
for gpuIndex, gpuMetrics := range deviceMetrics {
metricsByName := make(map[string][]dcgmMetric)
for _, metric := range gpuMetrics {
metricsByName[metric.name] = append(metricsByName[metric.name], metric)
}
s.settings.Logger.Sugar().Debugf("Got %d unique metrics: %v", len(metricsByName), metricsByName)
metrics := make(map[string]dcgmMetric)
for name, points := range metricsByName {
slices.SortStableFunc(points, func(a, b dcgmMetric) int {
Expand Down

0 comments on commit d7b2470

Please sign in to comment.