diff --git a/bpf/kepler.bpf.h b/bpf/kepler.bpf.h index e21715750b..d6e78a4e23 100644 --- a/bpf/kepler.bpf.h +++ b/bpf/kepler.bpf.h @@ -167,11 +167,30 @@ SEC(".rodata.config") __attribute__((btf_decl_tag( "Hardware Events Enabled"))) static volatile const int HW = 1; -// The sampling rate should be disabled by default because its impact on the -// measurements is unknown. +// Global parameters for tracking periods (in milli seconds) SEC(".rodata.config") -__attribute__(( - btf_decl_tag("Sample Rate"))) static volatile const int SAMPLE_RATE = 0; +__attribute__((btf_decl_tag( + "Active Time"))) static volatile const int ACTIVE_TIME = 20; + +// Global parameters for non-tracking periods (in milli seconds) +SEC(".rodata.config") +__attribute__((btf_decl_tag("Idle Time"))) static volatile const int IDLE_TIME = 80; + +// BPF map to track whether we are in the tracking period or not +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, u32); + __type(value, u32); + __uint(max_entries, 1); +} tracking_flag_map SEC(".maps"); + +// BPF map to store the timestamp when the tracking started +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, u64); + __uint(max_entries, 1); +} start_time_map SEC(".maps"); int counter_sched_switch = 0; @@ -306,7 +325,7 @@ static inline void do_page_cache_hit_increment(u32 curr_pid) process_metrics->page_cache_hit++; } -static inline int do_kepler_sched_switch_trace( +static inline int do_kepler_sched_switch_trace_old( u32 prev_pid, u32 next_pid, u32 prev_tgid, u32 next_tgid) { u32 cpu_id; @@ -317,24 +336,31 @@ static inline int do_kepler_sched_switch_trace( cpu_id = bpf_get_smp_processor_id(); - // Skip some samples to minimize overhead - if (SAMPLE_RATE > 0) { - if (counter_sched_switch > 0) { - // update hardware counters to be used when sample is taken - if (counter_sched_switch == 1) { - collect_metrics_and_reset_counters( - &buf, prev_pid, curr_ts, cpu_id); - // Add task on-cpu running start time - bpf_map_update_elem( - &pid_time_map, &next_pid, &curr_ts, - BPF_ANY); - // create new process metrics - register_new_process_if_not_exist(next_tgid); - } - counter_sched_switch--; + // Retrieve tracking flag and start time + u32 key = 0; + u32 *tracking_flag = bpf_map_lookup_elem(&tracking_flag_map, &key); + u64 *start_time = bpf_map_lookup_elem(&start_time_map, &key); + + if (tracking_flag && start_time) { + u64 elapsed_time = (curr_ts - *start_time) / 1000000ULL; + + // Update the tracking flag based on elapsed time + if (*tracking_flag && elapsed_time >= ACTIVE_TIME) { + // Stop tracking + *tracking_flag = 0; + // Reset start time + *start_time = curr_ts; + } else if (!*tracking_flag && elapsed_time >= IDLE_TIME) { + // Start tracking + *tracking_flag = 1; + // Reset start time + *start_time = curr_ts; + } + + // If we are not in the tracking period, return immediately + if (!*tracking_flag) { return 0; } - counter_sched_switch = SAMPLE_RATE; } collect_metrics_and_reset_counters(&buf, prev_pid, curr_ts, cpu_id); @@ -378,3 +404,195 @@ bpf_map_lookup_or_try_init(void *map, const void *key, const void *init) return bpf_map_lookup_elem(map, key); } + +typedef struct period_metrics_t { + u64 run_time_delta; + u64 cycles_delta; + u64 instr_delta; + u64 cache_miss_delta; + u64 period_duration_ns; +} period_metrics_t; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, period_metrics_t); + __uint(max_entries, MAP_SIZE); +} min_period_metrics SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, period_metrics_t); + __uint(max_entries, MAP_SIZE); +} max_period_metrics SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, process_metrics_t); + __uint(max_entries, MAP_SIZE); +} last_sample SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, u64); + __uint(max_entries, MAP_SIZE); +} last_interpolation_ts SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, u32); + __type(value, u64); + __uint(max_entries, MAP_SIZE); +} period_start_ts SEC(".maps"); + +// retain the last sample of the process metrics +static inline void update_period_statistics(struct process_metrics_t *curr_metrics, u32 tgid, u64 curr_ts) { + struct process_metrics_t *last = bpf_map_lookup_elem(&last_sample, &tgid); + struct period_metrics_t period = {0}; + struct period_metrics_t *min_val, *max_val; + u64 *period_start = bpf_map_lookup_elem(&period_start_ts, &tgid); + u32 key = 0; + + if (!period_start) { + bpf_map_update_elem(&period_start_ts, &tgid, &curr_ts, BPF_ANY); + return; + } + + period.period_duration_ns = curr_ts - *period_start; + + if (last) { + period.run_time_delta = curr_metrics->process_run_time - last->process_run_time; + period.cycles_delta = curr_metrics->cpu_cycles - last->cpu_cycles; + period.instr_delta = curr_metrics->cpu_instr - last->cpu_instr; + period.cache_miss_delta = curr_metrics->cache_miss - last->cache_miss; + } else { + period.run_time_delta = curr_metrics->process_run_time; + period.cycles_delta = curr_metrics->cpu_cycles; + period.instr_delta = curr_metrics->cpu_instr; + period.cache_miss_delta = curr_metrics->cache_miss; + } + + bpf_map_update_elem(&last_sample, &tgid, curr_metrics, BPF_ANY); + bpf_map_update_elem(&period_start_ts, &tgid, &curr_ts, BPF_ANY); + + min_val = bpf_map_lookup_elem(&min_period_metrics, &tgid); + max_val = bpf_map_lookup_elem(&max_period_metrics, &tgid); + + if (!min_val || !max_val) { + bpf_map_update_elem(&min_period_metrics, &tgid, &period, BPF_ANY); + bpf_map_update_elem(&max_period_metrics, &tgid, &period, BPF_ANY); + return; + } + + if (period.period_duration_ns > 1000000) { // threshold with 1ms period + if (period.run_time_delta < min_val->run_time_delta) + min_val->run_time_delta = period.run_time_delta; + if (period.run_time_delta > max_val->run_time_delta) + max_val->run_time_delta = period.run_time_delta; + + if (period.cycles_delta < min_val->cycles_delta) + min_val->cycles_delta = period.cycles_delta; + if (period.cycles_delta > max_val->cycles_delta) + max_val->cycles_delta = period.cycles_delta; + + if (period.instr_delta < min_val->instr_delta) + min_val->instr_delta = period.instr_delta; + if (period.instr_delta > max_val->instr_delta) + max_val->instr_delta = period.instr_delta; + + if (period.cache_miss_delta < min_val->cache_miss_delta) + min_val->cache_miss_delta = period.cache_miss_delta; + if (period.cache_miss_delta > max_val->cache_miss_delta) + max_val->cache_miss_delta = period.cache_miss_delta; + } +} + +// Interpolate the metrics during idle +static inline void interpolate_idle_metrics(u32 tgid, u64 curr_ts) { + struct process_metrics_t *curr_metrics = bpf_map_lookup_elem(&processes, &tgid); + struct period_metrics_t *min_val = bpf_map_lookup_elem(&min_period_metrics, &tgid); + struct period_metrics_t *max_val = bpf_map_lookup_elem(&max_period_metrics, &tgid); + u64 *last_ts = bpf_map_lookup_elem(&last_interpolation_ts, &tgid); + + if (!curr_metrics || !min_val || !max_val || !last_ts) + return; + + u64 time_since_last = curr_ts - *last_ts; + if (time_since_last < (IDLE_TIME * 1000000ULL)) + return; + + u64 avg_period_duration = (min_val->period_duration_ns + max_val->period_duration_ns) / 2; + if (avg_period_duration == 0) + return; + + u64 missed_periods = time_since_last / avg_period_duration; + if (missed_periods == 0) + return; + + u64 avg_runtime_delta = (min_val->run_time_delta + max_val->run_time_delta) / 2; + u64 avg_cycles_delta = (min_val->cycles_delta + max_val->cycles_delta) / 2; + u64 avg_instr_delta = (min_val->instr_delta + max_val->instr_delta) / 2; + u64 avg_cache_miss_delta = (min_val->cache_miss_delta + max_val->cache_miss_delta) / 2; + + curr_metrics->process_run_time += (avg_runtime_delta * missed_periods); + curr_metrics->cpu_cycles += (avg_cycles_delta * missed_periods); + curr_metrics->cpu_instr += (avg_instr_delta * missed_periods); + curr_metrics->cache_miss += (avg_cache_miss_delta * missed_periods); + + *last_ts = curr_ts; +} + +static inline int do_kepler_sched_switch_trace( + u32 prev_pid, u32 next_pid, u32 prev_tgid, u32 next_tgid) +{ + u32 cpu_id; + u64 curr_ts = bpf_ktime_get_ns(); + struct process_metrics_t *curr_tgid_metrics, *prev_tgid_metrics; + struct process_metrics_t buf = {}; + + cpu_id = bpf_get_smp_processor_id(); + + u32 key = 0; + u32 *tracking_flag = bpf_map_lookup_elem(&tracking_flag_map, &key); + u64 *start_time = bpf_map_lookup_elem(&start_time_map, &key); + + if (tracking_flag && start_time) { + u64 elapsed_time = (curr_ts - *start_time) / 1000000ULL; + + if (*tracking_flag && elapsed_time >= ACTIVE_TIME) { + *tracking_flag = 0; + *start_time = curr_ts; + bpf_map_update_elem(&last_interpolation_ts, &prev_tgid, &curr_ts, BPF_ANY); + } else if (!*tracking_flag && elapsed_time >= IDLE_TIME) { + *tracking_flag = 1; + *start_time = curr_ts; + } + + if (!*tracking_flag) { + interpolate_idle_metrics(prev_tgid, curr_ts); + return 0; + } + } + + collect_metrics_and_reset_counters(&buf, prev_pid, curr_ts, cpu_id); + + if (buf.process_run_time > 0) { + prev_tgid_metrics = bpf_map_lookup_elem(&processes, &prev_tgid); + if (prev_tgid_metrics) { + prev_tgid_metrics->process_run_time += buf.process_run_time; + prev_tgid_metrics->cpu_cycles += buf.cpu_cycles; + prev_tgid_metrics->cpu_instr += buf.cpu_instr; + prev_tgid_metrics->cache_miss += buf.cache_miss; + + update_period_statistics(prev_tgid_metrics, prev_tgid, curr_ts); + } + } + + bpf_map_update_elem(&pid_time_map, &next_pid, &curr_ts, BPF_ANY); + register_new_process_if_not_exist(prev_tgid); + + return 0; +} \ No newline at end of file diff --git a/pkg/bpf/exporter.go b/pkg/bpf/exporter.go index 14643370c5..94c05a3231 100644 --- a/pkg/bpf/exporter.go +++ b/pkg/bpf/exporter.go @@ -91,7 +91,13 @@ func (e *exporter) attach() error { // Set program global variables err = specs.RewriteConstants(map[string]interface{}{ - "SAMPLE_RATE": int32(config.GetBPFSampleRate()), + "ACTIVE_TIME": int32(config.GetBPFActiveSampleWindowMS()), + }) + if err != nil { + return fmt.Errorf("error rewriting program constants: %v", err) + } + err = specs.RewriteConstants(map[string]interface{}{ + "IDLE_TIME": int32(config.GetBPFIdleSampleWindowMS()), }) if err != nil { return fmt.Errorf("error rewriting program constants: %v", err) diff --git a/pkg/bpf/kepler_bpfeb.go b/pkg/bpf/kepler_bpfeb.go index 1f82307638..f82de9264a 100644 --- a/pkg/bpf/kepler_bpfeb.go +++ b/pkg/bpf/kepler_bpfeb.go @@ -12,6 +12,14 @@ import ( "github.com/cilium/ebpf" ) +type keplerPeriodMetricsT struct { + RunTimeDelta uint64 + CyclesDelta uint64 + InstrDelta uint64 + CacheMissDelta uint64 + PeriodDurationNs uint64 +} + type keplerProcessMetricsT struct { CgroupId uint64 Pid uint64 @@ -82,8 +90,15 @@ type keplerMapSpecs struct { CpuCyclesEventReader *ebpf.MapSpec `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.MapSpec `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.MapSpec `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.MapSpec `ebpf:"last_interpolation_ts"` + LastSample *ebpf.MapSpec `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.MapSpec `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.MapSpec `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.MapSpec `ebpf:"period_start_ts"` PidTimeMap *ebpf.MapSpec `ebpf:"pid_time_map"` Processes *ebpf.MapSpec `ebpf:"processes"` + StartTimeMap *ebpf.MapSpec `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.MapSpec `ebpf:"tracking_flag_map"` } // keplerObjects contains all objects after they have been loaded into the kernel. @@ -111,8 +126,15 @@ type keplerMaps struct { CpuCyclesEventReader *ebpf.Map `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.Map `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.Map `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.Map `ebpf:"last_interpolation_ts"` + LastSample *ebpf.Map `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.Map `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.Map `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.Map `ebpf:"period_start_ts"` PidTimeMap *ebpf.Map `ebpf:"pid_time_map"` Processes *ebpf.Map `ebpf:"processes"` + StartTimeMap *ebpf.Map `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.Map `ebpf:"tracking_flag_map"` } func (m *keplerMaps) Close() error { @@ -123,8 +145,15 @@ func (m *keplerMaps) Close() error { m.CpuCyclesEventReader, m.CpuInstructions, m.CpuInstructionsEventReader, + m.LastInterpolationTs, + m.LastSample, + m.MaxPeriodMetrics, + m.MinPeriodMetrics, + m.PeriodStartTs, m.PidTimeMap, m.Processes, + m.StartTimeMap, + m.TrackingFlagMap, ) } diff --git a/pkg/bpf/kepler_bpfeb.o b/pkg/bpf/kepler_bpfeb.o index be2eedd1df..d67563ac23 100644 Binary files a/pkg/bpf/kepler_bpfeb.o and b/pkg/bpf/kepler_bpfeb.o differ diff --git a/pkg/bpf/kepler_bpfel.go b/pkg/bpf/kepler_bpfel.go index 3fdd09bcda..9f94006d1a 100644 --- a/pkg/bpf/kepler_bpfel.go +++ b/pkg/bpf/kepler_bpfel.go @@ -12,6 +12,14 @@ import ( "github.com/cilium/ebpf" ) +type keplerPeriodMetricsT struct { + RunTimeDelta uint64 + CyclesDelta uint64 + InstrDelta uint64 + CacheMissDelta uint64 + PeriodDurationNs uint64 +} + type keplerProcessMetricsT struct { CgroupId uint64 Pid uint64 @@ -82,8 +90,15 @@ type keplerMapSpecs struct { CpuCyclesEventReader *ebpf.MapSpec `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.MapSpec `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.MapSpec `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.MapSpec `ebpf:"last_interpolation_ts"` + LastSample *ebpf.MapSpec `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.MapSpec `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.MapSpec `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.MapSpec `ebpf:"period_start_ts"` PidTimeMap *ebpf.MapSpec `ebpf:"pid_time_map"` Processes *ebpf.MapSpec `ebpf:"processes"` + StartTimeMap *ebpf.MapSpec `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.MapSpec `ebpf:"tracking_flag_map"` } // keplerObjects contains all objects after they have been loaded into the kernel. @@ -111,8 +126,15 @@ type keplerMaps struct { CpuCyclesEventReader *ebpf.Map `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.Map `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.Map `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.Map `ebpf:"last_interpolation_ts"` + LastSample *ebpf.Map `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.Map `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.Map `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.Map `ebpf:"period_start_ts"` PidTimeMap *ebpf.Map `ebpf:"pid_time_map"` Processes *ebpf.Map `ebpf:"processes"` + StartTimeMap *ebpf.Map `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.Map `ebpf:"tracking_flag_map"` } func (m *keplerMaps) Close() error { @@ -123,8 +145,15 @@ func (m *keplerMaps) Close() error { m.CpuCyclesEventReader, m.CpuInstructions, m.CpuInstructionsEventReader, + m.LastInterpolationTs, + m.LastSample, + m.MaxPeriodMetrics, + m.MinPeriodMetrics, + m.PeriodStartTs, m.PidTimeMap, m.Processes, + m.StartTimeMap, + m.TrackingFlagMap, ) } diff --git a/pkg/bpf/kepler_bpfel.o b/pkg/bpf/kepler_bpfel.o index 7f61cec6f9..69ece0cc5d 100644 Binary files a/pkg/bpf/kepler_bpfel.o and b/pkg/bpf/kepler_bpfel.o differ diff --git a/pkg/bpftest/bpf_suite_test.go b/pkg/bpftest/bpf_suite_test.go index 74b911431f..3998f945ef 100644 --- a/pkg/bpftest/bpf_suite_test.go +++ b/pkg/bpftest/bpf_suite_test.go @@ -330,7 +330,8 @@ var _ = Describe("BPF Exporter", func() { err = specs.RewriteConstants(map[string]interface{}{ "TEST": int32(1), - "SAMPLE_RATE": int32(1000), + "ACTIVE_TIME": int32(1000), + "IDLE_TIME": int32(0), }) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/bpftest/test_bpfeb.go b/pkg/bpftest/test_bpfeb.go index db3880c877..762e5c3965 100644 --- a/pkg/bpftest/test_bpfeb.go +++ b/pkg/bpftest/test_bpfeb.go @@ -12,6 +12,14 @@ import ( "github.com/cilium/ebpf" ) +type testPeriodMetricsT struct { + RunTimeDelta uint64 + CyclesDelta uint64 + InstrDelta uint64 + CacheMissDelta uint64 + PeriodDurationNs uint64 +} + type testProcessMetricsT struct { CgroupId uint64 Pid uint64 @@ -81,8 +89,15 @@ type testMapSpecs struct { CpuCyclesEventReader *ebpf.MapSpec `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.MapSpec `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.MapSpec `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.MapSpec `ebpf:"last_interpolation_ts"` + LastSample *ebpf.MapSpec `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.MapSpec `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.MapSpec `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.MapSpec `ebpf:"period_start_ts"` PidTimeMap *ebpf.MapSpec `ebpf:"pid_time_map"` Processes *ebpf.MapSpec `ebpf:"processes"` + StartTimeMap *ebpf.MapSpec `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.MapSpec `ebpf:"tracking_flag_map"` } // testObjects contains all objects after they have been loaded into the kernel. @@ -110,8 +125,15 @@ type testMaps struct { CpuCyclesEventReader *ebpf.Map `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.Map `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.Map `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.Map `ebpf:"last_interpolation_ts"` + LastSample *ebpf.Map `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.Map `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.Map `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.Map `ebpf:"period_start_ts"` PidTimeMap *ebpf.Map `ebpf:"pid_time_map"` Processes *ebpf.Map `ebpf:"processes"` + StartTimeMap *ebpf.Map `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.Map `ebpf:"tracking_flag_map"` } func (m *testMaps) Close() error { @@ -122,8 +144,15 @@ func (m *testMaps) Close() error { m.CpuCyclesEventReader, m.CpuInstructions, m.CpuInstructionsEventReader, + m.LastInterpolationTs, + m.LastSample, + m.MaxPeriodMetrics, + m.MinPeriodMetrics, + m.PeriodStartTs, m.PidTimeMap, m.Processes, + m.StartTimeMap, + m.TrackingFlagMap, ) } diff --git a/pkg/bpftest/test_bpfeb.o b/pkg/bpftest/test_bpfeb.o index f190a3ab7b..bb381f6c55 100644 Binary files a/pkg/bpftest/test_bpfeb.o and b/pkg/bpftest/test_bpfeb.o differ diff --git a/pkg/bpftest/test_bpfel.go b/pkg/bpftest/test_bpfel.go index 7317a75a1b..4a4e6d20da 100644 --- a/pkg/bpftest/test_bpfel.go +++ b/pkg/bpftest/test_bpfel.go @@ -12,6 +12,14 @@ import ( "github.com/cilium/ebpf" ) +type testPeriodMetricsT struct { + RunTimeDelta uint64 + CyclesDelta uint64 + InstrDelta uint64 + CacheMissDelta uint64 + PeriodDurationNs uint64 +} + type testProcessMetricsT struct { CgroupId uint64 Pid uint64 @@ -81,8 +89,15 @@ type testMapSpecs struct { CpuCyclesEventReader *ebpf.MapSpec `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.MapSpec `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.MapSpec `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.MapSpec `ebpf:"last_interpolation_ts"` + LastSample *ebpf.MapSpec `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.MapSpec `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.MapSpec `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.MapSpec `ebpf:"period_start_ts"` PidTimeMap *ebpf.MapSpec `ebpf:"pid_time_map"` Processes *ebpf.MapSpec `ebpf:"processes"` + StartTimeMap *ebpf.MapSpec `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.MapSpec `ebpf:"tracking_flag_map"` } // testObjects contains all objects after they have been loaded into the kernel. @@ -110,8 +125,15 @@ type testMaps struct { CpuCyclesEventReader *ebpf.Map `ebpf:"cpu_cycles_event_reader"` CpuInstructions *ebpf.Map `ebpf:"cpu_instructions"` CpuInstructionsEventReader *ebpf.Map `ebpf:"cpu_instructions_event_reader"` + LastInterpolationTs *ebpf.Map `ebpf:"last_interpolation_ts"` + LastSample *ebpf.Map `ebpf:"last_sample"` + MaxPeriodMetrics *ebpf.Map `ebpf:"max_period_metrics"` + MinPeriodMetrics *ebpf.Map `ebpf:"min_period_metrics"` + PeriodStartTs *ebpf.Map `ebpf:"period_start_ts"` PidTimeMap *ebpf.Map `ebpf:"pid_time_map"` Processes *ebpf.Map `ebpf:"processes"` + StartTimeMap *ebpf.Map `ebpf:"start_time_map"` + TrackingFlagMap *ebpf.Map `ebpf:"tracking_flag_map"` } func (m *testMaps) Close() error { @@ -122,8 +144,15 @@ func (m *testMaps) Close() error { m.CpuCyclesEventReader, m.CpuInstructions, m.CpuInstructionsEventReader, + m.LastInterpolationTs, + m.LastSample, + m.MaxPeriodMetrics, + m.MinPeriodMetrics, + m.PeriodStartTs, m.PidTimeMap, m.Processes, + m.StartTimeMap, + m.TrackingFlagMap, ) } diff --git a/pkg/bpftest/test_bpfel.o b/pkg/bpftest/test_bpfel.o index 5c8972722c..f2e3eb1a02 100644 Binary files a/pkg/bpftest/test_bpfel.o and b/pkg/bpftest/test_bpfel.o differ diff --git a/pkg/config/config.go b/pkg/config/config.go index d02a7e4296..19ea0c5dd5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -59,11 +59,12 @@ type KeplerConfig struct { MockACPIPowerPath string MaxLookupRetry int KubeConfig string - BPFSampleRate int EstimatorModel string EstimatorSelectFilter string CPUArchOverride string MachineSpecFilePath string + BPFActiveSampleWindowMS int + BPFIdleSampleWindowMS int } type MetricsConfig struct { CoreUsageMetric string @@ -154,10 +155,11 @@ func getKeplerConfig() KeplerConfig { MockACPIPowerPath: getConfig("MOCK_ACPI_POWER_PATH", ""), MaxLookupRetry: getIntConfig("MAX_LOOKUP_RETRY", defaultMaxLookupRetry), KubeConfig: getConfig("KUBE_CONFIG", defaultKubeConfig), - BPFSampleRate: getIntConfig("EXPERIMENTAL_BPF_SAMPLE_RATE", defaultBPFSampleRate), EstimatorModel: getConfig("ESTIMATOR_MODEL", defaultMetricValue), EstimatorSelectFilter: getConfig("ESTIMATOR_SELECT_FILTER", defaultMetricValue), // no filter CPUArchOverride: getConfig("CPU_ARCH_OVERRIDE", defaultCPUArchOverride), + BPFActiveSampleWindowMS: getIntConfig("EXPERIMENTAL_BPF_ACTIVE_SAMPLE_WINDOW_MS", 1000), + BPFIdleSampleWindowMS: getIntConfig("EXPERIMENTAL_BPF_IDLE_SAMPLE_WINDOW_MS", 0), } } @@ -261,7 +263,8 @@ func logBoolConfigs() { klog.V(5).Infof("EXPOSE_BPF_METRICS: %t", instance.Kepler.ExposeBPFMetrics) klog.V(5).Infof("EXPOSE_COMPONENT_POWER: %t", instance.Kepler.ExposeComponentPower) klog.V(5).Infof("EXPOSE_ESTIMATED_IDLE_POWER_METRICS: %t. This only impacts when the power is estimated using pre-prained models. Estimated idle power is meaningful only when Kepler is running on bare-metal or with a single virtual machine (VM) on the node.", instance.Kepler.ExposeIdlePowerMetrics) - klog.V(5).Infof("EXPERIMENTAL_BPF_SAMPLE_RATE: %d", instance.Kepler.BPFSampleRate) + klog.V(5).Infof("EXPERIMENTAL_BPF_ACTIVE_SAMPLE_WINDOW_MS: %d", instance.Kepler.BPFActiveSampleWindowMS) + klog.V(5).Infof("EXPERIMENTAL_BPF_IDLE_SAMPLE_WINDOW_MS: %d", instance.Kepler.BPFIdleSampleWindowMS) } } @@ -395,6 +398,21 @@ func SetGPUUsageMetric(metric string) { instance.Metrics.GPUUsageMetric = metric } +func GetBPFActiveSampleWindowMS() int { + ensureConfigInitialized() + return instance.Kepler.BPFActiveSampleWindowMS +} + +func GetBPFIdleSampleWindowMS() int { + ensureConfigInitialized() + return instance.Kepler.BPFIdleSampleWindowMS +} + +func GetDCGMHostEngineEndpoint() string { + ensureConfigInitialized() + return instance.DCGMHostEngineEndpoint +} + func (c *Config) getUnixName() (unix.Utsname, error) { var utsname unix.Utsname err := unix.Uname(&utsname) @@ -552,11 +570,6 @@ func ExposeIRQCounterMetrics() bool { return instance.Kepler.ExposeIRQCounterMetrics } -func GetBPFSampleRate() int { - ensureConfigInitialized() - return instance.Kepler.BPFSampleRate -} - func GetRedfishCredFilePath() string { ensureConfigInitialized() return instance.Redfish.CredFilePath diff --git a/pkg/sensors/accelerator/device/sources/dcgm.go b/pkg/sensors/accelerator/device/sources/dcgm.go index 9fbba5fdcb..bbeab92f8a 100644 --- a/pkg/sensors/accelerator/device/sources/dcgm.go +++ b/pkg/sensors/accelerator/device/sources/dcgm.go @@ -62,7 +62,7 @@ type GPUDcgm struct { } func init() { - if _, err := dcgm.Init(dcgm.Standalone, config.DCGMHostEngineEndpoint, isSocket); err != nil { + if _, err := dcgm.Init(dcgm.Standalone, config.GetDCGMHostEngineEndpoint(), isSocket); err != nil { klog.Errorf("Error initializing dcgm: %v", err) return } @@ -131,7 +131,7 @@ func (d *GPUDcgm) InitLib() (err error) { err = fmt.Errorf("could not init dcgm: %v", r) } }() - cleanup, err := dcgm.Init(dcgm.Standalone, config.DCGMHostEngineEndpoint, isSocket) + cleanup, err := dcgm.Init(dcgm.Standalone, config.GetDCGMHostEngineEndpoint(), isSocket) if err != nil { klog.Infof("There is no DCGM daemon running in the host: %s", err) // embedded mode is not recommended for production per https://github.com/NVIDIA/dcgm-exporter/issues/22#issuecomment-1321521995