core/scheduler: tweak clock sync (#778)

Few small tweaks to beacon node clock sync: - Remove waiting for minimum 10 samples. - Fix logging issue (mod vs div) - Update metric name - Add sync offset panel to Grafana dashboard category: misc ticket: #764 feature_flag: beacon_clock_sync
ObolNetwork · Jul 8, 2022 · e672e33 · e672e33
1 parent 6a1fe3b
commit e672e33
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 21 deletions.
diff --git a/core/scheduler/clocksync.go b/core/scheduler/clocksync.go
@@ -62,11 +62,10 @@ func newClockSyncer(ctx context.Context, eventsProvider eth2client.EventsProvide
 		newOffset := clock.Since(startTime)
 
 		offsets = append(offsets, newOffset)
-		if len(offsets) < offsetCount {
-			return
-		}
 
-		offsets = offsets[len(offsets)-offsetCount:] // Trim buffer to max offsetCount items.
+		if len(offsets) > offsetCount {
+			offsets = offsets[len(offsets)-offsetCount:] // Trim buffer to max offsetCount items.
+		}
 
 		clone := append([]time.Duration(nil), offsets...)
 		sort.Slice(clone, func(i, j int) bool {

diff --git a/core/scheduler/clocksync_internal_test.go b/core/scheduler/clocksync_internal_test.go
@@ -42,20 +42,14 @@ func TestClockSync(t *testing.T) {
 
 	var slot int
 
-	// Offset is zero until min 10 events
-	for i := 0; i < offsetCount-1; i++ {
+	// Offset is smallOffset
+	for i := 0; i < offsetCount; i++ {
 		clock.Advance(slotDuration)
 		slot++
 		provider.Push(slot)
-		require.Zero(t, syncOffset())
+		require.Equal(t, smallOffset, syncOffset())
 	}
 
-	clock.Advance(slotDuration)
-	slot++
-	provider.Push(slot)
-
-	require.Equal(t, smallOffset, syncOffset())
-
 	// Increase offset
 	clock.Advance(smallOffset)
 

diff --git a/core/scheduler/metrics.go b/core/scheduler/metrics.go
@@ -47,7 +47,7 @@ var (
 	syncMedianGauge = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: "core",
 		Subsystem: "scheduler",
-		Name:      "clock_sync_median_seconds",
+		Name:      "beacon_node_offset_seconds",
 		Help:      "The beacon node clock sync median offset in seconds",
 	})
 )

diff --git a/core/scheduler/scheduler.go b/core/scheduler/scheduler.go
@@ -467,7 +467,8 @@ func newSlotTicker(ctx context.Context, eth2Cl eth2Provider, clock clockwork.Clo
 				// Add offset to start time to account for beacon node clock skew.
 				delay += syncOffset()
 			}
-			if height/10 == 0 { // Log offset every minute or so.
+
+			if height%10 == 0 { // Log offset every minute or so.
 				log.Debug(ctx, "Beacon node clock sync: remote vs local next slot event",
 					z.Any("offset", syncOffset()),
 					z.Any("beacon_clock_sync_enabled", featureset.Enabled(featureset.BeaconClockSync)))

diff --git a/testutil/compose/define.go b/testutil/compose/define.go
@@ -257,7 +257,7 @@ func buildLocal(ctx context.Context, dir string) error {
 	log.Info(ctx, "Building local charon binary", z.Str("repo", repo), z.Str("target", target))
 
 	cmd := exec.CommandContext(ctx, "go", "build", "-o", target)
-	cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64")
+	cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	cmd.Dir = repo

diff --git a/testutil/compose/static/grafana/dash_simnet.json b/testutil/compose/static/grafana/dash_simnet.json
@@ -21,7 +21,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "iteration": 1649349193420,
+  "iteration": 1657289866291,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -177,6 +177,69 @@
       ],
       "type": "table"
     },
+    {
+      "description": "Difference of local vs remote next slot event",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 8,
+        "y": 1
+      },
+      "id": 21,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.4.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "exemplar": true,
+          "expr": "core_scheduler_beacon_node_offset_seconds{job=\"$node\"}",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Beacon Clock Sync",
+      "type": "stat"
+    },
     {
       "fieldConfig": {
         "defaults": {
@@ -230,8 +293,8 @@
       },
       "gridPos": {
         "h": 4,
-        "w": 8,
-        "x": 8,
+        "w": 7,
+        "x": 11,
         "y": 1
       },
       "id": 13,
@@ -315,8 +378,8 @@
       },
       "gridPos": {
         "h": 4,
-        "w": 8,
-        "x": 16,
+        "w": 6,
+        "x": 18,
         "y": 1
       },
       "id": 14,