fix: runner reaping (#3686)

backend/controller/controller.go

@@ -562,10 +562,11 @@ func (s *Service) RegisterRunner(ctx context.Context, stream *connect.ClientStre
 			return nil, connect.NewError(connect.CodeInvalidArgument, err)
 		}
 		// The created event does not matter if it is a new runner or not.
-		err = s.controllerState.Publish(&state.RunnerCreatedEvent{
+		err = s.controllerState.Publish(&state.RunnerRegisteredEvent{
 			Key:        runnerKey,
 			Endpoint:   msg.Endpoint,
 			Deployment: deploymentKey,
+			Time:       time.Now(),
 		})
 		if err != nil {
 			return nil, err
@@ -1137,7 +1138,7 @@ func (s *Service) reapStaleRunners(ctx context.Context) (time.Duration, error) {
 	for _, runner := range cs.Runners() {
 		if runner.LastSeen.Add(s.config.RunnerTimeout).Before(time.Now()) {
 			runnerKey := runner.Key
-			logger.Debugf("Reaping stale runner %s", runnerKey)
+			logger.Debugf("Reaping stale runner %s with last seen %v", runnerKey, runner.LastSeen.String())
 			if err := s.controllerState.Publish(&state.RunnerDeletedEvent{Key: runnerKey}); err != nil {
 				return 0, fmt.Errorf("failed to publish runner deleted event: %w", err)
 			}

backend/controller/state/controllerstate.go

@@ -1,7 +1,6 @@
 package state
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/alecthomas/types/optional"
@@ -55,26 +54,26 @@ type Runner struct {
 	Deployment model.DeploymentKey
 }
 
-var _ eventstream.Event[State] = (*RunnerCreatedEvent)(nil)
-var _ eventstream.Event[State] = (*RunnerHeartbeatEvent)(nil)
+var _ eventstream.Event[State] = (*RunnerRegisteredEvent)(nil)
 var _ eventstream.Event[State] = (*RunnerDeletedEvent)(nil)
 
-type RunnerCreatedEvent struct {
+type RunnerRegisteredEvent struct {
 	Key        model.RunnerKey
-	Create     time.Time
+	Time       time.Time
 	Endpoint   string
 	Module     string
 	Deployment model.DeploymentKey
 }
 
-func (r *RunnerCreatedEvent) Handle(t State) (State, error) {
+func (r *RunnerRegisteredEvent) Handle(t State) (State, error) {
 	if existing := t.runners[r.Key.String()]; existing != nil {
+		existing.LastSeen = r.Time
 		return t, nil
 	}
 	n := Runner{
 		Key:        r.Key,
-		Create:     r.Create,
-		LastSeen:   r.Create,
+		Create:     r.Time,
+		LastSeen:   r.Time,
 		Endpoint:   r.Endpoint,
 		Module:     r.Module,
 		Deployment: r.Deployment,
@@ -84,20 +83,6 @@ func (r *RunnerCreatedEvent) Handle(t State) (State, error) {
 	return t, nil
 }
 
-type RunnerHeartbeatEvent struct {
-	Key      model.RunnerKey
-	LastSeen time.Time
-}
-
-func (r *RunnerHeartbeatEvent) Handle(t State) (State, error) {
-	existing := t.runners[r.Key.String()]
-	if existing == nil {
-		return t, fmt.Errorf("runner %s not found", r.Key)
-	}
-	existing.LastSeen = r.LastSeen
-	return t, nil
-}
-
 type RunnerDeletedEvent struct {
 	Key model.RunnerKey
 }
@@ -106,7 +91,6 @@ func (r RunnerDeletedEvent) Handle(t State) (State, error) {
 	existing := t.runners[r.Key.String()]
 	if existing != nil {
 		delete(t.runners, r.Key.String())
-
 	}
 	return t, nil
 }

backend/controller/state/controllerstate_test.go

@@ -19,9 +19,9 @@ func TestRunnerState(t *testing.T) {
 	endpoint := "http://localhost:8080"
 	module := "test"
 	deploymentKey := model.NewDeploymentKey(module)
-	err := cs.Publish(&state.RunnerCreatedEvent{
+	err := cs.Publish(&state.RunnerRegisteredEvent{
 		Key:        key,
-		Create:     create,
+		Time:       create,
 		Endpoint:   endpoint,
 		Module:     module,
 		Deployment: deploymentKey,
@@ -36,9 +36,12 @@ func TestRunnerState(t *testing.T) {
 	assert.Equal(t, module, view.Runners()[0].Module)
 	assert.Equal(t, deploymentKey, view.Runners()[0].Deployment)
 	seen := time.Now()
-	err = cs.Publish(&state.RunnerHeartbeatEvent{
-		Key:      key,
-		LastSeen: seen,
+	err = cs.Publish(&state.RunnerRegisteredEvent{
+		Key:        key,
+		Time:       seen,
+		Endpoint:   endpoint,
+		Module:     module,
+		Deployment: deploymentKey,
 	})
 	assert.NoError(t, err)
 	view = cs.View()