From f4deea92b9cdbed0263e16110d82ce6d597df62b Mon Sep 17 00:00:00 2001 From: Oklahomer Date: Sat, 1 Jun 2019 16:35:45 +0900 Subject: [PATCH 1/3] Add customizable feature to judge if the error should be sent via alerter --- alerter/line/client.go | 2 +- runner.go | 116 ++++++++++++++++++----------- runner_test.go | 165 ++++++++++++++++++++++++++--------------- 3 files changed, 182 insertions(+), 101 deletions(-) diff --git a/alerter/line/client.go b/alerter/line/client.go index 4a4438c..8ca891c 100644 --- a/alerter/line/client.go +++ b/alerter/line/client.go @@ -47,7 +47,7 @@ func New(config *Config) *Client { // Alert sends alert message to notify critical state of caller. func (c *Client) Alert(ctx context.Context, botType sarah.BotType, err error) error { - msg := fmt.Sprintf("Critical error on %s: %s.", botType.String(), err.Error()) + msg := fmt.Sprintf("Error on %s: %s.", botType.String(), err.Error()) v := url.Values{"message": {msg}} req, err := http.NewRequest(http.MethodPost, Endpoint, strings.NewReader(v.Encode())) if err != nil { diff --git a/runner.go b/runner.go index e9401f0..62f5fae 100644 --- a/runner.go +++ b/runner.go @@ -143,6 +143,28 @@ func RegisterWorker(worker workers.Worker) { }) } +// RegisterAlertJudge registers given function, judgeFnc, that is called when a Bot raises an error. +// This function judges if the given error is worth being notified to administrators. +// If this function returns true, all registered alerters are called to notify such error state to the administrators. +// +// Bot/Adapter can raise an error via a function, func(error), that is passed to Run() as a third argument. +// When BotNonContinuableError is raised, go-sarah's core cancels Bot's context and thus failing Bot and related resources stop working. +// If one or more sarah.Alerters implementations are registered, such critical error is passed to the alerters and administrators will be notified. +// When other types of error are raised, the error is passed to the function registered via sarah.RegisterAlertJudge(). +// The function may return true when the error is worth being notified to administrators. +// +// Bot/Adapter's implementation should be simple. It should not handle serious errors by itself. +// Instead, it should simply raise an error and let core takes care of this. +// So if the calls to alerters should have some kind of rate limit, judgeFnc should internally handle such rate limit. +// In this way, each Bot/Adapter's implementation can be kept simple. +// go-sarah's core should always supervise and control its belonging Bots. +func RegisterAlertJudge(judgeFnc func(error) bool) { + options.register(func(r *runner) error { + r.alertJudge = judgeFnc + return nil + }) +} + // Run is a non-blocking function that starts running go-sarah's process with pre-registered options. // Workers, schedulers and other required resources for bot interaction starts running on this function call. // This returns error when bot interaction cannot start; No error is returned when process starts successfully. @@ -178,11 +200,13 @@ func newRunner(ctx context.Context, config *Config) (*runner, error) { config: config, bots: []Bot{}, worker: nil, + watcher: nil, commandProps: make(map[BotType][]*CommandProps), scheduledTaskProps: make(map[BotType][]*ScheduledTaskProps), scheduledTasks: make(map[BotType][]ScheduledTask), alerters: &alerters{}, scheduler: runScheduler(ctx, loc), + alertJudge: nil, } err = options.apply(r) @@ -221,6 +245,7 @@ type runner struct { scheduledTasks map[BotType][]ScheduledTask alerters *alerters scheduler scheduler + alertJudge func(error) bool } func (r *runner) botCommandProps(botType BotType) []*CommandProps { @@ -267,7 +292,7 @@ func (r *runner) run(ctx context.Context) { // This returns when bot stops. func (r *runner) runBot(runnerCtx context.Context, bot Bot) { log.Infof("Starting %s", bot.BotType()) - botCtx, errNotifier := superviseBot(runnerCtx, bot.BotType(), r.alerters) + botCtx, errNotifier := r.superviseBot(runnerCtx, bot.BotType()) // Setup config directory for this particular Bot. var configDir string @@ -366,6 +391,54 @@ func (r *runner) subscribeConfigDir(botCtx context.Context, bot Bot, configDir s } } +func (r *runner) superviseBot(runnerCtx context.Context, botType BotType) (context.Context, func(error)) { + botCtx, cancel := context.WithCancel(runnerCtx) + + sendAlert := func(err error) { + e := r.alerters.alertAll(runnerCtx, botType, err) + if e != nil { + log.Errorf("Failed to send alert for %s: %+v", botType, e) + } + } + + // A function that receives an escalated error from Bot. + // If critical error is sent, this cancels Bot context to finish its lifecycle. + // Bot itself MUST NOT kill itself, but the Runner does. Beware that Runner takes care of all related components' lifecycle. + handleError := func(err error) { + switch err.(type) { + case *BotNonContinuableError: + log.Errorf("Stop unrecoverable bot. BotType: %s. Error: %+v", botType, err) + cancel() + + go sendAlert(err) + + log.Infof("Stop supervising bot critical error due to context cancellation: %s.", botType) + + default: + if r.alertJudge != nil && r.alertJudge(err) { + go sendAlert(err) + } + + } + } + + // A function to be exposed to Bot/Adapter developers. + // When Bot/Adapter faces a critical state, it can call this function to let Runner judge the severity and stop Bot if necessary. + errNotifier := func(err error) { + select { + case <-botCtx.Done(): + // Bot context is already canceled by preceding error notification. Do nothing. + return + + default: + handleError(err) + + } + } + + return botCtx, errNotifier +} + func registerScheduledTask(botCtx context.Context, bot Bot, task ScheduledTask, taskScheduler scheduler) { err := taskScheduler.update(bot.BotType(), task, func() { executeScheduledTask(botCtx, bot, task) @@ -528,47 +601,6 @@ func executeScheduledTask(ctx context.Context, bot Bot, task ScheduledTask) { } } -func superviseBot(runnerCtx context.Context, botType BotType, alerters *alerters) (context.Context, func(error)) { - botCtx, cancel := context.WithCancel(runnerCtx) - - // A function that receives an escalated error from Bot. - // If critical error is sent, this cancels Bot context to finish its lifecycle. - // Bot itself MUST NOT kill itself, but the Runner does. Beware that Runner takes care of all related components' lifecycle. - handleError := func(err error) { - switch err.(type) { - case *BotNonContinuableError: - log.Errorf("Stop unrecoverable bot. BotType: %s. Error: %+v", botType, err) - cancel() - - go func() { - e := alerters.alertAll(runnerCtx, botType, err) - if e != nil { - log.Errorf("Failed to send alert for %s: %+v", botType, e) - } - }() - - log.Infof("Stop supervising bot critical error due to context cancellation: %s.", botType) - - } - } - - // A function to be exposed to Bot/Adapter developers. - // When Bot/Adapter faces a critical state, it can call this function to let Runner judge the severity and stop Bot if necessary. - errNotifier := func(err error) { - select { - case <-botCtx.Done(): - // Bot context is already canceled by preceding error notification. Do nothing. - return - - default: - handleError(err) - - } - } - - return botCtx, errNotifier -} - func setupInputReceiver(botCtx context.Context, bot Bot, worker workers.Worker) func(Input) error { continuousEnqueueErrCnt := 0 return func(input Input) error { diff --git a/runner_test.go b/runner_test.go index 1a49d97..4c8aa8d 100644 --- a/runner_test.go +++ b/runner_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "github.com/oklahomer/go-sarah/log" + "golang.org/x/xerrors" "io/ioutil" stdLogger "log" "os" @@ -288,6 +289,31 @@ func TestRegisterWorker(t *testing.T) { }) } +func TestRegisterAlertJudge(t *testing.T) { + SetupAndRun(func() { + judge := func(e error) bool { + return true + } + RegisterAlertJudge(judge) + r := &runner{} + + for _, v := range options.stashed { + err := v(r) + if err != nil { + t.Fatalf("Unexpected error is returned: %s.", err.Error()) + } + } + + if r.alertJudge == nil { + t.Fatal("alertJudge is not set.") + } + + if reflect.ValueOf(r.alertJudge).Pointer() != reflect.ValueOf(judge).Pointer() { + t.Error("Passed function is not set.") + } + }) +} + func TestRun(t *testing.T) { SetupAndRun(func() { config := &Config{ @@ -863,6 +889,87 @@ func Test_runner_subscribeConfigDir_WithCallback(t *testing.T) { } } +func Test_runner_superviseBot(t *testing.T) { + SetupAndRun(func() { + rootCxt := context.Background() + errs := []error{ + xerrors.New("first error"), + xerrors.New("second error"), + NewBotNonContinuableError("third error should stop Bot"), + } + alertedErr := make(chan error, len(errs)) + judgeCnt := 0 + r := runner{ + alerters: &alerters{ + &DummyAlerter{ + AlertFunc: func(_ context.Context, _ BotType, err error) error { + panic("Panic should not affect other alerters' behavior.") + }, + }, + &DummyAlerter{ + AlertFunc: func(_ context.Context, _ BotType, err error) error { + alertedErr <- err + return nil + }, + }, + }, + alertJudge: func(err error) bool { + judgeCnt++ + return judgeCnt%2 == 0 + }, + } + botCtx, errSupervisor := r.superviseBot(rootCxt, "DummyBotType") + + select { + case <-botCtx.Done(): + t.Error("Bot context should not be canceled at this point.") + + default: + // O.K. + + } + + // Raise all errors + for _, v := range errs { + errSupervisor(v) + } + + // Bot should be canceled + select { + case <-botCtx.Done(): + // O.K. + + case <-time.NewTimer(10 * time.Second).C: + t.Error("Bot context should be canceled at this point.") + + } + if e := botCtx.Err(); e != context.Canceled { + t.Errorf("botCtx.Err() must return context.Canceled, but was %#v", e) + } + + // Alerters should be called for BotNonContinuableError and the one causes true return value from alertJudge + time.Sleep(10 * time.Millisecond) + if len(alertedErr) != 2 { + t.Errorf("Alerters are not called as many times as expected: %d", len(alertedErr)) + } + + // See if a succeeding call block + nonBlocking := make(chan bool) + go func() { + errSupervisor(NewBotNonContinuableError("call after context cancellation should not block")) + nonBlocking <- true + }() + select { + case <-nonBlocking: + // O.K. + + case <-time.NewTimer(10 * time.Second).C: + t.Error("Call after context cancellation blocks.") + + } + }) +} + func Test_registerCommand(t *testing.T) { SetupAndRun(func() { command := &DummyCommand{} @@ -1232,64 +1339,6 @@ func Test_executeScheduledTask(t *testing.T) { }) } -func Test_superviseBot(t *testing.T) { - SetupAndRun(func() { - rootCxt := context.Background() - alerted := make(chan bool) - alerters := &alerters{ - &DummyAlerter{ - AlertFunc: func(_ context.Context, _ BotType, err error) error { - panic("Panic should not affect other alerters' behavior.") - }, - }, - &DummyAlerter{ - AlertFunc: func(_ context.Context, _ BotType, err error) error { - alerted <- true - return nil - }, - }, - } - botCtx, errSupervisor := superviseBot(rootCxt, "DummyBotType", alerters) - - select { - case <-botCtx.Done(): - t.Error("Bot context should not be canceled at this point.") - default: - // O.K. - } - - errSupervisor(NewBotNonContinuableError("should stop")) - - select { - case <-botCtx.Done(): - // O.K. - case <-time.NewTimer(10 * time.Second).C: - t.Error("Bot context should be canceled at this point.") - } - if e := botCtx.Err(); e != context.Canceled { - t.Errorf("botCtx.Err() must return context.Canceled, but was %#v", e) - } - select { - case <-alerted: - // O.K. - case <-time.NewTimer(10 * time.Second).C: - t.Error("Alert should be sent at this point.") - } - - nonBlocking := make(chan bool) - go func() { - errSupervisor(NewBotNonContinuableError("call after context cancellation should not block")) - nonBlocking <- true - }() - select { - case <-nonBlocking: - // O.K. - case <-time.NewTimer(10 * time.Second).C: - t.Error("Call after context cancellation blocks.") - } - }) -} - func Test_setupInputReceiver(t *testing.T) { SetupAndRun(func() { responded := make(chan bool, 1) From b4ae170edabe05ad38204ffa34c55134022d6569 Mon Sep 17 00:00:00 2001 From: Oklahomer Date: Sun, 2 Jun 2019 19:44:09 +0900 Subject: [PATCH 2/3] Enable custome supervisor to stop failing bot --- runner.go | 76 ++++++++++++++++------ runner_test.go | 170 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 218 insertions(+), 28 deletions(-) diff --git a/runner.go b/runner.go index 62f5fae..0fa05a0 100644 --- a/runner.go +++ b/runner.go @@ -143,24 +143,30 @@ func RegisterWorker(worker workers.Worker) { }) } -// RegisterAlertJudge registers given function, judgeFnc, that is called when a Bot raises an error. -// This function judges if the given error is worth being notified to administrators. -// If this function returns true, all registered alerters are called to notify such error state to the administrators. +// RegisterBotErrorSupervisor registers a given supervising function that is called when a Bot escalates an error. +// This function judges if the given error is worth being notified to administrators and if the Bot should stop. +// A developer may return *SupervisionDirective to tell such order. +// If the escalated error can simply be ignored, a nil value can be returned. // -// Bot/Adapter can raise an error via a function, func(error), that is passed to Run() as a third argument. -// When BotNonContinuableError is raised, go-sarah's core cancels Bot's context and thus failing Bot and related resources stop working. +// Bot/Adapter can escalate an error via a function, func(error), that is passed to Run() as a third argument. +// When BotNonContinuableError is escalated, go-sarah's core cancels failing Bot's context and thus the Bot and related resources stop working. // If one or more sarah.Alerters implementations are registered, such critical error is passed to the alerters and administrators will be notified. -// When other types of error are raised, the error is passed to the function registered via sarah.RegisterAlertJudge(). -// The function may return true when the error is worth being notified to administrators. +// When other types of error are escalated, the error is passed to the supervising function registered via sarah.RegisterBotErrorSupervisor(). +// The function may return *SupervisionDirective to tell how go-sarah's core should react. // // Bot/Adapter's implementation should be simple. It should not handle serious errors by itself. -// Instead, it should simply raise an error and let core takes care of this. -// So if the calls to alerters should have some kind of rate limit, judgeFnc should internally handle such rate limit. -// In this way, each Bot/Adapter's implementation can be kept simple. +// Instead, it should simply escalate an error every time when a noteworthy error occurs and let core judge how to react. +// For example, if the bot should stop when three reconnection trial fails in ten seconds, the scenario could be somewhat like below: +// 1. Bot escalates reconnection error, FooReconnectionFailureError, each time it fails to reconnect +// 2. Supervising function counts the error and ignores the first two occurrence +// 3. When the third error comes within ten seconds from the initial error escalation, return *SupervisionDirective with StopBot value of true +// +// Similarly, if there should be a rate limiter to limit the calls to alerters, the supervising function should take care of this instead of the failing Bot. +// Each Bot/Adapter's implementation can be kept simple in this way. // go-sarah's core should always supervise and control its belonging Bots. -func RegisterAlertJudge(judgeFnc func(error) bool) { +func RegisterBotErrorSupervisor(fnc func(BotType, error) *SupervisionDirective) { options.register(func(r *runner) error { - r.alertJudge = judgeFnc + r.superviseError = fnc return nil }) } @@ -206,7 +212,7 @@ func newRunner(ctx context.Context, config *Config) (*runner, error) { scheduledTasks: make(map[BotType][]ScheduledTask), alerters: &alerters{}, scheduler: runScheduler(ctx, loc), - alertJudge: nil, + superviseError: nil, } err = options.apply(r) @@ -245,7 +251,23 @@ type runner struct { scheduledTasks map[BotType][]ScheduledTask alerters *alerters scheduler scheduler - alertJudge func(error) bool + superviseError func(BotType, error) *SupervisionDirective +} + +// SupervisionDirective tells go-sarah's core how to react when a Bot escalates an error. +// A customized supervisor can be defined and registered via RegisterBotErrorSupervisor(). +type SupervisionDirective struct { + // StopBot tells the core to stop the failing Bot and related resources. + // When two or more Bots are registered and one or more Bots are to be still running after the failing Bot stops, + // internal workers and scheduler keep running. + // Directory watcher stops subscribing to corresponding Bot's configuration files' directory, + // but it still keeps running until the last Bot stops. + // + // When all Bots stop, then the core stops all resources. + StopBot bool + // AlertingErr is sent registered alerters and administrators will be notified. + // Set nil when such alert notification is not required. + AlertingErr error } func (r *runner) botCommandProps(botType BotType) []*CommandProps { @@ -401,6 +423,11 @@ func (r *runner) superviseBot(runnerCtx context.Context, botType BotType) (conte } } + stopBot := func() { + cancel() + log.Infof("Stop supervising bot's critical error due to its context cancellation: %s.", botType) + } + // A function that receives an escalated error from Bot. // If critical error is sent, this cancels Bot context to finish its lifecycle. // Bot itself MUST NOT kill itself, but the Runner does. Beware that Runner takes care of all related components' lifecycle. @@ -408,15 +435,26 @@ func (r *runner) superviseBot(runnerCtx context.Context, botType BotType) (conte switch err.(type) { case *BotNonContinuableError: log.Errorf("Stop unrecoverable bot. BotType: %s. Error: %+v", botType, err) - cancel() - go sendAlert(err) + stopBot() - log.Infof("Stop supervising bot critical error due to context cancellation: %s.", botType) + go sendAlert(err) default: - if r.alertJudge != nil && r.alertJudge(err) { - go sendAlert(err) + if r.superviseError != nil { + directive := r.superviseError(botType, err) + if directive == nil { + return + } + + if directive.StopBot { + log.Errorf("Stop bot due to given directive. BotType: %s. Reason: %+v", botType, err) + stopBot() + } + + if directive.AlertingErr != nil { + go sendAlert(directive.AlertingErr) + } } } diff --git a/runner_test.go b/runner_test.go index 4c8aa8d..9262da8 100644 --- a/runner_test.go +++ b/runner_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "reflect" "regexp" + "strconv" "sync" "testing" "time" @@ -289,12 +290,12 @@ func TestRegisterWorker(t *testing.T) { }) } -func TestRegisterAlertJudge(t *testing.T) { +func TestRegisterBotErrorSupervisor(t *testing.T) { SetupAndRun(func() { - judge := func(e error) bool { - return true + supervisor := func(_ BotType, _ error) *SupervisionDirective { + return nil } - RegisterAlertJudge(judge) + RegisterBotErrorSupervisor(supervisor) r := &runner{} for _, v := range options.stashed { @@ -304,11 +305,11 @@ func TestRegisterAlertJudge(t *testing.T) { } } - if r.alertJudge == nil { - t.Fatal("alertJudge is not set.") + if r.superviseError == nil { + t.Fatal("superviseError is not set.") } - if reflect.ValueOf(r.alertJudge).Pointer() != reflect.ValueOf(judge).Pointer() { + if reflect.ValueOf(r.superviseError).Pointer() != reflect.ValueOf(supervisor).Pointer() { t.Error("Passed function is not set.") } }) @@ -890,6 +891,150 @@ func Test_runner_subscribeConfigDir_WithCallback(t *testing.T) { } func Test_runner_superviseBot(t *testing.T) { + tests := []struct { + escalated error + directive *SupervisionDirective + shutdown bool + }{ + { + escalated: NewBotNonContinuableError("this should stop Bot"), + shutdown: true, + }, + { + escalated: xerrors.New("plain error"), + directive: nil, + shutdown: false, + }, + { + escalated: xerrors.New("plain error"), + directive: &SupervisionDirective{ + AlertingErr: xerrors.New("this is sent via alerter"), + StopBot: true, + }, + shutdown: true, + }, + { + escalated: xerrors.New("plain error"), + directive: &SupervisionDirective{ + AlertingErr: nil, + StopBot: true, + }, + shutdown: true, + }, + { + escalated: xerrors.New("plain error"), + directive: &SupervisionDirective{ + AlertingErr: xerrors.New("this is sent via alerter"), + StopBot: false, + }, + shutdown: false, + }, + { + escalated: xerrors.New("plain error"), + directive: &SupervisionDirective{ + AlertingErr: nil, + StopBot: false, + }, + shutdown: false, + }, + } + alerted := make(chan error, 1) + + for i, tt := range tests { + t.Run(strconv.Itoa(i+1), func(t *testing.T) { + r := &runner{ + alerters: &alerters{ + &DummyAlerter{ + AlertFunc: func(_ context.Context, _ BotType, err error) error { + panic("Panic should not affect other alerters' behavior.") + }, + }, + &DummyAlerter{ + AlertFunc: func(_ context.Context, _ BotType, err error) error { + alerted <- err + return nil + }, + }, + }, + superviseError: func(_ BotType, _ error) *SupervisionDirective { + return tt.directive + }, + } + rootCxt := context.Background() + botCtx, errSupervisor := r.superviseBot(rootCxt, "DummyBotType") + + // Make sure the Bot state is currently active + select { + case <-botCtx.Done(): + t.Error("Bot context should not be canceled at this point.") + + default: + // O.K. + + } + + // Escalate an error + errSupervisor(tt.escalated) + + if tt.shutdown { + // Bot should be canceled + select { + case <-botCtx.Done(): + // O.K. + + case <-time.NewTimer(1 * time.Second).C: + t.Error("Bot context should be canceled at this point.") + + } + if e := botCtx.Err(); e != context.Canceled { + t.Errorf("botCtx.Err() must return context.Canceled, but was %#v", e) + } + } + + if _, ok := tt.escalated.(*BotNonContinuableError); ok { + // When Bot escalate an non-continuable error, then alerter should be called. + select { + case e := <-alerted: + if e != tt.escalated { + t.Errorf("Unexpected error value is passed: %#v", e) + } + + case <-time.NewTimer(1 * time.Second).C: + t.Error("Alerter is not called.") + + } + } else if tt.directive != nil && tt.directive.AlertingErr != nil { + select { + case e := <-alerted: + if e != tt.directive.AlertingErr { + t.Errorf("Unexpected error value is passed: %#v", e) + } + + case <-time.NewTimer(1 * time.Second).C: + t.Error("Alerter is not called.") + + } + } + + // See if a succeeding call block + nonBlocking := make(chan bool) + go func() { + errSupervisor(xerrors.New("succeeding calls should never block")) + nonBlocking <- true + }() + select { + case <-nonBlocking: + // O.K. + + case <-time.NewTimer(10 * time.Second).C: + t.Error("Succeeding error escalation blocks.") + + } + }) + } +} + +func Test_runner_superviseBot1(t *testing.T) { SetupAndRun(func() { rootCxt := context.Background() errs := []error{ @@ -898,6 +1043,7 @@ func Test_runner_superviseBot(t *testing.T) { NewBotNonContinuableError("third error should stop Bot"), } alertedErr := make(chan error, len(errs)) + terminalErr := xerrors.New("this is the end") judgeCnt := 0 r := runner{ alerters: &alerters{ @@ -913,9 +1059,15 @@ func Test_runner_superviseBot(t *testing.T) { }, }, }, - alertJudge: func(err error) bool { + superviseError: func(_ BotType, _ error) *SupervisionDirective { judgeCnt++ - return judgeCnt%2 == 0 + if judgeCnt%2 == 0 { + return &SupervisionDirective{ + StopBot: true, + AlertingErr: terminalErr, + } + } + return nil }, } botCtx, errSupervisor := r.superviseBot(rootCxt, "DummyBotType") From d4233f4b14341df230c52a841b1aa8b9a6b1a3a5 Mon Sep 17 00:00:00 2001 From: Oklahomer Date: Sun, 2 Jun 2019 20:26:42 +0900 Subject: [PATCH 3/3] Remove old test --- runner_test.go | 88 -------------------------------------------------- 1 file changed, 88 deletions(-) diff --git a/runner_test.go b/runner_test.go index 9262da8..dcf1406 100644 --- a/runner_test.go +++ b/runner_test.go @@ -1034,94 +1034,6 @@ func Test_runner_superviseBot(t *testing.T) { } } -func Test_runner_superviseBot1(t *testing.T) { - SetupAndRun(func() { - rootCxt := context.Background() - errs := []error{ - xerrors.New("first error"), - xerrors.New("second error"), - NewBotNonContinuableError("third error should stop Bot"), - } - alertedErr := make(chan error, len(errs)) - terminalErr := xerrors.New("this is the end") - judgeCnt := 0 - r := runner{ - alerters: &alerters{ - &DummyAlerter{ - AlertFunc: func(_ context.Context, _ BotType, err error) error { - panic("Panic should not affect other alerters' behavior.") - }, - }, - &DummyAlerter{ - AlertFunc: func(_ context.Context, _ BotType, err error) error { - alertedErr <- err - return nil - }, - }, - }, - superviseError: func(_ BotType, _ error) *SupervisionDirective { - judgeCnt++ - if judgeCnt%2 == 0 { - return &SupervisionDirective{ - StopBot: true, - AlertingErr: terminalErr, - } - } - return nil - }, - } - botCtx, errSupervisor := r.superviseBot(rootCxt, "DummyBotType") - - select { - case <-botCtx.Done(): - t.Error("Bot context should not be canceled at this point.") - - default: - // O.K. - - } - - // Raise all errors - for _, v := range errs { - errSupervisor(v) - } - - // Bot should be canceled - select { - case <-botCtx.Done(): - // O.K. - - case <-time.NewTimer(10 * time.Second).C: - t.Error("Bot context should be canceled at this point.") - - } - if e := botCtx.Err(); e != context.Canceled { - t.Errorf("botCtx.Err() must return context.Canceled, but was %#v", e) - } - - // Alerters should be called for BotNonContinuableError and the one causes true return value from alertJudge - time.Sleep(10 * time.Millisecond) - if len(alertedErr) != 2 { - t.Errorf("Alerters are not called as many times as expected: %d", len(alertedErr)) - } - - // See if a succeeding call block - nonBlocking := make(chan bool) - go func() { - errSupervisor(NewBotNonContinuableError("call after context cancellation should not block")) - nonBlocking <- true - }() - select { - case <-nonBlocking: - // O.K. - - case <-time.NewTimer(10 * time.Second).C: - t.Error("Call after context cancellation blocks.") - - } - }) -} - func Test_registerCommand(t *testing.T) { SetupAndRun(func() { command := &DummyCommand{}