From 1e90e03aaa5ef2658f0f4dec93e3d39fcd4f6c94 Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Mon, 18 Dec 2023 23:06:55 +0200 Subject: [PATCH 1/8] timeoutSeconds config in .tbrc.yml for pulling imgs --- config/config.go | 12 ++++++++++++ engine/engine.go | 5 +++++ engine/service.go | 3 +++ 3 files changed, 20 insertions(+) diff --git a/config/config.go b/config/config.go index e2aac1b..cf217ad 100644 --- a/config/config.go +++ b/config/config.go @@ -45,6 +45,7 @@ type Config struct { Playlists map[string]playlist.Playlist `yaml:"playlists"` Overrides map[string]service.ServiceOverride `yaml:"overrides"` Registries []registry.Registry `yaml:"registries"` + TimeoutSeconds *int `yaml:timeoutSeconds` } // NOTE: This is deprecated and is only here for backwards compatibility. @@ -151,6 +152,11 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, return nil, errors.New(errkind.Invalid, "no registries defined", op) } + + if config.TimeoutSeconds != nil && *config.TimeoutSeconds < 5 || *config.TimeoutSeconds > 3600 { + return nil, errors.New(errkind.Invalid, fmt.Sprintf("Invalid timeoutSeconds field in .tbrc.yaml. Values must be between 5 and 3600 inclusive"), op) + } + // Validate and normalize all registries. tracker := progress.TrackerFromContext(ctx) for i, r := range config.Registries { @@ -284,6 +290,11 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, } } + timeoutSeconds := 3600 + if config.TimeoutSeconds != nil { + timeoutSeconds = *config.TimeoutSeconds + } + e, err := engine.New(engine.Options{ Workdir: tbRoot, Services: registryResult.Services, @@ -293,6 +304,7 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, BaseImages: registryResult.BaseImages, LoginStrategies: registryResult.LoginStrategies, DeviceList: deviceList, + TimeoutSeconds: timeoutSeconds, }) if err != nil { return nil, errors.Wrap(err, errors.Meta{Reason: "failed to initialize engine", Op: op}) diff --git a/engine/engine.go b/engine/engine.go index 6005235..f93bd61 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -28,6 +28,7 @@ type Engine struct { loginStrategies []string deviceList simulator.DeviceList concurrency int + timeoutSeconds int gitClient git.Git dockerClient *docker.Docker @@ -71,6 +72,9 @@ type Options struct { GitClient git.Git // DockerOptions is used to customize docker operations. DockerOptions docker.Options + // TimeoutSeconds is a limit to how long a docker image pull will last + // If no value is provided, it defaults to 3600 + TimeoutSeconds int } // New creates a new Engine instance. @@ -113,6 +117,7 @@ func New(opts Options) (*Engine, error) { baseImages: opts.BaseImages, loginStrategies: opts.LoginStrategies, deviceList: opts.DeviceList, + timeoutSeconds: opts.TimeoutSeconds, concurrency: opts.Concurrency, gitClient: opts.GitClient, dockerClient: dockerClient, diff --git a/engine/service.go b/engine/service.go index 69877f4..932a8d8 100644 --- a/engine/service.go +++ b/engine/service.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "time" "github.com/TouchBistro/goutils/errors" "github.com/TouchBistro/goutils/file" @@ -127,6 +128,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { Message: "Pulling docker base images", Count: len(e.baseImages), Concurrency: e.concurrency, + Timeout: time.Duration(e.timeoutSeconds) * time.Second, }, func(ctx context.Context, i int) error { img := e.baseImages[i] if err := e.dockerClient.PullImage(ctx, img); err != nil { @@ -154,6 +156,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { Message: "Pulling docker service images", Count: len(images), Concurrency: e.concurrency, + Timeout: time.Duration(e.timeoutSeconds) * time.Second, }, func(ctx context.Context, i int) error { img := images[i] if err := e.dockerClient.PullImage(ctx, img); err != nil { From 81bddf69cf05279f2dfa3aa8519c66b6f9f5e11c Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Mon, 18 Dec 2023 23:11:46 +0200 Subject: [PATCH 2/8] fix lint --- config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index cf217ad..00fc6d2 100644 --- a/config/config.go +++ b/config/config.go @@ -45,7 +45,7 @@ type Config struct { Playlists map[string]playlist.Playlist `yaml:"playlists"` Overrides map[string]service.ServiceOverride `yaml:"overrides"` Registries []registry.Registry `yaml:"registries"` - TimeoutSeconds *int `yaml:timeoutSeconds` + TimeoutSeconds *int `yaml:"timeoutSeconds"` } // NOTE: This is deprecated and is only here for backwards compatibility. @@ -154,7 +154,7 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, if config.TimeoutSeconds != nil && *config.TimeoutSeconds < 5 || *config.TimeoutSeconds > 3600 { - return nil, errors.New(errkind.Invalid, fmt.Sprintf("Invalid timeoutSeconds field in .tbrc.yaml. Values must be between 5 and 3600 inclusive"), op) + return nil, errors.New(errkind.Invalid, fmt.Sprintf("Invalid timeoutSeconds value '%d' in .tbrc.yaml. Values must be between 5 and 3600 inclusive", *config.TimeoutSeconds), op) } // Validate and normalize all registries. From e0aaa02813c0aff9a56c8c7ba6e0b0684d0a2b85 Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Tue, 19 Dec 2023 17:29:12 +0200 Subject: [PATCH 3/8] czat's feedback --- config/config.go | 16 ++++++++++------ engine/engine.go | 9 +++++---- engine/service.go | 10 +++++++--- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/config/config.go b/config/config.go index 00fc6d2..37aaff8 100644 --- a/config/config.go +++ b/config/config.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/TouchBistro/goutils/color" "github.com/TouchBistro/goutils/errors" @@ -45,7 +46,7 @@ type Config struct { Playlists map[string]playlist.Playlist `yaml:"playlists"` Overrides map[string]service.ServiceOverride `yaml:"overrides"` Registries []registry.Registry `yaml:"registries"` - TimeoutSeconds *int `yaml:"timeoutSeconds"` + TimeoutSeconds time.Duration `yaml:"timeoutSeconds"` } // NOTE: This is deprecated and is only here for backwards compatibility. @@ -153,8 +154,8 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, } - if config.TimeoutSeconds != nil && *config.TimeoutSeconds < 5 || *config.TimeoutSeconds > 3600 { - return nil, errors.New(errkind.Invalid, fmt.Sprintf("Invalid timeoutSeconds value '%d' in .tbrc.yaml. Values must be between 5 and 3600 inclusive", *config.TimeoutSeconds), op) + if config.TimeoutSeconds != 0 && (config.TimeoutSeconds < 5 || config.TimeoutSeconds > 3600) { + return nil, errors.New(errkind.Invalid, fmt.Sprintf("Invalid timeoutSeconds value '%d' in .tbrc.yaml. Values must be between 5 and 3600 inclusive", config.TimeoutSeconds), op) } // Validate and normalize all registries. @@ -290,9 +291,12 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, } } - timeoutSeconds := 3600 - if config.TimeoutSeconds != nil { - timeoutSeconds = *config.TimeoutSeconds + var timeoutSeconds time.Duration + if config.TimeoutSeconds != 0 { + timeoutSeconds = config.TimeoutSeconds + } else { + // default to 60 min timeout when not provided in .tbrc.yml + timeoutSeconds = 3600 } e, err := engine.New(engine.Options{ diff --git a/engine/engine.go b/engine/engine.go index f93bd61..84b7d1d 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -3,6 +3,7 @@ package engine import ( "os" "path/filepath" + "time" "github.com/TouchBistro/goutils/errors" "github.com/TouchBistro/tb/errkind" @@ -28,7 +29,7 @@ type Engine struct { loginStrategies []string deviceList simulator.DeviceList concurrency int - timeoutSeconds int + timeout time.Duration gitClient git.Git dockerClient *docker.Docker @@ -72,9 +73,9 @@ type Options struct { GitClient git.Git // DockerOptions is used to customize docker operations. DockerOptions docker.Options - // TimeoutSeconds is a limit to how long a docker image pull will last + // Timeout is a limit to how long an operation will last // If no value is provided, it defaults to 3600 - TimeoutSeconds int + Timeout time.Duration } // New creates a new Engine instance. @@ -117,7 +118,7 @@ func New(opts Options) (*Engine, error) { baseImages: opts.BaseImages, loginStrategies: opts.LoginStrategies, deviceList: opts.DeviceList, - timeoutSeconds: opts.TimeoutSeconds, + timeout: opts.Timeout, concurrency: opts.Concurrency, gitClient: opts.GitClient, dockerClient: dockerClient, diff --git a/engine/service.go b/engine/service.go index 932a8d8..a2ba470 100644 --- a/engine/service.go +++ b/engine/service.go @@ -8,7 +8,6 @@ import ( "io" "os" "path/filepath" - "time" "github.com/TouchBistro/goutils/errors" "github.com/TouchBistro/goutils/file" @@ -92,6 +91,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { Message: "Logging into services", Count: len(loginStrategies), Concurrency: e.concurrency, + Timeout: e.timeout, // Bail if one fails since there's no point on waiting on the others // since we can't proceed anyway. CancelOnError: true, @@ -114,6 +114,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { // Cleanup previous docker state err = progress.Run(ctx, progress.RunOptions{ Message: "Cleaning up previous docker state", + Timeout: e.timeout, }, func(ctx context.Context) error { return e.stopServices(ctx, op, services) }) @@ -128,7 +129,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { Message: "Pulling docker base images", Count: len(e.baseImages), Concurrency: e.concurrency, - Timeout: time.Duration(e.timeoutSeconds) * time.Second, + Timeout: e.timeout, }, func(ctx context.Context, i int) error { img := e.baseImages[i] if err := e.dockerClient.PullImage(ctx, img); err != nil { @@ -156,7 +157,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { Message: "Pulling docker service images", Count: len(images), Concurrency: e.concurrency, - Timeout: time.Duration(e.timeoutSeconds) * time.Second, + Timeout: e.timeout, }, func(ctx context.Context, i int) error { img := images[i] if err := e.dockerClient.PullImage(ctx, img); err != nil { @@ -182,6 +183,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { if len(buildServices) > 0 { err := progress.Run(ctx, progress.RunOptions{ Message: "Building docker images for services", + Timeout: e.timeout, }, func(ctx context.Context) error { return e.dockerClient.BuildServices(ctx, buildServices) }) @@ -199,6 +201,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { err := progress.Run(ctx, progress.RunOptions{ Message: "Performing pre-run step for services (this may take a long time)", Count: len(services), + Timeout: e.timeout, }, func(ctx context.Context) error { for _, s := range services { if s.PreRun == "" { @@ -228,6 +231,7 @@ func (e *Engine) Up(ctx context.Context, opts UpOptions) error { // Start services err = progress.Run(ctx, progress.RunOptions{ Message: "Starting services in the background", + Timeout: e.timeout, }, func(ctx context.Context) error { return e.dockerClient.UpServices(ctx, getServiceNames(services)) }) From 77f4efadb723afa1c2adecae446dcca30075a38b Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Tue, 19 Dec 2023 17:31:33 +0200 Subject: [PATCH 4/8] fix lint --- config/config.go | 8 ++++---- engine/app.go | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 37aaff8..6863990 100644 --- a/config/config.go +++ b/config/config.go @@ -291,12 +291,12 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, } } - var timeoutSeconds time.Duration + var timeout time.Duration if config.TimeoutSeconds != 0 { - timeoutSeconds = config.TimeoutSeconds + timeout = config.TimeoutSeconds } else { // default to 60 min timeout when not provided in .tbrc.yml - timeoutSeconds = 3600 + timeout = 3600 } e, err := engine.New(engine.Options{ @@ -308,7 +308,7 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, BaseImages: registryResult.BaseImages, LoginStrategies: registryResult.LoginStrategies, DeviceList: deviceList, - TimeoutSeconds: timeoutSeconds, + Timeout: timeout, }) if err != nil { return nil, errors.Wrap(err, errors.Meta{Reason: "failed to initialize engine", Op: op}) diff --git a/engine/app.go b/engine/app.go index 9566a3f..a0a78b1 100644 --- a/engine/app.go +++ b/engine/app.go @@ -101,6 +101,7 @@ func (e *Engine) AppiOSRun(ctx context.Context, appName string, opts AppiOSRunOp // Download the app appPath, err := progress.RunT(ctx, progress.RunOptions{ Message: fmt.Sprintf("Downloading iOS app %s", a.FullName()), + Timeout: e.timeout, }, func(ctx context.Context) (string, error) { return e.downloadApp(ctx, a, app.TypeiOS, op) }) From 384ac9d0857985efc98a50a646b0e04375c0cf31 Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Tue, 19 Dec 2023 17:35:25 +0200 Subject: [PATCH 5/8] add timeouts --- config/config.go | 17 +++++++++-------- engine/app.go | 1 + engine/service.go | 3 +++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/config/config.go b/config/config.go index 6863990..f1e497d 100644 --- a/config/config.go +++ b/config/config.go @@ -158,6 +158,14 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, return nil, errors.New(errkind.Invalid, fmt.Sprintf("Invalid timeoutSeconds value '%d' in .tbrc.yaml. Values must be between 5 and 3600 inclusive", config.TimeoutSeconds), op) } + var timeout time.Duration + if config.TimeoutSeconds != 0 { + timeout = config.TimeoutSeconds + } else { + // default to 60 min timeout when not provided in .tbrc.yml + timeout = 3600 + } + // Validate and normalize all registries. tracker := progress.TrackerFromContext(ctx) for i, r := range config.Registries { @@ -191,6 +199,7 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, err = progress.RunParallel(ctx, progress.RunParallelOptions{ Message: "Cloning/updating registries", Count: len(config.Registries), + Timeout: timeout, }, func(ctx context.Context, i int) error { r := config.Registries[i] if r.LocalPath != "" { @@ -291,14 +300,6 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, } } - var timeout time.Duration - if config.TimeoutSeconds != 0 { - timeout = config.TimeoutSeconds - } else { - // default to 60 min timeout when not provided in .tbrc.yml - timeout = 3600 - } - e, err := engine.New(engine.Options{ Workdir: tbRoot, Services: registryResult.Services, diff --git a/engine/app.go b/engine/app.go index a0a78b1..0420828 100644 --- a/engine/app.go +++ b/engine/app.go @@ -250,6 +250,7 @@ func (e *Engine) AppDesktopRun(ctx context.Context, appName string, opts AppDesk // Download the app appPath, err := progress.RunT(ctx, progress.RunOptions{ Message: fmt.Sprintf("Downloading Desktop app %s", a.FullName()), + Timeout: e.timeout, }, func(ctx context.Context) (string, error) { return e.downloadApp(ctx, a, app.TypeDesktop, op) }) diff --git a/engine/service.go b/engine/service.go index a2ba470..f197849 100644 --- a/engine/service.go +++ b/engine/service.go @@ -257,6 +257,7 @@ func (e *Engine) Down(ctx context.Context, opts DownOptions) error { } err = progress.Run(ctx, progress.RunOptions{ Message: "Stopping services", + Timeout: e.timeout, }, func(ctx context.Context) error { return e.stopServices(ctx, op, services) }) @@ -418,6 +419,7 @@ func (e *Engine) Nuke(ctx context.Context, opts NukeOptions) error { const op = errors.Op("engine.Engine.Nuke") return progress.Run(ctx, progress.RunOptions{ Message: "Cleaning up tb data", + Timeout: e.timeout, }, func(ctx context.Context) error { return e.nuke(ctx, opts, op) }) @@ -668,6 +670,7 @@ func (e *Engine) prepareGitRepos(ctx context.Context, op errors.Op, skipPull boo Message: "Cloning/pulling service git repos", Count: len(actions), Concurrency: e.concurrency, + Timeout: e.timeout, }, func(ctx context.Context, i int) error { a := actions[i] if a.clone { From eb59a55a473a71707e14fa25938004f1229eff33 Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Tue, 19 Dec 2023 18:05:45 +0200 Subject: [PATCH 6/8] docs + fix --- README.md | 4 ++++ config/config.go | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b153f70..3894f18 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,10 @@ Run `tb --help` to see the commands available. Run `tb --help` to get help `tb` can be configured through the `.tbrc.yml` file located in your home directory. `tb` will automatically create a basic `.tbrc.yml` for you if one doesn't exist. +### Timeout + +You can specify a timeout value in `.tbrc.yml`. This value will be used to kill any operation that exceeds the given time. All you need to do is set `timeoutSeconds: 1000` in your `.tbrc.yml`. Allowed values are 5 to 3600 inclusive. If `timeoutSeconds` is not specified or set to 0, `tb` will default to 3600 seconds (i.e 60 minutes). + ### Toggling experimental mode To enable experimental mode set the `experimental` field to `true`. Experimental mode will give you access to any new features that are still in the process of being tested. Please be aware that you may encounter bugs with these features as they have not yet been deemed ready for general use. diff --git a/config/config.go b/config/config.go index f1e497d..4dc582c 100644 --- a/config/config.go +++ b/config/config.go @@ -46,7 +46,7 @@ type Config struct { Playlists map[string]playlist.Playlist `yaml:"playlists"` Overrides map[string]service.ServiceOverride `yaml:"overrides"` Registries []registry.Registry `yaml:"registries"` - TimeoutSeconds time.Duration `yaml:"timeoutSeconds"` + TimeoutSeconds int `yaml:"timeoutSeconds"` } // NOTE: This is deprecated and is only here for backwards compatibility. @@ -160,10 +160,10 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, var timeout time.Duration if config.TimeoutSeconds != 0 { - timeout = config.TimeoutSeconds + timeout = time.Duration(config.TimeoutSeconds) * time.Second } else { // default to 60 min timeout when not provided in .tbrc.yml - timeout = 3600 + timeout = time.Duration(3600) * time.Second } // Validate and normalize all registries. From 305cf7a7a556f782fdb9e207d9e99c72a575f5e4 Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Tue, 19 Dec 2023 20:26:58 +0200 Subject: [PATCH 7/8] czat's feedback --- config/config.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 4dc582c..125048f 100644 --- a/config/config.go +++ b/config/config.go @@ -158,12 +158,10 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, return nil, errors.New(errkind.Invalid, fmt.Sprintf("Invalid timeoutSeconds value '%d' in .tbrc.yaml. Values must be between 5 and 3600 inclusive", config.TimeoutSeconds), op) } - var timeout time.Duration + // default to 60 min timeout when not provided in .tbrc.yml + timeout := 3600 * time.Second if config.TimeoutSeconds != 0 { timeout = time.Duration(config.TimeoutSeconds) * time.Second - } else { - // default to 60 min timeout when not provided in .tbrc.yml - timeout = time.Duration(3600) * time.Second } // Validate and normalize all registries. From fdc04130de22fcefa3dd009f01148f7a67f5216d Mon Sep 17 00:00:00 2001 From: Ismail Arafa Date: Tue, 19 Dec 2023 22:08:27 +0200 Subject: [PATCH 8/8] mins --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 125048f..20a4d19 100644 --- a/config/config.go +++ b/config/config.go @@ -159,7 +159,7 @@ func Init(ctx context.Context, config Config, opts InitOptions) (*engine.Engine, } // default to 60 min timeout when not provided in .tbrc.yml - timeout := 3600 * time.Second + timeout := 60 * time.Minute if config.TimeoutSeconds != 0 { timeout = time.Duration(config.TimeoutSeconds) * time.Second }