diff --git a/rules/builtins.build_defs b/rules/builtins.build_defs index 9d33ec83ae..75e4cf7ff6 100644 --- a/rules/builtins.build_defs +++ b/rules/builtins.build_defs @@ -12,7 +12,7 @@ def build_rule(name:str, cmd:str|dict='', test_cmd:str|dict='', srcs:list|dict=N test_timeout:int|str=0, pre_build:function=None, post_build:function=None, requires:list=None, provides:dict=None, licences:list=CONFIG.DEFAULT_LICENCES, test_outputs:list=None, system_srcs:list=None, stamp:bool=False, tag:str='', optional_outs:list=None, progress:bool=False, size:str=None, _urls:list=None, - internal_deps:list=None, pass_env:list=None): + internal_deps:list=None, pass_env:list=None, local:bool=False): pass diff --git a/rules/misc_rules.build_defs b/rules/misc_rules.build_defs index 0ae77f572c..de84885acc 100644 --- a/rules/misc_rules.build_defs +++ b/rules/misc_rules.build_defs @@ -6,7 +6,7 @@ def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs hashes:list=None, timeout:int=0, binary:bool=False, sandbox:bool=None, needs_transitive_deps:bool=False, output_is_complete:bool=True, test_only:bool&testonly=False, secrets:list|dict=None, requires:list=None, provides:dict=None, pre_build:function=None, - post_build:function=None, tools:list|dict=None, pass_env:list=None): + post_build:function=None, tools:list|dict=None, pass_env:list=None, local:bool=False): """A general build rule which allows the user to specify a command. Args: @@ -91,6 +91,8 @@ def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs to dynamically create new rules based on the output of another. pass_env: List of environment variables to be passed from outside. Any changes to them will be recorded in this target's hash and will hence force it to rebuild. + local: Forces the rule to be built locally; when remote execution is enabled it will not + be sent remotely but executed on the local machine. """ if out and outs: raise TypeError('Can\'t specify both "out" and "outs".') @@ -117,6 +119,7 @@ def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs provides = provides, test_only = test_only, pass_env = pass_env, + local = local, ) @@ -124,7 +127,7 @@ def gentest(name:str, test_cmd:str|dict, labels:list&features&tags=None, cmd:str deps:list=None, tools:list|dict=None, data:list=None, visibility:list=None, timeout:int=0, needs_transitive_deps:bool=False, flaky:bool|int=0, secrets:list|dict=None, no_test_output:bool=False, test_outputs:list=None, output_is_complete:bool=True, requires:list=None, - sandbox:bool=None, size:str=None): + sandbox:bool=None, size:str=None, local:bool=False): """A rule which creates a test with an arbitrary command. The command must return zero on success and nonzero on failure. Test results are written @@ -163,6 +166,8 @@ def gentest(name:str, test_cmd:str|dict, labels:list&features&tags=None, cmd:str including networking, process, IPC, etc. Only has an effect on Linux. If this is on by default then tests can opt out by setting this to False. size (str): Test size (enormous, large, medium or small). + local: Forces the rule to be built locally; when remote execution is enabled it will not + be sent remotely but executed on the local machine. """ return build_rule( name = name, @@ -187,6 +192,7 @@ def gentest(name:str, test_cmd:str|dict, labels:list&features&tags=None, cmd:str no_test_output = no_test_output, test_outputs = test_outputs, flaky = flaky, + local = local, ) diff --git a/src/build/incrementality.go b/src/build/incrementality.go index 141508ac20..452293cfb5 100644 --- a/src/build/incrementality.go +++ b/src/build/incrementality.go @@ -244,6 +244,7 @@ func ruleHash(state *core.BuildState, target *core.BuildTarget, runtime bool) [] hashOptionalBool(h, target.IsFilegroup) hashOptionalBool(h, target.IsHashFilegroup) hashOptionalBool(h, target.IsRemoteFile) + hashOptionalBool(h, target.Local) for _, require := range target.Requires { h.Write([]byte(require)) } diff --git a/src/build/incrementality_test.go b/src/build/incrementality_test.go index 44fb08e6db..3ad84b3f65 100644 --- a/src/build/incrementality_test.go +++ b/src/build/incrementality_test.go @@ -30,6 +30,7 @@ var KnownFields = map[string]bool{ "TestCommand": true, "TestCommands": true, "NeedsTransitiveDependencies": true, + "Local": true, "OptionalOutputs": true, "OutputIsComplete": true, "Requires": true, @@ -52,7 +53,6 @@ var KnownFields = map[string]bool{ // These only contribute to the runtime hash, not at build time. "Data": true, - "Containerise": true, "TestSandbox": true, "ContainerSettings": true, diff --git a/src/core/build_target.go b/src/core/build_target.go index 06421581a3..bb6b9affdc 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -109,6 +109,8 @@ type BuildTarget struct { // If true, the rule is given an env var at build time that contains the hash of its // transitive dependencies, which can be used to identify the output in a predictable way. Stamp bool + // If true, the target must be run locally (i.e. is not compatible with remote execution). + Local bool // Marks the target as a filegroup. IsFilegroup bool `print:"false"` // Marks the target as a hash_filegroup. diff --git a/src/core/state.go b/src/core/state.go index e731e7539e..031af1a968 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -234,18 +234,27 @@ func (state *BuildState) AddPendingTest(label BuildLabel) { // TaskQueues returns a set of channels to listen on for tasks of various types. // This should only be called once per state (otherwise you will not get a full set of tasks). -func (state *BuildState) TaskQueues() (parses <-chan LabelPair, builds, tests <-chan BuildLabel) { +func (state *BuildState) TaskQueues() (parses <-chan LabelPair, builds, tests, remoteBuilds, remoteTests <-chan BuildLabel) { p := make(chan LabelPair, 100) b := make(chan BuildLabel, 100) t := make(chan BuildLabel, 100) - go state.feedQueues(p, b, t) - return p, b, t + rb := make(chan BuildLabel, 100) + rt := make(chan BuildLabel, 100) + go state.feedQueues(p, b, t, rb, rt) + return p, b, t, rb, rt } // feedQueues feeds the build queues created in TaskQueues. // We retain the internal priority queue since it is unbounded size which is pretty important // for us not to deadlock. -func (state *BuildState) feedQueues(parses chan<- LabelPair, builds, tests chan<- BuildLabel) { +func (state *BuildState) feedQueues(parses chan<- LabelPair, builds, tests, remoteBuilds, remoteTests chan<- BuildLabel) { + anyRemote := state.Config.Remote.NumExecutors > 0 + queue := func(label BuildLabel, local, remote chan<- BuildLabel) chan<- BuildLabel { + if anyRemote && !state.Graph.Target(label).Local { + return remote + } + return local + } for { t, _ := state.pendingTasks.Get(1) task := t[0].(pendingTask) @@ -254,6 +263,8 @@ func (state *BuildState) feedQueues(parses chan<- LabelPair, builds, tests chan< close(parses) close(builds) close(tests) + close(remoteBuilds) + close(remoteTests) if state.results != nil { close(state.results) } @@ -265,10 +276,10 @@ func (state *BuildState) feedQueues(parses chan<- LabelPair, builds, tests chan< parses <- LabelPair{Label: task.Label, Dependent: task.Dependent, ForSubinclude: task.Type == SubincludeParse} case Build, SubincludeBuild: atomic.AddInt64(&state.progress.numRunning, 1) - builds <- task.Label + queue(task.Label, builds, remoteBuilds) <- task.Label case Test: atomic.AddInt64(&state.progress.numRunning, 1) - tests <- task.Label + queue(task.Label, tests, remoteTests) <- task.Label } } } diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index d505cc8aa6..66af535244 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -50,6 +50,7 @@ func createTarget(s *scope, args []pyObject) *core.BuildTarget { target.TestOnly = test || isTruthy(15) target.ShowProgress = isTruthy(36) target.IsRemoteFile = isTruthy(38) + target.Local = isTruthy(41) var size *core.Size if args[37] != None { diff --git a/src/parse/parse_step_test.go b/src/parse/parse_step_test.go index fa95a50eba..5a7df396bc 100644 --- a/src/parse/parse_step_test.go +++ b/src/parse/parse_step_test.go @@ -163,7 +163,7 @@ func assertPendingBuilds(t *testing.T, state *core.BuildState, targets ...string } func getAllPending(state *core.BuildState) ([]string, []string) { - parses, builds, tests := state.TaskQueues() + parses, builds, tests, _, _ := state.TaskQueues() state.Stop() var pendingParses, pendingBuilds []string for parses != nil || builds != nil || tests != nil { diff --git a/src/plz/plz.go b/src/plz/plz.go index abec0acab4..47de8efcc3 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -38,7 +38,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * // Start looking for the initial targets to kick the build off go findOriginalTasks(state, preTargets, targets, arch) - parses, builds, tests := state.TaskQueues() + parses, builds, tests, remoteBuilds, remoteTests := state.TaskQueues() // Start up all the build workers var wg sync.WaitGroup @@ -51,7 +51,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * } for i := 0; i < config.Remote.NumExecutors; i++ { go func(tid int) { - doTasks(tid, state, nil, builds, tests, arch, true) + doTasks(tid, state, nil, remoteBuilds, remoteTests, arch, true) wg.Done() }(config.Please.NumThreads + i) }