Failed to spawn a process -- buck2 tries to execute directory

[avdv]

In our project I declared an action:

    nix_build = cmd_args([
        "nix",
        "build",
        "--no-update-lock-file",
        "--no-use-registries",
        cmd_args("--out-link", out_link.as_output()),
        cmd_args(drv, format = "{}^*"),
    ], hidden = deps)
    ctx.actions.run(nix_build, category = "nix_build", identifier = package, local_only = True)

When running that action I got a mysterious error:

Action failed: toolchains//:ghc_package_db (nix_build word8)
Local command returned non-zero exit code <no exit code>
Reproduce locally: `env -- 'BUCK_SCRATCH_PATH=buck-out/v2/tmp/toolchains/904931f735703749/__ghc_package_db__/nix_build/w ...<omitted>... ge_db__/__action__1__/word8/out.link /nix/store/wxkgbc4hz6y85zvg2xs81mjg8ha6frff-word8-0.1.3.drv'^*' (run `buck2 log what-failed` to get the full command)`
stdout:
stderr:
Spawning executable `nix` failed: Failed to spawn a process

When trying to reproduce it locally, using the command from the log, it works:

$ env -- 'TMPDIR=/home/claudio/source/repo/buck-out/v2/tmp/toolchains/904931f735703749/_ghc_package_db__/nix_build/word8' 'BUCK_SCRATCH_PATH=buck-out/v2/tmp/toolchains/904931f735703749/_ghc_package_db__/nix_build/word8' BUCK2_DAEMON_UUID=e24cf324-c871-4855-99a9-a724821100c3' 'BUCK_BUILD_ID=b414deac-6ea2-4aa1-ac30-d3413d518fa1' nix build --no-update-lock-file --no-use-registries --out-link buck-out/v2/gen/toolchains/904931f735703749/__ghc_package_db__/__action__1__/word8/out.link /nix/store/wxkgbc4hz6y85zvg2xs81mjg8ha6frff-word8-0.1.3.drv'^*'
$ echo $status
0

Running buck2 under strace, revealed:

execve("/home/claudio/source/repo/nix", ["/home/claudio/source/repo/nix", "build", "--no-update-lock-file", "--no-use-registries", "--out-link", "buck-out/v2/gen/toolchains/904931f735703749/__ghc_package_db__/__action__1__/base-compat/out.link", "/nix/store/y3zshdaaxfzq64ikjd1w5h4hh65rxi9y-base-compat-0.12.3.drv^*"], 0x7f33e9c4d400 /* 173 vars */) = -1 EACCES (Permission denied)

So, there is indeed a directory called nix in the project root directory which it is trying to execute.

I think this is because the code introduced in 37f3d25 does only check if the file exists, not that it is an executable non-directory:

buck2/app/buck2_forkserver/src/run.rs

Lines 348 to 360 in 85f33c6

	pub fn maybe_absolutize_exe<'a>(
	exe: &'a (impl AsRef<Path> + ?Sized),
	spawned_process_cwd: &'_ AbsPath,
	) -> anyhow::Result<Cow<'a, Path>> {
	let exe = exe.as_ref();
	let abs = spawned_process_cwd.join(exe);
	if fs_util::try_exists(&abs).context("Error absolute-izing executable")? {
	return Ok(abs.into_path_buf().into());
	}
	Ok(exe.into())
	}

Also, it would have been nice to have a better error message that mentions the real cause of the error and the file it tried to execute.

My current workaround is to wrap command with env -- :

    nix_build = cmd_args([
        "env", "--",
        "nix",
        "build",
        "--no-update-lock-file",
        "--no-use-registries",
        cmd_args("--out-link", out_link.as_output()),
        cmd_args(drv, format = "{}^*"),
    ], hidden = deps)

That should work as long as nobody creates an env directory I guess... 😄

[cormacrelf]

Maybe your PATH on the remote executor contains "."? That might also explain it. It probably shouldn't.

[avdv]

Maybe your PATH on the remote executor contains "."? That might also explain it. It probably shouldn't.

All these actions run locally (local_only = True), and . is not in the PATH. But even if it was, I think it should not pick up the nix directory from the PATH when trying to locate a binary to run.

I checked what env does in this case, and apparently it just iterates over the directories in PATH and tries to execute the binary from there until it can start a process:

$ PATH=.:$PATH strace -s 200 -e %file env -- nix --version
...
execve("./nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 EACCES (Permission denied)
execve("/nix/store/lf0wpjrj8yx4gsmw2s3xfl58ixmqk8qa-bash-5.2-p15/bin/nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 ENOENT (No such file or directory)
execve("./nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 EACCES (Permission denied)
execve("/nix/store/vwkvhj69z4qqgmpa2lwm97kabf12p26r-coreutils-9.3/bin/nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 ENOENT (No such file or directory)
execve("./nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 EACCES (Permission denied)
execve("/nix/store/g5p3ky90xa05ggg5szyb0pbbl2vp7n03-gnused-4.9/bin/nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 ENOENT (No such file or directory)
execve("./nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 EACCES (Permission denied)
execve("/nix/store/whwwhd6ns271bj0ff86ap37i9r9kzi9c-git-2.42.0/bin/nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 ENOENT (No such file or directory)
execve("./nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = -1 EACCES (Permission denied)
execve("/nix/store/11glc2yk4jmycvk42q6kikpdcpbnbf6w-nix-2.17.1/bin/nix", ["nix", "--version"], 0x7ffcb10b62e0 /* 167 vars */) = 0

whereas Bash also iterates the directories in PATH, but stats the file and checks if it is executable before running it:

$ PATH=.:$PATH strace -fs 200 -e %file bash -c 'nix --version'
...
newfstatat(AT_FDCWD, "./nix", {st_mode=S_IFDIR|0755, st_size=106, ...}, 0) = 0
newfstatat(AT_FDCWD, "/nix/store/lf0wpjrj8yx4gsmw2s3xfl58ixmqk8qa-bash-5.2-p15/bin/nix", 0x7ffd26750d00, 0) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "./nix", {st_mode=S_IFDIR|0755, st_size=106, ...}, 0) = 0
newfstatat(AT_FDCWD, "/nix/store/vwkvhj69z4qqgmpa2lwm97kabf12p26r-coreutils-9.3/bin/nix", 0x7ffd26750d00, 0) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "./nix", {st_mode=S_IFDIR|0755, st_size=106, ...}, 0) = 0
newfstatat(AT_FDCWD, "/nix/store/g5p3ky90xa05ggg5szyb0pbbl2vp7n03-gnused-4.9/bin/nix", 0x7ffd26750d00, 0) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "./nix", {st_mode=S_IFDIR|0755, st_size=106, ...}, 0) = 0
newfstatat(AT_FDCWD, "/nix/store/whwwhd6ns271bj0ff86ap37i9r9kzi9c-git-2.42.0/bin/nix", 0x7ffd26750d00, 0) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "./nix", {st_mode=S_IFDIR|0755, st_size=106, ...}, 0) = 0
newfstatat(AT_FDCWD, "/nix/store/11glc2yk4jmycvk42q6kikpdcpbnbf6w-nix-2.17.1/bin/nix", {st_mode=S_IFREG|0555, st_size=2941200, ...}, 0) = 0
newfstatat(AT_FDCWD, "/nix/store/11glc2yk4jmycvk42q6kikpdcpbnbf6w-nix-2.17.1/bin/nix", {st_mode=S_IFREG|0555, st_size=2941200, ...}, 0) = 0
access("/nix/store/11glc2yk4jmycvk42q6kikpdcpbnbf6w-nix-2.17.1/bin/nix", X_OK) = 0
newfstatat(AT_FDCWD, "/nix/store/11glc2yk4jmycvk42q6kikpdcpbnbf6w-nix-2.17.1/bin/nix", {st_mode=S_IFREG|0555, st_size=2941200, ...}, 0) = 0
access("/nix/store/11glc2yk4jmycvk42q6kikpdcpbnbf6w-nix-2.17.1/bin/nix", R_OK) = 0
execve("/nix/store/11glc2yk4jmycvk42q6kikpdcpbnbf6w-nix-2.17.1/bin/nix", ["nix", "--version"], 0x68bbd0 /* 168 vars */) = 0