Merge branch 'busybox-w32'

This topic branch brings slightly experimental changes supporting Git
for Windows to use BusyBox-w32 to execute its shell scripts as well as
its test suite.

The test suite can be run by installing the test artifacts into a MinGit
that has busybox.exe (and using Git for Windows' SDK's Perl for now, as
the test suite requires Perl even when NO_PERL is set, go figure) by
using the `install-mingit-test-artifacts` Makefile target with the
DESTDIR variable pointing to the top-level directory of the MinGit
installation.

To facilitate running the test suite (without having `make` available,
as `make.exe` is not part of MinGit), this branch brings an experimental
patch to the `test-run-command` helper to run Git's test suite. It is
still very experimental, though: in this developer's tests it seemed
that the `poll()` emulation required for `run_parallel_processes()` to
work sometimes hiccups on Windows, causing infinite "hangs". It is also
possible that BusyBox itself has problems writing to the pipes opened by
`test-run-command` (and merging this branch will help investigate
further). Caveat emptor.

Signed-off-by: Johannes Schindelin <[email protected]>

Makefile

@@ -645,6 +645,7 @@ TEST_PROGRAMS_NEED_X += test-dump-untracked-cache
 TEST_PROGRAMS_NEED_X += test-fake-ssh
 TEST_PROGRAMS_NEED_X += test-genrandom
 TEST_PROGRAMS_NEED_X += test-hashmap
+TEST_PROGRAMS_NEED_X += test-helper
 TEST_PROGRAMS_NEED_X += test-index-version
 TEST_PROGRAMS_NEED_X += test-lazy-init-name-hash
 TEST_PROGRAMS_NEED_X += test-line-buffer

builtin/submodule--helper.c

@@ -930,7 +930,7 @@ static int update_clone_task_finished(int result,
 	const struct cache_entry *ce;
 	struct submodule_update_clone *suc = suc_cb;
 
-	int *idxP = *(int**)idx_task_cb;
+	int *idxP = idx_task_cb;
 	int idx = *idxP;
 	free(idxP);
 

compat/mingw.c

@@ -8,6 +8,7 @@
 #include "../cache.h"
 #include "win32/exit-process.h"
 #include "../config.h"
+#include "../string-list.h"
 
 #define HCAST(type, handle) ((type)(intptr_t)handle)
 
@@ -1126,7 +1127,7 @@ static const char *quote_arg_msvc(const char *arg)
 
 #include "quote.h"
 
-static const char *quote_arg_sh(const char *arg)
+static const char *quote_arg_msys2(const char *arg)
 {
 	struct strbuf buf = STRBUF_INIT;
 	const char *p2 = arg, *p;
@@ -1216,6 +1217,65 @@ static char *lookup_prog(const char *dir, int dirlen, const char *cmd,
 	return NULL;
 }
 
+static char *path_lookup(const char *cmd, int exe_only);
+
+static char *is_busybox_applet(const char *cmd)
+{
+	static struct string_list applets = STRING_LIST_INIT_DUP;
+	static char *busybox_path;
+	static int busybox_path_initialized;
+
+	/* Avoid infinite loop */
+	if (!strncasecmp(cmd, "busybox", 7) &&
+	    (!cmd[7] || !strcasecmp(cmd + 7, ".exe")))
+		return NULL;
+
+	if (!busybox_path_initialized) {
+		busybox_path = path_lookup("busybox.exe", 1);
+		busybox_path_initialized = 1;
+	}
+
+	/* Assume that sh is compiled in... */
+	if (!busybox_path || !strcasecmp(cmd, "sh"))
+		return xstrdup_or_null(busybox_path);
+
+	if (!applets.nr) {
+		struct child_process cp = CHILD_PROCESS_INIT;
+		struct strbuf buf = STRBUF_INIT;
+		char *p;
+
+		argv_array_pushl(&cp.args, busybox_path, "--help", NULL);
+
+		if (capture_command(&cp, &buf, 2048)) {
+			string_list_append(&applets, "");
+			return NULL;
+		}
+
+		/* parse output */
+		p = strstr(buf.buf, "Currently defined functions:\n");
+		if (!p) {
+			warning("Could not parse output of busybox --help");
+			string_list_append(&applets, "");
+			return NULL;
+		}
+		p = strchrnul(p, '\n');
+		for (;;) {
+			size_t len;
+
+			p += strspn(p, "\n\t ,");
+			len = strcspn(p, "\n\t ,");
+			if (!len)
+				break;
+			p[len] = '\0';
+			string_list_insert(&applets, p);
+			p = p + len + 1;
+		}
+	}
+
+	return string_list_has_string(&applets, cmd) ?
+		xstrdup(busybox_path) : NULL;
+}
+
 /*
  * Determines the absolute path of cmd using the split path in path.
  * If cmd contains a slash or backslash, no lookup is performed.
@@ -1244,6 +1304,9 @@ static char *path_lookup(const char *cmd, int exe_only)
 		path = sep + 1;
 	}
 
+	if (!prog && !isexe)
+		prog = is_busybox_applet(cmd);
+
 	return prog;
 }
 
@@ -1528,9 +1591,37 @@ static void kill_child_processes_on_signal(void)
 	LeaveCriticalSection(&pinfo_cs);
 }
 
+static int is_msys2_sh(const char *cmd)
+{
+	if (cmd && !strcmp(cmd, "sh")) {
+		static int ret = -1;
+		char *p;
+
+		if (ret >= 0)
+			return ret;
+
+		p = path_lookup(cmd, 0);
+		if (!p)
+			ret = 0;
+		else {
+			size_t len = strlen(p);
+			ret = len > 15 &&
+				is_dir_sep(p[len - 15]) &&
+				!strncasecmp(p + len - 14, "usr", 3) &&
+				is_dir_sep(p[len - 11]) &&
+				!strncasecmp(p + len - 10, "bin", 3) &&
+				is_dir_sep(p[len - 7]) &&
+				!strcasecmp(p + len - 6, "sh.exe");
+			free(p);
+		}
+		return ret;
+	}
+	return 0;
+}
+
 static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaenv,
-			      const char *dir,
-			      int prepend_cmd, int fhin, int fhout, int fherr)
+			      const char *dir, const char *prepend_cmd,
+			      int fhin, int fhout, int fherr)
 {
 	static int atexit_handler_initialized;
 	STARTUPINFOW si;
@@ -1542,7 +1633,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen
 	HANDLE cons;
 	const char *strace_env;
 	const char *(*quote_arg)(const char *arg) =
-		*argv && !strcmp("sh", *argv) ? quote_arg_sh : quote_arg_msvc;
+		is_msys2_sh(*argv) ? quote_arg_msys2 : quote_arg_msvc;
 
 	if (!atexit_handler_initialized) {
 		atexit_handler_initialized = 1;
@@ -1599,9 +1690,9 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen
 	/* concatenate argv, quoting args as we go */
 	strbuf_init(&args, 0);
 	if (prepend_cmd) {
-		char *quoted = (char *)quote_arg(cmd);
+		char *quoted = (char *)quote_arg(prepend_cmd);
 		strbuf_addstr(&args, quoted);
-		if (quoted != cmd)
+		if (quoted != prepend_cmd)
 			free(quoted);
 	}
 	for (; *argv; argv++) {
@@ -1678,7 +1769,8 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen
 	return (pid_t)pi.dwProcessId;
 }
 
-static pid_t mingw_spawnv(const char *cmd, const char **argv, int prepend_cmd)
+static pid_t mingw_spawnv(const char *cmd, const char **argv,
+			  const char *prepend_cmd)
 {
 	return mingw_spawnve_fd(cmd, argv, NULL, NULL, prepend_cmd, 0, 1, 2);
 }
@@ -1706,14 +1798,14 @@ pid_t mingw_spawnvpe(const char *cmd, const char **argv, char **deltaenv,
 				pid = -1;
 			}
 			else {
-				pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, 1,
+				pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, interpr,
 						       fhin, fhout, fherr);
 				free(iprog);
 			}
 			argv[0] = argv0;
 		}
 		else
-			pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, 0,
+			pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, NULL,
 					       fhin, fhout, fherr);
 		free(prog);
 	}
@@ -1739,7 +1831,7 @@ static int try_shell_exec(const char *cmd, char *const *argv)
 		ALLOC_ARRAY(argv2, argc + 1);
 		argv2[0] = (char *)cmd;	/* full path to the script file */
 		memcpy(&argv2[1], &argv[1], sizeof(*argv) * argc);
-		pid = mingw_spawnv(prog, argv2, 1);
+		pid = mingw_spawnv(prog, argv2, interpr);
 		if (pid >= 0) {
 			int status;
 			if (waitpid(pid, &status, 0) < 0)
@@ -1759,7 +1851,7 @@ int mingw_execv(const char *cmd, char *const *argv)
 	if (!try_shell_exec(cmd, argv)) {
 		int pid, status;
 
-		pid = mingw_spawnv(cmd, (const char **)argv, 0);
+		pid = mingw_spawnv(cmd, (const char **)argv, NULL);
 		if (pid < 0)
 			return -1;
 		if (waitpid(pid, &status, 0) < 0)

config.mak.uname

@@ -685,6 +685,72 @@ else
 		NO_CURL = YesPlease
 	endif
 endif
+ifeq (i686,$(uname_M))
+	MINGW_PREFIX := mingw32
+endif
+ifeq (x86_64,$(uname_M))
+	MINGW_PREFIX := mingw64
+endif
+
+	DESTDIR_WINDOWS = $(shell cygpath -aw '$(DESTDIR_SQ)')
+	DESTDIR_MIXED = $(shell cygpath -am '$(DESTDIR_SQ)')
+install-mingit-test-artifacts:
+	install -m755 -d '$(DESTDIR_SQ)/usr/bin'
+	printf '%s\n%s\n' >'$(DESTDIR_SQ)/usr/bin/perl' \
+		"#!/mingw64/bin/busybox sh" \
+		"exec \"$(shell cygpath -am /usr/bin/perl.exe)\" \"\$$@\""
+
+	install -m755 -d '$(DESTDIR_SQ)'
+	printf '%s%s\n%s\n%s\n%s\n%s\n' >'$(DESTDIR_SQ)/init.bat' \
+		"PATH=$(DESTDIR_WINDOWS)\\$(MINGW_PREFIX)\\bin;" \
+		"C:\\WINDOWS;C:\\WINDOWS\\system32" \
+		"@set GIT_TEST_INSTALLED=$(DESTDIR_MIXED)/$(MINGW_PREFIX)/bin" \
+		"@`echo "$(DESTDIR_WINDOWS)" | sed 's/:.*/:/'`" \
+		"@cd `echo "$(DESTDIR_WINDOWS)" | sed 's/^.://'`\\test-git\\t" \
+		"@echo Now, run 'helper\\test-run-command testsuite'"
+
+	install -m755 -d '$(DESTDIR_SQ)/$(MINGW_PREFIX)/bin'
+	install -m755 git.exe git-credential-store.exe git-fast-import.exe \
+		git-http-fetch.exe git-http-push.exe git-receive-pack.exe \
+		git-remote-http.exe git-remote-https.exe \
+		git-sh-i18n--envsubst.exe git-show-index.exe \
+		git-upload-pack.exe '$(DESTDIR_SQ)/$(MINGW_PREFIX)/bin'
+
+	install -m755 -d '$(DESTDIR_SQ)/$(MINGW_PREFIX)/libexec/git-core'
+	cp git-bisect git-difftool--helper git-filter-branch git-instaweb \
+		git-merge-octopus git-merge-one-file git-merge-resolve \
+		git-mergetool git-mergetool--lib git-parse-remote \
+		git-quiltimport git-rebase git-rebase--am \
+		git-rebase--interactive git-rebase--merge git-request-pull \
+		git-sh-i18n git-sh-setup git-stash git-submodule \
+		git-web--browse '$(DESTDIR_SQ)/$(MINGW_PREFIX)/libexec/git-core'
+
+	install -m755 -d '$(DESTDIR_SQ)/test-git'
+	sed 's/^\(NO_PERL\|NO_PYTHON\)=.*/\1=YesPlease/' \
+		<GIT-BUILD-OPTIONS >'$(DESTDIR_SQ)/test-git/GIT-BUILD-OPTIONS'
+
+	install -m755 -d '$(DESTDIR_SQ)/test-git/t/helper'
+	install -m755 $(TEST_PROGRAMS) '$(DESTDIR_SQ)/test-git/t/helper'
+	(cd t && $(TAR) cf - t[0-9][0-9][0-9][0-9] diff-lib) | \
+	(cd '$(DESTDIR_SQ)/test-git/t' && $(TAR) xf -)
+	install -m755 t/t556x_common t/*.sh '$(DESTDIR_SQ)/test-git/t'
+
+	install -m755 -d '$(DESTDIR_SQ)/test-git/templates'
+	(cd templates && $(TAR) cf - blt) | \
+	(cd '$(DESTDIR_SQ)/test-git/templates' && $(TAR) xf -)
+
+	# po/build/locale for t0200
+	install -m755 -d '$(DESTDIR_SQ)/test-git/po/build/locale'
+	(cd po/build/locale && $(TAR) cf - .) | \
+	(cd '$(DESTDIR_SQ)/test-git/po/build/locale' && $(TAR) xf -)
+
+	# git-daemon.exe for t5802, git-http-backend.exe for t5560
+	install -m755 git-daemon.exe git-http-backend.exe \
+		'$(DESTDIR_SQ)/$(MINGW_PREFIX)/bin'
+
+	# git-remote-testgit for t5801
+	install -m755 git-remote-testgit \
+		'$(DESTDIR_SQ)/$(MINGW_PREFIX)/libexec/git-core'
 endif
 ifeq ($(uname_S),QNX)
 	COMPAT_CFLAGS += -DSA_RESTART=0

connect.c

@@ -825,6 +825,9 @@ struct child_process *git_connect(int fd[2], const char *url,
 		child_process_init(conn);
 
 		strbuf_addstr(&cmd, prog);
+		/* Prefer the builtin */
+		if (starts_with(prog, "git-"))
+			cmd.buf[3] = ' ';
 		strbuf_addch(&cmd, ' ');
 		sq_quote_buf(&cmd, path);
 

git-sh-setup.sh

@@ -332,17 +332,30 @@ create_virtual_base() {
 # Platform specific tweaks to work around some commands
 case $(uname -s) in
 *MINGW*)
-	# Windows has its own (incompatible) sort and find
-	sort () {
-		/usr/bin/sort "$@"
-	}
-	find () {
-		/usr/bin/find "$@"
-	}
-	# git sees Windows-style pwd
-	pwd () {
-		builtin pwd -W
-	}
+	if test -x /usr/bin/sort
+	then
+		# Windows has its own (incompatible) sort; override
+		sort () {
+			/usr/bin/sort "$@"
+		}
+	fi
+	if test -x /usr/bin/find
+	then
+		# Windows has its own (incompatible) find; override
+		find () {
+			/usr/bin/find "$@"
+		}
+	fi
+	# On Windows, Git wants Windows paths. But /usr/bin/pwd spits out
+	# Unix-style paths. At least in Bash, we have a builtin pwd that
+	# understands the -W option to force "mixed" paths, i.e. with drive
+	# prefix but still with forward slashes. Let's use that, if available.
+	if type builtin >/dev/null 2>&1
+	then
+		pwd () {
+			builtin pwd -W
+		}
+	fi
 	is_absolute_path () {
 		case "$1" in
 		[/\\]* | [A-Za-z]:*)

run-command.c

@@ -1533,7 +1533,7 @@ static int pp_start_one(struct parallel_processes *pp)
 	if (start_command(&pp->children[i].process)) {
 		code = pp->start_failure(&pp->children[i].err,
 					 pp->data,
-					 &pp->children[i].data);
+					 pp->children[i].data);
 		strbuf_addbuf(&pp->buffered_output, &pp->children[i].err);
 		strbuf_reset(&pp->children[i].err);
 		if (code)
@@ -1601,7 +1601,7 @@ static int pp_collect_finished(struct parallel_processes *pp)
 
 		code = pp->task_finished(code,
 					 &pp->children[i].err, pp->data,
-					 &pp->children[i].data);
+					 pp->children[i].data);
 
 		if (code)
 			result = code;

t/.gitattributes

@@ -1,5 +1,6 @@
 t[0-9][0-9][0-9][0-9]/* -whitespace
 /diff-lib/* eol=lf
+/diff-lib/*.png binary
 /t0110/url-* binary
 /t3900/*.txt eol=lf
 /t3901/*.txt eol=lf

t/helper/.gitignore

@@ -10,6 +10,7 @@
 /test-scrap-cache-tree
 /test-genrandom
 /test-hashmap
+/test-helper
 /test-index-version
 /test-lazy-init-name-hash
 /test-line-buffer