external: fix submodule handling for parallel builds, submodule URL c…

…hanges.

If we change an upstream URL, all submodules break.  Users would need
to run 'git submodule sync'.  Note that the libbacktrace fix was merged
upstream so this is no longer necessary, but it's good for future changes.

Also, stress-testing reveals that git submodule fails locking
'.git/config' when run in paralell.  It also segfaults and other
problems.

This is my final attempt to fix submodules; I've wasted far too many
days on obscure problems it creates: I've already lost one copy of my
repo to apparently unfixable submodule preoblems.  The next "fix" will
be to simply import the source code so it works properly.

Reported-by: @jsarenik
Fixes: ElementsProject#1543
Signed-off-by: Rusty Russell <[email protected]>

external/Makefile

@@ -1,3 +1,10 @@
+SUBMODULES =					\
+	external/libsodium			\
+	external/libwally-core			\
+	external/jsmn				\
+	external/libbase58			\
+	external/libbacktrace
+
 LIBSODIUM_HEADERS := external/libsodium/src/libsodium/include/sodium.h
 LIBWALLY_HEADERS := external/libwally-core/include/wally_bip32.h		\
 			external/libwally-core/include/wally_core.h	\
@@ -21,20 +28,19 @@ EXTERNAL_INCLUDE_FLAGS :=					\
 
 EXTERNAL_LDLIBS := -Lexternal $(patsubst lib%.a,-l%,$(notdir $(EXTERNAL_LIBS)))
 
-# Might exist, but need updating.  Nuke and rebuild.
-submodcheck-%: FORCE
-	if git submodule status external/$* | grep -q '^[-+]'; then rm -rf external/$*; git submodule update --init external/$*; fi
+submodcheck: FORCE
+	@tools/refresh-submodules.sh $(SUBMODULES)
 
 # We build libsodium, since Ubuntu xenial has one too old.
 external/libsodium.a: external/libsodium/src/libsodium/libsodium.la
 	$(MAKE) -C external/libsodium DESTDIR=$$(pwd)/external install-exec
 
-external/libsodium/src/libsodium/include/sodium.h: submodcheck-libsodium
+external/libsodium/src/libsodium/include/sodium.h: submodcheck
 
 external/libsodium/src/libsodium/libsodium.la: external/libsodium/src/libsodium/include/sodium.h
 	cd external/libsodium && ./autogen.sh && ./configure CC="$(CC)" --enable-static=yes --host="$(MAKE_HOST)" --build="$(BUILD)" --enable-shared=no --enable-tests=no --prefix=/ --libdir=/ && $(MAKE)
 
-$(LIBWALLY_HEADERS) $(LIBSECP_HEADERS): submodcheck-libwally-core
+$(LIBWALLY_HEADERS) $(LIBSECP_HEADERS): submodcheck
 
 # libsecp included in libwally.
 # Wildcards here are magic.  See http://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file
@@ -44,7 +50,7 @@ external/libsecp256k1.% external/libwallycore.%: external/libwally-core/src/secp
 external/libwally-core/src/libwallycore.% external/libwally-core/src/secp256k1/libsecp256k1.%: $(LIBWALLY_HEADERS) $(LIBSECP_HEADERS)
 	cd external/libwally-core && ./tools/autogen.sh && ./configure CC="$(CC)" --enable-static=yes --host="$(MAKE_HOST)" --build="$(BUILD)" --enable-module-recovery --enable-shared=no --prefix=/ --libdir=/ && $(MAKE)
 
-external/jsmn/jsmn.h: submodcheck-jsmn
+external/jsmn/jsmn.h: submodcheck
 
 # If we tell Make that the above builds both, it runs it twice in
 # parallel.  So we lie :(
@@ -61,7 +67,7 @@ LIBBASE58_SRC := external/libbase58/base58.c
 
 $(LIBBASE58_SRC): $(LIBBASE58_HEADERS)
 
-$(LIBBASE58_HEADERS): submodcheck-libbase58
+$(LIBBASE58_HEADERS): submodcheck
 
 # Can't be inside submodule, as that makes git think it's dirty.
 external/base58.o: $(LIBBASE58_SRC) Makefile
@@ -70,7 +76,7 @@ external/base58.o: $(LIBBASE58_SRC) Makefile
 external/libbase58.a: external/base58.o
 	$(AR) rc $@ $<
 
-external/libbacktrace/backtrace.h: submodcheck-libbacktrace
+external/libbacktrace/backtrace.h: submodcheck
 
 # Need separate build dir: changes inside submodule make git think it's dirty.
 external/libbacktrace.a: external/libbacktrace/backtrace.h

tools/refresh-submodules.sh

@@ -0,0 +1,30 @@
+#! /bin/sh
+
+if [ $# = 0 ]; then
+    echo "Usage: $0 <submoduledir1>..." >&2
+    exit 1
+fi
+
+# git submodule can't run in parallel.  Really.
+echo $$ > .refresh-submodules.$$
+if ! mv -n .refresh-submodules.$$ .refresh-submodules; then
+    rm -f .refresh-submodules.$$
+    exit 0
+fi
+trap "rm -f .refresh-submodules" EXIT
+
+# Be a little careful here, since we do rm -rf!
+for m in "$@"; do
+    if ! grep -q "path = $m\$" .gitmodules; then
+	echo "$m is not a submodule!" >&2
+	exit 1
+    fi
+done
+
+# git submodule can segfault.  Really.
+if [ $(git submodule status "$@" | grep -c '^ ') != $# ]; then
+    echo Reinitializing submodules "$@"...
+    git submodule sync "$@"
+    rm -rf "$@"
+    git submodule update --init "$@"
+fi