Git external/sub project improvements

CHANGES.md

@@ -1,5 +1,7 @@
 # git master
 
+* [517](https://github.com/Eyescale/CMake/pull/517):
+  Clone sub projects in parallel
 * [510](https://github.com/Eyescale/CMake/pull/510):
   Also create project-all target for super project
 * [507](https://github.com/Eyescale/CMake/pull/507):

GitExternal.cmake

@@ -8,7 +8,7 @@
 #    update target to bump the tag to the master revision by
 #    recreating .gitexternals.
 #  * Provides function
-#      git_external(<directory> <giturl> <gittag> [VERBOSE,SHALLOW]
+#      git_external(<directory> <giturl> <gittag> [VERBOSE]
 #        [RESET <files>])
 #    which will check out directory in CMAKE_SOURCE_DIR (if relative)
 #    or in the given absolute path using the given repository and tag
@@ -18,11 +18,6 @@
 #  VERBOSE, when present, this option tells the function to output
 #    information about what operations are being performed by git on
 #    the repo.
-#  SHALLOW, when present, causes a shallow clone of depth 1 to be made
-#    of the specified repo. This may save considerable memory/bandwidth
-#    when only a specific branch of a repo is required and the full history
-#    is not required. Note that the SHALLOW option will only work for a branch
-#    or tag and cannot be used for an arbitrary SHA.
 #  OPTIONAL, when present, this option makes this operation optional.
 #    The function will output a warning and return if the repo could not be
 #    cloned.
@@ -69,15 +64,8 @@ macro(GIT_EXTERNAL_MESSAGE msg)
   endif()
 endmacro()
 
-# utility function for printing a list with custom separator
-function(JOIN VALUES GLUE OUTPUT)
-  string (REGEX REPLACE "([^\\]|^);" "\\1${GLUE}" _TMP_STR "${VALUES}")
-  string (REGEX REPLACE "[\\](.)" "\\1" _TMP_STR "${_TMP_STR}") #fixes escaping
-  set (${OUTPUT} "${_TMP_STR}" PARENT_SCOPE)
-endfunction()
-
 function(GIT_EXTERNAL DIR REPO tag)
-  cmake_parse_arguments(GIT_EXTERNAL_LOCAL "VERBOSE;SHALLOW;OPTIONAL" "" "RESET" ${ARGN})
+  cmake_parse_arguments(GIT_EXTERNAL_LOCAL "VERBOSE;OPTIONAL" "" "RESET" ${ARGN})
   set(TAG ${tag})
   if(GIT_EXTERNAL_TAG AND "${tag}" MATCHES "^[0-9a-f]+$")
     set(TAG ${GIT_EXTERNAL_TAG})
@@ -105,16 +93,9 @@ function(GIT_EXTERNAL DIR REPO tag)
 
   if(NOT EXISTS "${DIR}")
     # clone
-    set(_clone_options --recursive)
-    if(GIT_EXTERNAL_LOCAL_SHALLOW)
-      list(APPEND _clone_options --depth 1 --branch ${TAG})
-    else()
-      set(_msg_tag "[${TAG}]")
-    endif()
-    JOIN("${_clone_options}" " " _msg_text)
-    message(STATUS "git clone ${_msg_text} ${REPO} ${DIR} ${_msg_tag}")
+    message(STATUS "git clone --recursive ${REPO} ${DIR} [${TAG}]")
     execute_process(
-      COMMAND "${GIT_EXECUTABLE}" clone ${_clone_options} ${REPO} ${DIR}
+      COMMAND "${GIT_EXECUTABLE}" clone --recursive ${REPO} ${DIR}
       RESULT_VARIABLE nok ERROR_VARIABLE error
       WORKING_DIRECTORY "${GIT_EXTERNAL_DIR}")
     if(nok)
@@ -127,21 +108,8 @@ function(GIT_EXTERNAL DIR REPO tag)
     endif()
 
     # checkout requested tag
-    if(NOT GIT_EXTERNAL_LOCAL_SHALLOW)
-      execute_process(
-        COMMAND "${GIT_EXECUTABLE}" checkout -q "${TAG}"
-        RESULT_VARIABLE nok ERROR_VARIABLE error
-        WORKING_DIRECTORY "${DIR}")
-      if(nok)
-        message(FATAL_ERROR "git checkout ${TAG} in ${DIR} failed: ${error}\n")
-      endif()
-    endif()
-
-    # checkout requested tag
-    execute_process(
-      COMMAND "${GIT_EXECUTABLE}" checkout -q "${TAG}"
-      RESULT_VARIABLE nok ERROR_VARIABLE error
-      WORKING_DIRECTORY "${DIR}")
+    execute_process(COMMAND "${GIT_EXECUTABLE}" checkout -q "${TAG}"
+      RESULT_VARIABLE nok ERROR_VARIABLE error WORKING_DIRECTORY "${DIR}")
     if(nok)
       message(FATAL_ERROR "git checkout ${TAG} in ${DIR} failed: ${error}\n")
     endif()

SubProject.cmake

@@ -186,31 +186,35 @@ function(add_subproject name)
 endfunction()
 
 macro(git_subproject name url tag)
-  # enter early to catch all dependencies
-  common_graph_dep(${PROJECT_NAME} ${name} TRUE TRUE)
-  if(NOT DISABLE_SUBPROJECTS)
-    string(TOUPPER ${name} NAME)
-    if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND)
-      get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT)
-      if(NOT __included)
-        if(NOT EXISTS ${__common_source_dir}/${name})
-          # Always try first using Config mode, then Module mode.
-          find_package(${name} QUIET CONFIG)
-          if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND)
-            find_package(${name} QUIET MODULE)
+  if(__subprojects_collect)
+    list(APPEND __subprojects "${name} ${url} ${tag}")
+  else()
+    # enter early to catch all dependencies
+    common_graph_dep(${PROJECT_NAME} ${name} TRUE TRUE)
+    if(NOT DISABLE_SUBPROJECTS)
+      string(TOUPPER ${name} NAME)
+      if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND)
+        get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT)
+        if(NOT __included)
+          if(NOT EXISTS ${__common_source_dir}/${name})
+            # Always try first using Config mode, then Module mode.
+            find_package(${name} QUIET CONFIG)
+            if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND)
+              find_package(${name} QUIET MODULE)
+            endif()
+          endif()
+          if((NOT ${NAME}_FOUND AND NOT ${name}_FOUND) OR
+              ${NAME}_FOUND_SUBPROJECT)
+            # not found externally, add as sub project
+            git_external(${__common_source_dir}/${name} ${url} ${tag})
+            add_subproject(${name})
           endif()
-        endif()
-        if((NOT ${NAME}_FOUND AND NOT ${name}_FOUND) OR
-            ${NAME}_FOUND_SUBPROJECT)
-          # not found externally, add as sub project
-          git_external(${__common_source_dir}/${name} ${url} ${tag})
-          add_subproject(${name})
         endif()
       endif()
-    endif()
-    get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT)
-    if(__included)
-      list(APPEND __subprojects "${name} ${url} ${tag}")
+      get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT)
+      if(__included)
+        list(APPEND __subprojects "${name} ${url} ${tag}")
+      endif()
     endif()
   endif()
 endmacro()
@@ -219,7 +223,51 @@ endmacro()
 if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.gitsubprojects")
   subproject_install_packages(${PROJECT_NAME})
 
-  set(__subprojects) # appended on each git_subproject invocation
+  # Gather all subprojects
+  set(__subprojects_collect 1)
+  set(__subprojects) # all appended on each git_subproject invocation
+  include(.gitsubprojects)
+  set(__subprojects_collect)
+
+  # Clone all subprojects in parallel
+  set(__all_subprojects "${__subprojects}")
+  set(__clone_subprojects)
+  foreach(__subproject ${__subprojects})
+    string(REPLACE " " ";" __subproject_list ${__subproject})
+    list(GET __subproject_list 0 __name)
+    list(GET __subproject_list 1 __repo)
+    list(GET __subproject_list 2 __tag)
+    set(__dir "${__common_source_dir}/${__name}")
+
+    if(NOT EXISTS "${__dir}")
+      message(STATUS "git clone --recursive ${__repo} ${__name} [${__tag}]")
+      file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${__name}Clone.cmake"
+        "execute_process("
+        "  COMMAND \"${GIT_EXECUTABLE}\" clone --recursive ${__repo} ${__dir}"
+        "  RESULT_VARIABLE nok ERROR_VARIABLE error)\n"
+        "if(nok)\n"
+        "  message(FATAL_ERROR \"${__name} clone failed: \${error}\")\n"
+        "endif()\n"
+        "execute_process(COMMAND \"${GIT_EXECUTABLE}\" checkout -q ${__tag}"
+        "  RESULT_VARIABLE nok ERROR_VARIABLE error"
+        "  WORKING_DIRECTORY ${__dir})\n"
+        "if(nok)\n"
+        "  message(FATAL_ERROR \"git checkout ${__tag} in ${__dir} failed: \${error}\")\n"
+        "endif()\n")
+      list(APPEND __clone_subprojects COMMAND "${CMAKE_COMMAND}" -P
+        "${CMAKE_CURRENT_BINARY_DIR}/${__name}Clone.cmake")
+    endif()
+  endforeach()
+  if(__clone_subprojects)
+    execute_process(${__clone_subprojects}
+      RESULT_VARIABLE nok ERROR_VARIABLE error
+      WORKING_DIRECTORY "${__common_source_dir}")
+    if(nok)
+      message(FATAL_ERROR "Cloning of projects failed: ${error}")
+    endif()
+  endif()
+
+  set(__subprojects) # activate projects on each git_subproject invocation
   include(.gitsubprojects)
   if(__subprojects)
     get_property(__subproject_paths GLOBAL PROPERTY SUBPROJECT_PATHS)