From c7d4d46a8aac9b702c942bb90e56d81fe55a1edf Mon Sep 17 00:00:00 2001 From: Stefan Eilemann Date: Thu, 3 Nov 2016 08:12:03 +0100 Subject: [PATCH 1/2] Simplify GitExternals Remove unused SHALLOW option and prune code accordingly. --- GitExternal.cmake | 44 ++++++-------------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/GitExternal.cmake b/GitExternal.cmake index 8630673..1063610 100644 --- a/GitExternal.cmake +++ b/GitExternal.cmake @@ -8,7 +8,7 @@ # update target to bump the tag to the master revision by # recreating .gitexternals. # * Provides function -# git_external( [VERBOSE,SHALLOW] +# git_external( [VERBOSE] # [RESET ]) # which will check out directory in CMAKE_SOURCE_DIR (if relative) # or in the given absolute path using the given repository and tag @@ -18,11 +18,6 @@ # VERBOSE, when present, this option tells the function to output # information about what operations are being performed by git on # the repo. -# SHALLOW, when present, causes a shallow clone of depth 1 to be made -# of the specified repo. This may save considerable memory/bandwidth -# when only a specific branch of a repo is required and the full history -# is not required. Note that the SHALLOW option will only work for a branch -# or tag and cannot be used for an arbitrary SHA. # OPTIONAL, when present, this option makes this operation optional. # The function will output a warning and return if the repo could not be # cloned. @@ -69,15 +64,8 @@ macro(GIT_EXTERNAL_MESSAGE msg) endif() endmacro() -# utility function for printing a list with custom separator -function(JOIN VALUES GLUE OUTPUT) - string (REGEX REPLACE "([^\\]|^);" "\\1${GLUE}" _TMP_STR "${VALUES}") - string (REGEX REPLACE "[\\](.)" "\\1" _TMP_STR "${_TMP_STR}") #fixes escaping - set (${OUTPUT} "${_TMP_STR}" PARENT_SCOPE) -endfunction() - function(GIT_EXTERNAL DIR REPO tag) - cmake_parse_arguments(GIT_EXTERNAL_LOCAL "VERBOSE;SHALLOW;OPTIONAL" "" "RESET" ${ARGN}) + cmake_parse_arguments(GIT_EXTERNAL_LOCAL "VERBOSE;OPTIONAL" "" "RESET" ${ARGN}) set(TAG ${tag}) if(GIT_EXTERNAL_TAG AND "${tag}" MATCHES "^[0-9a-f]+$") set(TAG ${GIT_EXTERNAL_TAG}) @@ -105,16 +93,9 @@ function(GIT_EXTERNAL DIR REPO tag) if(NOT EXISTS "${DIR}") # clone - set(_clone_options --recursive) - if(GIT_EXTERNAL_LOCAL_SHALLOW) - list(APPEND _clone_options --depth 1 --branch ${TAG}) - else() - set(_msg_tag "[${TAG}]") - endif() - JOIN("${_clone_options}" " " _msg_text) - message(STATUS "git clone ${_msg_text} ${REPO} ${DIR} ${_msg_tag}") + message(STATUS "git clone --recursive ${REPO} ${DIR} [${TAG}]") execute_process( - COMMAND "${GIT_EXECUTABLE}" clone ${_clone_options} ${REPO} ${DIR} + COMMAND "${GIT_EXECUTABLE}" clone --recursive ${REPO} ${DIR} RESULT_VARIABLE nok ERROR_VARIABLE error WORKING_DIRECTORY "${GIT_EXTERNAL_DIR}") if(nok) @@ -127,21 +108,8 @@ function(GIT_EXTERNAL DIR REPO tag) endif() # checkout requested tag - if(NOT GIT_EXTERNAL_LOCAL_SHALLOW) - execute_process( - COMMAND "${GIT_EXECUTABLE}" checkout -q "${TAG}" - RESULT_VARIABLE nok ERROR_VARIABLE error - WORKING_DIRECTORY "${DIR}") - if(nok) - message(FATAL_ERROR "git checkout ${TAG} in ${DIR} failed: ${error}\n") - endif() - endif() - - # checkout requested tag - execute_process( - COMMAND "${GIT_EXECUTABLE}" checkout -q "${TAG}" - RESULT_VARIABLE nok ERROR_VARIABLE error - WORKING_DIRECTORY "${DIR}") + execute_process(COMMAND "${GIT_EXECUTABLE}" checkout -q "${TAG}" + RESULT_VARIABLE nok ERROR_VARIABLE error WORKING_DIRECTORY "${DIR}") if(nok) message(FATAL_ERROR "git checkout ${TAG} in ${DIR} failed: ${error}\n") endif() From 00731b6de346fe864aa38dbe4b36d394a5484116 Mon Sep 17 00:00:00 2001 From: Stefan Eilemann Date: Thu, 3 Nov 2016 08:12:39 +0100 Subject: [PATCH 2/2] Clone all subprojects of a project in parallel. This should speed up CI, especially latest. Exploits the fact that execute_process commands are executed in parallel. --- CHANGES.md | 2 ++ SubProject.cmake | 94 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9028319..bfe8732 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # git master +* [517](https://github.com/Eyescale/CMake/pull/517): + Clone sub projects in parallel * [510](https://github.com/Eyescale/CMake/pull/510): Also create project-all target for super project * [507](https://github.com/Eyescale/CMake/pull/507): diff --git a/SubProject.cmake b/SubProject.cmake index 3fea08a..4f2c9f2 100644 --- a/SubProject.cmake +++ b/SubProject.cmake @@ -186,31 +186,35 @@ function(add_subproject name) endfunction() macro(git_subproject name url tag) - # enter early to catch all dependencies - common_graph_dep(${PROJECT_NAME} ${name} TRUE TRUE) - if(NOT DISABLE_SUBPROJECTS) - string(TOUPPER ${name} NAME) - if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND) - get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT) - if(NOT __included) - if(NOT EXISTS ${__common_source_dir}/${name}) - # Always try first using Config mode, then Module mode. - find_package(${name} QUIET CONFIG) - if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND) - find_package(${name} QUIET MODULE) + if(__subprojects_collect) + list(APPEND __subprojects "${name} ${url} ${tag}") + else() + # enter early to catch all dependencies + common_graph_dep(${PROJECT_NAME} ${name} TRUE TRUE) + if(NOT DISABLE_SUBPROJECTS) + string(TOUPPER ${name} NAME) + if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND) + get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT) + if(NOT __included) + if(NOT EXISTS ${__common_source_dir}/${name}) + # Always try first using Config mode, then Module mode. + find_package(${name} QUIET CONFIG) + if(NOT ${NAME}_FOUND AND NOT ${name}_FOUND) + find_package(${name} QUIET MODULE) + endif() + endif() + if((NOT ${NAME}_FOUND AND NOT ${name}_FOUND) OR + ${NAME}_FOUND_SUBPROJECT) + # not found externally, add as sub project + git_external(${__common_source_dir}/${name} ${url} ${tag}) + add_subproject(${name}) endif() - endif() - if((NOT ${NAME}_FOUND AND NOT ${name}_FOUND) OR - ${NAME}_FOUND_SUBPROJECT) - # not found externally, add as sub project - git_external(${__common_source_dir}/${name} ${url} ${tag}) - add_subproject(${name}) endif() endif() - endif() - get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT) - if(__included) - list(APPEND __subprojects "${name} ${url} ${tag}") + get_property(__included GLOBAL PROPERTY ${name}_IS_SUBPROJECT) + if(__included) + list(APPEND __subprojects "${name} ${url} ${tag}") + endif() endif() endif() endmacro() @@ -219,7 +223,51 @@ endmacro() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.gitsubprojects") subproject_install_packages(${PROJECT_NAME}) - set(__subprojects) # appended on each git_subproject invocation + # Gather all subprojects + set(__subprojects_collect 1) + set(__subprojects) # all appended on each git_subproject invocation + include(.gitsubprojects) + set(__subprojects_collect) + + # Clone all subprojects in parallel + set(__all_subprojects "${__subprojects}") + set(__clone_subprojects) + foreach(__subproject ${__subprojects}) + string(REPLACE " " ";" __subproject_list ${__subproject}) + list(GET __subproject_list 0 __name) + list(GET __subproject_list 1 __repo) + list(GET __subproject_list 2 __tag) + set(__dir "${__common_source_dir}/${__name}") + + if(NOT EXISTS "${__dir}") + message(STATUS "git clone --recursive ${__repo} ${__name} [${__tag}]") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${__name}Clone.cmake" + "execute_process(" + " COMMAND \"${GIT_EXECUTABLE}\" clone --recursive ${__repo} ${__dir}" + " RESULT_VARIABLE nok ERROR_VARIABLE error)\n" + "if(nok)\n" + " message(FATAL_ERROR \"${__name} clone failed: \${error}\")\n" + "endif()\n" + "execute_process(COMMAND \"${GIT_EXECUTABLE}\" checkout -q ${__tag}" + " RESULT_VARIABLE nok ERROR_VARIABLE error" + " WORKING_DIRECTORY ${__dir})\n" + "if(nok)\n" + " message(FATAL_ERROR \"git checkout ${__tag} in ${__dir} failed: \${error}\")\n" + "endif()\n") + list(APPEND __clone_subprojects COMMAND "${CMAKE_COMMAND}" -P + "${CMAKE_CURRENT_BINARY_DIR}/${__name}Clone.cmake") + endif() + endforeach() + if(__clone_subprojects) + execute_process(${__clone_subprojects} + RESULT_VARIABLE nok ERROR_VARIABLE error + WORKING_DIRECTORY "${__common_source_dir}") + if(nok) + message(FATAL_ERROR "Cloning of projects failed: ${error}") + endif() + endif() + + set(__subprojects) # activate projects on each git_subproject invocation include(.gitsubprojects) if(__subprojects) get_property(__subproject_paths GLOBAL PROPERTY SUBPROJECT_PATHS)