microsoft · cbezault · Jun 12, 2019 · May 30, 2019 · May 30, 2019 · May 30, 2019
diff --git a/ports/icu/fix_parallel_build_on_windows.patch b/ports/icu/fix_parallel_build_on_windows.patch
@@ -0,0 +1,13 @@
+diff --git a/source/data/Makefile.in b/source/data/Makefile.in
+index 1140b69..936ef81 100644
+--- a/source/data/Makefile.in
++++ b/source/data/Makefile.in
+@@ -514,7 +514,7 @@ build-dir:
+ # The | is an order-only prerequisite. This helps when the -j option is used,
+ # and we don't want the files to be built before the directories are built.
+ ifneq ($(filter order-only,$(.FEATURES)),)
+-$(ALL_FILES) $(ALL_INDEX_SRC_FILES): | build-dir
++$(ALL_FILES) $(ALL_INDEX_SRC_FILES) $(SO_VERSION_DATA): | build-dir
+ endif
+
+ # Now, sections for building each kind of data.
diff --git a/ports/icu/portfile.cmake b/ports/icu/portfile.cmake
@@ -20,6 +20,7 @@ vcpkg_extract_source_archive(${ARCHIVE} ${CURRENT_BUILDTREES_DIR}/src/icu-${VERS
 vcpkg_apply_patches(SOURCE_PATH ${SOURCE_PATH}
     PATCHES ${CMAKE_CURRENT_LIST_DIR}/disable-escapestr-tool.patch
             ${CMAKE_CURRENT_LIST_DIR}/remove-MD-from-configure.patch
+            ${CMAKE_CURRENT_LIST_DIR}/fix_parallel_build_on_windows.patch
 )
 
 set(CONFIGURE_OPTIONS "--disable-samples --disable-tests")
@@ -94,8 +95,8 @@ else()
         message(STATUS "Configuring ${TARGET_TRIPLET}-rel")
         file(REMOVE_RECURSE ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
         file(MAKE_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
-        set(ENV{CFLAGS} "${ICU_RUNTIME} -O2 -Oi -Zi ${VCPKG_C_FLAGS} ${VCPKG_C_FLAGS_RELEASE}")
-        set(ENV{CXXFLAGS} "${ICU_RUNTIME} -O2 -Oi -Zi ${VCPKG_CXX_FLAGS} ${VCPKG_CXX_FLAGS_RELEASE}")
+        set(ENV{CFLAGS} "${ICU_RUNTIME} -O2 -Oi -Zi -FS ${VCPKG_C_FLAGS} ${VCPKG_C_FLAGS_RELEASE}")
+        set(ENV{CXXFLAGS} "${ICU_RUNTIME} -O2 -Oi -Zi -FS ${VCPKG_CXX_FLAGS} ${VCPKG_CXX_FLAGS_RELEASE}")
         set(ENV{LDFLAGS} "-DEBUG -INCREMENTAL:NO -OPT:REF -OPT:ICF")
         vcpkg_execute_required_process(
             COMMAND ${BASH} --noprofile --norc -c
@@ -110,8 +111,8 @@ else()
         message(STATUS "Configuring ${TARGET_TRIPLET}-dbg")
         file(REMOVE_RECURSE ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg)
         file(MAKE_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg)
-        set(ENV{CFLAGS} "${ICU_RUNTIME}d -Od -Zi -RTC1 ${VCPKG_C_FLAGS} ${VCPKG_C_FLAGS_DEBUG}")
-        set(ENV{CXXFLAGS} "${ICU_RUNTIME}d -Od -Zi -RTC1 ${VCPKG_CXX_FLAGS} ${VCPKG_CXX_FLAGS_DEBUG}")
+        set(ENV{CFLAGS} "${ICU_RUNTIME}d -Od -Zi -FS -RTC1 ${VCPKG_C_FLAGS} ${VCPKG_C_FLAGS_DEBUG}")
+        set(ENV{CXXFLAGS} "${ICU_RUNTIME}d -Od -Zi -FS -RTC1 ${VCPKG_CXX_FLAGS} ${VCPKG_CXX_FLAGS_DEBUG}")
         set(ENV{LDFLAGS} "-DEBUG")
         vcpkg_execute_required_process(
             COMMAND ${BASH} --noprofile --norc -c
@@ -129,8 +130,8 @@ unset(ENV{LDFLAGS})
 if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release")
     # Build release
     message(STATUS "Package ${TARGET_TRIPLET}-rel")
-    vcpkg_execute_required_process(
-        COMMAND ${BASH} --noprofile --norc -c "make && make install"
+    vcpkg_execute_build_process(
+        COMMAND ${BASH} --noprofile --norc -c "make -j ${VCPKG_CONCURRENCY} && make install"
         WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel"
         LOGNAME "build-${TARGET_TRIPLET}-rel")
     message(STATUS "Package ${TARGET_TRIPLET}-rel done")
@@ -139,8 +140,8 @@ endif()
 if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug")
     # Build debug
     message(STATUS "Package ${TARGET_TRIPLET}-dbg")
-    vcpkg_execute_required_process(
-        COMMAND ${BASH} --noprofile --norc -c "make && make install"
+    vcpkg_execute_build_process(
+        COMMAND ${BASH} --noprofile --norc -c "make -j ${VCPKG_CONCURRENCY} && make install"
         WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg"
         LOGNAME "build-${TARGET_TRIPLET}-dbg")
     message(STATUS "Package ${TARGET_TRIPLET}-dbg done")

diff --git a/scripts/cmake/vcpkg_build_cmake.cmake b/scripts/cmake/vcpkg_build_cmake.cmake
@@ -60,10 +60,6 @@ function(vcpkg_build_cmake)
         set(TARGET_PARAM)
     endif()
 
-    if(_bc_DISABLE_PARALLEL)
-        set(PARALLEL_ARG ${NO_PARALLEL_ARG})
-    endif()
-
     foreach(BUILDTYPE "debug" "release")
         if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL BUILDTYPE)
             if(BUILDTYPE STREQUAL "debug")
@@ -75,8 +71,6 @@ function(vcpkg_build_cmake)
             endif()
 
             message(STATUS "Building ${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
-            set(LOGPREFIX "${CURRENT_BUILDTREES_DIR}/${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
-            set(LOGS)
 
             if(_bc_ADD_BIN_TO_PATH)
                 set(_BACKUP_ENV_PATH "$ENV{PATH}")
@@ -91,121 +85,28 @@ function(vcpkg_build_cmake)
                     set(ENV{PATH} "${CURRENT_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin${_PATHSEP}$ENV{PATH}")
                 endif()
             endif()
-            execute_process(
-                COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
-                OUTPUT_FILE "${LOGPREFIX}-out.log"
-                ERROR_FILE "${LOGPREFIX}-err.log"
-                RESULT_VARIABLE error_code
-                WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-            if(error_code)
-                file(READ "${LOGPREFIX}-out.log" out_contents)
-                file(READ "${LOGPREFIX}-err.log" err_contents)
-
-                if(out_contents)
-                    list(APPEND LOGS "${LOGPREFIX}-out.log")
-                endif()
-                if(err_contents)
-                    list(APPEND LOGS "${LOGPREFIX}-err.log")
-                endif()
-
-                if(out_contents MATCHES "LINK : fatal error LNK1102:" OR out_contents MATCHES " fatal error C1060: ")
-                    # The linker ran out of memory during execution. We will try continuing once more, with parallelism disabled.
-                    message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} without parallelism because memory exceeded")
-                    execute_process(
-                        COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
-                        OUTPUT_FILE "${LOGPREFIX}-out-1.log"
-                        ERROR_FILE "${LOGPREFIX}-err-1.log"
-                        RESULT_VARIABLE error_code
-                        WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-
-                    if(error_code)
-                        file(READ "${LOGPREFIX}-out-1.log" out_contents)
-                        file(READ "${LOGPREFIX}-err-1.log" err_contents)
-
-                        if(out_contents)
-                            list(APPEND LOGS "${LOGPREFIX}-out-1.log")
-                        endif()
-                        if(err_contents)
-                            list(APPEND LOGS "${LOGPREFIX}-err-1.log")
-                        endif()
-                    endif()
-                elseif(out_contents MATCHES ": No such file or directory")
-                    # WSL workaround - WSL occassionally fails with no such file or directory. Detect if we are running in WSL and restart.
-                    execute_process(COMMAND "uname" "-r"
-                        OUTPUT_VARIABLE UNAME_R ERROR_VARIABLE UNAME_R
-                        OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
-
-                    if (UNAME_R MATCHES "Microsoft")
-                        set(ITERATION 0)
-                        while (ITERATION LESS 10 AND out_contents MATCHES ": No such file or directory")
-                            MATH(EXPR ITERATION "${ITERATION}+1")
-                            message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} because of wsl subsystem issue. Iteration: ${ITERATION}")
-                            execute_process(
-                                COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS}
-                                OUTPUT_FILE "${LOGPREFIX}-out-${ITERATION}.log"
-                                ERROR_FILE "${LOGPREFIX}-err-${ITERATION}.log"
-                                RESULT_VARIABLE error_code
-                                WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
 
-                            if(error_code)
-                                file(READ "${LOGPREFIX}-out-${ITERATION}.log" out_contents)
-                                file(READ "${LOGPREFIX}-err-${ITERATION}.log" err_contents)
-
-                                if(out_contents)
-                                    list(APPEND LOGS "${LOGPREFIX}-out-${ITERATION}.log")
-                                endif()
-                                if(err_contents)
-                                    list(APPEND LOGS "${LOGPREFIX}-err-${ITERATION}.log")
-                                endif()
-                            else()
-                                break()
-                            endif()
-                        endwhile()
-                    endif()
-                elseif(out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: ")
-                    # Antivirus workaround - occasionally files are locked and cause mt.exe to fail
-                    set(ITERATION 0)
-                    while (ITERATION LESS 3 AND (out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: "))
-                        MATH(EXPR ITERATION "${ITERATION}+1")
-                        message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} because of mt.exe file locking issue. Iteration: ${ITERATION}")
-                        execute_process(
-                            COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
-                            OUTPUT_FILE "${LOGPREFIX}-out-${ITERATION}.log"
-                            ERROR_FILE "${LOGPREFIX}-err-${ITERATION}.log"
-                            RESULT_VARIABLE error_code
-                            WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-
-                        if(error_code)
-                            file(READ "${LOGPREFIX}-out-${ITERATION}.log" out_contents)
-                            file(READ "${LOGPREFIX}-err-${ITERATION}.log" err_contents)
-
-                            if(out_contents)
-                                list(APPEND LOGS "${LOGPREFIX}-out-${ITERATION}.log")
-                            endif()
-                            if(err_contents)
-                                list(APPEND LOGS "${LOGPREFIX}-err-${ITERATION}.log")
-                            endif()
-                        else()
-                            break()
-                        endif()
-                    endwhile()
-                endif()
-
-                if(error_code)
-                    set(STRINGIFIED_LOGS)
-                    foreach(LOG ${LOGS})
-                        file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
-                        list(APPEND STRINGIFIED_LOGS "    ${NATIVE_LOG}\n")
-                    endforeach()
-                    set(_eb_COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG})
-                    set(_eb_WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-                    message(FATAL_ERROR
-                        "  Command failed: ${_eb_COMMAND}\n"
-                        "  Working Directory: ${_eb_WORKING_DIRECTORY}\n"
-                        "  See logs for more information:\n"
-                        ${STRINGIFIED_LOGS})
-                endif()
+            if (_bc_DISABLE_PARALLEL)
+                vcpkg_execute_build_process(
+                    COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
+                    WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE}
+                    LOGNAME "${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}"
+                )
+            elseif(NOT PARALLEL_ARG STREQUAL NO_PARALLEL_ARG)
+                vcpkg_execute_build_process(
+                    COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
+                    NO_PARALLEL_COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
+                    WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE}
+                    LOGNAME "${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}"
+                )
+            else()
+                vcpkg_execute_build_process(
+                    COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
+                    WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE}
+                    LOGNAME "${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}"
+                )
             endif()
+
             if(_bc_ADD_BIN_TO_PATH)
                 set(ENV{PATH} "${_BACKUP_ENV_PATH}")
             endif()

diff --git a/scripts/cmake/vcpkg_common_functions.cmake b/scripts/cmake/vcpkg_common_functions.cmake
@@ -7,6 +7,7 @@ include(vcpkg_extract_source_archive)
 include(vcpkg_extract_source_archive_ex)
 include(vcpkg_execute_required_process)
 include(vcpkg_execute_required_process_repeat)
+include(vcpkg_execute_build_process)
 include(vcpkg_find_acquire_program)
 include(vcpkg_fixup_cmake_targets)
 include(vcpkg_from_github)

diff --git a/scripts/cmake/vcpkg_execute_build_process.cmake b/scripts/cmake/vcpkg_execute_build_process.cmake
@@ -0,0 +1,69 @@
+# Usage: vcpkg_execute_build_process(COMMAND <cmd> [<args>...] [NO_PARALLEL_COMMAND <cmd> [<args>...]]
+#                                    WORKING_DIRECTORY </path/to/dir> LOGNAME <log_name>)
+function(vcpkg_execute_build_process)
+    cmake_parse_arguments(_ebp "" "WORKING_DIRECTORY;LOGNAME" "COMMAND;NO_PARALLEL_COMMAND" ${ARGN})
+
+    set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-out.log")
+    set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-err.log")
+
+    execute_process(
+        COMMAND ${_ebp_COMMAND}
+        WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
+        OUTPUT_FILE ${LOG_OUT}
+        ERROR_FILE ${LOG_ERR}
+        RESULT_VARIABLE error_code
+    )
+
+    if(error_code)
+        file(READ ${LOG_OUT} out_contents)
+        file(READ ${LOG_ERR} err_contents)
+
+        if(out_contents)
+            list(APPEND LOGS ${LOG_OUT})
+        endif()
+        if(err_contents)
+            list(APPEND LOGS ${LOG_ERR})
+        endif()
+
+        if(_ebp_NO_PARALLEL_COMMAND AND
+           (out_contents MATCHES "LINK : fatal error LNK1102:" OR out_contents MATCHES " fatal error C1060: "
+           OR err_contents MATCHES "LINK : fatal error LNK1102:" OR err_contents MATCHES " fatal error C1060: "))
+            # The linker ran out of memory during execution. We will try continuing once more, with parallelism disabled.
+            message(STATUS "Restarting Build without parallelism because memory exceeded")
+            set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-out-1.log")
+            set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-err-1.log")
+            execute_process(
+                COMMAND ${_ebp_NO_PARALLEL_COMMAND}
+                WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
+                OUTPUT_FILE ${LOG_OUT}
+                ERROR_FILE ${LOG_ERR}
+                RESULT_VARIABLE error_code
+            )
+
+            if(error_code)
+                file(READ ${LOG_OUT} out_contents)
+                file(READ ${LOG_ERR} err_contents)
+
+                if(out_contents)
+                    list(APPEND LOGS ${LOG_OUT})
+                endif()
+                if(err_contents)
+                    list(APPEND LOGS ${LOG_ERR})
+                endif()
+            endif()
+        endif()
+
+        if(error_code)
+            set(STRINGIFIED_LOGS)
+            foreach(LOG ${LOGS})
+                file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
+                list(APPEND STRINGIFIED_LOGS "    ${NATIVE_LOG}\n")
+            endforeach()
+            message(FATAL_ERROR
+                "  Command failed: ${_ebp_COMMAND}\n"
+                "  Working Directory: ${_ebp_WORKING_DIRECTORY}\n"
+                "  See logs for more information:\n"
+                ${STRINGIFIED_LOGS})
+        endif(error_code)
+    endif(error_code)
+endfunction(vcpkg_execute_build_process)
diff --git a/toolsrc/include/vcpkg/base/system.h b/toolsrc/include/vcpkg/base/system.h
@@ -28,4 +28,6 @@ namespace vcpkg::System
     const Optional<fs::path>& get_program_files_32_bit();
 
     const Optional<fs::path>& get_program_files_platform_bitness();
+
+    int get_num_logical_cores();
 }
diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp
@@ -596,6 +596,11 @@ namespace vcpkg
 #else
     void System::register_console_ctrl_handler() {}
 #endif
+
+    int System::get_num_logical_cores()
+    {
+        return std::thread::hardware_concurrency();
+    }
 }
 
 namespace vcpkg::Debug

diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
@@ -334,6 +334,23 @@ namespace vcpkg::Build
         return ret;
     }
 
+    static int get_concurrency()
+    {
+        static int concurrency = []{
+        auto user_defined_concurrency = System::get_environment_variable("VCPKG_MAX_CONCURRENCY");
+        if (user_defined_concurrency)
+        {
+            return std::stoi(user_defined_concurrency.value_or_exit(VCPKG_LINE_INFO));
+        }
+        else
+        {
+            return System::get_num_logical_cores() + 1;
+        }
+        }();
+
+        return concurrency;
+    }
+
     static ExtendedBuildResult do_build_package(const VcpkgPaths& paths,
                                                 const PreBuildInfo& pre_build_info,
                                                 const PackageSpec& spec,
@@ -372,6 +389,7 @@ namespace vcpkg::Build
             {"_VCPKG_DOWNLOAD_TOOL", to_string(config.build_package_options.download_tool)},
             {"FEATURES", Strings::join(";", config.feature_list)},
             {"ALL_FEATURES", all_features},
+            {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())},
         };
 
         if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value())