ARROW-4786: [C++/Python] Support better parallelisation in manylinux1…

… base build Author: Uwe L. Korn <[email protected]> Closes #3824 from xhochy/ARROW-4786 and squashes the following commits: 444b2f2 <Uwe L. Korn> ARROW-4786: Support better parallelisation in manylinux1 base build
apache · Mar 6, 2019 · 24e6ef0 · 24e6ef0
1 parent 8070d65
commit 24e6ef0
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 5 deletions.
diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
@@ -18,20 +18,21 @@
 
 BOOST_VERSION=1.66.0
 BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
 
 curl -sL https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz -o /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
 tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
 mkdir /arrow_boost
 pushd /boost_${BOOST_VERSION_UNDERSCORE}
 ./bootstrap.sh
-./b2 tools/bcp
+./b2 -j${NCORES} tools/bcp
 ./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build algorithm locale format variant multiprecision/cpp_int /arrow_boost
 popd
 
 pushd /arrow_boost
 ls -l
 ./bootstrap.sh
-./bjam dll-path="'\$ORIGIN/'" cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-regex install
+./bjam -j${NCORES} dll-path="'\$ORIGIN/'" cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-regex install
 popd
 rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE} arrow_boost
 # Boost always install header-only parts but they also take up quite some space.

diff --git a/python/manylinux1/scripts/build_brotli.sh b/python/manylinux1/scripts/build_brotli.sh
@@ -27,9 +27,9 @@ cmake -DCMAKE_BUILD_TYPE=release \
     "-DCMAKE_C_FLAGS=-fPIC" \
     -DCMAKE_INSTALL_PREFIX=/usr \
     -DBUILD_SHARED_LIBS=OFF \
+    -GNinja \
     ..
-make -j5
-make install
+ninja install
 popd
 popd
 rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION}
diff --git a/python/manylinux1/scripts/build_openssl.sh b/python/manylinux1/scripts/build_openssl.sh
@@ -17,12 +17,13 @@
 # under the License.
 
 OPENSSL_VERSION="1.0.2q"
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
 
 wget --no-check-certificate https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O openssl-${OPENSSL_VERSION}.tar.gz
 tar xf openssl-${OPENSSL_VERSION}.tar.gz
 pushd openssl-${OPENSSL_VERSION}
 ./config -fpic shared --prefix=/usr
-make -j5
+make -j${NCORES}
 make install
 popd
 rm -rf openssl-${OPENSSL_VERSION}.tar.gz openssl-${OPENSSL_VERSION}