diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub index 992e2cebf786..8f64e51aa04e 100644 --- a/dev-support/Jenkinsfile_GitHub +++ b/dev-support/Jenkinsfile_GitHub @@ -45,7 +45,10 @@ pipeline { // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure. TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite' EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes" - + // set build parallel + BUILD_THREAD = 4 + SUREFIRE_FIRST_PART_FORK_COUNT = '0.5C' + SUREFIRE_SECOND_PART_FORK_COUNT = '0.5C' // a global view of paths. parallel stages can land on the same host concurrently, so each // stage works in its own subdirectory. there is an "output" under each of these // directories, which we retrieve after the build is complete. diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index d9b1cf1fa7b7..af2bc8812829 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -119,6 +119,18 @@ function personality_parse_args delete_parameter "${i}" ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=} ;; + --build-thread=* + delete_parameter "${i}" + BUILD_THREAD=${i#*=} + ;; + --surefire-first-part-fork-count=* + delete_parameter "${i}" + SUREFIRE_FIRST_PART_FORK_COUNT=${i#*=} + ;; + --surefire-second-part-fork-count=* + delete_parameter "${i}" + SUREFIRE_SECOND_PART_FORK_COUNT=${i#*=} + ;; esac done } @@ -144,7 +156,13 @@ function personality_modules # At a few points, hbase modules can run build, test, etc. in parallel # Let it happen. Means we'll use more CPU but should be for short bursts. # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3 - extra="--threads=2 -DHBasePatchProcess" + if [[ -n "${BUILD_THREAD}" ]]; then + extra="--threads=${BUILD_THREAD}" + else + extra="--threads=2" + fi + + extra="${extra} -DHBasePatchProcess" if [[ "${PATCH_BRANCH}" = branch-1* ]]; then extra="${extra} -Dhttps.protocols=TLSv1.2" fi @@ -232,6 +250,15 @@ function personality_modules extra="${extra} -Dbuild.id=${BUILD_ID}" fi + # set forkCount + if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then + extra="${extra} -Dsurefire.firstPartForkCount=${SUREFIRE_FIRST_PART_FORK_COUNT}" + fi + + if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then + extra="${extra} -Dsurefire.secondPartForkCount=${SUREFIRE_SECOND_PART_FORK_COUNT}" + fi + # If the set of changed files includes CommonFSUtils then add the hbase-server # module to the set of modules (if not already included) to be tested for f in "${CHANGED_FILES[@]}" diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index 9652d1e471fa..8f48cf77c893 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -147,6 +147,16 @@ YETUS_ARGS+=("--github-use-emoji-vote") if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}") fi +# pass build parallelism in +if [[ -n "${BUILD_THREAD}" ]]; then + YETUS_ARGS+=("--build-thread=${BUILD_THREAD}") +fi +if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then + YETUS_ARGS+=("--surefire-first-part-fork-count=${SUREFIRE_FIRST_PART_FORK_COUNT}") +fi +if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then + YETUS_ARGS+=("--surefire-second-part-fork-count=${SUREFIRE_SECOND_PART_FORK_COUNT}") +fi echo "Launching yetus with command line:" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"