Skip to content
This repository has been archived by the owner on Apr 17, 2019. It is now read-only.

Commit

Permalink
Merge branch 'develop' into fix/vars_case
Browse files Browse the repository at this point in the history
Signed-off-by: Kitsu <[email protected]>
  • Loading branch information
l4l committed Jun 29, 2018
2 parents 8ae73c2 + f6a6c4d commit 66220fc
Show file tree
Hide file tree
Showing 18 changed files with 840 additions and 348 deletions.
15 changes: 15 additions & 0 deletions .jenkinsci/build-coverage.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env groovy

def checkCoverageConditions() {
// trigger coverage if branch is master, or it is a open PR commit, or a merge request
def branch_coverage = ['master']

if ( params.coverage ) {
return true
}
else {
return env.GIT_LOCAL_BRANCH in branch_coverage || INITIAL_COMMIT_PR == "true" || MERGE_CONDITIONS_SATISFIED == "true"
}
}

return this
160 changes: 95 additions & 65 deletions .jenkinsci/debug-build.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,20 @@ def doDebugBuild(coverageEnabled=false) {
def dPullOrBuild = load ".jenkinsci/docker-pull-or-build.groovy"
def manifest = load ".jenkinsci/docker-manifest.groovy"
def pCommit = load ".jenkinsci/previous-commit.groovy"
def parallelism = params.PARALLELISM
def setter = load ".jenkinsci/set-parallelism.groovy"
def parallelism = setter.setParallelism(params.PARALLELISM)
def platform = sh(script: 'uname -m', returnStdout: true).trim()
def previousCommit = pCommit.previousCommitOrCurrent()
def prevCommit = pCommit.previousCommitOrCurrent()
// params are always null unless job is started
// this is the case for the FIRST build only.
// So just set this to same value as default.
// This is a known bug. See https://issues.jenkins-ci.org/browse/JENKINS-41929
if (!parallelism) {
parallelism = 4
}
if (env.NODE_NAME.contains('arm7')) {
parallelism = 1
}
sh "docker network create ${env.IROHA_NETWORK}"

def iC = dPullOrBuild.dockerPullOrUpdate("${platform}-develop-build",
"${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/develop/Dockerfile",
"${env.GIT_RAW_BASE_URL}/${previousCommit}/docker/develop/Dockerfile",
"${env.GIT_RAW_BASE_URL}/${prevCommit}/docker/develop/Dockerfile",
"${env.GIT_RAW_BASE_URL}/develop/docker/develop/Dockerfile",
['PARALLELISM': parallelism])

if (GIT_LOCAL_BRANCH == 'develop' && manifest.manifestSupportEnabled()) {
manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:develop-build",
["${DOCKER_REGISTRY_BASENAME}:x86_64-develop-build",
Expand All @@ -42,6 +36,61 @@ def doDebugBuild(coverageEnabled=false) {
manifest.manifestPush("${DOCKER_REGISTRY_BASENAME}:develop-build", login, password)
}
}

iC.inside(""
+ " -e IROHA_POSTGRES_HOST=${env.IROHA_POSTGRES_HOST}"
+ " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
+ " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+ " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+ " -v ${CCACHE_DIR}:${CCACHE_DIR}") {
def scmVars = checkout scm
def cmakeOptions = ""

if ( coverageEnabled ) {
cmakeOptions = " -DCOVERAGE=ON "
}
env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
env.IROHA_HOME = "/opt/iroha"
env.IROHA_BUILD = "${env.IROHA_HOME}/build"

sh """
ccache --version
ccache --show-stats
ccache --zero-stats
ccache --max-size=5G
"""
sh """
cmake \
-DTESTING=ON \
-H. \
-Bbuild \
-DCMAKE_BUILD_TYPE=${params.build_type} \
-DIROHA_VERSION=${env.IROHA_VERSION} \
${cmakeOptions}
"""
sh "cmake --build build -- -j${parallelism}"
sh "ccache --show-stats"
}
}

// do not implement any checks as coverage runs only on x86_64
def doPreCoverageStep() {
sh "docker load -i ${JENKINS_DOCKER_IMAGE_DIR}/${DOCKER_IMAGE_FILE}"
def iC = docker.image("${DOCKER_AGENT_IMAGE}")
iC.inside() {
sh "cmake --build build --target coverage.init.info"
}
}

// run specified test categories
def doTestStep(testList) {
if (env.NODE_NAME.contains('x86_64')) {
sh "docker load -i ${JENKINS_DOCKER_IMAGE_DIR}/${DOCKER_IMAGE_FILE}"
}

def iC = docker.image("${DOCKER_AGENT_IMAGE}")
sh "docker network create ${env.IROHA_NETWORK}"

docker.image('postgres:9.5').withRun(""
+ " -e POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+ " -e POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
Expand All @@ -52,63 +101,44 @@ def doDebugBuild(coverageEnabled=false) {
+ " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
+ " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+ " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+ " --network=${env.IROHA_NETWORK}"
+ " -v /var/jenkins/ccache:${CCACHE_DIR}"
+ " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}") {

def scmVars = checkout scm
def cmakeOptions = ""
if ( coverageEnabled ) {
cmakeOptions = " -DCOVERAGE=ON "
}
env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
env.IROHA_HOME = "/opt/iroha"
env.IROHA_BUILD = "${env.IROHA_HOME}/build"

sh """
ccache --version
ccache --show-stats
ccache --zero-stats
ccache --max-size=5G
"""
sh """
cmake \
-DTESTING=ON \
-H. \
-Bbuild \
-DCMAKE_BUILD_TYPE=Debug \
-DIROHA_VERSION=${env.IROHA_VERSION} \
${cmakeOptions}
"""
sh "cmake --build build -- -j${parallelism}"
sh "ccache --show-stats"
if ( coverageEnabled ) {
sh "cmake --build build --target coverage.init.info"
}
def testExitCode = sh(script: """cd build && ctest --output-on-failure""", returnStatus: true)
if (testExitCode != 0) {
currentBuild.result = "UNSTABLE"
}
if ( coverageEnabled ) {
sh "cmake --build build --target cppcheck"
// Sonar
if (env.CHANGE_ID != null) {
sh """
sonar-scanner \
-Dsonar.github.disableInlineComments \
-Dsonar.github.repository='${DOCKER_REGISTRY_BASENAME}' \
-Dsonar.analysis.mode=preview \
-Dsonar.login=${SONAR_TOKEN} \
-Dsonar.projectVersion=${BUILD_TAG} \
-Dsonar.github.oauth=${SORABOT_TOKEN} \
-Dsonar.github.pullRequest=${CHANGE_ID}
"""
+ " --network=${env.IROHA_NETWORK}") {
def testExitCode = sh(script: """cd build && ctest --output-on-failure -R '${testList}' """, returnStatus: true)
if (testExitCode != 0) {
currentBuild.result = "UNSTABLE"
}
sh "cmake --build build --target coverage.info"
sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
}
}
}

// do cobertura analysis
// same as for doPreCoverageStep()
def doPostCoverageCoberturaStep() {
sh "docker load -i ${JENKINS_DOCKER_IMAGE_DIR}/${DOCKER_IMAGE_FILE}"
def iC = docker.image("${DOCKER_AGENT_IMAGE}")
iC.inside() {
sh "cmake --build build --target coverage.info"
sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
}
}

// do cppcheck analysis
// same as for doPreCoverageStep()
def doPostCoverageSonarStep() {
sh "docker load -i ${JENKINS_DOCKER_IMAGE_DIR}/${DOCKER_IMAGE_FILE}"
def iC = docker.image("${DOCKER_AGENT_IMAGE}")
iC.inside() {
sh "cmake --build build --target cppcheck"
sh """
sonar-scanner \
-Dsonar.github.disableInlineComments \
-Dsonar.github.repository='hyperledger/iroha' \
-Dsonar.analysis.mode=preview \
-Dsonar.login=${SONAR_TOKEN} \
-Dsonar.projectVersion=${BUILD_TAG} \
-Dsonar.github.oauth=${SORABOT_TOKEN} \
-Dsonar.github.pullRequest=${CHANGE_ID}
"""
}
}

Expand Down
26 changes: 16 additions & 10 deletions .jenkinsci/docker-cleanup.groovy
Original file line number Diff line number Diff line change
@@ -1,17 +1,23 @@
#!/usr/bin/env groovy

// remove docker network and stale images
def doDockerCleanup() {
sh """
# Check whether the image is the last-standing man
# i.e., no other tags exist for this image
docker rmi \$(docker images --no-trunc --format '{{.Repository}}:{{.Tag}}\\t{{.ID}}' | grep \$(docker images --no-trunc --format '{{.ID}}' ${iC.id}) | head -n -1 | cut -f 1) || true
"""
}

// cleanup docker network created for the test stage
def doDockerNetworkCleanup() {
sh "docker network rm ${env.IROHA_NETWORK}"
}

sh """
# Check whether the image is the last-standing man
# i.e., no other tags exist for this image
docker rmi \$(docker images --no-trunc --format '{{.Repository}}:{{.Tag}}\\t{{.ID}}' | grep \$(docker images --no-trunc --format '{{.ID}}' ${iC.id}) | head -n -1 | cut -f 1) || true
sleep 5
docker network rm $IROHA_NETWORK || true
#remove folder with iroha.deb package and Dockerfiles
rm -rf /tmp/${env.GIT_COMMIT}-${BUILD_NUMBER}
rm -rf /tmp/${env.GIT_COMMIT}
"""
// cleanup docker images which weren't used for more that 20 days and image for this PR in case of successful PR
def doStaleDockerImagesCleanup() {
sh "find ${JENKINS_DOCKER_IMAGE_DIR} -type f -mtime +20 -exec rm -f {} \\;"
sh "rm -f ${JENKINS_DOCKER_IMAGE_DIR}/${DOCKER_IMAGE_FILE}"
}

return this
44 changes: 33 additions & 11 deletions .jenkinsci/docker-pull-or-build.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -19,38 +19,60 @@ def buildOptionsString(options) {

def dockerPullOrUpdate(imageName, currentDockerfileURL, previousDockerfileURL, referenceDockerfileURL, buildOptions=null) {
buildOptions = buildOptionsString(buildOptions)
def uploadExitCode = 0
// GIT_PREVIOUS_COMMIT is null for first PR build
def commit = sh(script: "echo ${GIT_LOCAL_BRANCH} | md5sum | cut -c 1-8", returnStdout: true).trim()
DOCKER_IMAGE_FILE = commit

if (remoteFilesDiffer(currentDockerfileURL, previousDockerfileURL)) {
// Dockerfile has been changed compared to the previous commit
// Worst case scenario. We cannot count on the local cache
// because Dockerfile may contain apt-get entries that would try to update
// from invalid (stale) addresses
iC = docker.build("${DOCKER_REGISTRY_BASENAME}:${commit}-${BUILD_NUMBER}", "${buildOptions} --no-cache -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
if (env.NODE_NAME.contains('x86_64')) {
sh "docker save -o ${JENKINS_DOCKER_IMAGE_DIR}/${DOCKER_IMAGE_FILE} ${DOCKER_REGISTRY_BASENAME}:${commit}-${BUILD_NUMBER}"
}
}
else {
// upload reference image (hyperledger/iroha:develop-build) (required in all use-cases in this execution branch)
if (env.NODE_NAME.contains('x86_64')) {
uploadExitCode = sh(script: "docker load -i ${JENKINS_DOCKER_IMAGE_DIR}/${imageName}", returnStatus: true)
if (uploadExitCode != 0) {
sh "echo 'Reference image ${DOCKER_REGISTRY_BASENAME}:${imageName} doesn't exist on the EFS"
}
}
def pullExitCode = sh(script: "docker pull ${DOCKER_REGISTRY_BASENAME}:${imageName}", returnStatus: true)
if (pullExitCode != 0) {
sh "echo 'Reference image ${DOCKER_REGISTRY_BASENAME}:${imageName} doesn't exist on the dockerhub"
}
else {
// save reference docker image into the file when it is doesn't exist on the EFS
if (uploadExitCode != 0) {
sh "docker save -o ${JENKINS_DOCKER_IMAGE_DIR}/${imageName} ${DOCKER_REGISTRY_BASENAME}:${imageName}"
}
}
// first commit in this branch or Dockerfile modified
if (remoteFilesDiffer(currentDockerfileURL, referenceDockerfileURL)) {
// if we're lucky to build on the same agent, image will be built using cache
iC = docker.build("${DOCKER_REGISTRY_BASENAME}:${commit}-${BUILD_NUMBER}", "$buildOptions -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
// for x86 download reference image and start build
iC = docker.build("${DOCKER_REGISTRY_BASENAME}:${commit}-${BUILD_NUMBER}", "${buildOptions} -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
if (env.NODE_NAME.contains('x86_64')) {
sh "docker save -o /tmp/docker/${imageName} ${DOCKER_REGISTRY_BASENAME}:${commit}-${BUILD_NUMBER}"
}
}
else {
// try pulling image from Dockerhub, probably image is already there
def testExitCode = sh(script: "docker pull ${DOCKER_REGISTRY_BASENAME}:${imageName}", returnStatus: true)
if (testExitCode != 0) {
// image does not (yet) exist on Dockerhub. Build it
iC = docker.build("${DOCKER_REGISTRY_BASENAME}:${commit}-${BUILD_NUMBER}", "$buildOptions --no-cache -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
}
else {
// no difference found compared to both previous and reference Dockerfile
iC = docker.image("${DOCKER_REGISTRY_BASENAME}:${imageName}")
}
// now we get reference image from the file (pull from dockerhub)
iC = docker.image("${DOCKER_REGISTRY_BASENAME}:${imageName}")
DOCKER_IMAGE_FILE = imageName
}
}
if (GIT_LOCAL_BRANCH ==~ /develop|master/) {
docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
iC.push(imageName)
}
}
DOCKER_AGENT_IMAGE = iC.imageName()
return iC
}

Expand Down
12 changes: 12 additions & 0 deletions .jenkinsci/enums.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// types of tests provided by the developers (can be found at the CMakeLists.txt files)
enum TestTypes {
module(0), integration(1), system(2), cmake(3), regression(4), benchmark(5), framework(6)
TestTypes(int order) {
this.order = order
}
private final int order
int getOrder() {
order
}
}
return this
Loading

0 comments on commit 66220fc

Please sign in to comment.