Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update dev.external workflow for CRIU #4521

Merged
merged 2 commits into from
May 1, 2023
Merged

Update dev.external workflow for CRIU #4521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
acb136d
Update dev.external workflow for CRIU
llxia Apr 19, 2023
36cc4e5
Merge branch 'master' into test12
LongyuZhang May 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 111 additions & 0 deletions buildenv/jenkins/openjdk_tests
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,60 @@ timestamps{
}
parallel testJobs
}
} else if (!JOB_NAME.contains("_imagePull") && !JOB_NAME.contains("_imageUpload") && params.TARGET == "dev.external" ) {
// trigger xxx_imageUpload test job, then xxx_imagePull test job
def commonLabel = "sw.tool.podman&&sw.tool.container.criu"
if (params.LABEL_ADDITION) {
commonLabel += "&&${params.LABEL_ADDITION}"
}
def imageUploadMap = [
'x86-64_linux' : [
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.broadwell"],
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.amd"],
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.broadwell"],
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.amd"],
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.skylake"]
]
]
def imagePullMap = [
'x86-64_linux' : [
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22"],
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8"]
]
]
if (params.PLATFORM && imageUploadMap[params.PLATFORM] != null && imagePullMap[params.PLATFORM] != null) {
def imageUploadJobName = "${JOB_NAME}_imageUpload"
def imagePullJobName = "${JOB_NAME}_imagePull"
generateJobViaAutoGen(imageUploadJobName)
generateJobViaAutoGen(imagePullJobName)

def imageUploadJobs = [:]
def target = "testList TESTLIST=disabled.criu_pingPerf_testCreateRestoreImageAndPushToRegistry,disabled.criu-portable-checkpoint_test,disabled.criu-ubi-portable-checkpoint_test"
echo "Trigger ${imageUploadJobName} job ..."
for (int i = 0; i < imageUploadMap[params.PLATFORM].size(); i++) {
def labelAddition = imageUploadMap[params.PLATFORM][i].LABEL_ADDITION
def newParams = ["LABEL_ADDITION" : labelAddition, "TARGET" : "${target}", "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
def childParams = changeParam(params, newParams)
imageUploadJobs["${imageUploadJobName}_${i}]"] = {
build job: imageUploadJobName, parameters: childParams
}
}
parallel imageUploadJobs

def imagePullJobs = [:]
echo "Trigger ${imagePullJobName} job ..."
for (int i = 0; i < imagePullMap[params.PLATFORM].size(); i++) {
def labelAddition = imagePullMap[params.PLATFORM][i].LABEL_ADDITION
def newParams = ["LABEL_ADDITION" : labelAddition, "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
def childParams = changeParam(params, newParams)
imagePullJobs["${imagePullJobName}_${i}]"] = {
build job: imagePullJobName, parameters: childParams
}
}
parallel imagePullJobs
} else {
assert false : "Cannot find key PLATFORM: ${params.PLATFORM} in imageGenMap: ${imageGenMap} and/or imagePullMap: ${imagePullMap}."
}
} else {
if (PLATFORM_MAP.containsKey(params.PLATFORM)) {
env.EXTRA_OPTIONS = params.EXTRA_OPTIONS ? params.EXTRA_OPTIONS : ""
Expand Down Expand Up @@ -466,3 +520,60 @@ def forceCleanWS() {
}
}

def changeParam(paramsList, newParams) {
def childParams = []
// loop through all the params and change the parameters if needed
paramsList.each { param ->
def value = param.value.toString()
newParams.each { newParam ->

if (param.key == newParam.key) {
value = newParam.value.toString()
}
}
if (value == "true" || value == "false") {
childParams << booleanParam(name: param.key, value: value.toBoolean())
} else {
childParams << string(name: param.key, value: value)
}
}
return childParams
}

def generateJobViaAutoGen(testJobName) {
def JobHelper = library(identifier: 'openjdk-jenkins-helper@master').JobHelper
if (params.GENERATE_JOBS || !JobHelper.jobIsRunnable(testJobName as String)) {
def jobParams = []
jobParams << string(name: 'TEST_JOB_NAME', value: testJobName)
jobParams << string(name: 'ARCH_OS_LIST', value: params.PLATFORM)
jobParams << booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: false)

def jdk_impl = params.JDK_IMPL ?: ""
def jdk_version = params.JDK_VERSION ?: ""
if (testJobName.startsWith("Test_openjdk")) {
def tokens = testJobName.split('_');
def level = ""
def group = ""
if (tokens.size() > 3) {
if (!jdk_version) {
jdk_version = tokens[1]
}
if (!jdk_impl) {
jdk_impl = tokens[2]
}
if (tokens[3].contains(".")) {
level = tokens[3].split("\\.")[0]
group = tokens[3].split("\\.")[1]
if (level && group) {
jobParams << string(name: 'LEVELS', value: level)
jobParams << string(name: 'GROUPS', value: group)
}
}
}
}
jobParams << string(name: 'JDK_IMPL', value: jdk_impl)
jobParams << string(name: 'JDK_VERSIONS', value: jdk_version)

build job: 'Test_Job_Auto_Gen', parameters: jobParams, propagate: true
}
}
33 changes: 10 additions & 23 deletions external/criu/pingPerf.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ pingPerfZipPath=""
testJDKPath=""
jdkVersion=""
restoreImage="ol-instanton-test-pingperf-restore"
job_name=""
build_number=""
job_name=$JOB_NAME
build_number=$BUILD_NUMBER
docker_registry_dir=""
docker_os="ubi"
node_label_current_os=""
Expand Down Expand Up @@ -185,37 +185,24 @@ pushImage() {

restore_ready_checkpoint_image_folder="${DOCKER_REGISTRY_URL}/${job_name}/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${node_label_current_os}-${node_label_micro_architecture}"
tagged_restore_ready_checkpoint_image_num="${restore_ready_checkpoint_image_folder}:${build_number}"
tagged_restore_ready_checkpoint_image_latest="${restore_ready_checkpoint_image_folder}:latest"

# Push a docker image with build_num for records
echo "tag $restoreImage $tagged_restore_ready_checkpoint_image_num"
sudo podman tag $restoreImage $tagged_restore_ready_checkpoint_image_num
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_num}"
sudo podman push $tagged_restore_ready_checkpoint_image_num

if [[ "$job_name" == *"criu_image_upload"* ]]; then
# Push another copy as the nightly default latest
echo "tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest"
sudo podman tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_latest} to docker registry"
sudo podman push $tagged_restore_ready_checkpoint_image_latest
fi

dockerRegistryLogout
}

getImageNameList() {
if [ ! -z "${docker_registry_dir}" ]; then
echo "Testing image from specified docker_registry_dir: $docker_registry_dir"
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/${docker_registry_dir}")
else
echo "Testing images from nightly builds"
image_os_combo_list=($CRIU_XLINUX_COMBO_LIST)
for image_os_combo in ${image_os_combo_list[@]}
do
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/criu_image_upload/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${image_os_combo}:latest")
done
fi

echo "Testing images from job_name:build_number : ${job_name}:${build_number}"
image_os_combo_list=($CRIU_XLINUX_COMBO_LIST)
for image_os_combo in ${image_os_combo_list[@]}
do
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/${job_name}/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${image_os_combo}:${build_number}")
done
}

pullImageUnprivilegedRestore() {
Expand Down Expand Up @@ -291,9 +278,9 @@ setup() {
IFS=':' read -r -a dir_array <<< "$docker_registry_dir"
job_name=${dir_array[0]}
build_number=${dir_array[1]}
echo "job_name: $job_name"
echo "build_number: $build_number"
fi
echo "job_name: $job_name"

node_label_micro_architecture=""
node_label_current_os=""
Expand Down
1 change: 0 additions & 1 deletion external/criuSettings.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,3 @@

export CRIU_XLINUX_COMBO_LIST=sw.os.ubuntu.22-hw.arch.x86.broadwell sw.os.ubuntu.22-hw.arch.x86.amd sw.os.rhel.8-hw.arch.x86.broadwell sw.os.rhel.8-hw.arch.x86.amd sw.os.rhel.8-hw.arch.x86.skylake
# not available: sw.os.ubuntu.22-hw.arch.x86.skylake
export CRIU_DEFAULT_IMAGE_JOB_NAME=criu_image_upload
37 changes: 11 additions & 26 deletions external/external.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ test=derby
testtarget=""
platform="linux_x86-64"
portable="false"
job_name=""
build_number=""
job_name=$JOB_NAME
build_number=$BUILD_NUMBER
node_name=""
node_labels=""
node_label_micro_architecture=""
Expand Down Expand Up @@ -293,22 +293,12 @@ if [ $command_type == "run" ]; then

restore_ready_checkpoint_image_folder="${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}"
tagged_restore_ready_checkpoint_image_num="${restore_ready_checkpoint_image_folder}:${build_number}"
tagged_restore_ready_checkpoint_image_latest="${restore_ready_checkpoint_image_folder}:latest"

# Push a docker image with build_num for records
echo "tagged_restore_ready_checkpoint_image_num is $tagged_restore_ready_checkpoint_image_num"
$container_commit --change='ENTRYPOINT ["/bin/bash", "/test_restore.sh"]' $test-test $tagged_restore_ready_checkpoint_image_num
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_num}"
$container_push $tagged_restore_ready_checkpoint_image_num

# Push another copy as the nightly default latest
echo "Change Tag from build_number to latest"
$container_tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest

if [[ "$job_name" == *"$criu_default_image_job_name"* ]]; then
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_latest} to docker registry"
$container_push $tagged_restore_ready_checkpoint_image_latest
fi
$container_logout $docker_registry_url
else
echo "Docker Registry is not available on this Jenkins"
Expand Down Expand Up @@ -338,18 +328,14 @@ if [ $command_type == "load" ]; then
fi

restore_docker_image_name_list=()
if [ ! -z "${docker_registry_dir}" ]; then
echo "Testing image from specified DOCKER_REGISTRY_DIR"
restore_docker_image_name_list+=("${docker_registry_url}/${docker_registry_dir}")
else
echo "Testing images from nightly builds"
image_os_micro_architecture_list=($criu_combo_os_microarch_list)
for image_os_micro_architecture in ${image_os_micro_architecture_list[@]}
do
restore_docker_image_name_list+=("${docker_registry_url}/$criu_default_image_job_name/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${image_os_micro_architecture}:latest")
done
fi


echo "Testing images from nightly builds"
image_os_micro_architecture_list=($criu_combo_os_microarch_list)
for image_os_micro_architecture in ${image_os_micro_architecture_list[@]}
do
restore_docker_image_name_list+=("${docker_registry_url}/$job_name/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${image_os_micro_architecture}:${build_number}")
done

echo "The host machine OS is ${node_label_current_os}, and micro-architecture is ${node_label_micro_architecture}"
for restore_docker_image_name in ${restore_docker_image_name_list[@]}
do
Expand Down Expand Up @@ -390,6 +376,5 @@ if [ $command_type == "clean" ]; then
fi
$container_rm -f $test-test; $container_rmi -f adoptopenjdk-$test-test:${JDK_VERSION}-$package-$docker_os-${JDK_IMPL}-$build_type
$container_rm -f restore-test
$container_rmi -f ${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:latest
$container_rmi -f ${docker_registry_url}/${criu_default_image_job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:latest
$container_rmi -f ${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:${build_number}
fi