From c25305354199dd9b7e29d6a92a2301638559d811 Mon Sep 17 00:00:00 2001 From: driazati <9407960+driazati@users.noreply.github.com> Date: Fri, 27 May 2022 11:44:47 -0700 Subject: [PATCH] [skip ci][ci][docker] Prune all non-relevant images (#11491) Before this would leave around any image that could be used in CI. This PR changes it so that the `docker rmi` knows exactly which image is being used in CI so all others (even those that are being used in the same build but not currently on that node) are deleted This also adds some more logging so we can see what's going on and should help keep disk usage down. Co-authored-by: driazati --- Jenkinsfile | 88 +++++++++++++++++++++++++++++++---- jenkins/Build.groovy.j2 | 7 +++ jenkins/DockerBuild.groovy.j2 | 8 ++++ jenkins/Lint.groovy.j2 | 1 + jenkins/Prepare.groovy.j2 | 23 +++++++-- jenkins/Test.groovy.j2 | 15 +++++- jenkins/macros.j2 | 9 ++-- 7 files changed, 134 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ad04bf218e6e..6f434a8c813e 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-24T17:03:03.321649 +// Generated at 2022-05-27T11:07:02.305020 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -108,11 +108,7 @@ def per_exec_ws(folder) { def init_git() { checkout scm - // Clear out all Docker images that aren't going to be used - sh( - script: "docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' | { grep -vE '${ci_arm}|${ci_cpu}|${ci_gpu}|${ci_hexagon}|${ci_i386}|${ci_lint}|${ci_qemu}|${ci_wasm}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }", - label: 'Clean old Docker images', - ) + // Add more info about job node sh ( script: './tests/scripts/task_show_node_info.sh', @@ -143,6 +139,23 @@ def init_git() { } } +def docker_init(image) { + // Clear out all Docker images that aren't going to be used + sh( + script: """ + set -eux + docker image ls --all + IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}') + + echo -e "Found images:\\n\$IMAGES" + echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } + + docker image ls --all + """, + label: 'Clean old Docker images', + ) +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', @@ -304,6 +317,7 @@ def build_docker_images() { parallel 'ci-lint': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_lint') } @@ -311,6 +325,7 @@ def build_docker_images() { }, 'ci-cpu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_cpu') } @@ -318,6 +333,7 @@ def build_docker_images() { }, 'ci-gpu': { node('GPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_gpu') } @@ -325,6 +341,7 @@ def build_docker_images() { }, 'ci-qemu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_qemu') } @@ -332,6 +349,7 @@ def build_docker_images() { }, 'ci-i386': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_i386') } @@ -339,6 +357,7 @@ def build_docker_images() { }, 'ci-arm': { node('ARM') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_arm') } @@ -346,6 +365,7 @@ def build_docker_images() { }, 'ci-wasm': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_wasm') } @@ -353,6 +373,7 @@ def build_docker_images() { }, 'ci-hexagon': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_hexagon') } @@ -407,6 +428,7 @@ def lint() { 'Lint 1 of 2': { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { + docker_init(ci_lint) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -424,6 +446,7 @@ def lint() { 'Lint 2 of 2': { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { + docker_init(ci_lint) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -501,6 +524,7 @@ stage('Build') { if (!skip_ci) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") { + docker_init(ci_gpu) init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') @@ -547,6 +571,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") { + docker_init(ci_cpu) init_git() sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", @@ -586,6 +611,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") { + docker_init(ci_wasm) init_git() sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", @@ -610,6 +636,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") { + docker_init(ci_386) init_git() sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", @@ -643,6 +670,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") { + docker_init(ci_arm) init_git() sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", @@ -674,6 +702,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") { + docker_init(ci_qemu) init_git() sh ( script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", @@ -704,6 +733,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") { + docker_init(ci_hexagon) init_git() sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", @@ -748,6 +778,7 @@ def shard_run_unittest_GPU_1_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -813,6 +844,7 @@ def shard_run_unittest_GPU_2_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -864,6 +896,7 @@ def shard_run_unittest_GPU_3_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -912,6 +945,7 @@ def shard_run_integration_CPU_1_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { + docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -957,6 +991,7 @@ def shard_run_integration_CPU_2_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { + docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1002,6 +1037,7 @@ def shard_run_integration_CPU_3_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { + docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1047,6 +1083,7 @@ def shard_run_integration_CPU_4_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { + docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1092,6 +1129,7 @@ def shard_run_integration_CPU_5_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { + docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1137,6 +1175,7 @@ def shard_run_integration_CPU_6_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { + docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1183,6 +1222,7 @@ def shard_run_python_i386_1_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { + docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1229,6 +1269,7 @@ def shard_run_python_i386_2_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { + docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1274,6 +1315,7 @@ def shard_run_python_i386_3_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { + docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1319,6 +1361,7 @@ def shard_run_python_i386_4_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { + docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1364,6 +1407,7 @@ def shard_run_python_i386_5_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { + docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1410,6 +1454,7 @@ def shard_run_test_Hexagon_1_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { + docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1454,6 +1499,7 @@ def shard_run_test_Hexagon_2_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { + docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1497,6 +1543,7 @@ def shard_run_test_Hexagon_3_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { + docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1540,6 +1587,7 @@ def shard_run_test_Hexagon_4_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { + docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1583,6 +1631,7 @@ def shard_run_test_Hexagon_5_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { + docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1626,6 +1675,7 @@ def shard_run_test_Hexagon_6_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { + docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1669,6 +1719,7 @@ def shard_run_test_Hexagon_7_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { + docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1713,6 +1764,7 @@ def shard_run_integration_aarch64_1_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1757,6 +1809,7 @@ def shard_run_integration_aarch64_2_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1801,6 +1854,7 @@ def shard_run_integration_aarch64_3_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1845,6 +1899,7 @@ def shard_run_integration_aarch64_4_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1890,6 +1945,7 @@ def shard_run_topi_GPU_1_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1933,6 +1989,7 @@ def shard_run_topi_GPU_2_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1976,6 +2033,7 @@ def shard_run_topi_GPU_3_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2019,6 +2077,7 @@ def shard_run_topi_GPU_4_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2063,6 +2122,7 @@ def shard_run_frontend_GPU_1_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2106,6 +2166,7 @@ def shard_run_frontend_GPU_2_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2149,6 +2210,7 @@ def shard_run_frontend_GPU_3_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2192,6 +2254,7 @@ def shard_run_frontend_GPU_4_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2235,6 +2298,7 @@ def shard_run_frontend_GPU_5_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2278,6 +2342,7 @@ def shard_run_frontend_GPU_6_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { + docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2322,6 +2387,7 @@ def shard_run_topi_aarch64_1_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2370,6 +2436,7 @@ def shard_run_topi_aarch64_2_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2419,6 +2486,7 @@ def shard_run_frontend_aarch64_1_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2462,6 +2530,7 @@ def shard_run_frontend_aarch64_2_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { try { + docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2631,6 +2700,7 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { timeout(time: max_time, unit: 'MINUTES') { try { + docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu'], { sh( @@ -2675,6 +2745,7 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-qemu") { timeout(time: max_time, unit: 'MINUTES') { try { + docker_init(ci_qemu) init_git() withEnv(['PLATFORM=qemu'], { sh( @@ -2719,6 +2790,7 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") { timeout(time: max_time, unit: 'MINUTES') { try { + docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu'], { sh( @@ -2756,6 +2828,7 @@ stage('Test') { if (!skip_ci) { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") { + docker_init(ci_gpu) init_git() sh( script: """ @@ -2797,8 +2870,7 @@ stage('Test') { }, ) } -} -/* +}/* stage('Build packages') { parallel 'conda CPU': { node('CPU') { diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2 index 4b0b4ae2e2c8..7e19ce34e71e 100644 --- a/jenkins/Build.groovy.j2 +++ b/jenkins/Build.groovy.j2 @@ -62,6 +62,7 @@ stage('Build') { if (!skip_ci) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-gpu') }}) { + docker_init(ci_gpu) init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') @@ -79,6 +80,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-cpu') }}) { + docker_init(ci_cpu) init_git() sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", @@ -102,6 +104,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-wasm') }}) { + docker_init(ci_wasm) init_git() sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", @@ -126,6 +129,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-i386') }}) { + docker_init(ci_386) init_git() sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", @@ -143,6 +147,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('ARM-SMALL') { ws({{ m.per_exec_ws('tvm/build-arm') }}) { + docker_init(ci_arm) init_git() sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", @@ -160,6 +165,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-qemu') }}) { + docker_init(ci_qemu) init_git() sh ( script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", @@ -177,6 +183,7 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-hexagon') }}) { + docker_init(ci_hexagon) init_git() sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2 index 84bb8e3e376d..e9d80801a9d9 100644 --- a/jenkins/DockerBuild.groovy.j2 +++ b/jenkins/DockerBuild.groovy.j2 @@ -59,6 +59,7 @@ def build_docker_images() { parallel 'ci-lint': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_lint') } @@ -66,6 +67,7 @@ def build_docker_images() { }, 'ci-cpu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_cpu') } @@ -73,6 +75,7 @@ def build_docker_images() { }, 'ci-gpu': { node('GPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_gpu') } @@ -80,6 +83,7 @@ def build_docker_images() { }, 'ci-qemu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_qemu') } @@ -87,6 +91,7 @@ def build_docker_images() { }, 'ci-i386': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_i386') } @@ -94,6 +99,7 @@ def build_docker_images() { }, 'ci-arm': { node('ARM') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_arm') } @@ -101,6 +107,7 @@ def build_docker_images() { }, 'ci-wasm': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_wasm') } @@ -108,6 +115,7 @@ def build_docker_images() { }, 'ci-hexagon': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { + docker_init('none') init_git() build_image('ci_hexagon') } diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2 index 61c13cd407d0..40dad3aef7be 100644 --- a/jenkins/Lint.groovy.j2 +++ b/jenkins/Lint.groovy.j2 @@ -6,6 +6,7 @@ def lint() { num_shards=2, node='CPU-SMALL', ws='tvm/lint', + docker_image='ci_lint', ) %} sh ( diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2 index d7bf5e706b0b..2293c2b0a6c3 100644 --- a/jenkins/Prepare.groovy.j2 +++ b/jenkins/Prepare.groovy.j2 @@ -6,11 +6,7 @@ def per_exec_ws(folder) { def init_git() { checkout scm - // Clear out all Docker images that aren't going to be used - sh( - script: "docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %} | { grep -vE '{% for image in images %}{% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %}{% if not loop.last %}|{% endif %}{% endfor %}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }", - label: 'Clean old Docker images', - ) + // Add more info about job node sh ( script: './tests/scripts/task_show_node_info.sh', @@ -41,6 +37,23 @@ def init_git() { } } +def docker_init(image) { + // Clear out all Docker images that aren't going to be used + sh( + script: """ + set -eux + docker image ls --all + IMAGES=\$(docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %}) + + echo -e "Found images:\\n\$IMAGES" + echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } + + docker image ls --all + """, + label: 'Clean old Docker images', + ) +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/jenkins/Test.groovy.j2 b/jenkins/Test.groovy.j2 index a08c50905a05..9f949ae717c2 100644 --- a/jenkins/Test.groovy.j2 +++ b/jenkins/Test.groovy.j2 @@ -10,6 +10,7 @@ node="GPU", ws="tvm/ut-python-gpu", platform="gpu", + docker_image="ci_gpu", test_method_names=test_method_names, ) %} {% if shard_index == 1 %} @@ -44,6 +45,7 @@ num_shards=6, ws="tvm/integration-python-cpu", platform="cpu", + docker_image="ci_cpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} @@ -59,6 +61,7 @@ num_shards=5, ws="tvm/integration-python-i386", platform="i386", + docker_image="ci_i386", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }} @@ -78,6 +81,7 @@ node="CPU-SMALL", ws="tvm/test-hexagon", platform="hexagon", + docker_image="ci_hexagon", test_method_names=test_method_names, num_shards=7, ) %} @@ -98,6 +102,7 @@ node="ARM-SMALL", ws="tvm/ut-python-arm", platform="arm", + docker_image="ci_arm", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} @@ -114,6 +119,7 @@ num_shards=4, ws="tvm/topi-python-gpu", platform="gpu", + docker_image="ci_gpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} @@ -129,6 +135,7 @@ num_shards=6, ws="tvm/frontend-python-gpu", platform="gpu", + docker_image="ci_gpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} @@ -143,6 +150,7 @@ node="ARM-SMALL", ws="tvm/ut-python-arm", platform="arm", + docker_image="ci_arm", num_shards=2, test_method_names=test_method_names, ) %} @@ -163,6 +171,7 @@ node="ARM-SMALL", ws="tvm/frontend-python-arm", platform="arm", + docker_image="ci_arm", num_shards=2, test_method_names=test_method_names, ) %} @@ -191,6 +200,7 @@ stage('Test') { node="CPU-SMALL", ws="tvm/ut-python-cpu", platform="cpu", + docker_image="ci_cpu", ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} ci_setup(ci_cpu) @@ -207,6 +217,7 @@ stage('Test') { node="CPU-SMALL", ws="tvm/test-qemu", platform="qemu", + docker_image="ci_qemu", ) %} {{ m.download_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} add_microtvm_permissions() @@ -226,6 +237,7 @@ stage('Test') { node="CPU-SMALL", ws="tvm/frontend-python-cpu", platform="cpu", + docker_image="ci_cpu", ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }} ci_setup(ci_cpu) @@ -238,6 +250,7 @@ stage('Test') { if (!skip_ci) { node('GPU') { ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) { + docker_init(ci_gpu) init_git() {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} add_microtvm_permissions() @@ -256,4 +269,4 @@ stage('Test') { }, ) } -} +} \ No newline at end of file diff --git a/jenkins/macros.j2 b/jenkins/macros.j2 index 1c649e31fabf..5a641b73fea8 100644 --- a/jenkins/macros.j2 +++ b/jenkins/macros.j2 @@ -19,7 +19,7 @@ "workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}" {%- endmacro -%} -{% macro sharded_test_step(name, num_shards, node, ws, platform, test_method_names) %} +{% macro sharded_test_step(name, num_shards, node, ws, docker_image, platform, test_method_names) %} {% for shard_index in range(1, num_shards + 1) %} {% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %} @@ -28,6 +28,7 @@ def {{ method_name }}() { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { try { + docker_init({{ docker_image }}) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -51,11 +52,12 @@ def {{ method_name }}() { {% endfor %} {% endmacro %} -{% macro sharded_lint_step(name, num_shards, node, ws) %} +{% macro sharded_lint_step(name, num_shards, docker_image, node, ws) %} {% for shard_index in range(1, num_shards + 1) %} '{{ name }} {{ shard_index }} of {{ num_shards }}': { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { + docker_init({{ docker_image }}) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -71,13 +73,14 @@ def {{ method_name }}() { {% endmacro %} -{% macro test_step(name, node, ws, platform) %} +{% macro test_step(name, node, ws, docker_image, platform) %} '{{ name }}': { if (!skip_ci && is_docs_only_build != 1) { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { timeout(time: max_time, unit: 'MINUTES') { try { + docker_init({{ docker_image }}) init_git() withEnv(['PLATFORM={{ platform }}'], { {{ caller() | indent(width=12) | trim }}