Skip to content

Commit

Permalink
[GR-27858] [GR-27906] Align benchmarking methodology across benchmark…
Browse files Browse the repository at this point in the history 
… suites.

PullRequest: graal/7776
  • Loading branch information
@farquet
farquet committed Jan 22, 2021
2 parents e843ca8 + f66c1bf commit 8826307
Show file tree
Hide file tree
Showing 10 changed files with 252 additions and 249 deletions.
25 changes: 12 additions & 13 deletions compiler/ci_common/bench-aarch64.hocon
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,20 @@ aarch64.default-libgraal : ${aarch64.default} {
}
}

# TODO once GR-18514 is fixed, 'labsjdk-ce-11' should be replaced with 'labsjdk11'
builds += [
# jargraal
${aarch64.default} ${bench-dacapo} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, weekly], name: "bench-compiler-dacapo-11-linux-aarch64" }
${aarch64.default} ${bench-scala-dacapo} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, weekly], name: "bench-compiler-scala-dacapo-11-linux-aarch64", timelimit: "45:00" }
${aarch64.default} ${bench-renaissance-no-db-shootout} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, weekly], name: "bench-compiler-renaissance-11-linux-aarch64" }
${aarch64.default} ${bench-specjvm2008-Single} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, weekly], name: "bench-compiler-specjvm2008-Single-11-linux-aarch64" }
${aarch64.default} ${bench-specjbb2005} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, weekly], name: "bench-compiler-specjbb2005-11-linux-aarch64" }
${aarch64.default} ${bench-specjbb2015} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, weekly], name: "bench-compiler-specjbb2015-11-linux-aarch64" }
${aarch64.default} ${bench-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-dacapo-11-linux-aarch64" }
${aarch64.default} ${bench-scala-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-scala-dacapo-11-linux-aarch64", timelimit: "45:00" }
${aarch64.default} ${bench-renaissance-no-db-shootout} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-renaissance-11-linux-aarch64" }
${aarch64.default} ${bench-specjvm2008-Single} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjvm2008-Single-11-linux-aarch64" }
${aarch64.default} ${bench-specjbb2005} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjbb2005-11-linux-aarch64" }
${aarch64.default} ${bench-specjbb2015} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, weekly], name: "bench-compiler-specjbb2015-11-linux-aarch64" }

# libgraal
${aarch64.default-libgraal} ${bench-dacapo} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, daily], name: "bench-compiler-dacapo-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-scala-dacapo} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, daily], name: "bench-compiler-scala-dacapo-libgraal-11-linux-aarch64", timelimit: "45:00" }
${aarch64.default-libgraal} ${bench-renaissance-no-db-shootout} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, daily], name: "bench-compiler-renaissance-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjvm2008-Single} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, daily], name: "bench-compiler-specjvm2008-Single-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjbb2005} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, daily], name: "bench-compiler-specjbb2005-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjbb2015} ${aarch64-bench-notifications} ${labsjdk-ce-11} { targets: [bench, daily], name: "bench-compiler-specjbb2015-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, daily], name: "bench-compiler-dacapo-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-scala-dacapo} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, daily], name: "bench-compiler-scala-dacapo-libgraal-11-linux-aarch64", timelimit: "45:00" }
${aarch64.default-libgraal} ${bench-renaissance-no-db-shootout} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, daily], name: "bench-compiler-renaissance-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjvm2008-Single} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, daily], name: "bench-compiler-specjvm2008-Single-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjbb2005} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, daily], name: "bench-compiler-specjbb2005-libgraal-11-linux-aarch64" }
${aarch64.default-libgraal} ${bench-specjbb2015} ${aarch64-bench-notifications} ${labsjdk11} { targets: [bench, daily], name: "bench-compiler-specjbb2015-libgraal-11-linux-aarch64" }
]
164 changes: 78 additions & 86 deletions compiler/ci_common/bench-hwloc.hocon
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# hwloc (Hardware Locality) provides means to bind a process to NUMA nodes
hwloc.bench-common : {
packages: {hwloc: ">=1.9"}
environment: {
BASE_BENCH_RESULTS_FILE_NAME: "bench-results.json"
LOG_FILE_NAME : "-output.log"
MACHINE_NODE: undefined
BENCH_RESULTS_FILE_PATH: undefined
Expand All @@ -18,159 +16,153 @@ hwloc.bench-common : {
"*output.log"
]
}
hwloc.scala.bench-common : ${hwloc.bench-common} {
setup: ${hwloc.bench-common.setup}
teardown : ${hwloc.bench-common.teardown} [
["bench-uploader.py", "serial-node0-${BASE_BENCH_RESULTS_FILE_NAME}"]
["bench-uploader.py", "serial-node1-${BASE_BENCH_RESULTS_FILE_NAME}"]
]
}

hwloc.cmd.nobackground.suffix : ["|","tee", "-a", "${LOGDIR}/node${MACHINE_NODE}${LOG_FILE_NAME}"]
hwloc.cmd.suffix : ${hwloc.cmd.nobackground.suffix} ["&"]
hwloc.cmd.background : ${hwloc.cmd.nobackground.suffix} ["&"]

# command prefix for binding a command to a certain node
hwloc.node0 : ["hwloc-bind", "--cpubind", "node:0", "--membind", "node:0", "--"]
hwloc.node1 : ["hwloc-bind", "--cpubind", "node:1", "--membind", "node:1", "--"]
hwloc.node : ["hwloc-bind", "--cpubind", "node:${MACHINE_NODE}", "--membind", "node:${MACHINE_NODE}", "--"]

export.node0: [export, "MACHINE_NODE=0", "BENCH_RESULTS_FILE_PATH=node0-${BASE_BENCH_RESULTS_FILE_NAME}"]
export.node1: [export, "MACHINE_NODE=1", "BENCH_RESULTS_FILE_PATH=node1-${BASE_BENCH_RESULTS_FILE_NAME}"]

export.serial.node0: [export, "MACHINE_NODE=0", "BENCH_RESULTS_FILE_PATH=serial-node0-${BASE_BENCH_RESULTS_FILE_NAME}"]
export.serial.node1: [export, "MACHINE_NODE=1", "BENCH_RESULTS_FILE_PATH=serial-node1-${BASE_BENCH_RESULTS_FILE_NAME}"]


bench-awfy-hwloc: ${bench-awfy} ${hwloc.bench-common} {
setup: ${bench-awfy.setup} ${hwloc.bench-common.setup}
bench-micros-graal-whitebox-hwloc: ${bench-micros-graal-whitebox} {
setup: ${bench-micros-graal-whitebox.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-awfy} ${bench-arguments} ${hwloc.cmd.suffix}
${export.node1}
${hwloc.node} ${bench-jvmci.mx-awfy} ${bench-arguments} ${hwloc.cmd.suffix}
[wait]
${hwloc.node} ${bench-jvmci.mx-micros-graal-whitebox} ${bench-arguments}
]
}

bench-dacapo-hwloc: ${bench-dacapo} ${hwloc.bench-common} {
setup: ${bench-dacapo.setup} ${hwloc.bench-common.setup}
bench-micros-graal-dist-hwloc: ${bench-micros-graal-dist}{
setup: ${bench-micros-graal-dist.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-dacapo} ${bench-arguments} ${hwloc.cmd.suffix}
${export.node1}
${hwloc.node} ${bench-jvmci.mx-dacapo} ${bench-arguments} ${hwloc.cmd.suffix}
[wait]
${hwloc.node} ${bench-jvmci.mx-micros-graal-dist} ${bench-arguments}
]
}

bench-dacapo-timing-hwloc: ${bench-dacapo-timing} ${hwloc.bench-common} {
setup: ${bench-dacapo-timing.setup} ${hwloc.bench-common.setup}
bench-awfy-hwloc: ${bench-awfy} ${hwloc.bench-common} {
setup: ${bench-awfy.setup} ${hwloc.bench-common.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-dacapo-timing} ${bench-arguments} ${hwloc.cmd.suffix}
${hwloc.node} ${bench-jvmci.mx-awfy} ${bench-arguments} ${hwloc.cmd.background}
${export.node1}
${hwloc.node} ${bench-jvmci.mx-dacapo-timing} ${bench-arguments} ${hwloc.cmd.suffix}
${hwloc.node} ${bench-jvmci.mx-awfy} ${bench-arguments} ${hwloc.cmd.background}
[wait]
]
}

# Scala DaCapo-related targets
bench-hwloc-scala-all : {
mx-scala-dacapo : ${bench-jvmci.mx-benchmark} ["scala-dacapo:*"]
mx-scala-dacapo-timing : ${bench-jvmci.mx-benchmark} ["scala-dacapo-timing:*"]
}
bench-hwloc-without-scala-actors : {
mx-scala-dacapo : ${bench-jvmci.mx-benchmark} ["scala-dacapo:~actors"]
mx-scala-dacapo-timing : ${bench-jvmci.mx-benchmark} ["scala-dacapo-timing:~actors"]
}
bench-hwloc-only-scala-actors : {
mx-scala-dacapo : ${bench-jvmci.mx-benchmark} ["scala-dacapo:actors"]
mx-scala-dacapo-timing : ${bench-jvmci.mx-benchmark} ["scala-dacapo-timing:actors"]
bench-dacapo-hwloc: ${bench-dacapo} {
setup: ${bench-dacapo.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-dacapo} ${bench-arguments}
]
}


bench-scala-dacapo-hwloc: ${bench-scala-dacapo} ${hwloc.scala.bench-common} {
setup: ${bench-scala-dacapo.setup} ${hwloc.scala.bench-common.setup}
bench-dacapo-timing-hwloc: ${bench-dacapo-timing} {
setup: ${bench-dacapo-timing.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-hwloc-without-scala-actors.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.suffix}
${export.node1}
${hwloc.node} ${bench-hwloc-without-scala-actors.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.suffix}
[wait]
${export.serial.node0}
${hwloc.node} ${bench-hwloc-only-scala-actors.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.nobackground.suffix}
${export.serial.node1}
${hwloc.node} ${bench-hwloc-only-scala-actors.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.nobackground.suffix}
${hwloc.node} ${bench-jvmci.mx-dacapo-timing} ${bench-arguments}
]
}

bench-scala-dacapo-all-hwloc: ${bench-scala-dacapo} ${hwloc.bench-common} {
bench-scala-dacapo-hwloc: ${bench-scala-dacapo} ${hwloc.bench-common} {
setup: ${bench-scala-dacapo.setup} ${hwloc.bench-common.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-hwloc-scala-all.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.suffix}
${hwloc.node} ${bench-jvmci.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.background}
${export.node1}
${hwloc.node} ${bench-hwloc-scala-all.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.suffix}
${hwloc.node} ${bench-jvmci.mx-scala-dacapo} ${bench-arguments} ${hwloc.cmd.background}
[wait]
]
}

bench-scala-dacapo-timing-hwloc: ${bench-scala-dacapo-timing} ${hwloc.scala.bench-common} {
setup: ${bench-scala-dacapo-timing.setup} ${hwloc.scala.bench-common.setup}
bench-scala-dacapo-timing-hwloc: ${bench-scala-dacapo-timing} ${hwloc.bench-common} {
setup: ${bench-scala-dacapo-timing.setup} ${hwloc.bench-common.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-hwloc-without-scala-actors.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.suffix}
${hwloc.node} ${bench-jvmci.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.background}
${export.node1}
${hwloc.node} ${bench-hwloc-without-scala-actors.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.suffix}
${hwloc.node} ${bench-jvmci.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.background}
[wait]
${export.serial.node0}
${hwloc.node} ${bench-hwloc-only-scala-actors.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.nobackground.suffix}
${export.serial.node1}
${hwloc.node} ${bench-hwloc-only-scala-actors.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.nobackground.suffix}
]
}

bench-scala-dacapo-all-timing-hwloc: ${bench-scala-dacapo-timing} ${hwloc.bench-common} {
setup: ${bench-scala-dacapo-timing.setup} ${hwloc.bench-common.setup}
bench-renaissance-hwloc: ${bench-renaissance} {
setup: ${bench-renaissance.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-hwloc-scala-all.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.suffix}
${export.node1}
${hwloc.node} ${bench-hwloc-scala-all.mx-scala-dacapo-timing} ${bench-arguments} ${hwloc.cmd.suffix}
[wait]
${hwloc.node} ${bench-jvmci.mx-renaissance} ${bench-arguments}
]
}

# Renaissance-related targets
# Parallel executions of renaissance not yet supported : https://github.com/renaissance-benchmarks/renaissance/issues/13
# Still running on a single socket for better benchmark stability (local RAM and CPUs) and for consistency with other suites
bench-renaissance-hwloc: ${bench-renaissance} ${hwloc.bench-common} {
setup: ${bench-renaissance.setup} ${hwloc.bench-common.setup}

bench-renaissance-0-10-0-hwloc: ${bench-renaissance-0-10-0} {
setup: ${bench-renaissance.setup}
run: [
${export.node0}
${hwloc.node} ${bench-jvmci.mx-renaissance} ${bench-arguments}
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-renaissance} ${bench-arguments}
]
teardown: [
["bench-uploader.py", "node0-${BASE_BENCH_RESULTS_FILE_NAME}"]
}


bench-renaissance-legacy-hwloc: ${bench-renaissance-legacy} {
setup: ${bench-renaissance-legacy.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-renaissance-legacy} ${bench-arguments}
]
}


bench-renaissance-legacy-hwloc: ${bench-renaissance-legacy} ${hwloc.bench-common} {
setup: ${bench-renaissance-legacy.setup} ${hwloc.bench-common.setup}
bench-specjvm2008-Single-hwloc: ${bench-specjvm2008-Single} {
setup: ${bench-specjvm2008-Single.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-renaissance-legacy} ${bench-arguments} ${hwloc.cmd.suffix}
${export.node1}
${hwloc.node} ${bench-jvmci.mx-renaissance-legacy} ${bench-arguments} ${hwloc.cmd.suffix}
[wait]
${hwloc.node} ${bench-jvmci.mx-specjvm2008-Single} ${bench-arguments}
]
}


bench-specjvm2008-OneVM-hwloc: ${bench-specjvm2008-OneVM} {
setup: ${bench-specjvm2008-OneVM.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-specjvm2008-OneVM} ${bench-arguments}
]
}


bench-specjbb2005-hwloc: ${bench-specjbb2005} {
setup: ${bench-specjbb2005.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-specjbb2005} ${bench-arguments} ["input.ending_number_warehouses=77"]
]
}

bench-specjbb2015-hwloc: ${bench-specjbb2015} {
setup: ${bench-specjbb2015.setup}
run: [
[export, "LOGDIR=${PWD}"]
${export.node0}
${hwloc.node} ${bench-jvmci.mx-specjbb2015} ${bench-arguments}
]
}
Loading

0 comments on commit 8826307

Please sign in to comment.