From 9b1df7921c430d08c9a585e60b2185a3c6b737e4 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 21 Mar 2022 10:02:34 +0100 Subject: [PATCH 1/7] Index benchmark metadata Signed-off-by: Raul Sevilla --- utils/benchmark-operator.sh | 6 ++- utils/common.sh | 94 ++++++++++++++++----------------- workloads/kube-burner/README.md | 3 +- workloads/kube-burner/common.sh | 11 ++-- workloads/kube-burner/run.sh | 1 - 5 files changed, 55 insertions(+), 60 deletions(-) diff --git a/utils/benchmark-operator.sh b/utils/benchmark-operator.sh index 13189423..91af2691 100644 --- a/utils/benchmark-operator.sh +++ b/utils/benchmark-operator.sh @@ -45,13 +45,14 @@ remove_benchmark_operator() { } ############################################################################ -# Creates a benchmark and wait for it to complete +# Creates a benchmark, waits for it to complete and index benchmark metadata # Arguments: # Benchmark CR # Timeout in seconds ############################################################################ run_benchmark() { source ${ripsaw_tmp}/bin/activate + local start_date=$(date +%s) local rc=0 if ! ripsaw benchmark run -f ${1} -t ${2}; then rc=1 @@ -66,6 +67,7 @@ run_benchmark() { done remove_cli fi - deactivate + local benchmark_name=$(cat ${1} | python -c 'import yaml; import sys; print(yaml.safe_load(sys.stdin.read())["metadata"]["name"])') + gen_metadata ${benchmark_name} ${start_date} $(date +%s) return ${rc} } diff --git a/utils/common.sh b/utils/common.sh index 88560b65..79e0fd28 100644 --- a/utils/common.sh +++ b/utils/common.sh @@ -79,6 +79,11 @@ function check_pod_ready_state () { +gen_spreadsheet_helper() { + pip install oauth2client>=4.1.3 gspread + python3 $(dirname $(realpath ${BASH_SOURCE[0]}))/csv_gen.py --sheetname ${1}-$(date "+%Y-%m-%dT%H:%M:%S") -c ${2} --email ${3} --service-account ${4} +} + ############################################################################## # Imports a CSV file into a google spreadsheet # Arguments: @@ -87,12 +92,6 @@ function check_pod_ready_state () { # Gmail email address # Service account file ############################################################################## - -gen_spreadsheet_helper() { - pip install oauth2client>=4.1.3 gspread - python3 $(dirname $(realpath ${BASH_SOURCE[0]}))/csv_gen.py --sheetname ${1}-$(date "+%Y-%m-%dT%H:%M:%S") -c ${2} --email ${3} --service-account ${4} -} - gen_spreadsheet() { log "Installing requirements to generate spreadsheet" if [[ "${VIRTUAL_ENV}" != "" ]]; then @@ -116,46 +115,45 @@ gen_spreadsheet() { # start_date (epoch) # end_date (epoch) ############################################################################## - gen_metadata() { - BENCHMARK=$1 - START_DATE=$2 - END_DATE=$3 - - # construct all the required information - local VERSION_INFO=$(oc version -o json) - local INFRA_INFO=$(oc get infrastructure.config.openshift.io cluster -o json) - local PLATFORM=$(echo ${INFRA_INFO} | jq -r .spec.platformSpec.type) - if [[ ${PLATFORM} =~ "AWS" ]]; then - local CLUSTERTYPE=$(echo ${INFRA_INFO} | jq -r .status.platformStatus.aws.resourceTags[0].value) - fi - local CLUSTER_NAME=$(echo ${INFRA_INFO} | jq -r .status.infrastructureName) - local OCP_VERSION=$(echo ${VERSION_INFO} | jq -r .openshiftVersion) - local K8S_VERSION=$(echo ${VERSION_INFO} | jq -r .serverVersion.gitVersion) - local MASTER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/master= --no-headers | wc -l) - local WORKER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/worker= --no-headers | wc -l) - local INFRA_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/infra= --no-headers --ignore-not-found | wc -l) - local SDN_TYPE=$(oc get networks.operator.openshift.io cluster -o jsonpath="{.spec.defaultNetwork.type}") - if [[ ${PLATFORM} != "BareMetal" ]]; then - local MASTER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/master= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') - local WORKLOAD_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/workload= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') - local WORKER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/worker= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') - if [[ ${INFRA_NODES} -gt 0 ]]; then - local INFRA_NODES_TYPE=$(oc get node --ignore-not-found -l node-role.kubernetes.io/infra= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') - fi - fi - if [[ ${BENCHMARK} =~ "cyclictest" ]]; then - local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/cyclictest= --no-headers --ignore-not-found | wc -l) - elif [[ $BENCHMARK =~ "oslat" ]]; then - local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/oslat= --no-headers --ignore-not-found | wc -l) - elif [[ $BENCHMARK =~ "testpmd" ]]; then - local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/testpmd= --no-headers --ignore-not-found | wc -l) - else - local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/workload= --no-headers --ignore-not-found | wc -l) - fi - local TOTAL_NODES=$(oc get node --no-headers | wc -l) - local RESULT=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.state') - local UUID=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.uuid') + local BENCHMARK=$1 + local START_DATE=$2 + local END_DATE=$3 + + # construct all the required information + local VERSION_INFO=$(oc version -o json) + local INFRA_INFO=$(oc get infrastructure.config.openshift.io cluster -o json) + local PLATFORM=$(echo ${INFRA_INFO} | jq -r .spec.platformSpec.type) + if [[ ${PLATFORM} =~ "AWS" ]]; then + local CLUSTERTYPE=$(echo ${INFRA_INFO} | jq -r .status.platformStatus.aws.resourceTags[0].value) + fi + local CLUSTER_NAME=$(echo ${INFRA_INFO} | jq -r .status.infrastructureName) + local OCP_VERSION=$(echo ${VERSION_INFO} | jq -r .openshiftVersion) + local K8S_VERSION=$(echo ${VERSION_INFO} | jq -r .serverVersion.gitVersion) + local MASTER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/master= --no-headers | wc -l) + local WORKER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/worker= --no-headers | wc -l) + local INFRA_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/infra= --no-headers --ignore-not-found | wc -l) + local SDN_TYPE=$(oc get networks.operator.openshift.io cluster -o jsonpath="{.spec.defaultNetwork.type}") + if [[ ${PLATFORM} != "BareMetal" ]]; then + local MASTER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/master= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') + local WORKLOAD_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/workload= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') + local WORKER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/worker= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') + if [[ ${INFRA_NODES} -gt 0 ]]; then + local INFRA_NODES_TYPE=$(oc get node --ignore-not-found -l node-role.kubernetes.io/infra= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') + fi + fi + if [[ ${BENCHMARK} =~ "cyclictest" ]]; then + local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/cyclictest= --no-headers --ignore-not-found | wc -l) + elif [[ $BENCHMARK =~ "oslat" ]]; then + local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/oslat= --no-headers --ignore-not-found | wc -l) + elif [[ $BENCHMARK =~ "testpmd" ]]; then + local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/testpmd= --no-headers --ignore-not-found | wc -l) + else + local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/workload= --no-headers --ignore-not-found | wc -l) + fi + local TOTAL_NODES=$(oc get node --no-headers | wc -l) + local RESULT=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.state') + local UUID=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.uuid') # stupid indentation because bash won't find the closing EOF if it's not at the beginning of the line @@ -177,15 +175,15 @@ local METADATA=$(cat << EOF "total_nodes":"${TOTAL_NODES}", "sdn_type":"${SDN_TYPE}", "benchmark":"${BENCHMARK}", -"start_date":"${START_DATE}", +"timestamp":"${START_DATE}", "end_date":"${END_DATE}", "result":"${RESULT}" } EOF ) -# send the document to ES -curl -X POST -H "Content-type: application/json" ${ES_SERVER}/${ES_INDEX}/_doc -d "${METADATA}" + # send the document to ES + curl -X POST -H "Content-type: application/json" ${ES_SERVER}/${ES_INDEX}/_doc -d "${METADATA}" } diff --git a/workloads/kube-burner/README.md b/workloads/kube-burner/README.md index 35d6cb23..7bc90034 100644 --- a/workloads/kube-burner/README.md +++ b/workloads/kube-burner/README.md @@ -41,12 +41,11 @@ Workloads can be tweaked with the following environment variables: | **PRELOAD_PERIOD** | How long the preload stage will last | 2m | | **LOG_STREAMING** | Enable log streaming of kube-burner pod | true | | **CLEANUP** | Delete old namespaces for the selected workload before starting benchmark | false | -| **CLEANUP_WHEN_FINISH** | Delete workload's namespaces after running it | false | +| **CLEANUP_WHEN_FINISH** | Delete benchmark objects and workload's namespaces after running it | false | | **KUBE_BURNER_IMAGE** | Kube-burner container image | quay.io/cloud-bulldozer/kube-burner:v0.14.3 | | **LOG_LEVEL** | Kube-burner log level | info | | **PPROF_COLLECTION** | Collect and store pprof data locally | false | | **PPROF_COLLECTION_INTERVAL** | Intervals for which pprof data will be collected | 5m | -| **TEST_CLEANUP** | Remove benchmark CR at the end | true | | **POD_READY_THRESHOLD** | Pod ready latency threshold (only applies node-density and pod-density workloads). [More info](https://kube-burner.readthedocs.io/en/latest/measurements/#pod-latency-thresholds) | 5000ms | **Note**: You can use basic authentication for ES indexing using the notation `http(s)://[username]:[password]@[host]:[port]` in **ES_SERVER**. diff --git a/workloads/kube-burner/common.sh b/workloads/kube-burner/common.sh index cd5f196d..d92f59d9 100644 --- a/workloads/kube-burner/common.sh +++ b/workloads/kube-burner/common.sh @@ -75,13 +75,7 @@ run_workload() { local TMPCR=$(mktemp) envsubst < $1 > ${TMPCR} run_benchmark ${TMPCR} $((JOB_TIMEOUT + 600)) - local rc=$? - if [[ ${TEST_CLEANUP} == "true" ]]; then - log "Cleaning up benchmark" - kubectl delete -f ${TMPCR} - kubectl delete configmap -n benchmark-operator kube-burner-cfg-${UUID} - fi - return ${rc} + rc=$? } label_nodes() { @@ -142,6 +136,9 @@ check_running_benchmarks() { } cleanup() { + log "Cleaning up benchmark" + kubectl delete -f ${TMPCR} + kubectl delete configmap -n benchmark-operator kube-burner-cfg-${UUID} if ! oc delete ns -l kube-burner-uuid=${UUID} --grace-period=600 --timeout=30m; then log "Namespaces cleanup failure" rc=1 diff --git a/workloads/kube-burner/run.sh b/workloads/kube-burner/run.sh index 1a4120d4..a7edf360 100755 --- a/workloads/kube-burner/run.sh +++ b/workloads/kube-burner/run.sh @@ -73,7 +73,6 @@ if [[ ${PPROF_COLLECTION} == "true" ]] ; then get_pprof_secrets fi run_workload kube-burner-crd.yaml -rc=$? if [[ ${WORKLOAD} == node-density* ]]; then unlabel_nodes fi From fe017f98ec15826085fe07524b4aacd5db186bbd Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 21 Mar 2022 10:39:58 +0100 Subject: [PATCH 2/7] Add check for workload nodes Signed-off-by: Raul Sevilla --- utils/common.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/common.sh b/utils/common.sh index 79e0fd28..767dee5d 100644 --- a/utils/common.sh +++ b/utils/common.sh @@ -132,13 +132,16 @@ gen_metadata() { local K8S_VERSION=$(echo ${VERSION_INFO} | jq -r .serverVersion.gitVersion) local MASTER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/master= --no-headers | wc -l) local WORKER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/worker= --no-headers | wc -l) + local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/workload= --no-headers --ignore-not-found | wc -l) local INFRA_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/infra= --no-headers --ignore-not-found | wc -l) local SDN_TYPE=$(oc get networks.operator.openshift.io cluster -o jsonpath="{.spec.defaultNetwork.type}") if [[ ${PLATFORM} != "BareMetal" ]]; then local MASTER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/master= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') - local WORKLOAD_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/workload= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') local WORKER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/worker= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') - if [[ ${INFRA_NODES} -gt 0 ]]; then + if [[ ${WORKLOAD_NODES_COUNT} -gt 0 ]]; then + local WORKLOAD_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/workload= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') + fi + if [[ ${INFRA_NODES_COUNT} -gt 0 ]]; then local INFRA_NODES_TYPE=$(oc get node --ignore-not-found -l node-role.kubernetes.io/infra= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}') fi fi From abd3cdaa640fec0a5d51fd080f606a346b85667c Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 21 Mar 2022 10:53:47 +0100 Subject: [PATCH 3/7] Add informative log Signed-off-by: Raul Sevilla --- utils/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/common.sh b/utils/common.sh index 767dee5d..06aa988d 100644 --- a/utils/common.sh +++ b/utils/common.sh @@ -186,7 +186,8 @@ EOF ) # send the document to ES - curl -X POST -H "Content-type: application/json" ${ES_SERVER}/${ES_INDEX}/_doc -d "${METADATA}" + log "Indexing benchmark metadata to ${ES_SERVER}/${ES_INDEX}" + curl -sS -X POST -H "Content-type: application/json" ${ES_SERVER}/${ES_INDEX}/_doc -d "${METADATA}" } From a9714e4c70eecdd5cda84f744506a409bcfd4ba6 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 21 Mar 2022 11:00:11 +0100 Subject: [PATCH 4/7] Fix delete CR Signed-off-by: Raul Sevilla --- workloads/kube-burner/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workloads/kube-burner/common.sh b/workloads/kube-burner/common.sh index d92f59d9..09d700c4 100644 --- a/workloads/kube-burner/common.sh +++ b/workloads/kube-burner/common.sh @@ -72,7 +72,7 @@ run_workload() { rm -rf ${tmpdir} log "Deploying benchmark" set +e - local TMPCR=$(mktemp) + TMPCR=$(mktemp) envsubst < $1 > ${TMPCR} run_benchmark ${TMPCR} $((JOB_TIMEOUT + 600)) rc=$? From 1c993d098b3c92e7dd230150e60c11dda451a570 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 21 Mar 2022 11:26:44 +0100 Subject: [PATCH 5/7] Set static index variable Signed-off-by: Raul Sevilla --- workloads/network-perf/env.sh | 1 + workloads/scale-perf/env.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/workloads/network-perf/env.sh b/workloads/network-perf/env.sh index b7347679..bdbaa2fc 100644 --- a/workloads/network-perf/env.sh +++ b/workloads/network-perf/env.sh @@ -2,6 +2,7 @@ TEST_CLEANUP=${TEST_CLEANUP:-true} export ES_SERVER=${ES_SERVER:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443} +export ES_INDEX=ripsaw-uperf-results export METADATA_COLLECTION=${METADATA_COLLECTION:-true} export METADATA_TARGETED=${METADATA_TARGETED:-true} diff --git a/workloads/scale-perf/env.sh b/workloads/scale-perf/env.sh index 99dad005..e4b8281e 100644 --- a/workloads/scale-perf/env.sh +++ b/workloads/scale-perf/env.sh @@ -2,6 +2,7 @@ TEST_CLEANUP=${TEST_CLEANUP:-true} export UUID=${UUID:-$(uuidgen)} export ES_SERVER=${ES_SERVER:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443} +export ES_INDEX=openshift-cluster-timings export METADATA_COLLECTION=${METADATA_COLLECTION:-false} export CLOUD_NAME=${CLOUD_NAME:-test_cloud} if [[ -n $UUID ]]; then From fd080fa52efefc6cb25637d7c6fe54ce1aca9498 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 21 Mar 2022 14:00:02 +0100 Subject: [PATCH 6/7] Timestamp must be on milliseconds Signed-off-by: Raul Sevilla --- utils/benchmark-operator.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/benchmark-operator.sh b/utils/benchmark-operator.sh index 91af2691..8d095db6 100644 --- a/utils/benchmark-operator.sh +++ b/utils/benchmark-operator.sh @@ -52,7 +52,7 @@ remove_benchmark_operator() { ############################################################################ run_benchmark() { source ${ripsaw_tmp}/bin/activate - local start_date=$(date +%s) + local start_date=$(date +%s%3N) local rc=0 if ! ripsaw benchmark run -f ${1} -t ${2}; then rc=1 @@ -68,6 +68,6 @@ run_benchmark() { remove_cli fi local benchmark_name=$(cat ${1} | python -c 'import yaml; import sys; print(yaml.safe_load(sys.stdin.read())["metadata"]["name"])') - gen_metadata ${benchmark_name} ${start_date} $(date +%s) + gen_metadata ${benchmark_name} ${start_date} $(date +%s%3N) return ${rc} } From 533ff515e63901f90ab7881e3d6440fcbe2a6110 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 22 Mar 2022 11:25:37 +0100 Subject: [PATCH 7/7] Send node count as integer Signed-off-by: Raul Sevilla --- utils/common.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/common.sh b/utils/common.sh index 06aa988d..b373f9f7 100644 --- a/utils/common.sh +++ b/utils/common.sh @@ -171,11 +171,11 @@ local METADATA=$(cat << EOF "worker_nodes_type":"${WORKER_NODES_TYPE}", "infra_nodes_type":"${INFRA_NODES_TYPE}", "workload_nodes_type":"${INFRA_NODES_TYPE}", -"master_nodes_count":"${MASTER_NODES_COUNT}", -"worker_nodes_count":"${WORKER_NODES_COUNT}", -"infra_nodes_count":"${INFRA_NODES_COUNT}", +"master_nodes_count":${MASTER_NODES_COUNT}, +"worker_nodes_count":${WORKER_NODES_COUNT}, +"infra_nodes_count":${INFRA_NODES_COUNT}, "workload_nodes_count":${WORKLOAD_NODES_COUNT}, -"total_nodes":"${TOTAL_NODES}", +"total_nodes":${TOTAL_NODES}, "sdn_type":"${SDN_TYPE}", "benchmark":"${BENCHMARK}", "timestamp":"${START_DATE}",