Index benchmark metadata (#346)

* Index benchmark metadata Signed-off-by: Raul Sevilla <[email protected]> * Add check for workload nodes Signed-off-by: Raul Sevilla <[email protected]> * Add informative log Signed-off-by: Raul Sevilla <[email protected]> * Fix delete CR Signed-off-by: Raul Sevilla <[email protected]> * Set static index variable Signed-off-by: Raul Sevilla <[email protected]> * Timestamp must be on milliseconds Signed-off-by: Raul Sevilla <[email protected]> * Send node count as integer Signed-off-by: Raul Sevilla <[email protected]>
cloud-bulldozer · Mar 23, 2022 · 86c1842 · 86c1842
1 parent 310676d
commit 86c1842
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 65 deletions.
diff --git a/utils/benchmark-operator.sh b/utils/benchmark-operator.sh
@@ -45,13 +45,14 @@ remove_benchmark_operator() {
 }
 
 ############################################################################
-# Creates a benchmark and wait for it to complete
+# Creates a benchmark, waits for it to complete and index benchmark metadata
 # Arguments:
 #   Benchmark CR
 #   Timeout in seconds
 ############################################################################
 run_benchmark() {
   source ${ripsaw_tmp}/bin/activate
+  local start_date=$(date +%s%3N)
   local rc=0
   if ! ripsaw benchmark run -f ${1} -t ${2}; then
     rc=1
@@ -66,6 +67,7 @@ run_benchmark() {
     done
     remove_cli
   fi
-  deactivate
+  local benchmark_name=$(cat ${1} | python -c 'import yaml; import sys; print(yaml.safe_load(sys.stdin.read())["metadata"]["name"])')
+  gen_metadata ${benchmark_name} ${start_date} $(date +%s%3N)
   return ${rc}
 }
diff --git a/utils/common.sh b/utils/common.sh
@@ -79,6 +79,11 @@ function check_pod_ready_state () {
 
 
 
+gen_spreadsheet_helper() {
+  pip install oauth2client>=4.1.3 gspread
+  python3 $(dirname $(realpath ${BASH_SOURCE[0]}))/csv_gen.py --sheetname ${1}-$(date "+%Y-%m-%dT%H:%M:%S") -c ${2} --email ${3} --service-account ${4}
+}
+
 ##############################################################################
 # Imports a CSV file into a google spreadsheet
 # Arguments:
@@ -87,12 +92,6 @@ function check_pod_ready_state () {
 #   Gmail email address
 #   Service account file
 ##############################################################################
-
-gen_spreadsheet_helper() {
-  pip install oauth2client>=4.1.3 gspread
-  python3 $(dirname $(realpath ${BASH_SOURCE[0]}))/csv_gen.py --sheetname ${1}-$(date "+%Y-%m-%dT%H:%M:%S") -c ${2} --email ${3} --service-account ${4}
-}
-
 gen_spreadsheet() {
   log "Installing requirements to generate spreadsheet"
   if [[ "${VIRTUAL_ENV}" != "" ]]; then
@@ -116,46 +115,48 @@ gen_spreadsheet() {
 #   start_date (epoch)
 #   end_date (epoch)
 ##############################################################################
-
 gen_metadata() {
-        BENCHMARK=$1
-        START_DATE=$2
-        END_DATE=$3
-
-        # construct all the required information
-        local VERSION_INFO=$(oc version -o json)
-        local INFRA_INFO=$(oc get infrastructure.config.openshift.io cluster -o json)
-        local PLATFORM=$(echo ${INFRA_INFO} | jq -r .spec.platformSpec.type)
-	if [[ ${PLATFORM} =~ "AWS" ]]; then
-          local CLUSTERTYPE=$(echo ${INFRA_INFO} | jq -r .status.platformStatus.aws.resourceTags[0].value)
-	fi
-        local CLUSTER_NAME=$(echo ${INFRA_INFO} | jq -r .status.infrastructureName)
-        local OCP_VERSION=$(echo ${VERSION_INFO} | jq -r .openshiftVersion)
-        local K8S_VERSION=$(echo ${VERSION_INFO} | jq -r .serverVersion.gitVersion)
-        local MASTER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/master= --no-headers | wc -l)
-        local WORKER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/worker= --no-headers | wc -l)
-        local INFRA_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/infra= --no-headers --ignore-not-found | wc -l)
-        local SDN_TYPE=$(oc get networks.operator.openshift.io cluster -o jsonpath="{.spec.defaultNetwork.type}")
-        if [[ ${PLATFORM} != "BareMetal" ]]; then
-          local MASTER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/master= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
-          local WORKLOAD_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/workload= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
-          local WORKER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/worker= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
-          if [[ ${INFRA_NODES} -gt 0 ]]; then
-            local INFRA_NODES_TYPE=$(oc get node --ignore-not-found -l node-role.kubernetes.io/infra= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
-          fi
-        fi
-        if [[ ${BENCHMARK} =~ "cyclictest" ]]; then
-          local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/cyclictest= --no-headers --ignore-not-found | wc -l)
-        elif [[ $BENCHMARK =~ "oslat" ]]; then
-          local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/oslat= --no-headers --ignore-not-found | wc -l)
-        elif [[ $BENCHMARK =~ "testpmd" ]]; then
-          local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/testpmd= --no-headers --ignore-not-found | wc -l)
-        else
-          local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/workload= --no-headers --ignore-not-found | wc -l)
-        fi
-        local TOTAL_NODES=$(oc get node --no-headers | wc -l)
-        local RESULT=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.state')
-        local UUID=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.uuid')
+  local BENCHMARK=$1
+  local START_DATE=$2
+  local END_DATE=$3
+
+  # construct all the required information
+  local VERSION_INFO=$(oc version -o json)
+  local INFRA_INFO=$(oc get infrastructure.config.openshift.io cluster -o json)
+  local PLATFORM=$(echo ${INFRA_INFO} | jq -r .spec.platformSpec.type)
+  if [[ ${PLATFORM} =~ "AWS" ]]; then
+    local CLUSTERTYPE=$(echo ${INFRA_INFO} | jq -r .status.platformStatus.aws.resourceTags[0].value)
+  fi
+  local CLUSTER_NAME=$(echo ${INFRA_INFO} | jq -r .status.infrastructureName)
+  local OCP_VERSION=$(echo ${VERSION_INFO} | jq -r .openshiftVersion)
+  local K8S_VERSION=$(echo ${VERSION_INFO} | jq -r .serverVersion.gitVersion)
+  local MASTER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/master= --no-headers | wc -l)
+  local WORKER_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/worker= --no-headers | wc -l)
+  local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/workload= --no-headers --ignore-not-found | wc -l)
+  local INFRA_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/infra= --no-headers --ignore-not-found | wc -l)
+  local SDN_TYPE=$(oc get networks.operator.openshift.io cluster -o jsonpath="{.spec.defaultNetwork.type}")
+  if [[ ${PLATFORM} != "BareMetal" ]]; then
+    local MASTER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/master= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
+    local WORKER_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/worker= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
+    if [[ ${WORKLOAD_NODES_COUNT} -gt 0 ]]; then
+      local WORKLOAD_NODES_TYPE=$(oc get node -l node-role.kubernetes.io/workload= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
+    fi
+    if [[ ${INFRA_NODES_COUNT} -gt 0 ]]; then
+      local INFRA_NODES_TYPE=$(oc get node --ignore-not-found -l node-role.kubernetes.io/infra= --no-headers -o go-template='{{index (index .items 0).metadata.labels "beta.kubernetes.io/instance-type"}}')
+    fi
+  fi
+  if [[ ${BENCHMARK} =~ "cyclictest" ]]; then
+    local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/cyclictest= --no-headers --ignore-not-found | wc -l)
+  elif [[ $BENCHMARK =~ "oslat" ]]; then
+    local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/oslat= --no-headers --ignore-not-found | wc -l)
+  elif [[ $BENCHMARK =~ "testpmd" ]]; then
+    local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/testpmd= --no-headers --ignore-not-found | wc -l)
+  else
+    local WORKLOAD_NODES_COUNT=$(oc get node -l node-role.kubernetes.io/workload= --no-headers --ignore-not-found | wc -l)
+  fi
+  local TOTAL_NODES=$(oc get node --no-headers | wc -l)
+  local RESULT=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.state')
+  local UUID=$(oc get benchmark ${BENCHMARK} -o json | jq -r '.status.uuid')
 
 
 # stupid indentation because bash won't find the closing EOF if it's not at the beginning of the line
@@ -170,22 +171,23 @@ local METADATA=$(cat << EOF
 "worker_nodes_type":"${WORKER_NODES_TYPE}",
 "infra_nodes_type":"${INFRA_NODES_TYPE}",
 "workload_nodes_type":"${INFRA_NODES_TYPE}",
-"master_nodes_count":"${MASTER_NODES_COUNT}",
-"worker_nodes_count":"${WORKER_NODES_COUNT}",
-"infra_nodes_count":"${INFRA_NODES_COUNT}",
+"master_nodes_count":${MASTER_NODES_COUNT},
+"worker_nodes_count":${WORKER_NODES_COUNT},
+"infra_nodes_count":${INFRA_NODES_COUNT},
 "workload_nodes_count":${WORKLOAD_NODES_COUNT},
-"total_nodes":"${TOTAL_NODES}",
+"total_nodes":${TOTAL_NODES},
 "sdn_type":"${SDN_TYPE}",
 "benchmark":"${BENCHMARK}",
-"start_date":"${START_DATE}",
+"timestamp":"${START_DATE}",
 "end_date":"${END_DATE}",
 "result":"${RESULT}"
 }
 EOF
 )
 
-# send the document to ES
-curl -X POST -H "Content-type: application/json" ${ES_SERVER}/${ES_INDEX}/_doc -d "${METADATA}"
+  # send the document to ES
+  log "Indexing benchmark metadata to ${ES_SERVER}/${ES_INDEX}"
+  curl -sS -X POST -H "Content-type: application/json" ${ES_SERVER}/${ES_INDEX}/_doc -d "${METADATA}"
 }
 
 
diff --git a/workloads/kube-burner/README.md b/workloads/kube-burner/README.md
@@ -43,12 +43,11 @@ Workloads can be tweaked with the following environment variables:
 | **PRELOAD_PERIOD**   | How long the preload stage will last | 2m |
 | **LOG_STREAMING**    | Enable log streaming of kube-burner pod | true |
 | **CLEANUP**          | Delete old namespaces for the selected workload before starting benchmark | false |
-| **CLEANUP_WHEN_FINISH** | Delete workload's namespaces after running it | false |
+| **CLEANUP_WHEN_FINISH** | Delete benchmark objects and workload's namespaces after running it | false |
 | **KUBE_BURNER_IMAGE** | Kube-burner container image | quay.io/cloud-bulldozer/kube-burner:v0.14.3 |
 | **LOG_LEVEL**        | Kube-burner log level | info |
 | **PPROF_COLLECTION** | Collect and store pprof data locally | false |
 | **PPROF_COLLECTION_INTERVAL** | Intervals for which pprof data will be collected | 5m | 
-| **TEST_CLEANUP**     | Remove benchmark CR at the end | true |
 | **POD_READY_THRESHOLD** | Pod ready latency threshold (only applies node-density and pod-density workloads). [More info](https://kube-burner.readthedocs.io/en/latest/measurements/#pod-latency-thresholds) | 5000ms |
 
 **Note**: You can use basic authentication for ES indexing using the notation `http(s)://[username]:[password]@[host]:[port]` in **ES_SERVER**.

diff --git a/workloads/kube-burner/common.sh b/workloads/kube-burner/common.sh
@@ -72,16 +72,10 @@ run_workload() {
   rm -rf ${tmpdir}
   log "Deploying benchmark"
   set +e
-  local TMPCR=$(mktemp)
+  TMPCR=$(mktemp)
   envsubst < $1 > ${TMPCR}
   run_benchmark ${TMPCR} $((JOB_TIMEOUT + 600))
-  local rc=$?
-  if [[ ${TEST_CLEANUP} == "true" ]]; then
-    log "Cleaning up benchmark"
-    kubectl delete -f ${TMPCR}
-    kubectl delete configmap -n benchmark-operator kube-burner-cfg-${UUID}
-  fi
-  return ${rc}
+  rc=$?
 }
 
 label_nodes() {
@@ -142,6 +136,9 @@ check_running_benchmarks() {
 }
 
 cleanup() {
+  log "Cleaning up benchmark"
+  kubectl delete -f ${TMPCR}
+  kubectl delete configmap -n benchmark-operator kube-burner-cfg-${UUID}
   if ! oc delete ns -l kube-burner-uuid=${UUID} --grace-period=600 --timeout=30m; then
     log "Namespaces cleanup failure"
     rc=1

diff --git a/workloads/kube-burner/run.sh b/workloads/kube-burner/run.sh
@@ -88,7 +88,6 @@ if [[ ${PPROF_COLLECTION} == "true" ]] ; then
   get_pprof_secrets
 fi 
 run_workload kube-burner-crd.yaml
-rc=$?
 if [[ ${WORKLOAD} == node-density* ]]; then
   unlabel_nodes
 fi

diff --git a/workloads/network-perf/env.sh b/workloads/network-perf/env.sh
@@ -2,6 +2,7 @@
 
 TEST_CLEANUP=${TEST_CLEANUP:-true}
 export ES_SERVER=${ES_SERVER:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443}
+export ES_INDEX=ripsaw-uperf-results
 export METADATA_COLLECTION=${METADATA_COLLECTION:-true}
 export METADATA_TARGETED=${METADATA_TARGETED:-true}
 

diff --git a/workloads/scale-perf/env.sh b/workloads/scale-perf/env.sh
@@ -2,6 +2,7 @@
 TEST_CLEANUP=${TEST_CLEANUP:-true}
 export UUID=${UUID:-$(uuidgen)}
 export ES_SERVER=${ES_SERVER:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443}
+export ES_INDEX=openshift-cluster-timings
 export METADATA_COLLECTION=${METADATA_COLLECTION:-false}
 export CLOUD_NAME=${CLOUD_NAME:-test_cloud}
 if [[ -n $UUID ]]; then