From f1d1075f59e587746eb30a42c0723f8b42b95ed4 Mon Sep 17 00:00:00 2001 From: Hsuan-zong wu Date: Thu, 1 Feb 2024 10:17:36 +0800 Subject: [PATCH 1/2] [YUNIKORN-2374] Add Performance testing tools --- deployments/kwok-perf-test/.gitignore | 1 + deployments/kwok-perf-test/affinity.sh | 195 +++++++++++++++++++++++ deployments/kwok-perf-test/priority.sh | 120 ++++++++++++++ deployments/kwok-perf-test/readme.md | 175 ++++++++++++++++++++ deployments/kwok-perf-test/toleration.sh | 146 +++++++++++++++++ 5 files changed, 637 insertions(+) create mode 100644 deployments/kwok-perf-test/.gitignore create mode 100755 deployments/kwok-perf-test/affinity.sh create mode 100755 deployments/kwok-perf-test/priority.sh create mode 100644 deployments/kwok-perf-test/readme.md create mode 100755 deployments/kwok-perf-test/toleration.sh diff --git a/deployments/kwok-perf-test/.gitignore b/deployments/kwok-perf-test/.gitignore new file mode 100644 index 000000000..048910837 --- /dev/null +++ b/deployments/kwok-perf-test/.gitignore @@ -0,0 +1 @@ +./output \ No newline at end of file diff --git a/deployments/kwok-perf-test/affinity.sh b/deployments/kwok-perf-test/affinity.sh new file mode 100755 index 000000000..885e972d1 --- /dev/null +++ b/deployments/kwok-perf-test/affinity.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RANDOM=0 +OUTPUT_PATH="./output/affinity.yaml" +NUM_PODS=0 +NUM_NODES=0 +OPERATORS=("In" "NotIn") +OPERATORS_STATE=0 +j=0 + +show_help() { + cat << EOF +Invalid option: -$OPTARG +Usage: $0 + +Options: + -o, Specifies the location of the output yaml file (default is ./output/affinity.yaml) + +Arguments: + Number of pod to create (required). + Number of kwok nodes (required). +EOF +} + +# Process command-line options +while getopts ":o:" opt; do + case $opt in + o) + OUTPUT_PATH=$OPTARG + ;; + \?) + show_help + exit 1 + ;; + :) + show_help + exit 1 + ;; + esac +done + +# Shift the processed options out of the command-line arguments +shift $((OPTIND-1)) + +# Check if pod count and node count are provided +if [ $# -ne 2 ]; then + show_help + exit 1 +fi + +NUM_PODS=$1 +NUM_NODES=$2 + +# clear origin content in file +if [ -f "$OUTPUT_PATH" ]; then + # Clear the content of the file + echo "" > "$OUTPUT_PATH" + echo "The original content of the file located at $OUTPUT_PATH has been cleared." +else + echo "The file $OUTPUT_PATH does not exist." + mkdir -p "$(dirname "$OUTPUT_PATH")" + touch "$OUTPUT_PATH" + echo "The create $OUTPUT_PATH." +fi + +# create pods assigned with random node affinity +echo "Create $((NUM_PODS/2/2)) pods, each with random node affinity using the required rule." +echo "Create $((NUM_PODS/2/2)) pods, each with random node affinity using the preferred rule." +for (( ;j> "$OUTPUT_PATH" +apiVersion: v1 +kind: Pod +metadata: + name: nginx-$j + labels: + applicationId: nginx-$j +spec: + containers: + - name: sleep300 + image: "alpine:latest" + command: ["sleep", "0"] + ports: + - containerPort: 80 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: $operator + values: + - kwok-node-$randHost1 + - kwok-node-$randHost2 + tolerations: + - key: "kwok.x-k8s.io/node" + operator: "Exists" + effect: "NoSchedule" +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx-$((j+1)) + labels: + applicationId: nginx-$((j+1)) +spec: + containers: + - name: sleep300 + image: "alpine:latest" + command: ["sleep", "0"] + ports: + - containerPort: 80 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: $operator + values: + - kwok-node-$randHost3 + - kwok-node-$randHost4 + tolerations: + - key: "kwok.x-k8s.io/node" + operator: "Exists" + effect: "NoSchedule" +--- +EOF +done + +# create pods assigned with random pod affinity +echo "Create $((NUM_PODS-j)) pods, each with random node affinity using the preferred rule." +for (( ;j> "$OUTPUT_PATH" +apiVersion: v1 +kind: Pod +metadata: + name: nginx-$((j)) + labels: + applicationId: nginx-$((j)) +spec: + containers: + - name: sleep300 + image: "alpine:latest" + command: ["sleep", "0"] + ports: + - containerPort: 80 + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: applicationId + operator: $operator + values: + - nginx-$randAppID1 + - nginx-$randAppID2 + topologyKey: kubernetes.io/role + tolerations: + - key: "kwok.x-k8s.io/node" + operator: "Exists" + effect: "NoSchedule" +--- +EOF +done diff --git a/deployments/kwok-perf-test/priority.sh b/deployments/kwok-perf-test/priority.sh new file mode 100755 index 000000000..328ed863a --- /dev/null +++ b/deployments/kwok-perf-test/priority.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RANDOM=0 +OUTPUT_PATH="./output/priority.yaml" +NUM_PODS=0 +NUM_PRIORITY=0 + +show_help() { + cat << EOF +Invalid option: -$OPTARG +Usage: $0 + +Options: + -o, Specifies the location of the output yaml file (default is ./output/priority.yaml) + +Arguments: + Number of pod to create (required). + Number of priorityClass to create (required). +EOF +} + +# Process command-line options +while getopts ":o:" opt; do + case $opt in + o) + OUTPUT_PATH=$OPTARG + ;; + \?) + show_help + exit 1 + ;; + :) + show_help + exit 1 + ;; + esac +done + +# Shift the processed options out of the command-line arguments +shift $((OPTIND-1)) + +# Check if pod count and priorityClass count are provided +if [ $# -ne 2 ]; then + show_help + exit 1 +fi + +NUM_PODS=$1 +NUM_PRIORITY=$2 + +# clear origin content in file +if [ -f "$OUTPUT_PATH" ]; then + # Clear the content of the file + echo "" > "$OUTPUT_PATH" + echo "The original content of the file located at $OUTPUT_PATH has been cleared." +else + echo "The file $OUTPUT_PATH does not exist." + mkdir -p "$(dirname "$OUTPUT_PATH")" + touch "$OUTPUT_PATH" + echo "The create $OUTPUT_PATH." +fi + +# create PriorityClass +echo "Create $((NUM_PRIORITY)) PriorityClass and save then to a YAML file." +for (( i=0;i> "$OUTPUT_PATH" +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: priority-$i +value: $i +preemptionPolicy: Never +globalDefault: false +--- +EOF +done + +# create pods assigned with random priorityClass name +echo "Create $((NUM_PODS)) Pods and save them to a YAML file, with each Pod assigned a PriorityClass selected at random." +for (( j=0;j> "$OUTPUT_PATH" +apiVersion: v1 +kind: Pod +metadata: + name: nginx-$j +spec: + containers: + - name: sleep300 + image: "alpine:latest" + command: ["sleep", "0"] + ports: + - containerPort: 80 + tolerations: + - key: "kwok.x-k8s.io/node" + operator: "Exists" + effect: "NoSchedule" + priorityClassName: priority-$randPriority +--- +EOF +done + \ No newline at end of file diff --git a/deployments/kwok-perf-test/readme.md b/deployments/kwok-perf-test/readme.md new file mode 100644 index 000000000..86f15f282 --- /dev/null +++ b/deployments/kwok-perf-test/readme.md @@ -0,0 +1,175 @@ + + +## Performance testing tools + +For the [performance evaluation of YuniKorn with Kwok](https://yunikorn.apache.org/docs/next/performance/evaluate_perf_function_with_kwok), we've conducted four distinct performance tests: Throughput, Taint & Tolerations, Affinity & Anti-Affinity, and PriorityClass. This folder contains the tools used in these tests, enabling you to easily replicate the tests or fine-tune the performance evaluation process. + +### Prerequisites +For this test, we're using `Kwok in a cluster` setup, which means you'll need to have a Kubernetes cluster ready beforehand. + +### Introduction of tools +- kwok-setup.sh script allows for the swift establishment of Kwok within your cluster, with the capability to generate nodes based on the quantity you define. + Example: + ``` + $ ./kwok-setup.sh 3 + ... + customresourcedefinition.apiextensions.k8s.io/attaches.kwok.x-k8s.io created + customresourcedefinition.apiextensions.k8s.io/clusterattaches.kwok.x-k8s.io created + customresourcedefinition.apiextensions.k8s.io/clusterexecs.kwok.x-k8s.io created + ... + node/kwok-node-0 created + node/kwok-node-1 created + node/kwok-node-2 created + ``` + +- deploy-tool.sh is designed for throughput testing, allowing for rapid deployment creation and deletion while enabling you to set the desired number of replicas. + Example: + ``` + $ ./deploy-tool.sh + Usage: ./deploy-tool.sh [-d] [-i ] + + Options: + -d, --delete Delete the specified number of deployments. + -i, --interval Set the interval between deployments in seconds. + + Arguments: + Number of deployments to create or delete (required). + Number of replicas for each deployment (required). + + $ ./deploy-tool.sh 3 10 + Deploying 3 deployments with 10 replicas each. + deployment.apps/sleep-deployment-0 created + deployment.apps/sleep-deployment-1 created + deployment.apps/sleep-deployment-2 created + + $ ./deploy-tool.sh -d 3 10 + Deleting 3 deployments with 10 replicas each. + deployment.apps "sleep-deployment-0" deleted + deployment.apps "sleep-deployment-1" deleted + deployment.apps "sleep-deployment-2" deleted + ``` + +- toleration.sh offers two key functionalities: firstly, it allows for the tainting and untainting of nodes; secondly, it facilitates the creation of Taint & Toleration test cases using YAML templates. + Example: + ``` + $ ./toleration.sh + Usage: ./toleration.sh + + Commands: + -t, Taint kwok nodes with its index + -u, Untaint kwok nodes with its index + -o, Specifies the location of the output yaml file (default is ./output/priority.yaml) + + Arguments: + Number of pod to create (required). + Number of kwok nodes (required). + + $ ./toleration.sh -t 10 3 + node/kwok-node-0 tainted + node/kwok-node-1 tainted + node/kwok-node-2 tainted + + $ ./toleration.sh -u 10 3 + node/kwok-node-0 untainted + node/kwok-node-1 untainted + node/kwok-node-2 untainted + + $ ./toleration.sh 10 3 + The file ./output/toleration.yaml does not exist. + Create file ./output/toleration.yaml. + Create 10 Pods and save them to a YAML file, each Pod with a randomly assigned toleration. + $ cat ./output/toleration.yaml + apiVersion: v1 + kind: Pod + metadata: + name: nginx-0 + ... + tolerations: + ... + - key: key-0 + operator: "Exists" + ``` + +- affinity.sh is designed to generate YAML-based test cases for both Affinity & Anti-Affinity. The script covers tests for node affinity and pod affinity, where node affinity is further categorized into required and preferred rules. For optimal testing, the specified `` must be divisible by 4. + Example: + ``` + $ ./affinity.sh + Usage: ./affinity.sh + + Options: + -o, Specifies the location of the output yaml file (default is ./output/affinity.yaml) + + Arguments: + Number of pod to create (required). + Number of kwok nodes (required). + + $ ./affinity.sh 8 3 + The original content of the file located at ./output/affinity.yaml has been cleared. + Create 2 pods, each with random node affinity using the required rule. + Create 2 pods, each with random node affinity using the preferred rule. + Create 4 pods, each with random node affinity using the preferred rule. + $ cat output/affinity.yaml + apiVersion: v1 + kind: Pod + metadata: + name: nginx-0 + ... + affinity: + nodeAffinity: + ... + - key: kubernetes.io/hostname + operator: In/NotIn + ``` + +- priority.sh specializes in creating PriorityClass test cases, also utilizing YAML for test case definition. + Example: + ``` + $ ./priority.sh + Usage: ./priority.sh + + Options: + -o, Specifies the location of the output yaml file (default is ./output/priority.yaml) + + Arguments: + Number of pod to create (required). + Number of priorityClass to create (required). + + $ ./priority.sh 10 10 + The file ./output/priority.yaml does not exist. + The create ./output/priority.yaml. + Create 10 PriorityClass and save then to a YAML file. + Create 10 Pods and save them to a YAML file, with each Pod assigned a PriorityClass selected at random. + + $ cat output/priority.yaml + apiVersion: scheduling.k8s.io/v1 + kind: PriorityClass + metadata: + name: priority-0 + value: 0 + ... + --- + apiVersion: v1 + kind: Pod + metadata: + name: nginx-9 + spec: + ... + priorityClassName: priority-4 + ``` + diff --git a/deployments/kwok-perf-test/toleration.sh b/deployments/kwok-perf-test/toleration.sh new file mode 100755 index 000000000..55d4566d6 --- /dev/null +++ b/deployments/kwok-perf-test/toleration.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RANDOM=0 +OUTPUT_PATH="./output/toleration.yaml" +NUM_PODS=0 +NUM_NODES=0 +ACTION="" + +show_help() { + cat << EOF +Invalid option: -$OPTARG +Usage: $0 + +Commands: + -t, Taint kwok nodes with its index + -u, Untaint kwok nodes with its index + -o, Specifies the location of the output yaml file (default is ./output/priority.yaml) + +Arguments: + Number of pod to create (required). + Number of kwok nodes (required). +EOF +} + +taint(){ + for (( i=0;i "$OUTPUT_PATH" + echo "The original content of the file located at $OUTPUT_PATH has been cleared." + else + echo "The file $OUTPUT_PATH does not exist." + mkdir -p "$(dirname "$OUTPUT_PATH")" + touch "$OUTPUT_PATH" + echo "Create file $OUTPUT_PATH." + fi + + echo "Create $((NUM_PODS)) Pods and save them to a YAML file, each Pod with a randomly assigned toleration." + for (( j=0;j> "$OUTPUT_PATH" +apiVersion: v1 +kind: Pod +metadata: + name: nginx-$j +spec: + containers: + - name: sleep300 + image: "alpine:latest" + command: ["sleep", "0"] + ports: + - containerPort: 80 + tolerations: + - key: "kwok.x-k8s.io/node" + operator: "Exists" + effect: "NoSchedule" + - key: key-$randValue + operator: "Exists" +--- +EOF +done +} + +# Process command-line options +while getopts ":tuo:" opt; do + case $opt in + t) + ACTION="taint" + ;; + u) + ACTION="untaint" + ;; + o) + OUTPUT_PATH=$OPTARG + ;; + \?) + show_help + exit 1 + ;; + :) + show_help + exit 1 + ;; + esac +done + +# Shift the processed options out of the command-line arguments +shift $((OPTIND-1)) + +# Check if pod count and node count are provided +if [ $# -ne 2 ]; then + show_help + exit 1 +fi +NUM_PODS=$1 +NUM_NODES=$2 + +# Execute the chosen action +case $ACTION in + taint) + taint + exit 0 + ;; + untaint) + untaint + exit 0 + ;; + "") + createYaml + ;; + *) + show_help + ;; +esac + From 321985a8b9636fe10f21b15619302dffcba67042 Mon Sep 17 00:00:00 2001 From: Hsuan-zong wu Date: Thu, 1 Feb 2024 10:30:28 +0800 Subject: [PATCH 2/2] Simplified annotations --- deployments/kwok-perf-test/affinity.sh | 3 +-- deployments/kwok-perf-test/priority.sh | 3 +-- deployments/kwok-perf-test/toleration.sh | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/deployments/kwok-perf-test/affinity.sh b/deployments/kwok-perf-test/affinity.sh index 885e972d1..fff490973 100755 --- a/deployments/kwok-perf-test/affinity.sh +++ b/deployments/kwok-perf-test/affinity.sh @@ -67,9 +67,8 @@ fi NUM_PODS=$1 NUM_NODES=$2 -# clear origin content in file if [ -f "$OUTPUT_PATH" ]; then - # Clear the content of the file + # clear origin content in file echo "" > "$OUTPUT_PATH" echo "The original content of the file located at $OUTPUT_PATH has been cleared." else diff --git a/deployments/kwok-perf-test/priority.sh b/deployments/kwok-perf-test/priority.sh index 328ed863a..7c1cfa4f3 100755 --- a/deployments/kwok-perf-test/priority.sh +++ b/deployments/kwok-perf-test/priority.sh @@ -64,9 +64,8 @@ fi NUM_PODS=$1 NUM_PRIORITY=$2 -# clear origin content in file if [ -f "$OUTPUT_PATH" ]; then - # Clear the content of the file + # clear origin content in file echo "" > "$OUTPUT_PATH" echo "The original content of the file located at $OUTPUT_PATH has been cleared." else diff --git a/deployments/kwok-perf-test/toleration.sh b/deployments/kwok-perf-test/toleration.sh index 55d4566d6..266e8ee33 100755 --- a/deployments/kwok-perf-test/toleration.sh +++ b/deployments/kwok-perf-test/toleration.sh @@ -53,9 +53,8 @@ untaint(){ } createYaml(){ - # clear origin content in file if [ -f "$OUTPUT_PATH" ]; then - # Clear the content of the file + # clear origin content in file echo "" > "$OUTPUT_PATH" echo "The original content of the file located at $OUTPUT_PATH has been cleared." else