Skip to content

Commit

Permalink
e2e:update e2e test script and quick start script
Browse files Browse the repository at this point in the history
Signed-off-by: wangdepeng <[email protected]>
  • Loading branch information
village-way committed Mar 14, 2024
1 parent 57f162a commit 99696f9
Show file tree
Hide file tree
Showing 15 changed files with 528 additions and 341 deletions.
33 changes: 24 additions & 9 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ concurrency:
jobs:
verify:
name: verify
runs-on: self-hosted
runs-on: [self-hosted, ecs]
if: ${{ always() }}
env:
#GOPATH: ${{ github.workspace }}
Expand Down Expand Up @@ -38,7 +38,7 @@ jobs:
build:
name: build
needs: verify
runs-on: self-hosted
runs-on: [self-hosted, ecs]
steps:
- name: Checkout code
uses: actions/checkout@v3
Expand All @@ -54,7 +54,7 @@ jobs:
test:
name: Unit test
needs: build
runs-on: self-hosted
runs-on: [self-hosted, ecs]
steps:
- name: Checkout code
uses: actions/checkout@v3
Expand All @@ -68,7 +68,14 @@ jobs:
needs: build
env:
GOPROXY: "https://goproxy.cn,direct"
runs-on: self-hosted
runs-on: [self-hosted, ecs]
strategy:
fail-fast: false
matrix:
# Here support the latest three minor releases of Kubernetes, this can be considered to be roughly
# the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/
# Please remember to update the CI Schedule Workflow when we add a new version.
k8s: [ v1.25.3, v1.28.0, v1.29.0 ]
steps:
# Free up disk space on Ubuntu
- name: Free Disk Space (Ubuntu)
Expand All @@ -92,12 +99,20 @@ jobs:
with:
go-version: '1.20'
- name: Prepare e2e env
run: ./hack/prepare-e2e.sh
env: KIND_IMAGE=kindest/node:${{ matrix.k8s }}
run: hack/local-up-kosmos.sh
- name: Run e2e test
run: ./hack/rune2e.sh
env: ARTIFACTS_PATH=${{ github.workspace }}/e2e-test/${{ matrix.k8s }}/
run: hack/rune2e.sh
- name: Upload logs
uses: actions/upload-artifact@v3
if: failure()
if: always()
with:
name: kosmos-e2e-logs-${{ github.run_id }}
path: ${{ github.workspace }}/e2e-test/logs-*
name: kosmos-e2e-logs-${{ github.run_id }}_${{ matrix.k8s }}
path: ${{ github.workspace }}/e2e-test/${{ matrix.k8s }}/
- name: upload environments logs
if: always()
uses: actions/upload-artifact@v3
with:
name: kosmos_environments_log_${{ matrix.k8s }}
path: ${{ github.workspace }}/environments
25 changes: 25 additions & 0 deletions examples/nginx-demo.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
labels:
app: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
tolerations:
- key: kosmos.io/node
operator: Equal
value: "true"
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80
248 changes: 105 additions & 143 deletions hack/cluster.sh

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion hack/clustertemplete/kindconfig
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,5 @@ networking:
ipFamily: __IP_FAMILY__
disableDefaultCNI: true # disable kindnet
podSubnet: __POD_CIDR__
serviceSubnet: __SERVICE_CIDR__
serviceSubnet: __SERVICE_CIDR__
apiServerAddress: __HOST_IPADDRESS__
81 changes: 81 additions & 0 deletions hack/local-cleanup-kosmos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/usr/bin/env bash

set -o errexit
set -o nounset
set -o pipefail

function usage() {
echo "Usage:"
echo " hack/local-down-kosmos.sh [-k] [-h]"
echo "Args:"
echo " k: keep the local images"
echo " h: print help information"
}

keep_images="false"
while getopts 'kh' OPT; do
case $OPT in
k) keep_images="true";;
h)
usage
exit 0
;;
?)
usage
exit 1
;;
esac
done
HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"cluster-host"}
MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"cluster-member1"}
MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"cluster-member2"}
MEMBER_CLUSTER_3_NAME=${MEMBER_CLUSTER_3_NAME:-"cluster-member3"}
#step1 remove kind clusters
echo -e "\nStart removing kind clusters"
kind delete cluster --name "${HOST_CLUSTER_NAME}"
kind delete cluster --name "${MEMBER_CLUSTER_1_NAME}"
kind delete cluster --name "${MEMBER_CLUSTER_2_NAME}"
kind delete cluster --name "${MEMBER_CLUSTER_3_NAME}"
echo "Remove kind clusters successfully."

ROOT=$(dirname "${BASH_SOURCE[0]}")/..
CLUSTER_DIR="${ROOT}/environments"
source "${ROOT}/hack/cluster.sh"

#step2. remove kubeconfig
echo -e "\nStart removing kubeconfig, kindconfig, cailcoconfig"
HOST_CLUSTER_CONFIG=${HOST_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${HOST_CLUSTER_NAME}"}
MEMBER1_CLUSTER_CONFIG=${MEMBER_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${MEMBER_CLUSTER_1_NAME}"}
MEMBER2_CLUSTER_CONFIG=${MEMBER_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${MEMBER_CLUSTER_2_NAME}"}
MEMBER3_CLUSTER_CONFIG=${MEMBER_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${MEMBER_CLUSTER_3_NAME}"}
delete_cluster "${HOST_CLUSTER_CONFIG}" "${HOST_CLUSTER_CONFIG}"
delete_cluster "${MEMBER1_CLUSTER_CONFIG}" "${MEMBER1_CLUSTER_CONFIG}"
delete_cluster "${MEMBER2_CLUSTER_CONFIG}" "${MEMBER2_CLUSTER_CONFIG}"
delete_cluster "${MEMBER3_CLUSTER_CONFIG}" "${MEMBER3_CLUSTER_CONFIG}"

echo "Remove cluster configs successfully."

#step3. remove docker images
echo -e "\nStart removing images"
version="v0.2.0"
registry="ghcr.io/kosmos-io"
images=(
"${registry}/clusterlink-network-manager:${version}"
"${registry}/clusterlink-controller-manager:${version}"
"${registry}/clusterlink-elector:${version}"
"${registry}/clusterlink-operator:${version}"
"${registry}/clusterlink-agent:${version}"
"${registry}/clusterlink-proxy:${version}"
"${registry}/clustertree-cluster-manager:${version}"
"${registry}/scheduler:${version}"
)
if [[ "${keep_images}" == "false" ]] ; then
for ((i=0;i<${#images[*]};i++)); do
docker rmi ${images[i]} || true
done
echo "Remove images successfully."
else
echo "Skip removing images as required."
fi

echo -e "\nLocal Kosmos is removed successfully."
21 changes: 0 additions & 21 deletions hack/local-down-clusterlink.sh

This file was deleted.

41 changes: 0 additions & 41 deletions hack/local-up-clusterlink.sh

This file was deleted.

97 changes: 97 additions & 0 deletions hack/local-up-kosmos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/usr/bin/env bash

set -o errexit
set -o nounset
set -o pipefail


function usage() {
echo "Usage:"
echo " hack/local-up-kosmos.sh [HOST_IPADDRESS] [-h]"
echo "Args:"
echo " HOST_IPADDRESS: (required) if you want to export clusters' API server port to specific IP address"
echo " h: print help information"
}

while getopts 'h' OPT; do
case $OPT in
h)
usage
exit 0
;;
?)
usage
exit 1
;;
esac
done


KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
export KUBECONFIG=$KUBECONFIG_PATH/"config"

KIND_IMAGE=${KIND_IMAGE:-"kindest/node:v1.27.2"}
HOST_IPADDRESS=${1:-}
HOST_CLUSTER_NAME="cluster-host"
HOST_CLUSTER_POD_CIDR="10.233.64.0/18"
HOST_CLUSTER_SERVICE_CIDR="10.233.0.0/18"

MEMBER1_CLUSTER_NAME="cluster-member1"
MEMBER1_CLUSTER_POD_CIDR="10.234.64.0/18"
MEMBER1_CLUSTER_SERVICE_CIDR="10.234.0.0/18"

MEMBER2_CLUSTER_NAME="cluster-member2"
MEMBER2_CLUSTER_POD_CIDR="10.235.64.0/18"
MEMBER2_CLUSTER_SERVICE_CIDR="10.235.0.0/18"

MEMBER3_CLUSTER_NAME="cluster-member3"
MEMBER3_CLUSTER_POD_CIDR="10.236.64.0/18"
MEMBER3_CLUSTER_SERVICE_CIDR="10.236.0.0/18"

ROOT="$(dirname "${BASH_SOURCE[0]}")"
export VERSION="latest"
source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh"
source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh"
source "$(dirname "${BASH_SOURCE[0]}")/util.sh"

#step1. create host cluster and member clusters in parallel
# host IP address: script parameter ahead of macOS IP
if [[ -z "${HOST_IPADDRESS}" ]]; then
util::get_macos_ipaddress # Adapt for macOS
HOST_IPADDRESS=${MAC_NIC_IPADDRESS:-}
fi
make images GOOS="linux" --directory="${ROOT}"

make kosmosctl
os=$(go env GOOS)
arch=$(go env GOARCH)
export PATH=$PATH:"$ROOT"/_output/bin/"$os"/"$arch"

# prepare docker image
prepare_docker_image

#cluster cluster concurrent backend
create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $HOST_CLUSTER_NAME $HOST_CLUSTER_POD_CIDR $HOST_CLUSTER_SERVICE_CIDR &
create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $MEMBER1_CLUSTER_NAME $MEMBER1_CLUSTER_POD_CIDR $MEMBER1_CLUSTER_SERVICE_CIDR false &
create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $MEMBER2_CLUSTER_NAME $MEMBER2_CLUSTER_POD_CIDR $MEMBER2_CLUSTER_SERVICE_CIDR false &
create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $MEMBER3_CLUSTER_NAME $MEMBER3_CLUSTER_POD_CIDR $MEMBER3_CLUSTER_SERVICE_CIDR false &

# wait for finish
wait

#deploy cluster concurrent backend
deploy_cluster_by_ctl $HOST_CLUSTER_NAME "${ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig" "${ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig-nodeIp" &
load_cluster_images $MEMBER1_CLUSTER_NAME &
load_cluster_images $MEMBER2_CLUSTER_NAME &
load_cluster_images $MEMBER3_CLUSTER_NAME &

# wait for finish
wait

#join cluster
join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER1_CLUSTER_NAME "${ROOT}/environments/${HOST_CLUSTER_NAME}" "${ROOT}/environments/${MEMBER1_CLUSTER_NAME}"
join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER2_CLUSTER_NAME "${ROOT}/environments/${HOST_CLUSTER_NAME}" "${ROOT}/environments/${MEMBER2_CLUSTER_NAME}"
join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER3_CLUSTER_NAME "${ROOT}/environments/${HOST_CLUSTER_NAME}" "${ROOT}/environments/${MEMBER3_CLUSTER_NAME}"

#add leafnode test taint
addTaint $HOST_CLUSTER_NAME $MEMBER3_CLUSTER_NAME
Loading

0 comments on commit 99696f9

Please sign in to comment.