Skip to content

Commit

Permalink
E2E: Consoldiate docker and prefer bundled tests into new startup test (
Browse files Browse the repository at this point in the history
k3s-io#6851)

* Convert docker E2E to startup E2E
* Move preferedbundled into the e2e startup test

Signed-off-by: Derek Nola <[email protected]>
  • Loading branch information
dereknola committed Feb 6, 2023
1 parent d3dc5f0 commit 59a6743
Show file tree
Hide file tree
Showing 5 changed files with 235 additions and 144 deletions.
92 changes: 0 additions & 92 deletions tests/e2e/preferbundled/preferbundled_test.go

This file was deleted.

68 changes: 30 additions & 38 deletions tests/e2e/scripts/run_tests.sh
Original file line number Diff line number Diff line change
@@ -1,57 +1,49 @@
#!/bin/bash
servercount=${5:-3}
agentcount=${6:-1}
db=${7:-"etcd"}
k3s_version=${k3s_version}
k3s_channel=${k3s_channel:-"commit"}
hardened=${8:-""}

nodeOS=${1:-"generic/ubuntu2004"}
servercount=${2:-3}
agentcount=${3:-1}
db=${4:-"etcd"}
hardened=${5:-""}

E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB
E2E_REGISTRY=true && export E2E_REGISTRY

eval openvpn --daemon --config external.ovpn &>/dev/null &
sleep 10
cd
cd k3s && git pull --rebase origin master
/usr/local/go/bin/go mod tidy

ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s && git pull --rebase origin master'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s && /usr/local/go/bin/go mod tidy'
cd tests/e2e
OS=$(echo "$nodeOS"|cut -d'/' -f2)
echo "$OS"

ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e/dualstack && vagrant global-status | awk '/running/'|cut -c1-7| xargs -r -d '\n' -n 1 -- vagrant destroy -f"
vagrant global-status | awk '/running/'|cut -c1-7| xargs -r -d '\n' -n 1 -- vagrant destroy -f
E2E_RELEASE_CHANNEL="commit" && export E2E_RELEASE_CHANNEL

echo 'RUNNING DUALSTACK VALIDATION TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/dualstack && vagrant destroy -f'
echo 'RUNNING DUALSTACK TEST'
E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$nodeOS" -serverCount=1 -agentCount=1 -timeout=30m -json -ci |tee k3s_"$OS".log

echo 'RUNNING CLUSTER VALIDATION TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/validatecluster && vagrant destroy -f'
E2E_REGISTRY=true E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json -ci | tee -a k3s_"$OS".log

echo 'RUNNING SECRETS ENCRYPTION TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
/usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -timeout=1h -json -ci | tee -a k3s_"$OS".log

echo 'RUNNING SNAPSHOT RESTORE TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v snapshotrestore/snapshotrestore_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
echo 'RUN CLUSTER RESET TEST'
/usr/local/go/bin/go test -v clusterreset/clusterreset_test.go -nodeOS="$nodeOS" -serverCount=3 -agentCount=1 -timeout=30m -json -ci | tee -a createreport/k3s_"$OS".log

echo 'RUNNING SPLIT SERVER VALIDATION TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v splitserver/splitserver_test.go -nodeOS="$4" -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/splitserver && vagrant destroy -f'

E2E_RELEASE_VERSION=$k3s_version && export E2E_RELEASE_VERSION
E2E_RELEASE_CHANNEL=$k3s_channel && export E2E_RELEASE_CHANNEL
E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v splitserver/splitserver_test.go -nodeOS="$nodeOS" -timeout=30m -json -ci | tee -a k3s_"$OS".log

echo 'RUNNING CLUSTER UPGRADE TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/upgradecluster && vagrant destroy -f'
echo 'RUNNING STARTUP VALIDATION TEST'
/usr/local/go/bin/go test -v startup/startup_test.go -nodeOS="$nodeOS" -timeout=30m -json -ci | tee -a k3s_"$OS".log

echo 'RUN CLUSTER RESET TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v clusterreset/clusterreset_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/clusterreset && vagrant destroy -f'
echo 'RUNNING EXTERNAL IP TEST'
/usr/local/go/bin/go test -v externalip/externalip_test.go -nodeOS="$nodeOS" -serverCount=1 -agentCount=1 -timeout=30m -json -ci | tee -a k3s_"$OS".log

echo 'RUNNING DOCKER CRI VALIDATION TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v docker/docker_test.go -nodeOS="$4" -serverCount=1 -agentCount=1 -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/docker && vagrant destroy -f'
echo 'RUNNING SNAPSHOT AND RESTORE TEST'
/usr/local/go/bin/go test -v snapshotrestore/snapshotrestore_test.go -nodeOS="$nodeOS" -serverCount=1 -agentCount=1 -timeout=30m -json -ci | tee -a k3s_"$OS".log

echo 'RUNNING EXTERNALIP VALIDATION TEST'
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v externalip/externalip_test.go -nodeOS="$4" -serverCount=1 -agentCount=1 -timeout=30m -json" | tee -a testreport.log
ssh -i "$1" -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/dualstack && vagrant destroy -f'
E2E_RELEASE_CHANNEL="latest" && export E2E_RELEASE_CHANNEL
echo 'RUNNING CLUSTER UPGRADE TEST'
E2E_REGISTRY=true /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h -json -ci | tee -a k3s_"$OS".log
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
ENV['VAGRANT_NO_PARALLEL'] = 'no'
NODE_ROLES = (ENV['NODE_ROLES'] ||
["server-0", "agent-0"])
NODE_BOXES = (ENV['NODE_BOXES'] ||
['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
GITHUB_BRANCH = (ENV['GITHUB_BRANCH'] || "master")
RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "")
NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i
NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i
NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
['generic/ubuntu2004', 'generic/ubuntu2004'])
GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks
NETWORK_PREFIX = "10.10.10"
install_type = ""
Expand All @@ -21,32 +21,33 @@ def provision(vm, role, role_num, node_num)
load vagrant_defaults if File.exists?(vagrant_defaults)

defaultOSConfigure(vm)
dockerInstall(vm)
install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"

vm.provision "shell", inline: "ping -c 2 k3s.io"

if role.include?("server")
vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
k3s.args = "server"
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
k3s.args = "server "
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type} INSTALL_K3S_SKIP_START=true]
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
k3s.config = <<~YAML
token: vagrant
flannel-iface: eth1
node-external-ip: #{NETWORK_PREFIX}.100
prefer-bundled-bin: true
flannel-iface: eth1
YAML
end
elsif role.include?("agent")
vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
k3s.args = "agent"
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
k3s.args = "agent "
k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type} INSTALL_K3S_SKIP_START=true]
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
k3s.config = <<~YAML
server: "https://#{NETWORK_PREFIX}.100:6443"
token: vagrant
node-external-ip: #{node_ip}
flannel-iface: eth1
prefer-bundled-bin: true
YAML
end
end
Expand Down
Loading

0 comments on commit 59a6743

Please sign in to comment.