Skip to content

Commit

Permalink
Use Duplicate Files for Feature Test
Browse files Browse the repository at this point in the history
  • Loading branch information
Sheyla Trudo committed Sep 9, 2024
1 parent 0613fd5 commit 4e48340
Show file tree
Hide file tree
Showing 62 changed files with 3,536 additions and 542 deletions.
21 changes: 1 addition & 20 deletions .pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,13 @@ jobs:
displayName: CNI k8s E2E ${{ parameters.os }}
dependsOn: ${{ parameters.dependsOn }}
pool:
isCustom: true
type: linux
name: $(BUILD_POOL_NAME_DEFAULT)
variables:
ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output
ob_git_checkout: true
steps:
- checkout: ACNReviewChanges
clean: true

- task: AzureCLI@2
inputs:
azureSubscription: ${{ parameters.sub }}
scriptLocation: "inlineScript"
scriptType: "bash"
workingDirectory: $(ACN_DIR)
addSpnToEnvironment: true
inlineScript: |
set -e
Expand All @@ -42,8 +33,8 @@ jobs:
# https://github.com/kubernetes/sig-release/blob/master/release-engineering/artifacts.md#content-of-kubernetes-test-system-archtargz-on-example-of-kubernetes-test-linux-amd64targz-directories-removed-from-list
# explictly unzip and strip directories from ginkgo and e2e.test
tar -xvzf kubernetes-test-linux-amd64.tar.gz --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
displayName: "Setup Environment"
displayName: "Setup Environment"
- ${{ if contains(parameters.os, 'windows') }}:
- script: |
set -e
Expand All @@ -56,11 +47,9 @@ jobs:
kubectl exec -i -n kube-system $pod -- powershell "Restart-Service kubeproxy"
kubectl exec -i -n kube-system $pod -- powershell "Get-Service kubeproxy"
done
workingDirectory: $(ACN_DIR)
name: kubeproxy
displayName: Restart Kubeproxy on Windows nodes
retryCountOnTaskFailure: 3
- ${{ if eq(parameters.datapath, true) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -71,7 +60,6 @@ jobs:
os: ${{ parameters.os }}
processes: 8
attempts: 10

- ${{ if eq(parameters.dns, true) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -82,7 +70,6 @@ jobs:
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if eq(parameters.portforward, true) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -93,7 +80,6 @@ jobs:
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cni') ) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -104,7 +90,6 @@ jobs:
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cilium') ) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -115,7 +100,6 @@ jobs:
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if eq(parameters.hostport, true) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -126,7 +110,6 @@ jobs:
os: ${{ parameters.os }}
processes: 1 # Has a short serial test
attempts: 3

- ${{ if and(eq(parameters.hybridWin, true), eq(parameters.os, 'windows')) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -137,7 +120,6 @@ jobs:
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand All @@ -149,7 +131,6 @@ jobs:
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}:
- template: ../k8s-e2e/k8s-e2e-step-template.yaml
parameters:
Expand Down
1 change: 0 additions & 1 deletion .pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,3 @@ steps:
fi
name: ${{ parameters.name }}
displayName: k8s E2E - ${{ parameters.testName }}
workingDirectory: $(ACN_DIR)
165 changes: 165 additions & 0 deletions .pipelines/cni/k8s-e2e/k8s-e2e.jobs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
parameters:
clusterName: ""
os: ""
dependsOn: ""
sub: ""
cni: cni


jobs:
- job: CNI_${{ parameters.os }}
condition: and( not(canceled()), not(failed()) )
displayName: CNI k8s E2E ${{ parameters.os }}
dependsOn: ${{ parameters.dependsOn }}
pool:
isCustom: true
type: linux
name: $(BUILD_POOL_NAME_DEFAULT)
variables:
ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output
ob_git_checkout: true
steps:
- checkout: ACNReviewChanges
clean: true

- task: AzureCLI@2
inputs:
azureSubscription: ${{ parameters.sub }}
scriptLocation: "inlineScript"
scriptType: "bash"
workingDirectory: $(ACN_DIR)
addSpnToEnvironment: true
inlineScript: |
set -e
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
# sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx"
# pulling k8s version from AKS.
eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }} -n ${{ parameters.clusterName }} --query "currentKubernetesVersion")
echo $k8sVersion
curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz
# https://github.com/kubernetes/sig-release/blob/master/release-engineering/artifacts.md#content-of-kubernetes-test-system-archtargz-on-example-of-kubernetes-test-linux-amd64targz-directories-removed-from-list
# explictly unzip and strip directories from ginkgo and e2e.test
tar -xvzf kubernetes-test-linux-amd64.tar.gz --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
displayName: "Setup Environment"
retryCountOnTaskFailure: 5

- ${{ if contains(parameters.os, 'windows') }}:
- script: |
set -e
kubectl apply -f test/integration/manifests/load/privileged-daemonset-windows.yaml
kubectl rollout status -n kube-system ds privileged-daemonset
kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows -owide
pods=`kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows --no-headers | awk '{print $1}'`
for pod in $pods; do
kubectl exec -i -n kube-system $pod -- powershell "Restart-Service kubeproxy"
kubectl exec -i -n kube-system $pod -- powershell "Get-Service kubeproxy"
done
workingDirectory: $(ACN_DIR)
name: kubeproxy
displayName: Restart Kubeproxy on Windows nodes
retryCountOnTaskFailure: 3
- ${{ if eq(parameters.datapath, true) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: Datapath
name: datapath
ginkgoFocus: '(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api'
ginkgoSkip: 'SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6'
os: ${{ parameters.os }}
processes: 8
attempts: 10

- ${{ if eq(parameters.dns, true) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: DNS
name: dns
ginkgoFocus: '\[sig-network\].DNS.should'
ginkgoSkip: 'resolv|256 search'
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if eq(parameters.portforward, true) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: Kubectl Portforward
name: portforward
ginkgoFocus: '\[sig-cli\].Kubectl.Port'
ginkgoSkip: ''
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cni') ) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: Service Conformance
name: service
ginkgoFocus: 'Services.*\[Conformance\].*'
ginkgoSkip: ''
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cilium') ) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: Service Conformance|Cilium
name: service
ginkgoFocus: 'Services.*\[Conformance\].*'
ginkgoSkip: 'should serve endpoints on same port and different protocols' # Cilium does not support this feature. For more info on test: https://github.com/kubernetes/kubernetes/blame/e602e9e03cd744c23dde9fee09396812dd7bdd93/test/conformance/testdata/conformance.yaml#L1780-L1788
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if eq(parameters.hostport, true) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: Host Port
name: hostport
ginkgoFocus: '\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort'
ginkgoSkip: 'SCTP|exists conflict' # Skip slow 5 minute test
os: ${{ parameters.os }}
processes: 1 # Has a short serial test
attempts: 3

- ${{ if and(eq(parameters.hybridWin, true), eq(parameters.os, 'windows')) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: Hybrid Network
name: hybrid
ginkgoFocus: '\[sig-windows\].Hybrid'
ginkgoSkip: ''
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: DualStack Test
name: DualStack
clusterName: ${{ parameters.clusterName }}
ginkgoFocus: '\[Feature:IPv6DualStack\]'
ginkgoSkip: 'SCTP|session affinity'
os: ${{ parameters.os }}
processes: 8
attempts: 3

- ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}:
- template: k8s-e2e.steps.yaml@ACNTools
parameters:
testName: DualStack Test|Cilium
name: DualStack
clusterName: ${{ parameters.clusterName }}
ginkgoFocus: '\[Feature:IPv6DualStack\]'
ginkgoSkip: 'SCTP|session affinity|should function for service endpoints using hostNetwork' # Cilium dualstack has a known issue with this test https://github.com/cilium/cilium/issues/25135
os: ${{ parameters.os }}
processes: 8
attempts: 3

66 changes: 66 additions & 0 deletions .pipelines/cni/k8s-e2e/k8s-e2e.steps.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
parameters:
testName: ""
name: ""
ginkgoFocus: ""
ginkgoSkip: ""
os: ""
processes: "" # Number of parallel processes
attempts: ""


steps:
- script: |
set -ex
# ginkgoSkip cant handle only |LinuxOnly. Need to have check
if ${{ lower(and(ge(length(parameters.ginkgoSkip), 1), eq(parameters.os, 'windows'))) }}
then
SKIP="|LinuxOnly"
elif ${{ lower(eq(parameters.os, 'windows')) }}
then
SKIP="LinuxOnly"
fi
# Taint Linux nodes so that windows tests do not run on them
if ${{ lower(eq(parameters.os, 'windows')) }}
then
kubectl rollout status -n kube-system deployment/konnectivity-agent --timeout=3m
kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule
fi
# Taint Windows nodes so that Linux tests do not run on them
if ${{ lower(eq(parameters.os, 'linux')) }}
then
kubectl taint nodes -l kubernetes.azure.com/mode=user node-role.kubernetes.io/control-plane:NoSchedule
fi
# Depreciating flags. Change once k8s minimum version supported is > 1.24
# nodes -> procs
# flakeAttempts -> flake-attempts
# dryRun -> dry-run
./ginkgo --nodes=${{ parameters.processes }} \
./e2e.test -- \
--num-nodes=2 \
--provider=skeleton \
--ginkgo.focus='${{ parameters.ginkgoFocus }}' \
--ginkgo.skip="${{ parameters.ginkgoSkip }}$SKIP" \
--ginkgo.flakeAttempts=${{ parameters.attempts }} \
--ginkgo.v \
--node-os-distro=${{ parameters.os }} \
--kubeconfig=$HOME/.kube/config
# Untaint Linux nodes once testing is complete
if ${{ lower(eq(parameters.os, 'windows')) }}
then
kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule-
fi
# Untaint Windows nodes once testing is complete
if ${{ lower(eq(parameters.os, 'linux')) }}
then
kubectl taint nodes -l kubernetes.azure.com/mode=user node-role.kubernetes.io/control-plane:NoSchedule-
fi
name: ${{ parameters.name }}
displayName: k8s E2E - ${{ parameters.testName }}
workingDirectory: $(ACN_DIR)
Loading

0 comments on commit 4e48340

Please sign in to comment.