Skip to content

Commit

Permalink
github workflow e2e test
Browse files Browse the repository at this point in the history
Signed-off-by: liheng.zms <[email protected]>
  • Loading branch information
zmberg committed Apr 1, 2022
1 parent e35e4f9 commit 8241ae1
Show file tree
Hide file tree
Showing 24 changed files with 514 additions and 214 deletions.
147 changes: 147 additions & 0 deletions .github/workflows/e2e-1.23.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
name: E2E-1.23

on:
push:
branches:
- master
- release-*
pull_request: {}
workflow_dispatch: {}

env:
# Common versions
GO_VERSION: '1.17'
KIND_IMAGE: 'kindest/node:v1.23.3'
KIND_CLUSTER_NAME: 'ci-testing'

jobs:

rollout:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/[email protected]
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
config: ./test/kind-conf.yaml
- name: Build image
run: |
export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
docker build --pull --no-cache . -t $IMAGE
kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
- name: Install Kruise Rollout
run: |
set -ex
kubectl cluster-info
IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
set -e
if [ "$PODS" -eq "1" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
kubectl get node -o yaml
kubectl get all -n kruise-rollout -o yaml
set -e
if [ "$PODS" -eq "1" ]; then
echo "Wait for kruise-rollout ready successfully"
else
echo "Timeout to wait for kruise-rollout ready"
exit 1
fi
- name: Run E2E Tests
run: |
export KUBECONFIG=/home/runner/.kube/config
make ginkgo
set +e
./bin/ginkgo -timeout 60m -v --focus='\[rollouts\] (Rollout)' test/e2e
retVal=$?
restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
if [ "${restartCount}" -eq "0" ];then
echo "Kruise-rollout has not restarted"
else
kubectl get pod -n kruise-rollout --no-headers
echo "Kruise-rollout has restarted, abort!!!"
kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
exit 1
fi
exit $retVal
batchrelease:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/[email protected]
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
config: ./test/kind-conf.yaml
- name: Build image
run: |
export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
docker build --pull --no-cache . -t $IMAGE
kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
- name: Install Kruise Rollout
run: |
set -ex
kubectl cluster-info
IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
set -e
if [ "$PODS" -eq "1" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
kubectl get node -o yaml
kubectl get all -n kruise-rollout -o yaml
set -e
if [ "$PODS" -eq "1" ]; then
echo "Wait for kruise-rollout ready successfully"
else
echo "Timeout to wait for kruise-rollout ready"
exit 1
fi
- name: Run E2E Tests
run: |
export KUBECONFIG=/home/runner/.kube/config
make ginkgo
set +e
./bin/ginkgo -timeout 60m -v --focus='\[rollouts\] (BatchRelease)' test/e2e
retVal=$?
restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
if [ "${restartCount}" -eq "0" ];then
echo "Kruise-rollout has not restarted"
else
kubectl get pod -n kruise-rollout --no-headers
echo "Kruise-rollout has restarted, abort!!!"
kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
exit 1
fi
exit $retVal
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ KUSTOMIZE = $(shell pwd)/bin/kustomize
kustomize: ## Download kustomize locally if necessary.
$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])

GINKGO = $(shell pwd)/bin/ginkgo
ginkgo: ## Download ginkgo locally if necessary.
$(call go-get-tool,$(GINKGO),github.com/onsi/ginkgo/[email protected])

# go-get-tool will 'go get' any package $2 and install it to $1.
PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
define go-get-tool
Expand All @@ -106,7 +110,7 @@ TMP_DIR=$$(mktemp -d) ;\
cd $$TMP_DIR ;\
go mod init tmp ;\
echo "Downloading $(2)" ;\
GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
rm -rf $$TMP_DIR ;\
}
endef
3 changes: 2 additions & 1 deletion api/v1alpha1/rollout_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ type RolloutStatus struct {
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// CanaryRevision the hash of the canary pod template
// +optional
CanaryRevision string `json:"canaryRevision,omitempty"`
//CanaryRevision string `json:"canaryRevision,omitempty"`
// StableRevision indicates the revision pods that has successfully rolled out
StableRevision string `json:"stableRevision,omitempty"`
// Conditions a list of conditions a rollout can have.
Expand Down Expand Up @@ -274,6 +274,7 @@ const (
// +kubebuilder:printcolumn:name="CANARY_STEP",type="integer",JSONPath=".status.canaryStatus.currentStepIndex",description="The rollout canary status step"
// +kubebuilder:printcolumn:name="CANARY_STATE",type="string",JSONPath=".status.canaryStatus.currentStepState",description="The rollout canary status step state"
// +kubebuilder:printcolumn:name="MESSAGE",type="string",JSONPath=".status.message",description="The rollout canary status message"
// +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=".metadata.creationTimestamp"

// Rollout is the Schema for the rollouts API
type Rollout struct {
Expand Down
11 changes: 6 additions & 5 deletions config/crd/bases/rollouts.kruise.io_rollouts.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ spec:
jsonPath: .status.message
name: MESSAGE
type: string
- jsonPath: .metadata.creationTimestamp
name: AGE
type: date
name: v1alpha1
schema:
openAPIV3Schema:
Expand Down Expand Up @@ -172,9 +175,6 @@ spec:
status:
description: RolloutStatus defines the observed state of Rollout
properties:
canaryRevision:
description: CanaryRevision the hash of the canary pod template
type: string
canaryStatus:
description: Canary describes the state of the canary rollout
properties:
Expand Down Expand Up @@ -272,8 +272,9 @@ spec:
Phase is the rollout phase.
type: string
stableRevision:
description: StableRevision indicates the revision pods that has successfully
rolled out
description: CanaryRevision the hash of the canary pod template CanaryRevision
string `json:"canaryRevision,omitempty"` StableRevision indicates
the revision pods that has successfully rolled out
type: string
type: object
type: object
Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/batchrelease/batchrelease_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -840,7 +840,7 @@ func setPhase(release *v1alpha1.BatchRelease, phase v1alpha1.RolloutPhase) *v1al
case v1alpha1.RolloutPhaseInitial, v1alpha1.RolloutPhaseHealthy:
default:
r.Status.ObservedWorkloadReplicas = 100
r.Status.ObservedReleasePlanHash = hashReleasePlanBatches(&release.Spec.ReleasePlan)
r.Status.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan)
}
return r
}
Expand All @@ -850,7 +850,7 @@ func setState(release *v1alpha1.BatchRelease, state v1alpha1.BatchReleaseBatchSt
r.Status.Phase = v1alpha1.RolloutPhaseProgressing
r.Status.CanaryStatus.CurrentBatchState = state
r.Status.ObservedWorkloadReplicas = 100
r.Status.ObservedReleasePlanHash = hashReleasePlanBatches(&release.Spec.ReleasePlan)
r.Status.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan)
return r
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/controller/batchrelease/workloads"
"github.com/openkruise/rollouts/pkg/util"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -170,10 +171,9 @@ func refreshStatus(release *v1alpha1.BatchRelease, newStatus *v1alpha1.BatchRele
newStatus.CanaryStatus.UpdatedReplicas = workloadInfo.Status.UpdatedReplicas
newStatus.CanaryStatus.UpdatedReadyReplicas = workloadInfo.Status.UpdatedReadyReplicas
}
planHash := hashReleasePlanBatches(&release.Spec.ReleasePlan)
if newStatus.ObservedReleasePlanHash != planHash {
newStatus.ObservedReleasePlanHash = planHash
}
}
if len(newStatus.ObservedReleasePlanHash) == 0 {
newStatus.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan)
}
}

Expand All @@ -182,7 +182,7 @@ func isPlanTerminating(release *v1alpha1.BatchRelease, status *v1alpha1.BatchRel
}

func isPlanChanged(plan *v1alpha1.ReleasePlan, status *v1alpha1.BatchReleaseStatus) bool {
return status.ObservedReleasePlanHash != hashReleasePlanBatches(plan) && status.Phase == v1alpha1.RolloutPhaseProgressing
return status.ObservedReleasePlanHash != util.HashReleasePlanBatches(plan) && status.Phase == v1alpha1.RolloutPhaseProgressing
}

func isPlanUnhealthy(plan *v1alpha1.ReleasePlan, status *v1alpha1.BatchReleaseStatus) bool {
Expand Down
14 changes: 3 additions & 11 deletions pkg/controller/batchrelease/batchrelease_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,8 @@ limitations under the License.
package batchrelease

import (
"crypto/sha256"
"encoding/hex"
"encoding/json"

"github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/util"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog/v2"
Expand All @@ -39,12 +36,6 @@ func HasTerminatingCondition(status v1alpha1.BatchReleaseStatus) bool {
return false
}

func hashReleasePlanBatches(releasePlan *v1alpha1.ReleasePlan) string {
by, _ := json.Marshal(releasePlan.Batches)
md5Hash := sha256.Sum256(by)
return hex.EncodeToString(md5Hash[:])
}

func initializeStatusIfNeeds(status *v1alpha1.BatchReleaseStatus) {
if len(status.Phase) == 0 {
resetStatus(status)
Expand Down Expand Up @@ -76,13 +67,14 @@ func signalRecalculate(release *v1alpha1.BatchRelease, newStatus *v1alpha1.Batch
currentBatch := int32(0)
if release.Spec.ReleasePlan.BatchPartition != nil {
// ensure current batch upper bound
currentBatch = integer.Int32Min(*release.Spec.ReleasePlan.BatchPartition, currentBatch)
currentBatch = integer.Int32Min(*release.Spec.ReleasePlan.BatchPartition, int32(len(release.Spec.ReleasePlan.Batches)-1))
}

klog.Infof("BatchRelease(%v) canary batch changed from %v to %v when the release plan changed",
client.ObjectKeyFromObject(release), newStatus.CanaryStatus.CurrentBatch, currentBatch)
newStatus.CanaryStatus.CurrentBatch = currentBatch
newStatus.CanaryStatus.CurrentBatchState = v1alpha1.UpgradingBatchState
newStatus.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan)
}

func resetStatus(status *v1alpha1.BatchReleaseStatus) {
Expand Down
3 changes: 2 additions & 1 deletion pkg/controller/rollout/batchrelease/batchrelease.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
// but rather the ability to interact with the BatchRelease controller through the BatchRelease CRD to achieve a batch release
type BatchRelease interface {
// Verify will create batchRelease or update batchRelease steps configuration
Verify(index int32) error
// isLatest(Bool) determines if the batchRelease configuration is the latest
Verify(index int32) (bool, error)

// 1. Promote release workload in step(index), 1<=index<=len(step)
// 2. Promote will resume stable workload if the last batch(index=-1) is finished
Expand Down
22 changes: 11 additions & 11 deletions pkg/controller/rollout/batchrelease/inner_batchrelease.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func NewInnerBatchController(c client.Client, rollout *rolloutv1alpha1.Rollout)
return r
}

func (r *innerBatchRelease) Verify(index int32) error {
func (r *innerBatchRelease) Verify(index int32) (bool, error) {
index = index - 1
batch := &rolloutv1alpha1.BatchRelease{}
err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch)
Expand All @@ -68,42 +68,42 @@ func (r *innerBatchRelease) Verify(index int32) error {
br := createBatchRelease(r.rollout, r.batchName)
if err = r.Create(context.TODO(), br); err != nil && !errors.IsAlreadyExists(err) {
klog.Errorf("rollout(%s/%s) create BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
return err
return false, err
}
data := util.DumpJSON(br)
klog.Infof("rollout(%s/%s) create BatchRelease(%s) success", r.rollout.Namespace, r.rollout.Name, data)
return nil
return false, nil
} else if err != nil {
klog.Errorf("rollout(%s/%s) fetch BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
return err
return false, err
}

// check whether batchRelease configuration is the latest
newBr := createBatchRelease(r.rollout, r.batchName)
if reflect.DeepEqual(batch.Spec.ReleasePlan.Batches, newBr.Spec.ReleasePlan.Batches) {
klog.Infof("rollout(%s/%s) batchRelease is initialize done", r.rollout.Namespace, r.rollout.Name)
return nil
klog.Infof("rollout(%s/%s) batchRelease(generation:%d) configuration is the latest", r.rollout.Namespace, r.rollout.Name, batch.Generation)
return true, nil
}

// update batchRelease to the latest version
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
if err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil {
if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
if err = r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil {
klog.Errorf("error getting updated BatchRelease(%s/%s) from client", batch.Namespace, batch.Name)
return err
}
batch.Spec.ReleasePlan.Batches = newBr.Spec.ReleasePlan.Batches
batch.Spec.ReleasePlan.BatchPartition = utilpointer.Int32Ptr(index)
if err := r.Client.Update(context.TODO(), batch); err != nil {
if err = r.Client.Update(context.TODO(), batch); err != nil {
return err
}
return nil
}); err != nil {
klog.Errorf("rollout(%s/%s) update batchRelease configuration failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
return err
return false, err
}
data := util.DumpJSON(batch)
klog.Infof("rollout(%s/%s) update batchRelease configuration(%s) to the latest", r.rollout.Namespace, r.rollout.Name, data)
return nil
return false, nil
}

func (r *innerBatchRelease) FetchBatchRelease() (*rolloutv1alpha1.BatchRelease, error) {
Expand Down
Loading

0 comments on commit 8241ae1

Please sign in to comment.