Skip to content

Commit

Permalink
add advanced deployment controller
Browse files Browse the repository at this point in the history
Signed-off-by: mingzhou.swx <[email protected]>
  • Loading branch information
mingzhou.swx committed Jan 5, 2023
1 parent 3165f4e commit 0babeee
Show file tree
Hide file tree
Showing 18 changed files with 1,592 additions and 162 deletions.
110 changes: 110 additions & 0 deletions .github/workflows/e2e-advanced-deployment-1.19.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
name: E2E-Advanced-Deployment-1.19

on:
push:
branches:
- master
- release-*
pull_request: {}
workflow_dispatch: {}

env:
# Common versions
GO_VERSION: '1.17'
KIND_IMAGE: 'kindest/node:v1.19.16'
KIND_CLUSTER_NAME: 'ci-testing'

jobs:

rollout:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/[email protected]
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
config: ./test/kind-conf.yaml
- name: Build image
run: |
export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
docker build --pull --no-cache . -t $IMAGE
kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
- name: Install Kruise
run: |
set -ex
kubectl cluster-info
make helm
helm repo add openkruise https://openkruise.github.io/charts/
helm repo update
helm install kruise openkruise/kruise
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
set -e
if [ "$PODS" -eq "2" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
set -e
if [ "$PODS" -eq "2" ]; then
echo "Wait for kruise-manager ready successfully"
else
echo "Timeout to wait for kruise-manager ready"
exit 1
fi
- name: Install Kruise Rollout
run: |
set -ex
kubectl cluster-info
IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
set -e
if [ "$PODS" -eq "1" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
kubectl get node -o yaml
kubectl get all -n kruise-rollout -o yaml
set -e
if [ "$PODS" -eq "1" ]; then
echo "Wait for kruise-rollout ready successfully"
else
echo "Timeout to wait for kruise-rollout ready"
exit 1
fi
- name: Run E2E Tests
run: |
export KUBECONFIG=/home/runner/.kube/config
make ginkgo
set +e
./bin/ginkgo -timeout 60m -v --focus='Advanced Deployment' test/e2e
retVal=$?
# kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
if [ "${restartCount}" -eq "0" ];then
echo "Kruise-rollout has not restarted"
else
kubectl get pod -n kruise-rollout --no-headers
echo "Kruise-rollout has restarted, abort!!!"
kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
exit 1
fi
exit $retVal
110 changes: 110 additions & 0 deletions .github/workflows/e2e-advanced-deployment-1.23.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
name: E2E-Advanced-Deployment-1.23

on:
push:
branches:
- master
- release-*
pull_request: {}
workflow_dispatch: {}

env:
# Common versions
GO_VERSION: '1.17'
KIND_IMAGE: 'kindest/node:v1.23.3'
KIND_CLUSTER_NAME: 'ci-testing'

jobs:

rollout:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/[email protected]
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
config: ./test/kind-conf.yaml
- name: Build image
run: |
export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
docker build --pull --no-cache . -t $IMAGE
kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
- name: Install Kruise
run: |
set -ex
kubectl cluster-info
make helm
helm repo add openkruise https://openkruise.github.io/charts/
helm repo update
helm install kruise openkruise/kruise
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
set -e
if [ "$PODS" -eq "2" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
set -e
if [ "$PODS" -eq "2" ]; then
echo "Wait for kruise-manager ready successfully"
else
echo "Timeout to wait for kruise-manager ready"
exit 1
fi
- name: Install Kruise Rollout
run: |
set -ex
kubectl cluster-info
IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
set -e
if [ "$PODS" -eq "1" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
kubectl get node -o yaml
kubectl get all -n kruise-rollout -o yaml
set -e
if [ "$PODS" -eq "1" ]; then
echo "Wait for kruise-rollout ready successfully"
else
echo "Timeout to wait for kruise-rollout ready"
exit 1
fi
- name: Run E2E Tests
run: |
export KUBECONFIG=/home/runner/.kube/config
make ginkgo
set +e
./bin/ginkgo -timeout 60m -v --focus='Advanced Deployment' test/e2e
retVal=$?
# kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
if [ "${restartCount}" -eq "0" ];then
echo "Kruise-rollout has not restarted"
else
kubectl get pod -n kruise-rollout --no-headers
echo "Kruise-rollout has restarted, abort!!!"
kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
exit 1
fi
exit $retVal
12 changes: 5 additions & 7 deletions api/v1alpha1/deployment_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,14 @@ type DeploymentStrategy struct {
type RollingStyleType string

const (
// PartitionRollingStyleType means rolling in batches just like CloneSet, and will NOT create any extra Deployment;
PartitionRollingStyleType RollingStyleType = "Partition"
// CanaryRollingStyleType means rolling in canary way, and will create a canary Deployment.
CanaryRollingStyleType RollingStyleType = "Canary"
// PartitionRollingStyle means rolling in batches just like CloneSet, and will NOT create any extra Deployment;
PartitionRollingStyle RollingStyleType = "Partition"
// CanaryRollingStyle means rolling in canary way, and will create a canary Deployment.
CanaryRollingStyle RollingStyleType = "Canary"
)

// DeploymentExtraStatus is extra status field for Advanced Deployment
type DeploymentExtraStatus struct {
// ObservedGeneration record the generation of deployment this status observed.
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// UpdatedReadyReplicas the number of pods that has been updated and ready.
UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
// ExpectedUpdatedReplicas is an absolute number calculated based on Partition
Expand All @@ -52,7 +50,7 @@ type DeploymentExtraStatus struct {
}

func SetDefaultDeploymentStrategy(strategy *DeploymentStrategy) {
if strategy.RollingStyle == CanaryRollingStyleType {
if strategy.RollingStyle == CanaryRollingStyle {
return
}
if strategy.RollingUpdate == nil {
Expand Down
1 change: 1 addition & 0 deletions config/default/manager_auth_proxy_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,5 @@ spec:
- "--health-probe-bind-address=:8081"
- "--metrics-bind-address=127.0.0.1:8080"
- "--leader-elect"
- "--feature-gates=AdvancedDeployment=true"
- "--v=3"
1 change: 1 addition & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ spec:
- /manager
args:
- --leader-elect
- --feature-gates=AdvancedDeployment=true
image: controller:latest
name: manager
securityContext:
Expand Down
5 changes: 5 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
kruisev1beta1 "github.com/openkruise/kruise-api/apps/v1beta1"
rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
br "github.com/openkruise/rollouts/pkg/controller/batchrelease"
"github.com/openkruise/rollouts/pkg/controller/deployment"
"github.com/openkruise/rollouts/pkg/controller/rollout"
"github.com/openkruise/rollouts/pkg/controller/rollouthistory"
utilclient "github.com/openkruise/rollouts/pkg/util/client"
Expand Down Expand Up @@ -116,6 +117,10 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "rollouthistory")
os.Exit(1)
}
if err = deployment.Add(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "advanceddeployment")
os.Exit(1)
}

//+kubebuilder:scaffold:builder
setupLog.Info("setup webhook")
Expand Down
34 changes: 23 additions & 11 deletions pkg/controller/deployment/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@ import (
"encoding/json"
"flag"
"reflect"
"time"

appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
appslisters "k8s.io/client-go/listers/apps/v1"
corelisters "k8s.io/client-go/listers/core/v1"
toolscache "k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
Expand All @@ -51,9 +52,13 @@ import (
)

func init() {
flag.IntVar(&concurrentReconciles, "deployment-workers", concurrentReconciles, "Max concurrent workers for StatefulSet controller.")
flag.IntVar(&concurrentReconciles, "deployment-workers", concurrentReconciles, "Max concurrent workers for advanced deployment controller.")
}

const (
DefaultRetryDuration = 2 * time.Second
)

var (
concurrentReconciles = 3
)
Expand All @@ -75,10 +80,6 @@ func Add(mgr manager.Manager) error {
// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
cacher := mgr.GetCache()
podInformer, err := cacher.GetInformerForKind(context.TODO(), v1.SchemeGroupVersion.WithKind("Pod"))
if err != nil {
return nil, err
}
dInformer, err := cacher.GetInformerForKind(context.TODO(), appsv1.SchemeGroupVersion.WithKind("Deployment"))
if err != nil {
return nil, err
Expand All @@ -91,7 +92,6 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
// Lister
dLister := appslisters.NewDeploymentLister(dInformer.(toolscache.SharedIndexInformer).GetIndexer())
rsLister := appslisters.NewReplicaSetLister(rsInformer.(toolscache.SharedIndexInformer).GetIndexer())
podLister := corelisters.NewPodLister(podInformer.(toolscache.SharedIndexInformer).GetIndexer())

// Client & Recorder
genericClient := clientutil.GetGenericClientWithName("advanced-deployment-controller")
Expand All @@ -107,7 +107,6 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
eventRecorder: recorder,
dLister: dLister,
rsLister: rsLister,
podLister: podLister,
}
return &ReconcileDeployment{Client: mgr.GetClient(), controllerFactory: factory}, nil
}
Expand Down Expand Up @@ -179,8 +178,22 @@ func (r *ReconcileDeployment) Reconcile(_ context.Context, request reconcile.Req
return reconcile.Result{}, nil
}

errList := field.ErrorList{}
err = dc.syncDeployment(context.Background(), deployment)
return ctrl.Result{}, err
if err != nil {
errList = append(errList, field.InternalError(field.NewPath("syncDeployment"), err))
}
err = dc.patchExtraStatus(deployment)
if err != nil {
errList = append(errList, field.InternalError(field.NewPath("patchExtraStatus"), err))
}
err = deploymentutil.DeploymentRolloutSatisfied(deployment, dc.strategy.Partition)
if err != nil {
klog.V(3).Infof("Deployment %v is still rolling: %v", klog.KObj(deployment), err)
return reconcile.Result{RequeueAfter: DefaultRetryDuration}, errList.ToAggregate()
}

return ctrl.Result{}, errList.ToAggregate()
}

type controllerFactory DeploymentController
Expand All @@ -201,7 +214,7 @@ func (f *controllerFactory) NewController(deployment *appsv1.Deployment) *Deploy
}

// We do NOT process such deployment with canary rolling style
if strategy.RollingStyle == rolloutsv1alpha1.CanaryRollingStyleType {
if strategy.RollingStyle == rolloutsv1alpha1.CanaryRollingStyle {
return nil
}

Expand All @@ -214,7 +227,6 @@ func (f *controllerFactory) NewController(deployment *appsv1.Deployment) *Deploy
eventRecorder: f.eventRecorder,
dLister: f.dLister,
rsLister: f.rsLister,
podLister: f.podLister,
strategy: strategy,
}
}
Loading

0 comments on commit 0babeee

Please sign in to comment.