Merge pull request #21 from kuizhiqing/main

first milestone
PaddlePaddle · Mar 3, 2021 · 548d729 · 548d729
2 parents 8357573 + f7ca61b
commit 548d729
Show file tree

Hide file tree

Showing 11 changed files with 75 additions and 21 deletions.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= registry.baidubce.com/kuizhiqing/paddle-operator:v1
+IMG ?= registry.baidubce.com/paddle-operator/controller:v1
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:maxDescLen=0,trivialVersions=true,preserveUnknownFields=false"
 

diff --git a/charts/paddle-operator/values.yaml b/charts/paddle-operator/values.yaml
@@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-image: registry.baidubce.com/kuizhiqing/paddle-operator:v1
+image: registry.baidubce.com/paddle-operator/controller:v1
 
 controllernamespace: paddle-system
 jobnamespace: paddle-system
diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -14,5 +14,5 @@ configMapGenerator:
 
 images:
 - name: controller
-  newName: registry.baidubce.com/kuizhiqing/paddle-operator
+  newName: registry.baidubce.com/paddle-operator/controller
   newTag: v1
diff --git a/controllers/paddlejob_controller.go b/controllers/paddlejob_controller.go
@@ -195,6 +195,30 @@ func (r *PaddleJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		}
 	}
 
+	if len(pdj.Status.PS.Refs) == pdj.Spec.PS.Replicas && len(pdj.Status.Worker.Refs) == pdj.Spec.Worker.Replicas {
+		if pdj.Spec.Intranet == pdv1.Service {
+			if len(pdj.Status.PS.Refs)+len(pdj.Status.Worker.Refs) != len(svcs.Items) {
+				return ctrl.Result{}, nil
+			}
+		}
+		if err := r.Get(ctx, types.NamespacedName{Name: pdj.Name, Namespace: pdj.Namespace}, &corev1.ConfigMap{}); err == nil || !apierrors.IsNotFound(err) {
+			return ctrl.Result{}, err
+		}
+		cm := constructConfigMap(&pdj, childPods)
+		if cm == nil {
+			return ctrl.Result{Requeue: true}, nil
+		}
+		if err := ctrl.SetControllerReference(&pdj, cm, r.Scheme); err != nil {
+			log.Error(err, "make reference failed")
+			return ctrl.Result{Requeue: true}, nil
+		}
+		err := r.createResource(ctx, &pdj, cm)
+		if apierrors.IsConflict(err) {
+			return ctrl.Result{Requeue: true}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
 	if len(pdj.Status.PS.Refs) == pdj.Spec.PS.Replicas && len(pdj.Status.Worker.Refs) == pdj.Spec.Worker.Replicas {
 		if err := r.Get(ctx, types.NamespacedName{Name: pdj.Name, Namespace: pdj.Namespace}, &corev1.ConfigMap{}); err == nil || !apierrors.IsNotFound(err) {
 			return ctrl.Result{}, err

diff --git a/controllers/paddlejob_helper.go b/controllers/paddlejob_helper.go
@@ -204,7 +204,14 @@ func constructPod(pdj *pdv1.PaddleJob, resType string, idx int) (pod *corev1.Pod
 	if pdj.Spec.Intranet == pdv1.Service {
 		pod.Spec.Containers[0].Ports = append(pod.Spec.Containers[0].Ports, corev1.ContainerPort{ContainerPort: pdv1.PADDLE_PORT})
 	}
-	pod.Spec.RestartPolicy = "Never"
+
+	if pod.Spec.RestartPolicy == "" {
+		if resType == pdv1.ResourceWorker && pdj.Spec.Intranet == pdv1.Service {
+			pod.Spec.RestartPolicy = "OnFailure"
+		} else {
+			pod.Spec.RestartPolicy = "Never"
+		}
+	}
 
 	return pod
 }

diff --git a/deploy/examples/resnet.yaml b/deploy/examples/resnet.yaml
@@ -0,0 +1,29 @@
+apiVersion: batch.paddlepaddle.org/v1
+kind: PaddleJob
+metadata:
+  name: resnet
+spec:
+  cleanPodPolicy: Never
+  worker:
+    replicas: 2
+    template:
+      spec:
+        containers:
+          - name: resnet
+            image: registry.baidubce.com/paddle-operator/demo-resnet:v1
+            command:
+            - python
+            args:
+            - "-m"
+            - "paddle.distributed.launch"
+            - "train_fleet.py"
+            volumeMounts:
+            - mountPath: /dev/shm
+              name: dshm
+            resources:
+              limits:
+                nvidia.com/gpu: 1
+        volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
diff --git a/deploy/examples/wide_and_deep.yaml b/deploy/examples/wide_and_deep.yaml
@@ -9,16 +9,13 @@ spec:
     replicas: 2
     template:
       spec:
-        restartPolicy: "Never"
-        schedulerName: volcano
         containers:
           - name: paddle
-            image: registry.baidubce.com/kuizhiqing/demo-wide-and-deep:v1
+            image: registry.baidubce.com/paddle-operator/demo-wide-and-deep:v1
   ps:
     replicas: 2
     template:
       spec:
-        restartPolicy: "Never"
         containers:
           - name: paddle
-            image: registry.baidubce.com/kuizhiqing/demo-wide-and-deep:v1
+            image: registry.baidubce.com/paddle-operator/demo-wide-and-deep:v1
diff --git a/deploy/examples/wide_and_deep_service.yaml b/deploy/examples/wide_and_deep_service.yaml
@@ -3,21 +3,19 @@ kind: PaddleJob
 metadata:
   name: wide-ande-deep-service
 spec:
-  intranet: Service
+  intranet: Service # default: PodIP
   cleanPodPolicy: Nerver
   worker:
     replicas: 2
     template:
       spec:
-        restartPolicy: "Never"
         containers:
           - name: paddle
-            image: registry.baidubce.com/kuizhiqing/demo-wide-and-deep:v1
+            image: registry.baidubce.com/paddle-operator/demo-wide-and-deep:v1
   ps:
     replicas: 2
     template:
       spec:
-        restartPolicy: "Never"
         containers:
           - name: paddle
-            image: registry.baidubce.com/kuizhiqing/demo-wide-and-deep:v1
+            image: registry.baidubce.com/paddle-operator/demo-wide-and-deep:v1
diff --git a/deploy/examples/wide_and_deep_volcano.yaml b/deploy/examples/wide_and_deep_volcano.yaml
@@ -10,24 +10,23 @@ spec:
     replicas: 2
     template:
       spec:
-        restartPolicy: "Never"
         schedulerName: volcano
         containers:
           - name: paddle
-            image: registry.baidubce.com/kuizhiqing/demo-wide-and-deep:v1
+            image: registry.baidubce.com/paddle-operator/demo-wide-and-deep:v1
   ps:
     replicas: 2
     template:
       spec:
-        restartPolicy: "Never"
         containers:
           - name: paddle
-            image: registry.baidubce.com/kuizhiqing/demo-wide-and-deep:v1
+            image: registry.baidubce.com/paddle-operator/demo-wide-and-deep:v1
+
 
 ---
 apiVersion: scheduling.volcano.sh/v1beta1
 kind: PodGroup
 metadata:
-  name: wide-ande-deep
+  name: wide-ande-deep # name should be the same with PaddleJob
 spec:
   minMember: 4
diff --git a/deploy/v1/operator.yaml b/deploy/v1/operator.yaml
@@ -245,7 +245,7 @@ spec:
         - --namespace=paddle-system
         command:
         - /manager
-        image: registry.baidubce.com/kuizhiqing/paddle-operator:v1
+        image: registry.baidubce.com/paddle-operator/controller:v1
         imagePullPolicy: Always
         livenessProbe:
           httpGet:

diff --git a/deploy/v1beta1/operator.yaml b/deploy/v1beta1/operator.yaml
@@ -245,7 +245,7 @@ spec:
         - --namespace=paddle-system
         command:
         - /manager
-        image: registry.baidubce.com/kuizhiqing/paddle-operator:v1
+        image: registry.baidubce.com/paddle-operator/controller:v1
         imagePullPolicy: Always
         livenessProbe:
           httpGet: