Add possibility to return non-final error

controller/controller.go

@@ -783,7 +783,7 @@ func (ctrl *ProvisionController) processNextClaimWorkItem() bool {
 			return fmt.Errorf("expected string in workqueue but got %#v", obj)
 		}
 
-		if err := ctrl.syncClaimHandler(key); err != nil {
+		if _, err := ctrl.syncClaimHandler(key); err != nil {
 			if ctrl.failedProvisionThreshold == 0 {
 				glog.Warningf("Retrying syncing claim %q, failure %v", key, ctrl.claimQueue.NumRequeues(obj))
 				ctrl.claimQueue.AddRateLimited(obj)
@@ -855,15 +855,15 @@ func (ctrl *ProvisionController) processNextVolumeWorkItem() bool {
 }
 
 // syncClaimHandler gets the claim from informer's cache then calls syncClaim
-func (ctrl *ProvisionController) syncClaimHandler(key string) error {
+func (ctrl *ProvisionController) syncClaimHandler(key string) (ProvisioningState, error) {
 	objs, err := ctrl.claimsIndexer.ByIndex(uidIndex, key)
 	if err != nil {
-		return err
+		return ProvisioningFinished, err
 	}
 	if len(objs) == 0 {
 		// TODO: handle deleted PVCs
 		utilruntime.HandleError(fmt.Errorf("claim %q in work queue no longer exists", key))
-		return nil
+		return ProvisioningFinished, nil
 	}
 	claimObj := objs[0]
 	return ctrl.syncClaim(claimObj)
@@ -885,19 +885,19 @@ func (ctrl *ProvisionController) syncVolumeHandler(key string) error {
 
 // syncClaim checks if the claim should have a volume provisioned for it and
 // provisions one if so.
-func (ctrl *ProvisionController) syncClaim(obj interface{}) error {
+func (ctrl *ProvisionController) syncClaim(obj interface{}) (ProvisioningState, error) {
 	claim, ok := obj.(*v1.PersistentVolumeClaim)
 	if !ok {
-		return fmt.Errorf("expected claim but got %+v", obj)
+		return ProvisioningFinished, fmt.Errorf("expected claim but got %+v", obj)
 	}
 
 	if ctrl.shouldProvision(claim) {
 		startTime := time.Now()
-		err := ctrl.provisionClaimOperation(claim)
+		status, err := ctrl.provisionClaimOperation(claim)
 		ctrl.updateProvisionStats(claim, err, startTime)
-		return err
+		return status, err
 	}
-	return nil
+	return ProvisioningFinished, nil
 }
 
 // syncVolume checks if the volume should be deleted and deletes if so
@@ -1044,7 +1044,7 @@ func (ctrl *ProvisionController) updateDeleteStats(volume *v1.PersistentVolume,
 // provisionClaimOperation attempts to provision a volume for the given claim.
 // Returns error, which indicates whether provisioning should be retried
 // (requeue the claim) or not
-func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVolumeClaim) error {
+func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVolumeClaim) (ProvisioningState, error) {
 	// Most code here is identical to that found in controller.go of kube's PV controller...
 	claimClass := util.GetPersistentVolumeClaimClass(claim)
 	operation := fmt.Sprintf("provision %q class %q", claimToClaimKey(claim), claimClass)
@@ -1058,48 +1058,48 @@ func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVol
 	if err == nil && volume != nil {
 		// Volume has been already provisioned, nothing to do.
 		glog.Info(logOperation(operation, "persistentvolume %q already exists, skipping", pvName))
-		return nil
+		return ProvisioningFinished, nil
 	}
 
 	// Prepare a claimRef to the claim early (to fail before a volume is
 	// provisioned)
 	claimRef, err := ref.GetReference(scheme.Scheme, claim)
 	if err != nil {
 		glog.Error(logOperation(operation, "unexpected error getting claim reference: %v", err))
-		return nil
+		return ProvisioningNoChange, nil
 	}
 
 	provisioner, parameters, err := ctrl.getStorageClassFields(claimClass)
 	if err != nil {
 		glog.Error(logOperation(operation, "error getting claim's StorageClass's fields: %v", err))
-		return nil
+		return ProvisioningFinished, nil
 	}
 	if provisioner != ctrl.provisionerName {
 		// class.Provisioner has either changed since shouldProvision() or
 		// annDynamicallyProvisioned contains different provisioner than
 		// class.Provisioner.
 		glog.Error(logOperation(operation, "unknown provisioner %q requested in claim's StorageClass", provisioner))
-		return nil
+		return ProvisioningFinished, nil
 	}
 
 	// Check if this provisioner can provision this claim.
 	if err = ctrl.canProvision(claim); err != nil {
 		ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "ProvisioningFailed", err.Error())
 		glog.Error(logOperation(operation, "failed to provision volume: %v", err))
-		return nil
+		return ProvisioningFinished, nil
 	}
 
 	reclaimPolicy := v1.PersistentVolumeReclaimDelete
 	if ctrl.kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.8.0")) {
 		reclaimPolicy, err = ctrl.fetchReclaimPolicy(claimClass)
 		if err != nil {
-			return err
+			return ProvisioningFinished, err
 		}
 	}
 
 	mountOptions, err := ctrl.fetchMountOptions(claimClass)
 	if err != nil {
-		return err
+		return ProvisioningFinished, err
 	}
 
 	var selectedNode *v1.Node
@@ -1111,7 +1111,7 @@ func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVol
 			if err != nil {
 				err = fmt.Errorf("failed to get target node: %v", err)
 				ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "ProvisioningFailed", err.Error())
-				return err
+				return ProvisioningNoChange, err
 			}
 		}
 
@@ -1120,7 +1120,7 @@ func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVol
 		if err != nil {
 			err = fmt.Errorf("failed to get AllowedTopologies from StorageClass: %v", err)
 			ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "ProvisioningFailed", err.Error())
-			return err
+			return ProvisioningNoChange, err
 		}
 	}
 
@@ -1136,16 +1136,21 @@ func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVol
 
 	ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, "Provisioning", fmt.Sprintf("External provisioner is provisioning volume for claim %q", claimToClaimKey(claim)))
 
-	volume, err = ctrl.provisioner.Provision(options)
+	result := ProvisioningFinished
+	if p, ok := ctrl.provisioner.(ProvisionerExt); ok {
+		volume, result, err = p.ProvisionExt(options)
+	} else {
+		volume, err = ctrl.provisioner.Provision(options)
+	}
 	if err != nil {
 		if ierr, ok := err.(*IgnoredError); ok {
 			// Provision ignored, do nothing and hope another provisioner will provision it.
 			glog.Info(logOperation(operation, "volume provision ignored: %v", ierr))
-			return nil
+			return ProvisioningFinished, nil
 		}
 		err = fmt.Errorf("failed to provision volume with StorageClass %q: %v", claimClass, err)
 		ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "ProvisioningFailed", err.Error())
-		return err
+		return result, err
 	}
 
 	glog.Info(logOperation(operation, "volume %q provisioned", volume.Name))
@@ -1218,7 +1223,7 @@ func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVol
 	}
 
 	glog.Info(logOperation(operation, "succeeded"))
-	return nil
+	return ProvisioningFinished, nil
 }
 
 // deleteVolumeOperation attempts to delete the volume backing the given

controller/volume.go

@@ -62,6 +62,42 @@ type BlockProvisioner interface {
 	SupportsBlock() bool
 }
 
+// ProvisionerExt is an optional interface implemented by provisioners that
+// can return enhanced error code from provisioner.
+type ProvisionerExt interface {
+	// ProvisionExt creates a volume i.e. the storage asset and returns a PV object
+	// for the volume. The provisioner can return an error (e.g. timeout) and state
+	// ProvisioningInBackground to tell the controller that provisioning may be in
+	// progress after ProvisionExt() finishes. The controller will call ProvisionExt()
+	// again with the same parameters, assuming that the provisioner continues
+	// provisioning the volume. The provisioner must return either final error (with
+	// ProvisioningFinished) or success eventually, otherwise the controller will try
+	// forever (unless FailedProvisionThreshold is set).
+	ProvisionExt(options VolumeOptions) (*v1.PersistentVolume, ProvisioningState, error)
+}
+
+// ProvisioningState is state of volume provisioning. It tells the controller if
+// provisioning could be in progress in the background after ProvisionExt() call
+// returns or the provisioning is 100% finished (either with success or error).
+type ProvisioningState string
+
+const (
+	// ProvisioningInBackground tells the controller that provisioning may be in
+	// progress in background after ProvisionExt call finished.
+	ProvisioningInBackground ProvisioningState = "Background"
+	// ProvisioningFinished tells the controller that provisioning for sure does
+	// not continue in background, error code of ProvisionExt() is final.
+	ProvisioningFinished ProvisioningState = "Finished"
+	// ProvisioningNoChange tells the controller that provisioning state is the same as
+	// before the call - either ProvisioningInBackground or ProvisioningFinished from
+	// the previous ProvisionExt(). This state is typically returned by a provisioner
+	// before it could reach storage backend - the provisioner could not check status
+	// of provisioning and previous state applies. If this state is returned from the
+	// first ProvisionExt call, ProvisioningFinished is assumed (the provisioning
+	// could not even start).
+	ProvisioningNoChange ProvisioningState = "NoChange"
+)
+
 // IgnoredError is the value for Delete to return to indicate that the call has
 // been ignored and no action taken. In case multiple provisioners are serving
 // the same storage class, provisioners may ignore PVs they are not responsible