diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h
index 6df41f609e055..7615a0b7ea08c 100644
--- a/paddle/fluid/platform/device_context.h
+++ b/paddle/fluid/platform/device_context.h
@@ -162,14 +162,9 @@ class CUDAContext {
             << "Please recompile or reinstall Paddle with compatible CUDNN "
                "version.";
       }
+      PADDLE_ENFORCE_CUDA_SUCCESS(dynload::cudnnCreate(&cudnn_handle_));
       PADDLE_ENFORCE_CUDA_SUCCESS(
-          dynload::cudnnCreate(&cudnn_handle_),
-          platform::errors::Fatal(
-              "Failed to create Cudnn handle in DeviceContext"));
-      PADDLE_ENFORCE_CUDA_SUCCESS(
-          dynload::cudnnSetStream(cudnn_handle_, RawStream()),
-          platform::errors::Fatal(
-              "Failed to set stream for Cudnn handle in DeviceContext"));
+          dynload::cudnnSetStream(cudnn_handle_, RawStream()));
     } else {
       cudnn_handle_ = nullptr;
     }
@@ -177,9 +172,7 @@ class CUDAContext {
 
   void DestoryCuDNNContext() {
     if (cudnn_handle_) {
-      PADDLE_ENFORCE_CUDA_SUCCESS(
-          dynload::cudnnDestroy(cudnn_handle_),
-          platform::errors::Fatal("Failed to destory Cudnn handle"));
+      PADDLE_ENFORCE_CUDA_SUCCESS(dynload::cudnnDestroy(cudnn_handle_));
     }
     cudnn_handle_ = nullptr;
   }
diff --git a/paddle/fluid/platform/stream/cuda_stream.cc b/paddle/fluid/platform/stream/cuda_stream.cc
index 67f8989b32bcc..6d3b065037600 100644
--- a/paddle/fluid/platform/stream/cuda_stream.cc
+++ b/paddle/fluid/platform/stream/cuda_stream.cc
@@ -30,13 +30,10 @@ bool CUDAStream::Init(const Place& place, const Priority& priority) {
   CUDADeviceGuard guard(boost::get<CUDAPlace>(place_).device);
   if (priority == Priority::kHigh) {
     PADDLE_ENFORCE_CUDA_SUCCESS(
-        cudaStreamCreateWithPriority(&stream_, kDefaultFlag, -1),
-        platform::errors::Fatal("High priority cuda stream creation failed."));
+        cudaStreamCreateWithPriority(&stream_, kDefaultFlag, -1));
   } else if (priority == Priority::kNormal) {
     PADDLE_ENFORCE_CUDA_SUCCESS(
-        cudaStreamCreateWithPriority(&stream_, kDefaultFlag, 0),
-        platform::errors::Fatal(
-            "Normal priority cuda stream creation failed."));
+        cudaStreamCreateWithPriority(&stream_, kDefaultFlag, 0));
   }
   callback_manager_.reset(new StreamCallbackManager(stream_));
   VLOG(3) << "CUDAStream Init stream: " << stream_
@@ -49,9 +46,7 @@ void CUDAStream::Destroy() {
   Wait();
   WaitCallback();
   if (stream_) {
-    PADDLE_ENFORCE_CUDA_SUCCESS(
-        cudaStreamDestroy(stream_),
-        platform::errors::Fatal("Cuda stream destruction failed."));
+    PADDLE_ENFORCE_CUDA_SUCCESS(cudaStreamDestroy(stream_));
   }
   stream_ = nullptr;
 }
@@ -67,10 +62,7 @@ void CUDAStream::Wait() const {
   }
 #endif
 
-  PADDLE_ENFORCE_CUDA_SUCCESS(
-      e_sync, platform::errors::Fatal(
-                  "cudaStreamSynchronize raises error: %s, errono: %d",
-                  cudaGetErrorString(e_sync), static_cast<int>(e_sync)));
+  PADDLE_ENFORCE_CUDA_SUCCESS(e_sync);
 }
 
 }  // namespace stream
diff --git a/paddle/fluid/platform/stream/cuda_stream.h b/paddle/fluid/platform/stream/cuda_stream.h
index 1fc55ce660ec5..4272d5fd0b14c 100644
--- a/paddle/fluid/platform/stream/cuda_stream.h
+++ b/paddle/fluid/platform/stream/cuda_stream.h
@@ -52,21 +52,15 @@ class CUDAStream final {
   template <typename Callback>
   void RecordEvent(cudaEvent_t ev, Callback callback) const {
     callback();
-    PADDLE_ENFORCE_CUDA_SUCCESS(
-        cudaEventRecord(ev, stream_),
-        platform::errors::Fatal("CUDA event recording failed."));
+    PADDLE_ENFORCE_CUDA_SUCCESS(cudaEventRecord(ev, stream_));
   }
 
   void RecordEvent(cudaEvent_t ev) const {
-    PADDLE_ENFORCE_CUDA_SUCCESS(
-        cudaEventRecord(ev, stream_),
-        platform::errors::Fatal("CUDA event recording failed."));
+    PADDLE_ENFORCE_CUDA_SUCCESS(cudaEventRecord(ev, stream_));
   }
 
   void WaitEvent(cudaEvent_t ev) const {
-    PADDLE_ENFORCE_CUDA_SUCCESS(
-        cudaStreamWaitEvent(stream_, ev, 0),
-        platform::errors::Fatal("Failed to wait event."));
+    PADDLE_ENFORCE_CUDA_SUCCESS(cudaStreamWaitEvent(stream_, ev, 0));
   }
 
   void Wait() const;