diff --git a/java-client/src/main/java/org/apache/pegasus/client/retry/DefaultRetryPolicy.java b/java-client/src/main/java/org/apache/pegasus/client/retry/DefaultRetryPolicy.java
index 8bdef05c7f..10b7958029 100644
--- a/java-client/src/main/java/org/apache/pegasus/client/retry/DefaultRetryPolicy.java
+++ b/java-client/src/main/java/org/apache/pegasus/client/retry/DefaultRetryPolicy.java
@@ -38,10 +38,11 @@ public RetryAction shouldRetry(int retries, long deadlineNanos, Duration timeout
     if (now >= deadlineNanos) {
       return new RetryAction(RetryDecision.FAIL, Duration.ZERO, "request deadline reached");
     }
-    long timeoutMs = timeout.toMillis();
-    long retryDelayMs =
+    long timeoutNanos = timeout.toNanos();
+    long retryDelayNanos =
         Math.min(
-            timeoutMs > 3 ? timeoutMs / 3 : 1, TimeUnit.NANOSECONDS.toMillis(deadlineNanos - now));
-    return new RetryAction(RetryDecision.RETRY, Duration.ofNanos(retryDelayMs), "");
+            timeoutNanos > 3000000 ? timeoutNanos / 3000000 : 1,
+            TimeUnit.NANOSECONDS.toNanos(deadlineNanos - now));
+    return new RetryAction(RetryDecision.RETRY, Duration.ofNanos(retryDelayNanos), "");
   }
 }
diff --git a/java-client/src/main/java/org/apache/pegasus/client/retry/ExponentialBackoffRetryPolicy.java b/java-client/src/main/java/org/apache/pegasus/client/retry/ExponentialBackoffRetryPolicy.java
index 4bebe97a44..42ecf9a36c 100644
--- a/java-client/src/main/java/org/apache/pegasus/client/retry/ExponentialBackoffRetryPolicy.java
+++ b/java-client/src/main/java/org/apache/pegasus/client/retry/ExponentialBackoffRetryPolicy.java
@@ -53,13 +53,14 @@ public RetryAction shouldRetry(int retries, long deadlineNanos, Duration timeout
     if (now >= deadlineNanos) {
       return new RetryAction(RetryDecision.FAIL, Duration.ZERO, "request deadline reached");
     }
-    long normalIntervalMs =
-        retryBaseIntervalMs * RETRY_BACKOFF[Math.min(retries, RETRY_BACKOFF.length - 1)];
+    long normalIntervalNanos =
+        retryBaseIntervalMs * RETRY_BACKOFF[Math.min(retries, RETRY_BACKOFF.length - 1)] * 1000000;
     // 1% possible jitter
-    long jitterNanos = (long) (normalIntervalMs * ThreadLocalRandom.current().nextFloat() * 0.01f);
+    long jitterNanos =
+        (long) (normalIntervalNanos * ThreadLocalRandom.current().nextFloat() * 0.01f);
     long retryIntervalNanos =
         Math.min(
-            normalIntervalMs + jitterNanos, TimeUnit.NANOSECONDS.toMillis(deadlineNanos - now));
+            normalIntervalNanos + jitterNanos, TimeUnit.NANOSECONDS.toNanos(deadlineNanos - now));
     return new RetryAction(RetryDecision.RETRY, Duration.ofNanos(retryIntervalNanos), "");
   }
 }