diff --git a/src/main/java/org/embulk/input/sftp/SftpFileInput.java b/src/main/java/org/embulk/input/sftp/SftpFileInput.java
index 620300b..e1a80d5 100644
--- a/src/main/java/org/embulk/input/sftp/SftpFileInput.java
+++ b/src/main/java/org/embulk/input/sftp/SftpFileInput.java
@@ -264,6 +264,12 @@ else if (files.isFile()) {
                         @Override
                         public boolean isRetryableException(Exception exception)
                         {
+                            if (exception.getCause() != null && exception.getCause().getCause() != null) {
+                                Throwable cause = exception.getCause().getCause();
+                                if (cause.getMessage() != null && cause.getMessage().contains("Auth fail")) {
+                                    throw new ConfigException(exception);
+                                }
+                            }
                             if (exception instanceof ConfigException) {
                                 return false;
                             }
@@ -288,14 +294,6 @@ public void onRetry(Exception exception, int retryCount, int retryLimit, int ret
                         public void onGiveup(Exception firstException, Exception lastException)
                                 throws RetryGiveupException
                         {
-                            // Generally, Auth fail should be caught and throw ConfigException when first retry. But this library is a bit unstable.
-                            // So we throw ConfigException after all retries are completed
-                            if (lastException.getCause() != null && lastException.getCause().getCause() != null) {
-                                Throwable cause = lastException.getCause().getCause();
-                                if (cause.getMessage().contains("Auth fail")) {
-                                    throw new ConfigException(lastException);
-                                }
-                            }
                         }
                     });
         }
diff --git a/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java b/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java
index 65d49b5..ee21be8 100644
--- a/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java
+++ b/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java
@@ -56,6 +56,7 @@
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 public class TestSftpFileInputPlugin
 {
@@ -248,6 +249,36 @@ public List<TaskReport> run(TaskSource taskSource, int taskCount)
         assertEquals(SftpFileInput.getRelativePath(task, Optional.of(expected.get(1).get(0))), configDiff.get(String.class, "last_path"));
     }
 
+    @Test
+    public void testListFilesAuthFail() throws Exception
+    {
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv", true);
+        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv", true);
+
+        PluginTask task = config.deepCopy().set("user", "wrong_user").loadConfig(PluginTask.class);
+
+        plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            {
+                assertEquals(2, taskCount);
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        Method listFilesByPrefix = SftpFileInput.class.getDeclaredMethod("listFilesByPrefix", PluginTask.class);
+        listFilesByPrefix.setAccessible(true);
+        try {
+            listFilesByPrefix.invoke(plugin, task);
+            fail();
+        }
+        catch (Exception ex) {
+            Throwable cause = ex.getCause();
+            assertEquals(cause.getClass(), ConfigException.class);
+            assertEquals(cause.getCause().getCause().getCause().getMessage(), "Auth fail");
+        }
+    }
+
     @Test
     public void testListFilesWithPathPrefixPointToFile() throws Exception
     {