diff --git a/cmd/subcommand/cmd_host.go b/cmd/subcommand/cmd_host.go
index 1da3547..d1ff45c 100644
--- a/cmd/subcommand/cmd_host.go
+++ b/cmd/subcommand/cmd_host.go
@@ -35,6 +35,7 @@ func (c *hostCommand) run(ctx *cli.Context) error {
 
 	metricNames, err := c.client.ListHostMetricNames(c.host)
 	if err != nil {
+		fileutil.RemoveDir(exportDir)
 		return err
 	}
 
diff --git a/cmd/subcommand/cmd_service.go b/cmd/subcommand/cmd_service.go
index 4baf17e..8242a40 100644
--- a/cmd/subcommand/cmd_service.go
+++ b/cmd/subcommand/cmd_service.go
@@ -48,6 +48,7 @@ func (c *serviceCommand) run(ctx *cli.Context) error {
 	}
 
 	if len(metricNames) == 0 {
+		fileutil.RemoveDir(exportDir)
 		return fmt.Errorf("There was no service metric available for export.")
 	}
 
diff --git a/internal/fileutil/fileutil.go b/internal/fileutil/fileutil.go
index 829ca37..006d1fb 100644
--- a/internal/fileutil/fileutil.go
+++ b/internal/fileutil/fileutil.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/tukaelu/sabadashi/internal/exporter"
 )
@@ -21,11 +22,30 @@ func CreateExportDir(name, from, to string) (string, error) {
 			return "", err
 		}
 	} else {
-		return "", fmt.Errorf("Directory '%s' is already exists. Please try again after making a backup", dirPath)
+		now := time.Now().Format("20060102T150405")
+		renameTo := filepath.Join(
+			cwd,
+			name,
+			fmt.Sprintf("%s_%s_%s", from, to, now),
+		)
+		if err := os.Rename(dirPath, renameTo); err != nil {
+			return "", fmt.Errorf("Failed to rename an already existing directory. ('%s' to '%s')", dirPath, renameTo)
+		}
+		fmt.Printf("Renamed an already existing directory to '%s'.\n", renameTo)
+
+		if err := os.MkdirAll(dirPath, 0755); err != nil {
+			return "", err
+		}
 	}
 	return dirPath, nil
 }
 
+func RemoveDir(exportDir string) {
+	if err := os.Remove(exportDir); err != nil {
+		fmt.Printf("Failed to delete directory. (path: %s)", exportDir)
+	}
+}
+
 func GetExportFilePath(dir, name, ext string) string {
 	return filepath.Join(
 		dir,