[FIXED] #2417

server/filestore.go

@@ -4145,7 +4145,19 @@ func (fs *fileStore) Delete() error {
 	if err := fs.Stop(); err != nil {
 		return err
 	}
-	return os.RemoveAll(fs.fcfg.StoreDir)
+
+	err := os.RemoveAll(fs.fcfg.StoreDir)
+	if err == nil {
+		return nil
+	}
+	ttl := time.Now().Add(time.Second)
+	for time.Now().Before(ttl) {
+		time.Sleep(10 * time.Millisecond)
+		if err = os.RemoveAll(fs.fcfg.StoreDir); err == nil {
+			return nil
+		}
+	}
+	return err
 }
 
 // Lock should be held.

server/filestore_test.go

@@ -2675,6 +2675,41 @@ func TestFileStoreStreamStateDeleted(t *testing.T) {
 	}
 }
 
+// We have reports that sometimes under load a stream could complain about a storage directory
+// not being empty.
+func TestFileStoreStreamDeleteDirNotEmpty(t *testing.T) {
+	storeDir := createDir(t, JetStreamStoreDir)
+	defer removeDir(t, storeDir)
+
+	fs, err := newFileStore(FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: "zzz", Storage: FileStorage})
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer fs.Stop()
+
+	subj, toStore := "foo", uint64(10)
+	for i := uint64(1); i <= toStore; i++ {
+		msg := []byte(fmt.Sprintf("[%08d] Hello World!", i))
+		if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil {
+			t.Fatalf("Error storing msg: %v", err)
+		}
+	}
+
+	ready := make(chan bool)
+	go func() {
+		g := path.Join(storeDir, "g")
+		ready <- true
+		for i := 0; i < 100; i++ {
+			ioutil.WriteFile(g, []byte("OK"), defaultFilePerms)
+		}
+	}()
+
+	<-ready
+	if err := fs.Delete(); err != nil {
+		t.Fatalf("Delete returned an error: %v", err)
+	}
+}
+
 func TestFileStoreConsumerPerf(t *testing.T) {
 	// Comment out to run, holding place for now.
 	t.SkipNow()