diff --git a/packages/mongodb-memory-server-core/src/MongoMemoryReplSet.ts b/packages/mongodb-memory-server-core/src/MongoMemoryReplSet.ts
index 4cfe812d0..8a66374b1 100644
--- a/packages/mongodb-memory-server-core/src/MongoMemoryReplSet.ts
+++ b/packages/mongodb-memory-server-core/src/MongoMemoryReplSet.ts
@@ -353,11 +353,13 @@ export class MongoMemoryReplSet extends EventEmitter implements ManagerAdvanced
     await ensureAsync()
       .then(() => this.initAllServers())
       .then(() => this._initReplSet())
-      .catch((err) => {
+      .catch(async (err) => {
         if (!debug.enabled('MongoMS:MongoMemoryReplSet')) {
           console.warn('Starting the ReplSet failed, enable debug for more information');
         }
 
+        await this.stop(false); // still try to close the instance that was spawned, without cleanup for investigation
+
         this.stateChange(MongoMemoryReplSetStates.stopped);
 
         throw err;
diff --git a/packages/mongodb-memory-server-core/src/MongoMemoryServer.ts b/packages/mongodb-memory-server-core/src/MongoMemoryServer.ts
index 5ddb0b062..b86e0c9ce 100644
--- a/packages/mongodb-memory-server-core/src/MongoMemoryServer.ts
+++ b/packages/mongodb-memory-server-core/src/MongoMemoryServer.ts
@@ -276,11 +276,13 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
 
     this.stateChange(MongoMemoryServerStates.starting);
 
-    await this._startUpInstance(forceSamePort).catch((err) => {
+    await this._startUpInstance(forceSamePort).catch(async (err) => {
       if (!debug.enabled('MongoMS:MongoMemoryServer')) {
         console.warn('Starting the instance failed, enable debug for more information');
       }
 
+      await this.stop(false); // still try to close the instance that was spawned, without cleanup for investigation
+
       this.stateChange(MongoMemoryServerStates.stopped);
 
       throw err;