diff --git a/CHANGELOG.md b/CHANGELOG.md
index e54bfb56..de5c4a5e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Only filter the list of available serial ports if a port has not been specified via CLI option or configuration file (#693)
+
 ### Removed
 
 ## [3.2.0]
diff --git a/espflash/src/cli/serial.rs b/espflash/src/cli/serial.rs
index 993efd5d..ab71630b 100644
--- a/espflash/src/cli/serial.rs
+++ b/espflash/src/cli/serial.rs
@@ -33,13 +33,14 @@ pub fn get_serial_port_info(
     // doesn't work (on Windows) with "dummy" device paths like `COM4`. That's
     // the reason we need to handle Windows/Posix differently.
 
-    let ports = detect_usb_serial_ports(matches.list_all_ports).unwrap_or_default();
-
     if let Some(serial) = &matches.port {
+        let ports = detect_usb_serial_ports(true).unwrap_or_default();
         find_serial_port(&ports, serial)
     } else if let Some(serial) = &config.connection.serial {
+        let ports = detect_usb_serial_ports(true).unwrap_or_default();
         find_serial_port(&ports, serial)
     } else {
+        let ports = detect_usb_serial_ports(matches.list_all_ports).unwrap_or_default();
         let (port, matches) = select_serial_port(ports, config, matches.confirm_port)?;
 
         match &port.port_type {