diff --git a/components/esp_hw_support/include/esp_private/sleep_modem.h b/components/esp_hw_support/include/esp_private/sleep_modem.h
index 7e11d88a9e6a..7601aeebd82b 100644
--- a/components/esp_hw_support/include/esp_private/sleep_modem.h
+++ b/components/esp_hw_support/include/esp_private/sleep_modem.h
@@ -42,6 +42,13 @@ void mac_bb_power_up_cb_execute(void);
 
 #if SOC_PM_SUPPORT_PMU_MODEM_STATE
 
+/**
+ * @brief The retention action in the modem state of WiFi PHY module
+ *
+ * @param restore  true for restore the PHY context, false for backup the PHY context
+ */
+void sleep_modem_wifi_do_phy_retention(bool restore);
+
 /**
  * @brief Get WiFi modem state
  *
diff --git a/components/esp_hw_support/sleep_modem.c b/components/esp_hw_support/sleep_modem.c
index 438ce2809928..6f8994d2e6a3 100644
--- a/components/esp_hw_support/sleep_modem.c
+++ b/components/esp_hw_support/sleep_modem.c
@@ -13,6 +13,7 @@
 #include "esp_attr.h"
 #include "esp_sleep.h"
 #include "soc/soc_caps.h"
+#include "esp_private/pm_impl.h"
 #include "esp_private/sleep_modem.h"
 #include "esp_private/sleep_retention.h"
 #include "sdkconfig.h"
@@ -26,6 +27,7 @@
 
 #if SOC_PM_SUPPORT_PMU_MODEM_STATE
 #include "soc/pmu_reg.h"
+#include "esp_private/esp_pau.h"
 #endif
 
 static __attribute__((unused)) const char *TAG = "sleep_modem";
@@ -153,10 +155,17 @@ typedef struct {
 typedef struct sleep_modem_config {
     struct {
         void    *phy_link;
+        union {
+            struct {
+                uint32_t modem_state_phy_done: 1;
+                uint32_t reserved: 31;
+            };
+            uint32_t flags;
+        };
     } wifi;
 } sleep_modem_config_t;
 
-static sleep_modem_config_t s_sleep_modem = { .wifi.phy_link = NULL };
+static sleep_modem_config_t s_sleep_modem = { .wifi.phy_link = NULL, .wifi.flags = 0 };
 
 static __attribute__((unused)) esp_err_t sleep_modem_wifi_modem_state_init(void)
 {
@@ -229,6 +238,7 @@ static __attribute__((unused)) esp_err_t sleep_modem_wifi_modem_state_init(void)
         if (err == ESP_OK) {
             pau_regdma_set_modem_link_addr(link);
             s_sleep_modem.wifi.phy_link = link;
+            s_sleep_modem.wifi.flags = 0;
         }
     }
     return err;
@@ -239,6 +249,19 @@ static __attribute__((unused)) void sleep_modem_wifi_modem_state_deinit(void)
     if (s_sleep_modem.wifi.phy_link) {
         regdma_link_destroy(s_sleep_modem.wifi.phy_link, 0);
         s_sleep_modem.wifi.phy_link = NULL;
+        s_sleep_modem.wifi.flags = 0;
+    }
+}
+
+void IRAM_ATTR sleep_modem_wifi_do_phy_retention(bool restore)
+{
+    if (restore) {
+        if (s_sleep_modem.wifi.modem_state_phy_done == 1) {
+            pau_regdma_trigger_modem_link_restore();
+        }
+    } else {
+        pau_regdma_trigger_modem_link_backup();
+        s_sleep_modem.wifi.modem_state_phy_done = 1;
     }
 }
 
@@ -269,16 +292,27 @@ uint32_t IRAM_ATTR sleep_modem_reject_triggers(void)
     return reject_triggers;
 }
 
+static __attribute__((unused)) bool IRAM_ATTR sleep_modem_wifi_modem_state_skip_light_sleep(void)
+{
+    bool skip = false;
+#if SOC_PM_SUPPORT_PMU_MODEM_STATE
+    skip = (s_sleep_modem.wifi.phy_link != NULL) && (s_sleep_modem.wifi.modem_state_phy_done == 0);
+#endif
+    return skip;
+}
+
 esp_err_t sleep_modem_configure(int max_freq_mhz, int min_freq_mhz, bool light_sleep_enable)
 {
 #if CONFIG_ESP_WIFI_ENHANCED_LIGHT_SLEEP
     extern int esp_wifi_internal_mac_sleep_configure(bool, bool);
     if (light_sleep_enable) {
         if (sleep_modem_wifi_modem_state_init() == ESP_OK) {
+            esp_pm_register_skip_light_sleep_callback(sleep_modem_wifi_modem_state_skip_light_sleep);
             esp_wifi_internal_mac_sleep_configure(light_sleep_enable, true); /* require WiFi to enable automatically receives the beacon */
         }
     } else {
         esp_wifi_internal_mac_sleep_configure(light_sleep_enable, false); /* require WiFi to disable automatically receives the beacon */
+        esp_pm_unregister_skip_light_sleep_callback(sleep_modem_wifi_modem_state_skip_light_sleep);
         sleep_modem_wifi_modem_state_deinit();
     }
 #endif
diff --git a/components/esp_phy/src/phy_init.c b/components/esp_phy/src/phy_init.c
index d9b072399652..69fb80f9315c 100644
--- a/components/esp_phy/src/phy_init.c
+++ b/components/esp_phy/src/phy_init.c
@@ -42,7 +42,6 @@
 #include "soc/dport_reg.h"
 #elif CONFIG_IDF_TARGET_ESP32C6
 #include "esp_private/sleep_modem.h"
-#include "esp_private/esp_pau.h"
 #endif
 #include "hal/efuse_hal.h"
 
@@ -253,7 +252,7 @@ void esp_phy_enable(void)
             extern bool pm_mac_modem_rf_already_enabled(void);
             if (!pm_mac_modem_rf_already_enabled()) {
                 if (sleep_modem_wifi_modem_state_enabled()) {
-                    pau_regdma_trigger_modem_link_restore();
+                    sleep_modem_wifi_do_phy_retention(true);
                 } else {
                     phy_wakeup_init();
                 }
@@ -287,7 +286,7 @@ void esp_phy_disable(void)
 #endif
 #if SOC_PM_SUPPORT_PMU_MODEM_STATE && CONFIG_ESP_WIFI_ENHANCED_LIGHT_SLEEP
         if (sleep_modem_wifi_modem_state_enabled()) {
-            pau_regdma_trigger_modem_link_backup();
+            sleep_modem_wifi_do_phy_retention(false);
         } else
 #endif /* SOC_PM_SUPPORT_PMU_MODEM_STATE && CONFIG_ESP_WIFI_ENHANCED_LIGHT_SLEEP */
         {
diff --git a/components/esp_pm/pm_impl.c b/components/esp_pm/pm_impl.c
index c020dd100430..d4ff1804a834 100644
--- a/components/esp_pm/pm_impl.c
+++ b/components/esp_pm/pm_impl.c
@@ -100,7 +100,7 @@ static uint32_t s_mode_mask;
 
 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
 
-#define PERIPH_SKIP_LIGHT_SLEEP_NO 1
+#define PERIPH_SKIP_LIGHT_SLEEP_NO 2
 
 /* Indicates if light sleep shoule be skipped by peripherals. */
 static skip_light_sleep_cb_t s_periph_skip_light_sleep_cb[PERIPH_SKIP_LIGHT_SLEEP_NO];