Fix/plugin server release lock

CHANGELOG.md

@@ -131,6 +131,8 @@
   [#10691](https://github.com/Kong/kong/pull/10691)
 - Fix a typo of mlcache option `shm_set_tries`.
   [#10712](https://github.com/Kong/kong/pull/10712)
+- Fix an issue where slow start up of Go plugin server causes dead lock.
+  [#10561](https://github.com/Kong/kong/pull/10561)
 - Tracing: fix an issue that caused the `sampled` flag of incoming propagation
   headers to be handled incorrectly and only affect some spans.
   [#10655](https://github.com/Kong/kong/pull/10655)

kong/runloop/plugin_servers/init.lua

@@ -40,6 +40,9 @@ else
   resp_get_headers = NOOP
 end
 
+local SLEEP_STEP = 0.1
+local WAIT_TIME = 10
+local MAX_WAIT_STEPS = WAIT_TIME / SLEEP_STEP
 
 --- keep request data a bit longer, into the log timer
 local save_for_later = {}
@@ -192,9 +195,17 @@ function get_instance_id(plugin_name, conf)
   local key = type(conf) == "table" and kong.plugin.get_id() or plugin_name
   local instance_info = running_instances[key]
 
+  local wait_count = 0
   while instance_info and not instance_info.id do
     -- some other thread is already starting an instance
-    ngx_sleep(0)
+    -- prevent busy-waiting
+    ngx_sleep(SLEEP_STEP)
+
+    -- to prevent a potential dead loop when someone failed to release the ID
+    wait_count = wait_count + 1
+    if wait_count > MAX_WAIT_STEPS then
+      return nil, "Could not claim instance_id for " .. plugin_name .. " (key: " .. key .. ")"
+    end
     instance_info = running_instances[key]
   end
 

kong/runloop/plugin_servers/mp_rpc.lua

@@ -336,6 +336,7 @@ function Rpc:handle_event(plugin_name, conf, phase)
 
   if err then
     if err == "not ready" then
+      self.reset_instance(plugin_name, conf)
       return handle_not_ready(plugin_name)
     end
     if err and str_find(err:lower(), "no plugin instance", 1, true) then

kong/runloop/plugin_servers/pb_rpc.lua

@@ -401,6 +401,7 @@ function Rpc:handle_event(plugin_name, conf, phase)
 
   if not res or res == "" then
     if err == "not ready" then
+      self.reset_instance(plugin_name, conf)
       return handle_not_ready(plugin_name)
     end
     if err and (str_find(err:lower(), "no plugin instance", 1, true)