From 794f4804cfff0af5c214dfb77f0b2dd10545b625 Mon Sep 17 00:00:00 2001 From: Marco Palladino Date: Fri, 22 Jul 2016 17:54:16 -0700 Subject: [PATCH] cache-locks (#1402) Closes #264 --- kong/core/cluster.lua | 19 +++-- kong/core/reports.lua | 6 +- kong/templates/nginx_kong.lua | 1 + kong/tools/07-cache | 0 kong/tools/database_cache.lua | 37 +++++++-- .../07-cache/database_cache_spec.lua | 78 +++++++++++++++++++ .../kong/plugins/database-cache/handler.lua | 35 +++++++++ .../kong/plugins/database-cache/schema.lua | 3 + 8 files changed, 166 insertions(+), 13 deletions(-) create mode 100644 kong/tools/07-cache create mode 100644 spec/02-integration/07-cache/database_cache_spec.lua create mode 100644 spec/fixtures/kong/plugins/database-cache/handler.lua create mode 100644 spec/fixtures/kong/plugins/database-cache/schema.lua diff --git a/kong/core/cluster.lua b/kong/core/cluster.lua index d5a8d87a059b..70efcc82b19b 100644 --- a/kong/core/cluster.lua +++ b/kong/core/cluster.lua @@ -1,5 +1,6 @@ local cache = require "kong.tools.database_cache" local singletons = require "kong.singletons" +local ngx_log = ngx.log local resty_lock local status, res = pcall(require, "resty.lock") @@ -14,7 +15,7 @@ local ASYNC_AUTOJOIN_RETRIES = 20 -- Try for max a minute (3s * 20) local function create_timer(at, cb) local ok, err = ngx.timer.at(at, cb) if not ok then - ngx.log(ngx.ERR, "[cluster] failed to create timer: ", err) + ngx_log(ngx.ERR, "[cluster] failed to create timer: ", err) end end @@ -24,25 +25,29 @@ local function async_autojoin(premature) -- If this node is the only node in the cluster, but other nodes are present, then try to join them -- This usually happens when two nodes are started very fast, and the first node didn't write his -- information into the datastore yet. When the second node starts up, there is nothing to join yet. - local lock = resty_lock:new("cluster_autojoin_locks", { + local lock, err = resty_lock:new("cluster_autojoin_locks", { exptime = ASYNC_AUTOJOIN_INTERVAL - 0.001 }) + if not lock then + ngx_log(ngx.ERR, "could not create lock: ", err) + return + end local elapsed = lock:lock("async_autojoin") if elapsed and elapsed == 0 then -- If the current member count on this node's cluster is 1, but there are more than 1 active nodes in -- the DAO, then try to join them local count, err = singletons.dao.nodes:count() if err then - ngx.log(ngx.ERR, tostring(err)) + ngx_log(ngx.ERR, tostring(err)) elseif count > 1 then local members, err = singletons.serf:members() if err then - ngx.log(ngx.ERR, tostring(err)) + ngx_log(ngx.ERR, tostring(err)) elseif #members < 2 then -- Trigger auto-join local _, err = singletons.serf:autojoin() if err then - ngx.log(ngx.ERR, tostring(err)) + ngx_log(ngx.ERR, tostring(err)) end else return -- The node is already in the cluster and no need to continue @@ -74,12 +79,12 @@ local function send_keepalive(premature) name = singletons.serf.node_name } if err then - ngx.log(ngx.ERR, tostring(err)) + ngx_log(ngx.ERR, tostring(err)) elseif #nodes == 1 then local node = nodes[1] local _, err = singletons.dao.nodes:update(node, node, {ttl=singletons.configuration.cluster_ttl_on_failure}) if err then - ngx.log(ngx.ERR, tostring(err)) + ngx_log(ngx.ERR, tostring(err)) end end end diff --git a/kong/core/reports.lua b/kong/core/reports.lua index db7feb86eac6..1f9cba47b83b 100644 --- a/kong/core/reports.lua +++ b/kong/core/reports.lua @@ -107,9 +107,13 @@ end ping_handler = function(premature) if premature then return end - local lock = resty_lock:new("reports_locks", { + local lock, err = resty_lock:new("reports_locks", { exptime = ping_interval - 0.001 }) + if not lock then + log_error("could not create lock: ", err) + return + end local elapsed, err = lock:lock("ping") if not elapsed then diff --git a/kong/templates/nginx_kong.lua b/kong/templates/nginx_kong.lua index 0535736b37f6..57f80653e234 100644 --- a/kong/templates/nginx_kong.lua +++ b/kong/templates/nginx_kong.lua @@ -38,6 +38,7 @@ lua_shared_dict cache ${{MEM_CACHE_SIZE}}; lua_shared_dict reports_locks 100k; lua_shared_dict cluster_locks 100k; lua_shared_dict cluster_autojoin_locks 100k; +lua_shared_dict cache_locks 100k; lua_shared_dict cassandra 1m; lua_shared_dict cassandra_prepared 5m; lua_socket_log_errors off; diff --git a/kong/tools/07-cache b/kong/tools/07-cache new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/kong/tools/database_cache.lua b/kong/tools/database_cache.lua index 7e77d238e3da..a95a79c4a1aa 100644 --- a/kong/tools/database_cache.lua +++ b/kong/tools/database_cache.lua @@ -1,5 +1,7 @@ +local resty_lock = require "resty.lock" local cjson = require "cjson" local cache = ngx.shared.cache +local ngx_log = ngx.log local CACHE_KEYS = { APIS = "apis", @@ -121,20 +123,45 @@ end function _M.get_or_set(key, cb) local value, err - -- Try to get + + -- Try to get the value from the cache + value = _M.get(key) + if value then return value end + + local lock, err = resty_lock:new("cache_locks", { + exptime = 10, + timeout = 5 + }) + if not lock then + ngx_log(ngx.ERR, "could not create lock: ", err) + return + end + + -- The value is missing, acquire a lock + local elapsed, err = lock:lock(key) + if not elapsed then + ngx_log(ngx.ERR, "failed to acquire cache lock: ", err) + end + + -- Lock acquired. Since in the meantime another worker may have + -- populated the value we have to check again value = _M.get(key) if not value then -- Get from closure value, err = cb() - if err then - return nil, err - elseif value then + if value then local ok, err = _M.set(key, value) if not ok then - ngx.log(ngx.ERR, err) + ngx_log(ngx.ERR, err) end end end + + local ok, err = lock:unlock() + if not ok then + ngx_log(ngx.ERR, "failed to unlock: ", err) + end + return value end diff --git a/spec/02-integration/07-cache/database_cache_spec.lua b/spec/02-integration/07-cache/database_cache_spec.lua new file mode 100644 index 000000000000..c9ee25bb6990 --- /dev/null +++ b/spec/02-integration/07-cache/database_cache_spec.lua @@ -0,0 +1,78 @@ +local helpers = require "spec.helpers" +local cjson = require "cjson" +local meta = require "kong.meta" + +describe("Resolver", function() + local client + setup(function() + + assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://mockbin.com" + }) + + assert(helpers.start_kong({ + ["custom_plugins"] = "database-cache", + lua_package_path = "?/init.lua;./kong/?.lua;./spec/fixtures/?.lua" + })) + + -- Add the plugin + local admin_client = helpers.admin_client() + local res = assert(admin_client:send { + method = "POST", + path = "/apis/mockbin.com/plugins/", + body = { + name = "database-cache" + }, + headers = { + ["Content-Type"] = "application/json" + } + }) + assert.res_status(201, res) + admin_client:close() + end) + + teardown(function() + helpers.kill_all() + end) + + it("avoids dog-pile effect", function() + local function make_request(premature, sleep_time) + local client = helpers.proxy_client() + local res = assert(client:send { + method = "GET", + path = "/status/200?sleep="..sleep_time, + headers = { + ["Host"] = "mockbin.com" + } + }) + res:read_body() + client:close() + end + + assert(ngx.timer.at(0, make_request, 2)) + assert(ngx.timer.at(0, make_request, 5)) + assert(ngx.timer.at(0, make_request, 1)) + + helpers.wait_until(function() + local admin_client = helpers.admin_client() + local res = assert(admin_client:send { + method = "GET", + path = "/cache/invocations" + }) + local body = res:read_body() + admin_client:close() + return cjson.decode(body).message == 3 + end, 10) + + -- Invocation are 3, but lookups should be 1 + local admin_client = helpers.admin_client() + local res = assert(admin_client:send { + method = "GET", + path = "/cache/lookups" + }) + local body = res:read_body() + admin_client:close() + assert.equal(1, cjson.decode(body).message) + end) +end) diff --git a/spec/fixtures/kong/plugins/database-cache/handler.lua b/spec/fixtures/kong/plugins/database-cache/handler.lua new file mode 100644 index 000000000000..cb393d9cd184 --- /dev/null +++ b/spec/fixtures/kong/plugins/database-cache/handler.lua @@ -0,0 +1,35 @@ +local BasePlugin = require "kong.plugins.base_plugin" +local cache = require "kong.tools.database_cache" + +local INVOCATIONS = "invocations" +local LOOKUPS = "lookups" + +local DatabaseCacheHandler = BasePlugin:extend() + +DatabaseCacheHandler.PRIORITY = 1000 + +function DatabaseCacheHandler:new() + DatabaseCacheHandler.super.new(self, "database-cache") +end + +function DatabaseCacheHandler:init_worker() + DatabaseCacheHandler.super.init_worker(self) + + cache.rawset(INVOCATIONS, 0) + cache.rawset(LOOKUPS, 0) +end + +function DatabaseCacheHandler:access(conf) + DatabaseCacheHandler.super.access(self) + + cache.get_or_set("pile_effect", function() + cache.incr(LOOKUPS, 1) + -- Adds some delay + ngx.sleep(tonumber(ngx.req.get_uri_args().sleep)) + return true + end) + + cache.incr(INVOCATIONS, 1) +end + +return DatabaseCacheHandler diff --git a/spec/fixtures/kong/plugins/database-cache/schema.lua b/spec/fixtures/kong/plugins/database-cache/schema.lua new file mode 100644 index 000000000000..d02ea8dc964c --- /dev/null +++ b/spec/fixtures/kong/plugins/database-cache/schema.lua @@ -0,0 +1,3 @@ +return { + fields = {} +} \ No newline at end of file