Skip to content

Commit

Permalink
feat(datadog) new metrics and more flexible configuration
Browse files Browse the repository at this point in the history 
- Now each metric supports configurable stat type,
  sample rate and customer identifier if applicable.
- Custom prefix for stats's name
- Custom tags for Datadog.
- New metrics `upstream_latency`, `kong_latency` and `status_count_per_user`.
- Code style format
  • Loading branch information
Shashi Ranjan committed Jun 6, 2017
1 parent 9e7506e commit cd1864d
Show file tree
Hide file tree
Showing 8 changed files with 805 additions and 159 deletions.
2 changes: 2 additions & 0 deletions kong-0.10.3-0.rockspec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,8 @@ build = {
["kong.plugins.loggly.handler"] = "kong/plugins/loggly/handler.lua",
["kong.plugins.loggly.schema"] = "kong/plugins/loggly/schema.lua",

["kong.plugins.datadog.migrations.cassandra"] = "kong/plugins/datadog/migrations/cassandra.lua",
["kong.plugins.datadog.migrations.postgres"] = "kong/plugins/datadog/migrations/postgres.lua",
["kong.plugins.datadog.handler"] = "kong/plugins/datadog/handler.lua",
["kong.plugins.datadog.schema"] = "kong/plugins/datadog/schema.lua",
["kong.plugins.datadog.statsd_logger"] = "kong/plugins/datadog/statsd_logger.lua",
Expand Down
144 changes: 97 additions & 47 deletions kong/plugins/datadog/handler.lua
View file
Original file line number Diff line number Diff line change
@@ -1,74 +1,124 @@
local BasePlugin = require "kong.plugins.base_plugin"
local BasePlugin = require "kong.plugins.base_plugin"
local basic_serializer = require "kong.plugins.log-serializers.basic"
local statsd_logger = require "kong.plugins.datadog.statsd_logger"
local statsd_logger = require "kong.plugins.datadog.statsd_logger"

local ngx_log = ngx.log
local ngx_timer_at = ngx.timer.at
local string_gsub = string.gsub
local pairs = pairs
local format = string.format
local NGX_ERR = ngx.ERR

local DatadogHandler = BasePlugin:extend()

local DatadogHandler = BasePlugin:extend()
DatadogHandler.PRIORITY = 1

local ngx_timer_at = ngx.timer.at
local string_gsub = string.gsub
local ipairs = ipairs

local gauges = {
request_size = function (api_name, message, logger, tags)
local stat = api_name..".request.size"
logger:gauge(stat, message.request.size, 1, tags)
end,
response_size = function (api_name, message, logger, tags)
local stat = api_name..".response.size"
logger:gauge(stat, message.response.size, 1, tags)
end,
status_count = function (api_name, message, logger, tags)
local stat = api_name..".request.status."..message.response.status
logger:counter(stat, 1, 1, tags)
end,
latency = function (api_name, message, logger, tags)
local stat = api_name..".latency"
logger:gauge(stat, message.latencies.request, 1, tags)
local metrics = {
status_count = function (api_name, message, metric_config, logger)
local fmt = format("%s.request.status", api_name,
message.response.status)

logger:send_statsd(format("%s.%s", fmt, message.response.status),
1, logger.stat_types.counter,
metric_config.sample_rate, metric_config.tags)

logger:send_statsd(format("%s.%s", fmt, "total"), 1,
logger.stat_types.counter,
metric_config.sample_rate, metric_config.tags)
end,
request_count = function (api_name, message, logger, tags)
local stat = api_name..".request.count"
logger:counter(stat, 1, 1, tags)
unique_users = function (api_name, message, metric_config, logger)
local consumer_id = message.consumer
and message.consumer[metric_config.consumer_identifier]

if consumer_id then
local stat = format("%s.user.uniques", api_name)

logger:send_statsd(stat, consumer_id, logger.stat_types.set,
nil, metric_config.tags)
end
end,
unique_users = function (api_name, message, logger, tags)
if message.authenticated_entity ~= nil and message.authenticated_entity.consumer_id ~= nil then
local stat = api_name..".user.uniques"
logger:set(stat, message.authenticated_entity.consumer_id, tags)
request_per_user = function (api_name, message, metric_config, logger)
local consumer_id = message.consumer
and message.consumer[metric_config.consumer_identifier]

if consumer_id then
local stat = format("%s.user.%s.request.count", api_name,
string_gsub(consumer_id, "-", "_"))

logger:send_statsd(stat, 1, logger.stat_types.counter,
metric_config.sample_rate, metric_config.tags)
end
end,
request_per_user = function (api_name, message, logger, tags)
if message.authenticated_entity ~= nil and message.authenticated_entity.consumer_id ~= nil then
local stat = api_name.."."..string_gsub(message.authenticated_entity.consumer_id, "-", "_")..".request.count"
logger:counter(stat, 1, 1, tags)
status_count_per_user = function (api_name, message, metric_config, logger)
local consumer_id = message.consumer
and message.consumer[metric_config.consumer_identifier]

if consumer_id then
local fmt = format("%s.user.%s.request.status", api_name,
string_gsub(consumer_id, "-", "_"))

logger:send_statsd(format("%s.%s", fmt, message.response.status),
1, logger.stat_types.counter,
metric_config.sample_rate, metric_config.tags)

logger:send_statsd(format("%s.%s", fmt, "total"),
1, logger.stat_types.counter,
metric_config.sample_rate, metric_config.tags)
end
end,
upstream_latency = function (api_name, message, logger, tags)
local stat = api_name..".upstream_latency"
logger:gauge(stat, message.latencies.proxy, 1, tags)
end
}


local function log(premature, conf, message)
if premature then return end
if premature then
return
end

local api_name = string_gsub(message.api.name, "%.", "_")
local stat_name = {
request_size = api_name .. ".request.size",
response_size = api_name .. ".response.size",
latency = api_name .. ".latency",
upstream_latency = api_name .. ".upstream_latency",
kong_latency = api_name .. ".kong_latency",
request_count = api_name .. ".request.count",
}
local stat_value = {
request_size = message.request.size,
response_size = message.response.size,
latency = message.latencies.request,
upstream_latency = message.latencies.proxy,
kong_latency = message.latencies.kong,
request_count = 1,
}

local logger, err = statsd_logger:new(conf)
if err then
ngx.log(ngx.ERR, "failed to create Statsd logger: ", err)
ngx_log(NGX_ERR, "failed to create Statsd logger: ", err)
return
end

local api_name = string_gsub(message.api.name, "%.", "_")
for _, metric in ipairs(conf.metrics) do
local gauge = gauges[metric]
if gauge then

gauge(api_name, message, logger, conf.tags[metric])

for _, metric_config in pairs(conf.metrics) do
local metric = metrics[metric_config.name]

if metric then
metric(api_name, message, metric_config, logger)

else
local stat_name = stat_name[metric_config.name]
local stat_value = stat_value[metric_config.name]

logger:send_statsd(stat_name, stat_value,
logger.stat_types[metric_config.stat_type],
metric_config.sample_rate, metric_config.tags)
end
end

logger:close_socket()
end


function DatadogHandler:new()
DatadogHandler.super.new(self, "datadog")
end
Expand All @@ -79,7 +129,7 @@ function DatadogHandler:log(conf)

local ok, err = ngx_timer_at(0, log, conf, message)
if not ok then
ngx.log(ngx.ERR, "failed to create timer: ", err)
ngx_log(NGX_ERR, "failed to create timer: ", err)
end
end

Expand Down
93 changes: 93 additions & 0 deletions kong/plugins/datadog/migrations/cassandra.lua
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
return {
{
name = "2017-02-09-160000_datadog_schema_changes",
up = function(_, _, factory)

local plugins, err = factory.plugins:find_all { name = "datadog" }
if err then
return err
end

local default_metrics = {
request_count = {
name = "request_count",
stat_type = "counter",
sample_rate = 1,
tags = { "app:kong" },
},
latency = {
name = "latency",
stat_type = "timer",
tags = { "app:kong" },
},
request_size = {
name = "request_size",
stat_type = "timer",
tags = { "app:kong" },
},
status_count = {
name = "status_count",
stat_type = "counter",
sample_rate = 1,
tags = { "app:kong" },
},
response_size = {
name = "response_size",
stat_type = "timer",
tags = { "app:kong" },
},
unique_users = {
name = "unique_users",
stat_type = "set",
consumer_identifier = "custom_id",
tags = { "app:kong" },
},
request_per_user = {
name = "request_per_user",
stat_type = "counter",
sample_rate = 1,
consumer_identifier = "custom_id",
tags = { "app:kong" },
},
upstream_latency = {
name = "upstream_latency",
stat_type = "timer",
tags = { "app:kong" },
},
kong_latency = {
name = "kong_latency",
stat_type = "timer",
tags = {"app:kong"},
},
status_count_per_user = {
name = "status_count_per_user",
stat_type = "counter",
sample_rate = 1,
consumer_identifier = "custom_id",
tags = { "app:kong" },
}
}

for _, plugin in ipairs(plugins) do
local new_metrics = {}
plugin.config.tags = nil
plugin.config.timeout = nil

if plugin.config.metrics then
for _, metric in ipairs(plugin.config.metrics) do
table.insert(new_metrics, default_metrics[metric])
end

plugin.config.metrics = new_metrics
plugin.config.timeout = nil
plugin.config.prefix = "kong"

local _, err = factory.plugins:update(plugin, plugin, { full = true })
if err then
return err
end
end
end
end
}
}
100 changes: 100 additions & 0 deletions kong/plugins/datadog/migrations/postgres.lua
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
return {
{
name = "2017-02-09-160000_datadog_schema_changes",
up = function(_, _, dao)
local rows, err = dao.plugins:find_all {name = "datadog"}
if err then
return err
end

local default_metrics = {
request_count = {
name = "request_count",
stat_type = "counter",
sample_rate = 1,
tags = { "app:kong" },
},
latency = {
name = "latency",
stat_type = "timer",
tags = { "app:kong" },
},
request_size = {
name = "request_size",
stat_type = "timer",
tags = { "app:kong" },
},
status_count = {
name = "status_count",
stat_type = "counter",
sample_rate = 1,
tags = { "app:kong" },
},
response_size = {
name = "response_size",
stat_type = "timer",
tags = { "app:kong" },
},
unique_users = {
name = "unique_users",
stat_type = "set",
consumer_identifier = "custom_id",
tags = { "app:kong" },
},
request_per_user = {
name = "request_per_user",
stat_type = "counter",
sample_rate = 1,
consumer_identifier = "custom_id",
tags = { "app:kong" },
},
upstream_latency = {
name = "upstream_latency",
stat_type = "timer",
tags = { "app:kong" }
},
kong_latency = {
name = "kong_latency",
stat_type = "timer",
tags = { "app:kong" },
},
status_count_per_user = {
name = "status_count_per_user",
stat_type = "counter",
sample_rate = 1,
consumer_identifier = "custom_id",
tags = { "app:kong" },
},
}

for i = 1, #rows do
local row = rows[i]

local _, err = dao.plugins:delete(row)
if err then return err end
local new_metrics = {}
if row.config.metrics then
for _, metric in ipairs(row.config.metrics) do
table.insert(new_metrics, default_metrics[metric])
end
end

local _, err = dao.plugins:insert {
name = "datadog",
api_id = row.api_id,
enabled = row.enabled,
config = {
host = row.config.host,
port = row.config.port,
metrics = new_metrics,
prefix = "kong",
},
}

if err then
return err
end
end
end
}
}
Loading

0 comments on commit cd1864d

Please sign in to comment.