From 55d29dcfcc01742168319f295b043ccfc8cdaa0b Mon Sep 17 00:00:00 2001 From: Yuansheng Date: Tue, 13 Oct 2020 11:15:21 +0800 Subject: [PATCH 1/2] feat(http-logger): support for specified the log formats via admin API . curl http://****/apisix/admin/plugin_metadata/http-logger -d ' { "log_format": { "host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr" } }' when we enabled plugin http-logger, we will get the message body like: {"host":"localhost","@timestamp":"2020-09-23T18:29:07-04:00","client_ip":"127.0.0.1","route_id":"1"} {"host":"localhost","@timestamp":"2020-09-23T18:29:07-04:00","client_ip":"127.0.0.1","route_id":"1"} --- apisix/plugin.lua | 16 ++++ apisix/plugins/http-logger.lua | 84 ++++++++++++++--- bin/apisix | 2 +- doc/zh-cn/plugins/http-logger.md | 28 +++++- t/plugin/http-logger-log-format.t | 147 ++++++++++++++++++++++++++++++ t/plugin/http-logger-new-line.t | 83 +++++++++++++++++ 6 files changed, 347 insertions(+), 13 deletions(-) create mode 100644 t/plugin/http-logger-log-format.t diff --git a/apisix/plugin.lua b/apisix/plugin.lua index f992dc308a3e..3077b9460953 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -25,6 +25,7 @@ local type = type local local_plugins = core.table.new(32, 0) local ngx = ngx local tostring = tostring +local error = error local local_plugins_hash = core.table.new(0, 32) local stream_local_plugins = core.table.new(32, 0) local stream_local_plugins_hash = core.table.new(0, 32) @@ -352,6 +353,21 @@ end function _M.init_worker() _M.load() + + local plugin_metadatas, err = core.config.new("/plugin_metadata", + {automatic = true} + ) + if not plugin_metadatas then + error("failed to create etcd instance for fetching /plugin_metadatas : " + .. err) + end + + _M.plugin_metadatas = plugin_metadatas +end + + +function _M.plugin_metadata(name) + return _M.plugin_metadatas:get(name) end diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index 4694b605fd55..e96df8bd2675 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -14,17 +14,26 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local core = require("apisix.core") -local log_util = require("apisix.utils.log-util") + local batch_processor = require("apisix.utils.batch-processor") -local plugin_name = "http-logger" -local ngx = ngx +local log_util = require("apisix.utils.log-util") +local core = require("apisix.core") +local http = require("resty.http") +local url = require("net.url") +local plugin = require("apisix.plugin") +local ngx = ngx local tostring = tostring -local http = require "resty.http" -local url = require "net.url" -local buffers = {} +local pairs = pairs local ipairs = ipairs + +local plugin_name = "http-logger" +local buffers = {} +local lru_log_format = core.lrucache.new({ + ttl = 300, count = 512 +}) + + local schema = { type = "object", properties = { @@ -45,11 +54,24 @@ local schema = { } +local metadata_schema = { + type = "object", + properties = { + log_format = { + type = "object", + default = {}, + }, + }, + additionalProperties = false, +} + + local _M = { version = 0.1, priority = 410, name = plugin_name, schema = schema, + metadata_schema = metadata_schema, } @@ -117,12 +139,52 @@ local function send_http_data(conf, log_message) end -function _M.log(conf) - local entry = log_util.get_full_log(ngx, conf) +local function gen_log_format(metadata) + local log_format = {} + if metadata == nil then + return log_format + end + + for k, var_name in pairs(metadata.value.log_format) do + if var_name:sub(1, 1) == "$" then + log_format[k] = {true, var_name:sub(2)} + else + log_format[k] = {false, var_name} + end + end + core.log.info("log_format: ", core.json.delay_encode(log_format)) + return log_format +end + + +function _M.log(conf, ctx) + local metadata = plugin.plugin_metadata(plugin_name) + core.log.info("metadata: ", core.json.delay_encode(metadata)) + + local entry + local log_format = lru_log_format(metadata or "", nil, gen_log_format, + metadata) + if core.table.nkeys(log_format) > 0 then + entry = core.table.new(0, core.table.nkeys(log_format)) + for k, var_attr in pairs(log_format) do + if var_attr[1] then + entry[k] = ctx.var[var_attr[2]] + else + entry[k] = var_attr[2] + end + end + + local matched_route = ctx.matched_route and ctx.matched_route.value + if matched_route then + entry.service_id = matched_route.service_id + entry.route_id = matched_route.id + end + else + entry = log_util.get_full_log(ngx, conf) + end if not entry.route_id then - core.log.error("failed to obtain the route id for http logger") - return + entry.route_id = "no-matched" end local log_buffer = buffers[entry.route_id] diff --git a/bin/apisix b/bin/apisix index b37caf47d898..6184993356f7 100755 --- a/bin/apisix +++ b/bin/apisix @@ -1021,7 +1021,7 @@ local function init_etcd(show_output) for _, dir_name in ipairs({"/routes", "/upstreams", "/services", "/plugins", "/consumers", "/node_status", "/ssl", "/global_rules", "/stream_routes", - "/proto"}) do + "/proto", "/plugin_metadata"}) do local key = (etcd_conf.prefix or "") .. dir_name .. "/" local base64_encode = require("base64").encode diff --git a/doc/zh-cn/plugins/http-logger.md b/doc/zh-cn/plugins/http-logger.md index 77910071cb79..55694322272b 100644 --- a/doc/zh-cn/plugins/http-logger.md +++ b/doc/zh-cn/plugins/http-logger.md @@ -51,7 +51,7 @@ ## 如何开启 -1. 这是有关如何为特定路由启用 http-logger 插件的示例。 +这是有关如何为特定路由启用 http-logger 插件的示例。 ```shell curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' @@ -82,6 +82,32 @@ HTTP/1.1 200 OK hello, world ``` +## 插件元数据设置 + +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ | +| log_format | object | 可选 | | | 以 Hash 对象方式声明日志格式。对 value 部分，仅支持字符串。如果是以`$`开头，则表明是要获取 [Nginx 内置变量](http://nginx.org/en/docs/varindex.html)。特别的，该设置是全局生效的，意味着指定 log_format 后，将对所有绑定 http-logger 的 Route 或 Service 生效。 | + +### 设置日志格式示例 + +```shell +curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/http-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "log_format": { + "host": "$host", + "@timestamp": "$time_iso8601", + "client_ip": "$remote_addr" + } +}' +``` + +在日志收集处，将得到类似下面的日志： + +```shell +{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"} +{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"} +``` + ## 禁用插件 在插件配置中删除相应的 json 配置以禁用 http-logger。APISIX 插件是热重载的，因此无需重新启动 APISIX： diff --git a/t/plugin/http-logger-log-format.t b/t/plugin/http-logger-log-format.t new file mode 100644 index 000000000000..d092eebba66f --- /dev/null +++ b/t/plugin/http-logger-log-format.t @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level('info'); +repeat_each(1); +no_long_string(); +no_root_location(); + +run_tests; + +__DATA__ + +=== TEST 1: add plugin metadata +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/http-logger', + ngx.HTTP_PUT, + [[{ + "log_format": { + "host": "$host", + "@timestamp": "$time_iso8601", + "client_ip": "$remote_addr" + } + }]], + [[{ + "node": { + "value": { + "log_format": { + "host": "$host", + "@timestamp": "$time_iso8601", + "client_ip": "$remote_addr" + } + } + }, + "action": "set" + }]] + ) + + ngx.status = code + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 2: sanity, batch_max_size=1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1980/log", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2, + "concat_method": "new_line" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 3: hit route and report http logger +--- request +GET /hello +--- response_body +hello world +--- wait: 0.5 +--- no_error_log +[error] +--- error_log +request log: { + + + +=== TEST 4: remove plugin metadata +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/http-logger', + ngx.HTTP_DELETE + ) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] diff --git a/t/plugin/http-logger-new-line.t b/t/plugin/http-logger-new-line.t index 14c2daaba782..ce5517d28d2f 100644 --- a/t/plugin/http-logger-new-line.t +++ b/t/plugin/http-logger-new-line.t @@ -221,3 +221,86 @@ qr/"upstream":"127.0.0.1:1982"/ "upstream":"127.0.0.1:1982" "upstream":"127.0.0.1:1982" "upstream":"127.0.0.1:1982" + + + +=== TEST 8: set in global rule +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1980/log", + "batch_max_size": 3, + "max_retry_count": 3, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 1, + "concat_method": "new_line" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 9: not hit route, and report log +--- config +location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + for i = 1, 5 do + t('/not_hit_route', ngx.HTTP_GET) + end + + ngx.sleep(3) + ngx.say("done") + } +} +--- request +GET /t +--- timeout: 10 +--- no_error_log +[error] + + + +=== TEST 10: delete the global rule +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_DELETE + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] From 31cc7efd72125375726adc8ebab303788788c10f Mon Sep 17 00:00:00 2001 From: Yuansheng Date: Tue, 13 Oct 2020 12:55:19 +0800 Subject: [PATCH 2/2] perf: use `string.byte` replace `string.sub`, avoid temporary variable. --- apisix/plugins/http-logger.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index e96df8bd2675..b18efba41d16 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -25,6 +25,7 @@ local ngx = ngx local tostring = tostring local pairs = pairs local ipairs = ipairs +local str_byte = string.byte local plugin_name = "http-logger" @@ -59,7 +60,11 @@ local metadata_schema = { properties = { log_format = { type = "object", - default = {}, + default = { + ["host"] = "$host", + ["@timestamp"] = "$time_iso8601", + ["client_ip"] = "$remote_addr", + }, }, }, additionalProperties = false, @@ -146,7 +151,7 @@ local function gen_log_format(metadata) end for k, var_name in pairs(metadata.value.log_format) do - if var_name:sub(1, 1) == "$" then + if var_name:byte(1, 1) == str_byte("/") then log_format[k] = {true, var_name:sub(2)} else log_format[k] = {false, var_name}