Skip to content

Commit

Permalink
feat(log-rorate): log-rotate plugin support max_size (apache#7749)
Browse files Browse the repository at this point in the history
Co-authored-by: 罗泽轩 <[email protected]>
Co-authored-by: Fei Han <[email protected]>
Co-authored-by: qihaiyan <[email protected]>
  • Loading branch information
4 people authored and Liu-Junlin committed Nov 4, 2022
1 parent 562d54e commit e75b73a
Show file tree
Hide file tree
Showing 5 changed files with 211 additions and 62 deletions.
121 changes: 62 additions & 59 deletions apisix/plugins/log-rotate.lua
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ local plugin = require("apisix.plugin")
local process = require("ngx.process")
local signal = require("resty.signal")
local shell = require("resty.shell")
local ipairs = ipairs
local ngx = ngx
local ngx_time = ngx.time
local ngx_update_time = ngx.update_time
Expand All @@ -43,6 +44,7 @@ local local_conf
local plugin_name = "log-rotate"
local INTERVAL = 60 * 60 -- rotate interval (unit: second)
local MAX_KEPT = 24 * 7 -- max number of log files will be kept
local MAX_SIZE = -1 -- max size of file will be rotated
local COMPRESSION_FILE_SUFFIX = ".tar.gz" -- compression file suffix
local rotate_time
local default_logs
Expand Down Expand Up @@ -123,34 +125,22 @@ local function tab_sort_comp(a, b)
end


local function scan_log_folder()
local t = {
access = {},
error = {},
}
local function scan_log_folder(log_file_name)
local t = {}

local log_dir, access_name = get_log_path_info("access.log")
local _, error_name = get_log_path_info("error.log")

if enable_compression then
access_name = access_name .. COMPRESSION_FILE_SUFFIX
error_name = error_name .. COMPRESSION_FILE_SUFFIX
end
local log_dir, _ = get_log_path_info(log_file_name)

for file in lfs.dir(log_dir) do
local n = get_last_index(file, "__")
if n ~= nil then
local log_type = file:sub(n + 2)
if log_type == access_name then
tab_insert(t.access, file)
elseif log_type == error_name then
tab_insert(t.error, file)
if log_type == log_file_name then
tab_insert(t, file)
end
end
end

tab_sort(t.access, tab_sort_comp)
tab_sort(t.error, tab_sort_comp)
tab_sort(t, tab_sort_comp)
return t, log_dir
end

Expand Down Expand Up @@ -219,18 +209,62 @@ local function init_default_logs(logs_info, log_type)
end


local function file_size(file)
local attr = lfs.attributes(file)
if attr then
return attr.size
end
return 0
end


local function rotate_file(files, now_time, max_kept)
for _, file in ipairs(files) do
local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time)
local new_file = rename_file(default_logs[file], now_date)
if not new_file then
return
end

local pid = process.get_master_pid()
core.log.warn("send USR1 signal to master process [", pid, "] for reopening log file")
local ok, err = signal.kill(pid, signal.signum("USR1"))
if not ok then
core.log.error("failed to send USR1 signal for reopening log file: ", err)
end

if enable_compression then
compression_file(new_file)
end

-- clean the oldest file
local log_list, log_dir = scan_log_folder(file)
for i = max_kept + 1, #log_list do
local path = log_dir .. log_list[i]
local ok, err = os_remove(path)
if err then
core.log.error("remove old log file: ", path, " err: ", err, " res:", ok)
end
end
end
end


local function rotate()
local interval = INTERVAL
local max_kept = MAX_KEPT
local max_size = MAX_SIZE
local attr = plugin.plugin_attr(plugin_name)
if attr then
interval = attr.interval or interval
max_kept = attr.max_kept or max_kept
max_size = attr.max_size or max_size
enable_compression = attr.enable_compression or enable_compression
end

core.log.info("rotate interval:", interval)
core.log.info("rotate max keep:", max_kept)
core.log.info("rotate max size:", max_size)

if not default_logs then
-- first init default log filepath and filename
Expand All @@ -248,53 +282,22 @@ local function rotate()
return
end

if now_time < rotate_time then
-- did not reach the rotate time
core.log.info("rotate time: ", rotate_time, " now time: ", now_time)
return
end
if now_time >= rotate_time then
local files = {DEFAULT_ACCESS_LOG_FILENAME, DEFAULT_ERROR_LOG_FILENAME}
rotate_file(files, now_time, max_kept)

local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time)
local access_new_file = rename_file(default_logs[DEFAULT_ACCESS_LOG_FILENAME], now_date)
local error_new_file = rename_file(default_logs[DEFAULT_ERROR_LOG_FILENAME], now_date)
if not access_new_file and not error_new_file then
-- reset rotate time
rotate_time = rotate_time + interval
return
end

core.log.warn("send USR1 signal to master process [",
process.get_master_pid(), "] for reopening log file")
local ok, err = signal.kill(process.get_master_pid(), signal.signum("USR1"))
if not ok then
core.log.error("failed to send USR1 signal for reopening log file: ", err)
end

if enable_compression then
compression_file(access_new_file)
compression_file(error_new_file)
end

-- clean the oldest file
local log_list, log_dir = scan_log_folder()
for i = max_kept + 1, #log_list.error do
local path = log_dir .. log_list.error[i]
ok, err = os_remove(path)
if err then
core.log.error("remove old error file: ", path, " err: ", err, " res:", ok)
elseif max_size > 0 then
local access_log_file_size = file_size(default_logs[DEFAULT_ACCESS_LOG_FILENAME].file)
local error_log_file_size = file_size(default_logs[DEFAULT_ERROR_LOG_FILENAME].file)
if access_log_file_size >= max_size then
rotate_file({DEFAULT_ACCESS_LOG_FILENAME}, now_time, max_kept)
end
end

for i = max_kept + 1, #log_list.access do
local path = log_dir .. log_list.access[i]
ok, err = os_remove(path)
if err then
core.log.error("remove old error file: ", path, " err: ", err, " res:", ok)
if error_log_file_size >= max_size then
rotate_file({DEFAULT_ERROR_LOG_FILENAME}, now_time, max_kept)
end
end

-- reset rotate time
rotate_time = rotate_time + interval
end


Expand Down
1 change: 1 addition & 0 deletions conf/config-default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,7 @@ plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
max_size: -1 # max size bytes of log files to be rotated, size check would be skipped with a value less than 0
enable_compression: false # enable log file compression(gzip) or not, default false
skywalking:
service_name: APISIX
Expand Down
8 changes: 5 additions & 3 deletions docs/en/latest/plugins/log-rotate.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ You can configure how often the logs are rotated and how many logs to keep. When
|--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------|
| interval | integer | True | 60 * 60 | Time in seconds specifying how often to rotate the logs. |
| max_kept | integer | True | 24 * 7 | Maximum number of historical logs to keep. If this number is exceeded, older logs are deleted. |
| max_size | integer | False | -1 | Max size(Bytes) of log files to be rotated, size check would be skipped with a value less than 0 or time is up specified by interval. |
| enable_compression | boolean | False | false | When set to `true`, compresses the log file (gzip). Requires `tar` to be installed. |

## Enabling the Plugin
Expand All @@ -51,9 +52,10 @@ plugins:

plugin_attr:
log-rotate:
interval: 3600
max_kept: 168
enable_compression: false
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
max_size: -1 # max size of log files will be kept
enable_compression: false # enable log file compression(gzip) or not, default false
```
## Example usage
Expand Down
2 changes: 2 additions & 0 deletions docs/zh/latest/plugins/log-rotate.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ description: 云原生 API 网关 Apache APISIX log-rotate 插件用于定期切
| ------------------ | ------- | ------ | ------- | ------------- | ---------------------------------------------------------------------------- |
| interval | integer || 60 * 60 | | 每间隔多长时间切分一次日志,以秒为单位。 |
| max_kept | integer || 24 * 7 | | 最多保留多少份历史日志,超过指定数量后,自动删除老文件。 |
| max_size | integer || -1 | | 日志文件超过指定大小时进行切分,单位为 Byte 。如果 `max_size` 小于 0 或者根据 `interval` 计算的时间到达时,将不会根据 `max_size` 切分日志。 |
| enable_compression | boolean || false | [false, true] | 当设置为 `true` 时,启用日志文件压缩。该功能需要在系统中安装 `tar`|

开启该插件后,就会按照参数自动切分日志文件了。比如以下示例是根据 `interval: 10``max_kept: 10` 得到的样本。
Expand Down Expand Up @@ -92,6 +93,7 @@ plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
max_size: -1 # max size of log files will be kept
enable_compression: false # enable log file compression(gzip) or not, default false
```
Expand Down
141 changes: 141 additions & 0 deletions t/plugin/log-rotate3.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

use t::APISIX 'no_plan';

repeat_each(1);
no_long_string();
no_shuffle();
no_root_location();

add_block_preprocessor(sub {
my ($block) = @_;

if (!defined $block->yaml_config) {
my $yaml_config = <<_EOC_;
apisix:
node_listen: 1984
admin_key: ~
plugins:
- log-rotate
plugin_attr:
log-rotate:
interval: 86400
max_size: 9
max_kept: 3
enable_compression: false
_EOC_

$block->set_value("yaml_config", $yaml_config);
}

if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
$block->set_value("no_error_log", "[error]");
}

if (!defined $block->request) {
$block->set_value("request", "GET /t");
}

});

run_tests;

__DATA__
=== TEST 1: log rotate by max_size
--- config
location /t {
content_by_lua_block {
ngx.log(ngx.ERR, "start xxxxxx")
ngx.sleep(2)
local has_split_access_file = false
local has_split_error_file = false
local lfs = require("lfs")
for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do
if string.match(file_name, "__access.log$") then
has_split_access_file = true
end
if string.match(file_name, "__error.log$") then
has_split_error_file = true
end
end
if not has_split_access_file and has_split_error_file then
ngx.status = 200
else
ngx.status = 500
end
}
}
=== TEST 2: in current log
--- config
location /t {
content_by_lua_block {
ngx.sleep(0.1)
ngx.log(ngx.WARN, "start xxxxxx")
ngx.say("done")
}
}
--- response_body
done
--- error_log
start xxxxxx
=== TEST 3: check file changes
--- config
location /t {
content_by_lua_block {
ngx.sleep(1)
local default_logs = {}
for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do
if string.match(file_name, "__error.log$") or string.match(file_name, "__access.log$") then
local filepath = ngx.config.prefix() .. "/logs/" .. file_name
local attr = lfs.attributes(filepath)
if attr then
default_logs[filepath] = { change = attr.change, size = attr.size }
end
end
end
ngx.sleep(1)
local passed = false
for filepath, origin_attr in pairs(default_logs) do
local check_attr = lfs.attributes(filepath)
if check_attr.change == origin_attr.change and check_attr.size == origin_attr.size then
passed = true
else
passed = false
break
end
end
if passed then
ngx.say("passed")
end
}
}
--- response_body
passed

0 comments on commit e75b73a

Please sign in to comment.