From 4d28e1e389811db86738b4df2463940dab8d5012 Mon Sep 17 00:00:00 2001 From: Nick Muerdter Date: Sun, 19 Feb 2017 08:47:12 -0700 Subject: [PATCH] Fix some logrotate issues. - The logs for the root perpd process weren't being reopened during log rotation. - If API Umbrella was running under a different user (not "api-umbrella"), then the logrotate process would lead to files being created which couldn't be written to, which could eventually lock up the Ruby web processes (they would seem to lock up once the last file they were being written to got moved to a gzip file). This issues impacted the vagrant development environment (where we run everything as the vagrant users) if things were running for more than a couple days. --- .../files/etc/logrotate.d/api-umbrella | 4 ---- build/package_dependencies.sh | 15 +++++++++--- src/api-umbrella/cli/reopen_logs.lua | 24 ++++++++++++------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/build/package/files/etc/logrotate.d/api-umbrella b/build/package/files/etc/logrotate.d/api-umbrella index a0855916f..823304864 100644 --- a/build/package/files/etc/logrotate.d/api-umbrella +++ b/build/package/files/etc/logrotate.d/api-umbrella @@ -3,7 +3,6 @@ /opt/api-umbrella/var/log/*/current /opt/api-umbrella/var/log/nginx/*.log /opt/api-umbrella/var/log/rsyslog/*.log { daily rotate 90 - create 644 api-umbrella api-umbrella missingok compress delaycompress @@ -18,7 +17,6 @@ /opt/api-umbrella/var/log/nginx/*.log.gz { daily rotate 90 - create 644 api-umbrella api-umbrella missingok nocompress notifempty @@ -33,7 +31,6 @@ /opt/api-umbrella/var/log/rsyslog/*.log.gz { daily rotate 21 - create 644 api-umbrella api-umbrella missingok nocompress notifempty @@ -48,7 +45,6 @@ /opt/api-umbrella/var/log/elasticsearch/*.log /opt/api-umbrella/var/log/trafficserver/*.blog /opt/api-umbrella/var/log/trafficserver/*.log /opt/api-umbrella/var/log/trafficserver/*.out { daily rotate 90 - create 644 api-umbrella api-umbrella missingok compress delaycompress diff --git a/build/package_dependencies.sh b/build/package_dependencies.sh index e35ab18ea..eee36648f 100644 --- a/build/package_dependencies.sh +++ b/build/package_dependencies.sh @@ -39,14 +39,17 @@ if [ -f /etc/redhat-release ]; then # ElasticSearch java-1.8.0-openjdk-headless - # For getopt, should no longer be necessary in ElasticSearch 2: - # https://github.com/elastic/elasticsearch/pull/12165 - $util_linux_package which # init.d script helpers initscripts + # For kill used in stop/reopen-logs commands. + $util_linux_package + + # For pstree used in reopen-logs command. + psmisc + # For pkill/pgrep used for legacy status/stop commands. $procps_package ) @@ -142,6 +145,12 @@ elif [ -f /etc/debian_version ]; then sysvinit-utils lsb-base + # For kill used in stop/reopen-logs commands. + procps + + # For pstree used in reopen-logs command. + psmisc + # For pkill/pgrep used for legacy status/stop commands. procps ) diff --git a/src/api-umbrella/cli/reopen_logs.lua b/src/api-umbrella/cli/reopen_logs.lua index 09d135faa..11002f9b9 100644 --- a/src/api-umbrella/cli/reopen_logs.lua +++ b/src/api-umbrella/cli/reopen_logs.lua @@ -3,19 +3,27 @@ local read_config = require "api-umbrella.cli.read_config" local run_command = require "api-umbrella.utils.run_command" local status = require "api-umbrella.cli.status" -local function reopen_perp_logs(perp_base) - local _, output, err = run_command("perpls -g -b " .. perp_base) +local function reopen_perp_logs(parent_pid) + -- Use pstree and parse the output to find all the log processes under the + -- root process. + -- + -- We use this instead of perpctl for finding the log processes, since + -- perpctl doesn't seem to have a way to send signals to the root perpd's log + -- process (just the services underneath perpd). Since we also want to be + -- sure to reopen perpd's logs, we need to use this approach. + local _, output, err = run_command("pstree -p -A " .. parent_pid) if err then print("Failed to reopen logs for perp\n" .. err) os.exit(1) end + local log_process_name = "svlogd" for line in string.gmatch(output, "[^\r\n]+") do - local service_status, service = string.match(line, "^%[(.) .-%]%s+(%S+)") - if service_status == "+" then - local _, _, reload_err = run_command("perpctl -L -b " .. perp_base .. " hup " .. service) + local log_pid = string.match(line, log_process_name .. "%((%d+)%)") + if log_pid then + local _, _, reload_err = run_command("kill -s HUP " .. log_pid) if reload_err then - print("Failed to reopen logs for " .. service .. "\n" .. reload_err) + print("Failed to reopen logs for " .. log_pid .. "\n" .. reload_err) os.exit(1) end end @@ -39,7 +47,7 @@ local function reopen_rsyslog(perp_base) end return function() - local running = status() + local running, pid = status() if not running then print("api-umbrella is stopped") os.exit(1) @@ -48,7 +56,7 @@ return function() local config = read_config() local perp_base = path.join(config["etc_dir"], "perp") - reopen_perp_logs(perp_base) + reopen_perp_logs(pid) if config["_service_router_enabled?"] then reopen_nginx(perp_base)