gprocess: add fatal signal logging with a backtrace

Signed-off-by: Balazs Scheidler <[email protected]>

lib/CMakeLists.txt

@@ -153,6 +153,7 @@ set (LIB_HEADERS
     serialize.h
     service-management.h
     seqnum.h
+    stackdump.h
     str-format.h
     str-utils.h
     syslog-names.h
@@ -249,6 +250,7 @@ set(LIB_SOURCES
     scratch-buffers.c
     serialize.c
     service-management.c
+    stackdump.c
     str-format.c
     str-utils.c
     syslog-names.c

lib/Makefile.am

@@ -172,6 +172,7 @@ pkginclude_HEADERS			+= \
 	lib/service-management.h	\
 	lib/seqnum.h			\
 	lib/signal-handler.h		\
+	lib/stackdump.h			\
 	lib/str-format.h		\
 	lib/str-utils.h			\
 	lib/syslog-names.h		\
@@ -264,6 +265,7 @@ lib_libsyslog_ng_la_SOURCES		= \
 	lib/scratch-buffers.c		\
 	lib/serialize.c			\
 	lib/service-management.c	\
+	lib/stackdump.c			\
 	lib/str-format.c		\
 	lib/str-utils.c			\
 	lib/syslog-names.c		\

lib/gprocess.c

@@ -27,6 +27,7 @@
 #include "messages.h"
 #include "reloc.h"
 #include "console.h"
+#include "stackdump.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -1306,6 +1307,14 @@ g_process_perform_supervise(void)
   exit(0);
 }
 
+
+static void
+g_process_setup_fatal_signal_handler(void)
+{
+  stackdump_setup_signal(SIGSEGV);
+  stackdump_setup_signal(SIGABRT);
+}
+
 /**
  * g_process_start:
  *
@@ -1409,6 +1418,8 @@ g_process_start(void)
       g_assert_not_reached();
     }
 
+  g_process_setup_fatal_signal_handler();
+
   /* daemon process, we should return to the caller to perform work */
   /* Only call setsid() for backgrounded processes. */
   if (process_opts.mode != G_PM_FOREGROUND)

lib/stackdump.c

@@ -0,0 +1,246 @@
+/*
+ * Copyright (c) 2024 Balázs Scheidler <[email protected]>
+ * Copyright (c) 2024 Axoflow
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+#include "stackdump.h"
+#include "console.h"
+
+#include <execinfo.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#ifdef __linux__
+
+/* this is Linux only for now */
+
+static void
+_stackdump_print_stack(gpointer stack_pointer)
+{
+  guint8 *p = stack_pointer;
+
+  for (gint i = 0; i < 16; i++)
+    {
+      gchar line[51] = {0};
+      for (gint j = 0; j < 8; j++)
+        {
+          sprintf(&line[j*3], "%02x ", (guint) *(p+j));
+        }
+      line[8*3] = ' ';
+      for (gint j = 8; j < 16; j++)
+        {
+          sprintf(&line[j*3 + 1], "%02x ", (guint) *(p+j));
+        }
+
+      console_printf("Stack %p: %s", p, line);
+      p += 16;
+    }
+}
+
+/* should not do any allocation to allow this to work even if our heap is corrupted */
+static void
+_stackdump_print_maps(void)
+{
+  int fd;
+
+  console_printf("Maps file follows");
+  fd = open("/proc/self/maps", O_RDONLY);
+  if (fd < 0)
+    {
+      console_printf("Error opening /proc/self/maps");
+      return;
+    }
+
+  gchar buf[1024];
+  int rc;
+  gchar *p, *eol;
+  gint avail, end = 0;
+
+  while (1)
+    {
+      avail = sizeof(buf) - end;
+      rc = read(fd, buf + end, avail);
+
+      if (rc < 0)
+        break;
+
+      end += rc;
+
+      if (rc == 0)
+        break;
+
+      p = buf;
+      while (*p && p < (buf + end))
+        {
+          eol = memchr(p, '\n', buf + end - p);
+          if (eol)
+            {
+              *eol = 0;
+              console_printf("%s", p);
+              p = eol + 1;
+            }
+          else
+            {
+              end = end - (p - buf);
+              memmove(buf, p, end);
+              break;
+            }
+        }
+    }
+  if (end)
+    console_printf("%.*s", end, buf);
+  close(fd);
+}
+
+static inline void
+_stackdump_print_backtrace(void)
+{
+  void *bt[256];
+  gint count;
+
+  count = backtrace(bt, 256);
+  console_printf("Raw backtrace dump, count=%d", count);
+  for (gint i = 0; i < count; i++)
+    {
+      console_printf("[%d]: %p", i, bt[i]);
+    }
+  if (count)
+    {
+      gchar **symbols;
+
+      console_printf("Symbol backtrace dump, count=%d", count);
+      symbols = backtrace_symbols(bt, count);
+      for (gint i = 0; i < count; i++)
+        {
+          console_printf("[%d]: %s", i, symbols[i]);
+        }
+    }
+}
+
+
+#ifdef __x86_64__
+/****************************************************************************
+ *
+ *
+ * x86_64 support
+ *
+ *
+ ****************************************************************************/
+
+void
+_stackdump_print_registers(struct sigcontext *p)
+{
+  console_printf(
+    "Registers: "
+    "rax=%016lx rbx=%016lx rcx=%016lx rdx=%016lx rsi=%016lx rdi=%016lx "
+    "rbp=%016lx rsp=%016lx r8=%016lx r9=%016lx r10=%016lx r11=%016lx "
+    "r12=%016lx r13=%016lx r14=%016lx r15=%016lx rip=%016lx",
+    p->rax, p->rbx, p->rcx, p->rdx, p->rsi, p->rdi, p->rbp, p->rsp, p->r8, p->r9, p->r10, p->r11, p->r12, p->r13, p->r14,
+    p->r15, p->rip);
+  _stackdump_print_stack((gpointer) p->rsp);
+}
+
+#elif __x86__
+/****************************************************************************
+ *
+ *
+ * i386 support
+ *
+ *
+ ****************************************************************************/
+
+void
+_stackdump_print_registers(struct sigcontext *p)
+{
+  console_printf(
+    "Registers: eax=%08lx ebx=%08lx ecx=%08lx edx=%08lx esi=%08lx edi=%08lx ebp=%08lx esp=%08lx eip=%08lx",
+    p->eax, p->ebx, p->ecx, p->edx, p->esi, p->edi, p->ebp, p->esp, p->eip);
+  _stackdump_print_stack((gpointer) p->esp);
+}
+
+#else
+/****************************************************************************
+ *
+ *
+ * unsupported platform
+ *
+ *
+ ****************************************************************************/
+
+static void
+_stackdump_print_registers(struct sigcontext *p)
+{
+}
+
+#endif
+
+
+static inline void
+_stackdump_log(struct sigcontext *p)
+{
+  /* the order is important here, even if it might be illogical.  The
+   * backtrace function is the most fragile (as backtrace_symbols() may
+   * allocate memory).  Let's log everything else first, and then we can
+   * produce the backtrace, which is potentially causing another crash.  */
+
+  _stackdump_print_registers(p);
+  _stackdump_print_maps();
+  _stackdump_print_backtrace();
+}
+
+static void
+_fatal_signal_handler(int signo, siginfo_t *info, void *uc)
+{
+  struct ucontext_t *ucontext = (struct ucontext_t *) uc;
+  struct sigcontext *p = (struct sigcontext *) &ucontext->uc_mcontext;
+  struct sigaction act;
+
+  memset(&act, 0, sizeof(act));
+  act.sa_handler = SIG_DFL;
+  sigaction(signo, &act, NULL);
+
+  console_printf("Fatal signal received, stackdump follows, signal=%d", signo);
+  _stackdump_log(p);
+  /* let's get a stacktrace as well */
+  kill(getpid(), signo);
+}
+
+void
+stackdump_setup_signal(gint signal_number)
+{
+  struct sigaction act;
+
+  memset(&act, 0, sizeof(act));
+  act.sa_flags = SA_SIGINFO;
+  act.sa_sigaction = _fatal_signal_handler;
+
+  sigaction(signal_number, &act, NULL);
+}
+
+#else
+
+void
+stackdump_setup_signal(gint signal_number)
+{
+}
+
+#endif

lib/stackdump.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2024 Balázs Scheidler <[email protected]>
+ * Copyright (c) 2024 Axoflow
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+#ifndef STACKDUMP_H_INCLUDED
+#define STACKDUMP_H_INCLUDED
+
+#include "syslog-ng.h"
+
+void stackdump_setup_signal(gint signal_number);
+
+#endif

tests/copyright/policy

@@ -109,6 +109,7 @@ lib/stats/stats-compat\.h
 lib/stats/tests/test_stats_prometheus\.c
 lib/stats/tests/test_stats_cluster_key_builder\.c
 lib/timeutils/timeutils\.h
+lib/stackdump\.[ch]
 lib/logscheduler\.[ch]
 lib/logscheduler-pipe\.[ch]
 lib/logmsg/tests/dump_logmsg\.c