Fix printf funcs on memory pressure with floats (#1275)

Cosmopolitan's printf-family functions will currently crash if one tries
formatting a floating point number with a larger precision (large enough
that gdtoa attempts to allocate memory to format the number) while under
memory pressure (i.e. when malloc fails) because gdtoa fails to check if
malloc fails.

The added tests (which would previously crash under cosmopolitan without
this patch) show how to reproduce the issue.

This patch fixes this, and adds the aforementioned tests.

libc/stdio/fmt.c

@@ -1145,6 +1145,8 @@ int __fmt(void *fn, void *arg, const char *format, va_list va, int *wrote) {
           s = s0 =
               gdtoa(fpb.fpi, fpb.ex, fpb.bits, &fpb.kind, 3, prec, &decpt, &se);
         }
+        if (s0 == NULL)
+          return -1;
         if (decpt == 9999 || decpt == -32768) {
         FormatDecpt9999Or32768:
           if (s0)
@@ -1258,6 +1260,8 @@ int __fmt(void *fn, void *arg, const char *format, va_list va, int *wrote) {
           s = s0 = gdtoa(fpb.fpi, fpb.ex, fpb.bits, &fpb.kind, prec ? 2 : 0,
                          prec, &decpt, &se);
         }
+        if (s0 == NULL)
+          return -1;
         if (decpt == 9999 || decpt == -32768)
           goto FormatDecpt9999Or32768;
         c = se - s;
@@ -1304,6 +1308,8 @@ int __fmt(void *fn, void *arg, const char *format, va_list va, int *wrote) {
           s = s0 = gdtoa(fpb.fpi, fpb.ex, fpb.bits, &fpb.kind, prec ? 2 : 0,
                          prec, &decpt, &se);
         }
+        if (s0 == NULL)
+          return -1;
         if (decpt == 9999 || decpt == -32768)
           goto FormatDecpt9999Or32768;
       FormatExpo:

test/libc/stdio/snprintf_enomem_test.c

@@ -0,0 +1,61 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Gabriel Ravier                                                │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/struct/rlimit.h"
+#include "libc/errno.h"
+#include "libc/stdio/stdio.h"
+#include "libc/sysv/consts/rlim.h"
+#include "libc/sysv/consts/rlimit.h"
+#include "libc/testlib/testlib.h"
+
+static void limit_memory_to_1mb() {
+  struct rlimit limit = {};
+  ASSERT_GE(getrlimit(RLIMIT_AS, &limit), 0);
+
+  if (limit.rlim_max > 1000000 || limit.rlim_max == RLIM_INFINITY) {
+    limit.rlim_max = 1000000;
+    limit.rlim_cur = limit.rlim_max;
+    ASSERT_GE(setrlimit(RLIMIT_AS, &limit), 0);
+  }
+}
+
+static void check_double_format_enomem(const char *fmt) {
+  errno = 0;
+  int result = printf(fmt, 1.0);
+  ASSERT_LE(result, 0);
+  ASSERT_EQ(errno, ENOMEM);
+}
+
+static void check_long_double_format_enomem(const char *fmt) {
+  errno = 0;
+  int result = printf(fmt, 1.0L);
+  ASSERT_LE(result, 0);
+  ASSERT_EQ(errno, ENOMEM);
+}
+
+TEST(snprintf, enomemFloat) {
+  limit_memory_to_1mb();
+
+  check_double_format_enomem("%.1000000f");
+  check_double_format_enomem("%.1000000g");
+  check_double_format_enomem("%.1000000e");
+
+  check_long_double_format_enomem("%.1000000Lf");
+  check_long_double_format_enomem("%.1000000Lg");
+  check_long_double_format_enomem("%.1000000Le");
+}

third_party/gdtoa/dmisc.c

@@ -49,6 +49,8 @@ __gdtoa_rv_alloc(int i, ThInfo **PTI)
 	     j <<= 1)
 		k++;
 	r = (int *)__gdtoa_Balloc(k, PTI);
+	if (r == NULL)
+		return NULL;
 	*r = k;
 	return (char *)(r + 1);
 }

third_party/gdtoa/dtoa.c

@@ -246,6 +246,9 @@ dtoa(double d0, int mode, int ndigits, int *decpt, int *sign, char **rve)
 			i = 1;
 	}
 	s = s0 = __gdtoa_rv_alloc(i, &TI);
+	if (s0 == NULL)
+		goto ret1;
+
 	if (mode > 1 && Rounding != 1)
 		leftright = 0;
 	if (ilim >= 0 && ilim <= Quick_max && try_quick) {
@@ -614,7 +617,8 @@ dtoa(double d0, int mode, int ndigits, int *decpt, int *sign, char **rve)
 		--s;
 ret1:
 	__gdtoa_Bfree(b, &TI);
-	*s = 0;
+	if (s != NULL)
+		*s = 0;
 	*decpt = k + 1;
 	if (rve)
 		*rve = s;

third_party/gdtoa/gdtoa.c

@@ -286,6 +286,8 @@ gdtoa(const FPI *fpi, int be, ULong *bits, int *kindp, int mode, int ndigits, in
 			i = 1;
 	}
 	s = s0 = __gdtoa_rv_alloc(i, &TI);
+	if (s0 == NULL)
+		goto ret1;
 	if (mode <= 1)
 		rdir = 0;
 	else if ( (rdir = fpi->rounding - 1) !=0) {
@@ -673,10 +675,12 @@ gdtoa(const FPI *fpi, int be, ULong *bits, int *kindp, int mode, int ndigits, in
 		__gdtoa_Bfree(mhi, &TI);
 	}
 ret1:
-	while(s > s0 && s[-1] == '0')
-		--s;
+	if (s != NULL)
+		while(s > s0 && s[-1] == '0')
+			--s;
 	__gdtoa_Bfree(b, &TI);
-	*s = 0;
+	if (s != NULL)
+		*s = 0;
 	*decpt = k + 1;
 	if (rve)
 		*rve = s;

third_party/gdtoa/misc.c

@@ -129,12 +129,16 @@ __gdtoa_Balloc(int k, ThInfo **PTI)
 	} else {
 		x = 1 << k;
 		rv = malloc(sizeof(Bigint) + (x-1)*sizeof(ULong));
+		if (rv == NULL)
+			goto ret;
 		rv->k = k;
 		rv->maxwds = x;
 	}
+	rv->sign = rv->wds = 0;
+
+ret:
 	if (TI == &TI0)
 		__gdtoa_unlock();
-	rv->sign = rv->wds = 0;
 	return rv;
 }