Make SSL handshakes much faster

This change boosts SSL handshake performance from 2,627 to ~10,000 per
second which is the same level of performance as NGINX at establishing
secure connections. That's impressive if we consider that redbean is a
forking frontend application server. This was accomplished by:

  1. Enabling either SSL session caching or SSL tickets. We choose to
     use tickets since they reduce network round trips too and that's
     a more important metric than wrk'ing localhost.

  2. Fixing mbedtls_mpi_sub_abs() which is the most frequently called
     function. It's called about 12,000 times during an SSL handshake
     since it's the basis of most arithmetic operations like addition
     and for some strange reason it was designed to make two needless
     copies in addition to calling malloc and free. That's now fixed.

  3. Improving TLS output buffering during the SSL handshake only, so
     that only a single is write and read system call is needed until
     blocking on the ping pong.

redbean will now do a better job wiping sensitive memory from a child
process as soon as it's not needed. The nice thing about fork is it's
much faster than reverse proxying so the goal is to use the different
address spaces along with setuid() to minimize the risk that a server
key will be compromised in the event that application code is hacked.

libc/calls/sigenter-xnu.c

@@ -20,6 +20,7 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/struct/siginfo.h"
 #include "libc/calls/ucontext.h"
+#include "libc/intrin/repstosb.h"
 #include "libc/str/str.h"
 
 /**
@@ -401,7 +402,14 @@ noasan static void xnuthreadstate2linux(
   mc->fs = xnuss->__fs;
   mc->eflags = xnuss->__rflags;
   uc->uc_flags = xnuss->__rflags;
-  memcpy(&mc->r8, &xnuss->__r8, 8 * sizeof(int64_t));
+  mc->r8 = xnuss->__r8;
+  mc->r9 = xnuss->__r9;
+  mc->r10 = xnuss->__r10;
+  mc->r11 = xnuss->__r11;
+  mc->r12 = xnuss->__r12;
+  mc->r13 = xnuss->__r13;
+  mc->r14 = xnuss->__r14;
+  mc->r15 = xnuss->__r15;
 }
 
 noasan static void linuxthreadstate2xnu(
@@ -420,7 +428,21 @@ noasan static void linuxthreadstate2xnu(
   xnuss->__fs = mc->fs;
   xnuss->__rflags = mc->eflags;
   xnuss->__rflags = uc->uc_flags;
-  memcpy(&xnuss->__r8, &mc->r8, 8 * sizeof(int64_t));
+  xnuss->__r8 = mc->r8;
+  xnuss->__r9 = mc->r9;
+  xnuss->__r10 = mc->r10;
+  xnuss->__r11 = mc->r11;
+  xnuss->__r12 = mc->r12;
+  xnuss->__r13 = mc->r13;
+  xnuss->__r14 = mc->r14;
+  xnuss->__r15 = mc->r15;
+}
+
+noasan static void CopyFpXmmRegs(void *d, const void *s) {
+  size_t i;
+  for (i = 0; i < (8 + 16) * 16; i += 16) {
+    __builtin_memcpy((char *)d + i, (const char *)s + i, 16);
+  }
 }
 
 noasan static void xnussefpustate2linux(
@@ -433,8 +455,7 @@ noasan static void xnussefpustate2linux(
   fs->rdp = xnufs->__fpu_dp;
   fs->mxcsr = xnufs->__fpu_mxcsr;
   fs->mxcr_mask = xnufs->__fpu_mxcsrmask;
-  /* copy st0-st7 as well as xmm0-xmm15 */
-  memcpy(fs->st, &xnufs->__fpu_stmm0, (8 + 16) * sizeof(uint128_t));
+  CopyFpXmmRegs(fs->st, &xnufs->__fpu_stmm0);
 }
 
 noasan static void linuxssefpustate2xnu(
@@ -447,8 +468,7 @@ noasan static void linuxssefpustate2xnu(
   xnufs->__fpu_dp = fs->rdp;
   xnufs->__fpu_mxcsr = fs->mxcsr;
   xnufs->__fpu_mxcsrmask = fs->mxcr_mask;
-  /* copy st0-st7 as well as xmm0-xmm15 */
-  memcpy(&xnufs->__fpu_stmm0, fs->st, (8 + 16) * sizeof(uint128_t));
+  CopyFpXmmRegs(&xnufs->__fpu_stmm0, fs->st);
 }
 
 noasan void __sigenter_xnu(void *fn, int infostyle, int sig,
@@ -462,10 +482,9 @@ noasan void __sigenter_xnu(void *fn, int infostyle, int sig,
   } g;
   rva = __sighandrvas[sig & (NSIG - 1)];
   if (rva >= kSigactionMinRva) {
-    memset(&g, 0, sizeof(g));
+    repstosb(&g, 0, sizeof(g));
     if (xnuctx) {
-      memcpy(&g.uc.uc_sigmask, &xnuctx->uc_sigmask,
-             MIN(sizeof(g.uc.uc_sigmask), sizeof(xnuctx->uc_sigmask)));
+      g.uc.uc_sigmask.__bits[0] = xnuctx->uc_sigmask;
       g.uc.uc_stack.ss_sp = xnuctx->uc_stack.ss_sp;
       g.uc.uc_stack.ss_flags = xnuctx->uc_stack.ss_flags;
       g.uc.uc_stack.ss_size = xnuctx->uc_stack.ss_size;

libc/intrin/mpsadbw.c

@@ -40,5 +40,5 @@ void(mpsadbw)(uint16_t c[8], const uint8_t b[16], const uint8_t a[16],
       r[i] += ABS(b[(control & 4) + i + j] - a[(control & 3) * 4 + j]);
     }
   }
-  memcpy(c, r, 16);
+  __builtin_memcpy(c, r, 16);
 }

libc/intrin/pabsb.c

@@ -30,5 +30,5 @@ void(pabsb)(uint8_t a[16], const int8_t b[16]) {
   for (i = 0; i < 16; ++i) {
     r[i] = ABS(b[i]);
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pabsd.c

@@ -30,5 +30,5 @@ void(pabsd)(uint32_t a[4], const int32_t b[4]) {
   for (i = 0; i < 4; ++i) {
     r[i] = b[i] >= 0 ? b[i] : -(uint32_t)b[i];
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pabsw.c

@@ -30,5 +30,5 @@ void(pabsw)(uint16_t a[8], const int16_t b[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = ABS(b[i]);
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/packsswb.c

@@ -34,5 +34,5 @@ void(packsswb)(int8_t a[16], const int16_t b[8], const int16_t c[8]) {
   int8_t r[16];
   for (i = 0; i < 8; ++i) r[i + 0] = MIN(INT8_MAX, MAX(INT8_MIN, b[i]));
   for (i = 0; i < 8; ++i) r[i + 8] = MIN(INT8_MAX, MAX(INT8_MIN, c[i]));
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/packusdw.c

@@ -30,5 +30,5 @@ void(packusdw)(uint16_t a[8], const int32_t b[4], const int32_t c[4]) {
   uint16_t r[8];
   for (i = 0; i < 4; ++i) r[i + 0] = MIN(UINT16_MAX, MAX(UINT16_MIN, b[i]));
   for (i = 0; i < 4; ++i) r[i + 4] = MIN(UINT16_MAX, MAX(UINT16_MIN, c[i]));
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/packuswb.c

@@ -34,5 +34,5 @@ void(packuswb)(uint8_t a[16], const int16_t b[8], const int16_t c[8]) {
   uint8_t r[16];
   for (i = 0; i < 8; ++i) r[i + 0] = MIN(UINT8_MAX, MAX(UINT8_MIN, b[i]));
   for (i = 0; i < 8; ++i) r[i + 8] = MIN(UINT8_MAX, MAX(UINT8_MIN, c[i]));
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddb.c

@@ -31,5 +31,5 @@ void(paddb)(int8_t a[16], const int8_t b[16], const int8_t c[16]) {
   unsigned i;
   int8_t r[16];
   for (i = 0; i < 16; ++i) r[i] = b[i] + c[i];
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddd.c

@@ -33,5 +33,5 @@ void(paddd)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) {
   for (i = 0; i < 4; ++i) {
     r[i] = b[i] + c[i];
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddq.c

@@ -31,5 +31,5 @@ void(paddq)(uint64_t a[2], const uint64_t b[2], const uint64_t c[2]) {
   unsigned i;
   uint64_t r[2];
   for (i = 0; i < 2; ++i) r[i] = b[i] + c[i];
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddsb.c

@@ -35,5 +35,5 @@ void(paddsb)(int8_t a[16], const int8_t b[16], const int8_t c[16]) {
   for (i = 0; i < 16; ++i) {
     r[i] = MIN(INT8_MAX, MAX(INT8_MIN, b[i] + c[i]));
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddsw.c

@@ -35,5 +35,5 @@ void(paddsw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = MIN(SHRT_MAX, MAX(SHRT_MIN, b[i] + c[i]));
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddusb.c

@@ -35,5 +35,5 @@ void(paddusb)(uint8_t a[16], const uint8_t b[16], const uint8_t c[16]) {
   for (i = 0; i < 16; ++i) {
     r[i] = MIN(UINT8_MAX, b[i] + c[i]);
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddusw.c

@@ -35,5 +35,5 @@ void(paddusw)(uint16_t a[8], const uint16_t b[8], const uint16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = MIN(UINT16_MAX, b[i] + c[i]);
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/paddw.c

@@ -35,5 +35,5 @@ void(paddw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = b[i] + c[i];
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/palignr.c

@@ -36,8 +36,8 @@
  */
 void(palignr)(void *c, const void *b, const void *a, unsigned long i) {
   char t[48];
-  memcpy(t, a, 16);
-  memcpy(t + 16, b, 16);
-  memset(t + 32, 0, 16);
-  memcpy(c, t + MIN(i, 32), 16);
+  __builtin_memcpy(t, a, 16);
+  __builtin_memcpy(t + 16, b, 16);
+  __builtin_memset(t + 32, 0, 16);
+  __builtin_memcpy(c, t + MIN(i, 32), 16);
 }

libc/intrin/pavgb.c

@@ -33,5 +33,5 @@ void(pavgb)(uint8_t a[16], const uint8_t b[16], const uint8_t c[16]) {
   for (i = 0; i < 16; ++i) {
     r[i] = (b[i] + c[i] + 1) >> 1;
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pavgw.c

@@ -33,5 +33,5 @@ void(pavgw)(uint16_t a[8], const uint16_t b[8], const uint16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = (b[i] + c[i] + 1) >> 1;
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pcmpeqb.c

@@ -31,5 +31,5 @@ void(pcmpeqb)(uint8_t a[16], const uint8_t b[16], const uint8_t c[16]) {
   unsigned i;
   uint8_t r[16];
   for (i = 0; i < 16; ++i) r[i] = -(b[i] == c[i]);
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pcmpeqd.c

@@ -31,5 +31,5 @@ void(pcmpeqd)(int32_t a[4], const int32_t b[4], const int32_t c[4]) {
   unsigned i;
   int32_t r[4];
   for (i = 0; i < 4; ++i) r[i] = -(b[i] == c[i]);
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pcmpeqw.c

@@ -31,5 +31,5 @@ void(pcmpeqw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   unsigned i;
   int16_t r[8];
   for (i = 0; i < 8; ++i) r[i] = -(b[i] == c[i]);
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pcmpgtb.c

@@ -33,5 +33,5 @@ void(pcmpgtb)(int8_t a[16], const int8_t b[16], const int8_t c[16]) {
   unsigned i;
   int8_t r[16];
   for (i = 0; i < 16; ++i) r[i] = -(b[i] > c[i]);
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pcmpgtd.c

@@ -31,5 +31,5 @@ void(pcmpgtd)(int32_t a[4], const int32_t b[4], const int32_t c[4]) {
   unsigned i;
   int32_t r[4];
   for (i = 0; i < 4; ++i) r[i] = -(b[i] > c[i]);
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pcmpgtw.c

@@ -31,5 +31,5 @@ void(pcmpgtw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   unsigned i;
   int16_t r[8];
   for (i = 0; i < 8; ++i) r[i] = -(b[i] > c[i]);
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/phaddd.c

@@ -34,5 +34,5 @@ void(phaddd)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) {
   t[1] = b[2] + b[3];
   t[2] = c[0] + c[1];
   t[3] = c[2] + c[3];
-  memcpy(a, t, sizeof(t));
+  __builtin_memcpy(a, t, sizeof(t));
 }

libc/intrin/phaddw.c

@@ -38,5 +38,5 @@ void(phaddw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   t[5] = c[2] + c[3];
   t[6] = c[4] + c[5];
   t[7] = c[6] + c[7];
-  memcpy(a, t, sizeof(t));
+  __builtin_memcpy(a, t, sizeof(t));
 }

libc/intrin/phsubd.c

@@ -34,5 +34,5 @@ void(phsubd)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) {
   t[1] = b[2] - b[3];
   t[2] = c[0] - c[1];
   t[3] = c[2] - c[3];
-  memcpy(a, t, sizeof(t));
+  __builtin_memcpy(a, t, sizeof(t));
 }

libc/intrin/phsubw.c

@@ -38,5 +38,5 @@ void(phsubw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   t[5] = c[2] - c[3];
   t[6] = c[4] - c[5];
   t[7] = c[6] - c[7];
-  memcpy(a, t, sizeof(t));
+  __builtin_memcpy(a, t, sizeof(t));
 }

libc/intrin/pmaxsw.c

@@ -34,5 +34,5 @@ void(pmaxsw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = MAX(b[i], c[i]);
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pminsw.c

@@ -34,5 +34,5 @@ void(pminsw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = MIN(b[i], c[i]);
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pmulhrsw.c

@@ -31,5 +31,5 @@ void(pmulhrsw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   unsigned i;
   int16_t r[8];
   for (i = 0; i < 8; ++i) r[i] = (((b[i] * c[i]) >> 14) + 1) >> 1;
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pmulhuw.c

@@ -37,5 +37,5 @@ void(pmulhuw)(uint16_t a[8], const uint16_t b[8], const uint16_t c[8]) {
     x >>= 16;
     r[i] = x;
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pmulhw.c

@@ -33,5 +33,5 @@ void(pmulhw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = ((b[i] * c[i]) & 0xffff0000) >> 16;
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pmulld.c

@@ -34,5 +34,5 @@ void(pmulld)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) {
   for (i = 0; i < 4; ++i) {
     r[i] = b[i] * c[i];
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pmullw.c

@@ -33,5 +33,5 @@ void(pmullw)(int16_t a[8], const int16_t b[8], const int16_t c[8]) {
   for (i = 0; i < 8; ++i) {
     r[i] = b[i] * c[i];
   }
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pshufb.c

@@ -33,5 +33,5 @@ void(pshufb)(uint8_t a[16], const uint8_t b[16], const uint8_t c[16]) {
   unsigned i;
   uint8_t r[16];
   for (i = 0; i < 16; ++i) r[i] = (c[i] & 0x80) ? 0 : b[c[i] & 0x0F];
-  memcpy(a, r, 16);
+  __builtin_memcpy(a, r, 16);
 }

libc/intrin/pshufd.c

@@ -30,5 +30,5 @@ void(pshufd)(int32_t b[4], const int32_t a[4], uint8_t m) {
   t[1] = a[(m & 0b00001100) >> 2];
   t[2] = a[(m & 0b00110000) >> 4];
   t[3] = a[(m & 0b11000000) >> 6];
-  memcpy(b, t, 16);
+  __builtin_memcpy(b, t, 16);
 }

libc/intrin/pslld.c

@@ -32,6 +32,6 @@ void(pslld)(uint32_t a[4], const uint32_t b[4], unsigned char c) {
       a[i] = b[i] << c;
     }
   } else {
-    memset(a, 0, 16);
+    __builtin_memset(a, 0, 16);
   }
 }

libc/intrin/pslldq.c

@@ -32,5 +32,5 @@ void(pslldq)(uint8_t b[16], const uint8_t a[16], unsigned long n) {
   if (n > 16) n = 16;
   for (i = 0; i < n; ++i) t[i] = 0;
   for (i = 0; i < 16 - n; ++i) t[n + i] = a[i];
-  memcpy(b, t, 16);
+  __builtin_memcpy(b, t, 16);
 }

libc/intrin/pslldv.c

@@ -30,6 +30,6 @@ void(pslldv)(uint32_t a[4], const uint32_t b[4], const uint64_t c[2]) {
       a[i] = b[i] << c[0];
     }
   } else {
-    memset(a, 0, 16);
+    __builtin_memset(a, 0, 16);
   }
 }

libc/intrin/psllq.c

@@ -32,6 +32,6 @@ void(psllq)(uint64_t a[2], const uint64_t b[2], unsigned char c) {
       a[i] = b[i] << c;
     }
   } else {
-    memset(a, 0, 16);
+    __builtin_memset(a, 0, 16);
   }
 }

libc/intrin/psllqv.c

@@ -30,6 +30,6 @@ void(psllqv)(uint64_t a[2], const uint64_t b[2], const uint64_t c[2]) {
       a[i] = b[i] << c[0];
     }
   } else {
-    memset(a, 0, 16);
+    __builtin_memset(a, 0, 16);
   }
 }

libc/intrin/psllw.c

@@ -32,6 +32,6 @@ void(psllw)(uint16_t a[8], const uint16_t b[8], unsigned char c) {
       a[i] = b[i] << c;
     }
   } else {
-    memset(a, 0, 16);
+    __builtin_memset(a, 0, 16);
   }
 }

libc/intrin/psllwv.c

@@ -31,6 +31,6 @@ void(psllwv)(uint16_t a[8], const uint16_t b[8], const uint64_t c[2]) {
       a[i] = b[i] << c[0];
     }
   } else {
-    memset(a, 0, 16);
+    __builtin_memset(a, 0, 16);
   }
 }