src: fix CSPRNG when length exceeds INT_MAX

CSPRNG implicitly casts the size_t length argument to a signed int when
calling RAND_bytes(), which leaves it up to the caller to ensure that
the length argument actually fits into such a signed int. However, not
all call sites explicitly ensure that, which could lead to subtle bugs.

In OpenSSL 3, use RAND_bytes_ex() instead, which does not require
casting the length to a signed int.

In OpenSSL 1.1.1, RAND_bytes_ex() is not supported, thus we have to
process blocks of size INT_MAX one by one.

PR-URL: nodejs#47515
Reviewed-By: Richard Lau <[email protected]>
Reviewed-By: James M Snell <[email protected]>

src/crypto/crypto_util.cc

@@ -61,10 +61,20 @@ int VerifyCallback(int preverify_ok, X509_STORE_CTX* ctx) {
 }
 
 MUST_USE_RESULT CSPRNGResult CSPRNG(void* buffer, size_t length) {
+  unsigned char* buf = static_cast<unsigned char*>(buffer);
   do {
-    if (1 == RAND_status())
-      if (1 == RAND_bytes(static_cast<unsigned char*>(buffer), length))
+    if (1 == RAND_status()) {
+#if OPENSSL_VERSION_MAJOR >= 3
+      if (1 == RAND_bytes_ex(nullptr, buf, length, 0)) return {true};
+#else
+      while (length > INT_MAX && 1 == RAND_bytes(buf, INT_MAX)) {
+        buf += INT_MAX;
+        length -= INT_MAX;
+      }
+      if (length <= INT_MAX && 1 == RAND_bytes(buf, static_cast<int>(length)))
         return {true};
+#endif
+    }
 #if OPENSSL_VERSION_MAJOR >= 3
     const auto code = ERR_peek_last_error();
     // A misconfigured OpenSSL 3 installation may report 1 from RAND_poll()