build: encode non-ASCII Latin1 characters as one byte in JS2C

Previously we had two encodings for JS files:

1. If a file contains only ASCII characters, encode it as a one-byte
  string (interpreted as uint8_t array during loading).
2. If a file contains any characters with code point above 127,
  encode it as a two-byte string (interpreted as uint16_t array
  during loading).

This was done because V8 only supports Latin-1 and UTF16 encoding
as underlying representation for strings. To store the JS code
as external strings to save encoding cost and memory overhead
we need to follow the representations supported by V8.
Notice that there is a gap in the Latin1 range (128-255) that we
encoded as two-byte, which was an undocumented TODO for a long
time. That was fine previously because then files that contained
code points beyond the 0-127 range contained code points >255.
Now we have undici which contains code points in the range 0-255
(minus a replaceable code point >255). So this patch adds handling
for the 128-255 range to reduce the size overhead caused by encoding
them as two-byte. This could reduce the size of the binary by
~500KB and helps future files with this kind of code points.

Drive-by: replace `’` with `'` in undici.js to make it a Latin-1
only string. That could be removed if undici updates itself to
replace this character in the comment.

PR-URL: #51605
Reviewed-By: Daniel Lemire <[email protected]>
Reviewed-By: Ethan Arrowood <[email protected]>

node.gyp

@@ -73,6 +73,7 @@
       'src/connection_wrap.cc',
       'src/dataqueue/queue.cc',
       'src/debug_utils.cc',
+      'src/embedded_data.cc',
       'src/encoding_binding.cc',
       'src/env.cc',
       'src/fs_event_wrap.cc',
@@ -192,6 +193,7 @@
       'src/dataqueue/queue.h',
       'src/debug_utils.h',
       'src/debug_utils-inl.h',
+      'src/embeded_data.h',
       'src/encoding_binding.h',
       'src/env_properties.h',
       'src/env.h',
@@ -1196,11 +1198,14 @@
         'deps/simdutf/simdutf.gyp:simdutf#host',
       ],
       'include_dirs': [
-        'tools'
+        'tools',
+        'src',
       ],
       'sources': [
         'tools/js2c.cc',
-        'tools/executable_wrapper.h'
+        'tools/executable_wrapper.h',
+        'src/embedded_data.h',
+        'src/embedded_data.cc',
       ],
       'conditions': [
         [ 'node_shared_libuv=="false"', {

src/embedded_data.cc

@@ -0,0 +1,33 @@
+#include "embedded_data.h"
+#include <vector>
+
+namespace node {
+std::string ToOctalString(const uint8_t ch) {
+  // We can print most printable characters directly. The exceptions are '\'
+  // (escape characters), " (would end the string), and ? (trigraphs). The
+  // latter may be overly conservative: we compile with C++17 which doesn't
+  // support trigraphs.
+  if (ch >= ' ' && ch <= '~' && ch != '\\' && ch != '"' && ch != '?') {
+    return std::string(1, static_cast<char>(ch));
+  }
+  // All other characters are blindly output as octal.
+  const char c0 = '0' + ((ch >> 6) & 7);
+  const char c1 = '0' + ((ch >> 3) & 7);
+  const char c2 = '0' + (ch & 7);
+  return std::string("\\") + c0 + c1 + c2;
+}
+
+std::vector<std::string> GetOctalTable() {
+  size_t size = 1 << 8;
+  std::vector<std::string> code_table(size);
+  for (size_t i = 0; i < size; ++i) {
+    code_table[i] = ToOctalString(static_cast<uint8_t>(i));
+  }
+  return code_table;
+}
+
+const std::string& GetOctalCode(uint8_t index) {
+  static std::vector<std::string> table = GetOctalTable();
+  return table[index];
+}
+}  // namespace node

src/embedded_data.h

@@ -0,0 +1,17 @@
+#ifndef SRC_EMBEDDED_DATA_H_
+#define SRC_EMBEDDED_DATA_H_
+
+#include <cinttypes>
+#include <string>
+
+// This file must not depend on node.h or other code that depends on
+// the full Node.js implementation because it is used during the
+// compilation of the Node.js implementation itself (especially js2c).
+
+namespace node {
+
+const std::string& GetOctalCode(uint8_t index);
+
+}  // namespace node
+
+#endif  // SRC_EMBEDDED_DATA_H_

src/node_snapshotable.cc

@@ -8,6 +8,7 @@
 #include "base_object-inl.h"
 #include "blob_serializer_deserializer-inl.h"
 #include "debug_utils-inl.h"
+#include "embedded_data.h"
 #include "encoding_binding.h"
 #include "env-inl.h"
 #include "json_parser.h"
@@ -747,35 +748,6 @@ static std::string FormatSize(size_t size) {
   return buf;
 }
 
-std::string ToOctalString(const uint8_t ch) {
-  // We can print most printable characters directly. The exceptions are '\'
-  // (escape characters), " (would end the string), and ? (trigraphs). The
-  // latter may be overly conservative: we compile with C++17 which doesn't
-  // support trigraphs.
-  if (ch >= ' ' && ch <= '~' && ch != '\\' && ch != '"' && ch != '?') {
-    return std::string(1, static_cast<char>(ch));
-  }
-  // All other characters are blindly output as octal.
-  const char c0 = '0' + ((ch >> 6) & 7);
-  const char c1 = '0' + ((ch >> 3) & 7);
-  const char c2 = '0' + (ch & 7);
-  return std::string("\\") + c0 + c1 + c2;
-}
-
-std::vector<std::string> GetOctalTable() {
-  size_t size = 1 << 8;
-  std::vector<std::string> code_table(size);
-  for (size_t i = 0; i < size; ++i) {
-    code_table[i] = ToOctalString(static_cast<uint8_t>(i));
-  }
-  return code_table;
-}
-
-const std::string& GetOctalCode(uint8_t index) {
-  static std::vector<std::string> table = GetOctalTable();
-  return table[index];
-}
-
 template <typename T>
 void WriteByteVectorLiteral(std::ostream* ss,
                             const T* vec,

tools/js2c.cc

@@ -11,6 +11,7 @@
 #include <string>
 #include <string_view>
 #include <vector>
+#include "embedded_data.h"
 #include "executable_wrapper.h"
 #include "simdutf.h"
 #include "uv.h"
@@ -396,11 +397,14 @@ const std::string& GetCode(uint16_t index) {
 
 #ifdef NODE_JS2C_USE_STRING_LITERALS
 const char* string_literal_def_template = "static const %s *%s_raw = ";
+constexpr std::string_view latin1_string_literal_start =
+    "reinterpret_cast<const uint8_t*>(\"";
 constexpr std::string_view ascii_string_literal_start =
     "reinterpret_cast<const uint8_t*>(R\"JS2C1b732aee(";
 constexpr std::string_view utf16_string_literal_start =
     "reinterpret_cast<const uint16_t*>(uR\"JS2C1b732aee(";
-constexpr std::string_view string_literal_end = ")JS2C1b732aee\");";
+constexpr std::string_view latin1_string_literal_end = "\");";
+constexpr std::string_view utf_string_literal_end = ")JS2C1b732aee\");";
 #else
 const char* array_literal_def_template = "static const %s %s_raw[] = ";
 constexpr std::string_view array_literal_start = "{\n";
@@ -424,9 +428,15 @@ constexpr std::string_view array_literal_end = "\n};\n\n";
 // If NODE_JS2C_USE_STRING_LITERALS is defined, the data is output as C++
 // raw strings (i.e. R"JS2C1b732aee(...)JS2C1b732aee") rather than as an
 // array. This speeds up compilation for gcc/clang.
+enum class CodeType {
+  kAscii,   // Code points are all within 0-127
+  kLatin1,  // Code points are all within 0-255
+  kTwoByte,
+};
 template <typename T>
 Fragment GetDefinitionImpl(const std::vector<char>& code,
-                           const std::string& var) {
+                           const std::string& var,
+                           CodeType type) {
   constexpr bool is_two_byte = std::is_same_v<T, uint16_t>;
   static_assert(is_two_byte || std::is_same_v<T, char>);
 
@@ -440,11 +450,14 @@ Fragment GetDefinitionImpl(const std::vector<char>& code,
 
 #ifdef NODE_JS2C_USE_STRING_LITERALS
   const char* literal_def_template = string_literal_def_template;
-  size_t def_size = 512 + code.size();
+  // For code that contains Latin-1 characters, be conservative and assume
+  // they all need escaping: one "\" and three digits.
+  size_t unit = type == CodeType::kLatin1 ? 4 : 1;
+  size_t def_size = 512 + code.size() * unit;
 #else
   const char* literal_def_template = array_literal_def_template;
   constexpr size_t unit =
-      (is_two_byte ? 5 : 3) + 1;  // 0-65536 or 0-127 and a ","
+      (is_two_byte ? 5 : 3) + 1;  // 0-65536 or 0-255 and a ","
   size_t def_size = 512 + count * unit;
 #endif
 
@@ -456,16 +469,56 @@ Fragment GetDefinitionImpl(const std::vector<char>& code,
   assert(cur != 0);
 
 #ifdef NODE_JS2C_USE_STRING_LITERALS
-  constexpr std::string_view start_string_view =
-      is_two_byte ? utf16_string_literal_start : ascii_string_literal_start;
+  std::string_view start_string_view;
+  switch (type) {
+    case CodeType::kAscii:
+      start_string_view = ascii_string_literal_start;
+      break;
+    case CodeType::kLatin1:
+      start_string_view = latin1_string_literal_start;
+      break;
+    case CodeType::kTwoByte:
+      start_string_view = utf16_string_literal_start;
+      break;
+  }
 
   memcpy(
       result.data() + cur, start_string_view.data(), start_string_view.size());
   cur += start_string_view.size();
 
-  memcpy(result.data() + cur, code.data(), code.size());
-  cur += code.size();
+  if (type != CodeType::kLatin1) {
+    memcpy(result.data() + cur, code.data(), code.size());
+    cur += code.size();
+  } else {
+    const uint8_t* ptr = reinterpret_cast<const uint8_t*>(code.data());
+    for (size_t i = 0; i < count; ++i) {
+      // Avoid using snprintf on large chunks of data because it's much slower.
+      // It's fine to use it on small amount of data though.
+      uint8_t ch = ptr[i];
+      if (ch > 127) {
+        Debug("In %s, found non-ASCII Latin-1 character at %zu: %d\n",
+              var.c_str(),
+              i,
+              ch);
+      }
+      const std::string& str = GetOctalCode(ch);
+      memcpy(result.data() + cur, str.c_str(), str.size());
+      cur += str.size();
+    }
+  }
 
+  std::string_view string_literal_end;
+  switch (type) {
+    case CodeType::kAscii:
+      string_literal_end = utf_string_literal_end;
+      break;
+    case CodeType::kLatin1:
+      string_literal_end = latin1_string_literal_end;
+      break;
+    case CodeType::kTwoByte:
+      string_literal_end = utf_string_literal_end;
+      break;
+  }
   memcpy(result.data() + cur,
          string_literal_end.data(),
          string_literal_end.size());
@@ -476,10 +529,10 @@ Fragment GetDefinitionImpl(const std::vector<char>& code,
          array_literal_start.size());
   cur += array_literal_start.size();
 
-  const std::vector<T>* codepoints;
-
-  std::vector<uint16_t> utf16_codepoints;
+  // Avoid using snprintf on large chunks of data because it's much slower.
+  // It's fine to use it on small amount of data though.
   if constexpr (is_two_byte) {
+    std::vector<uint16_t> utf16_codepoints;
     utf16_codepoints.resize(count);
     size_t utf16_count = simdutf::convert_utf8_to_utf16(
         code.data(),
@@ -488,19 +541,25 @@ Fragment GetDefinitionImpl(const std::vector<char>& code,
     assert(utf16_count != 0);
     utf16_codepoints.resize(utf16_count);
     Debug("static size %zu\n", utf16_count);
-    codepoints = &utf16_codepoints;
+    for (size_t i = 0; i < utf16_count; ++i) {
+      const std::string& str = GetCode(utf16_codepoints[i]);
+      memcpy(result.data() + cur, str.c_str(), str.size());
+      cur += str.size();
+    }
   } else {
-    // The code is ASCII, so no need to translate.
-    codepoints = &code;
-  }
-
-  for (size_t i = 0; i < codepoints->size(); ++i) {
-    // Avoid using snprintf on large chunks of data because it's much slower.
-    // It's fine to use it on small amount of data though.
-    const std::string& str = GetCode(static_cast<uint16_t>((*codepoints)[i]));
-
-    memcpy(result.data() + cur, str.c_str(), str.size());
-    cur += str.size();
+    const uint8_t* ptr = reinterpret_cast<const uint8_t*>(code.data());
+    for (size_t i = 0; i < count; ++i) {
+      uint16_t ch = static_cast<uint16_t>(ptr[i]);
+      if (ch > 127) {
+        Debug("In %s, found non-ASCII Latin-1 character at %zu: %d\n",
+              var.c_str(),
+              i,
+              ch);
+      }
+      const std::string& str = GetCode(ch);
+      memcpy(result.data() + cur, str.c_str(), str.size());
+      cur += str.size();
+    }
   }
 
   memcpy(
@@ -520,17 +579,80 @@ Fragment GetDefinitionImpl(const std::vector<char>& code,
   return result;
 }
 
-Fragment GetDefinition(const std::string& var, const std::vector<char>& code) {
-  Debug("GetDefinition %s, code size %zu ", var.c_str(), code.size());
-  bool is_one_byte = simdutf::validate_ascii(code.data(), code.size());
-  Debug("with %s\n", is_one_byte ? "1-byte chars" : "2-byte chars");
+bool Simplify(const std::vector<char>& code,
+              const std::string& var,
+              std::vector<char>* simplified) {
+  // Allowlist files to avoid false positives.
+  // TODO(joyeecheung): this could be removed if undici updates itself
+  // to replace "’" with "'" though we could still keep this skeleton in
+  // place for future hot fixes that are verified by humans.
+  if (var != "internal_deps_undici_undici") {
+    return false;
+  }
 
-  if (is_one_byte) {
-    Debug("static size %zu\n", code.size());
-    return GetDefinitionImpl<char>(code, var);
-  } else {
-    return GetDefinitionImpl<uint16_t>(code, var);
+  size_t code_size = code.size();
+  simplified->reserve(code_size);
+  const uint8_t* ptr = reinterpret_cast<const uint8_t*>(code.data());
+  size_t simplified_count = 0;
+  for (size_t i = 0; i < code_size; ++i) {
+    switch (ptr[i]) {
+      case 226: {  // ’ [ 226, 128, 153 ] -> '
+        if (i + 2 < code_size && ptr[i + 1] == 128 && ptr[i + 2] == 153) {
+          simplified->push_back('\'');
+          i += 2;
+          simplified_count++;
+          break;
+        }
+      }
+      default: {
+        simplified->push_back(code[i]);
+        break;
+      }
+    }
   }
+
+  if (simplified_count > 0) {
+    Debug("Simplified %d characters, ", simplified_count);
+    Debug("old size %d, new size %d\n", code_size, simplified->size());
+    return true;
+  }
+  return false;
+}
+
+Fragment GetDefinition(const std::string& var, const std::vector<char>& code) {
+  Debug("GetDefinition %s, code size %zu\n", var.c_str(), code.size());
+  bool is_ascii = simdutf::validate_ascii(code.data(), code.size());
+
+  if (is_ascii) {
+    Debug("ASCII-only, static size %zu\n", code.size());
+    return GetDefinitionImpl<char>(code, var, CodeType::kAscii);
+  }
+
+  std::vector<char> latin1(code.size());
+  auto result = simdutf::convert_utf8_to_latin1_with_errors(
+      code.data(), code.size(), latin1.data());
+  if (!result.error) {
+    latin1.resize(result.count);
+    Debug("Latin-1-only, old size %zu, new size %zu\n",
+          code.size(),
+          latin1.size());
+    return GetDefinitionImpl<char>(latin1, var, CodeType::kLatin1);
+  }
+
+  // Since V8 only supports Latin-1 and UTF16 as underlying representation
+  // we have to encode all files containing two-byte characters as UTF16.
+  // While some files do need two-byte characters, some just
+  // unintentionally have them. Replace certain characters that are known
+  // to have sane one-byte equivalent to save space.
+  std::vector<char> simplified;
+  if (Simplify(code, var, &simplified)) {  // Changed.
+    Debug("%s is simplified, re-generate definition\n", var.c_str());
+    return GetDefinition(var, simplified);
+  }
+
+  // Simplification did not turn the code into 1-byte string. Just
+  // use the original.
+  return GetDefinitionImpl<uint16_t>(code, var, CodeType::kTwoByte);
 }
 
 int AddModule(const std::string& filename,