From da8ec3d106018175290ba91aa07f9dd994388828 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 3 Dec 2018 00:49:12 +0800 Subject: [PATCH] process: specialize building and storage of process.config Instead of treating config.gypi as a JavaScript file, specialize the processing in js2c and make the serialized result a real JSON string (with 'true' and 'false' converted to boolean values) so we don't have to use a custom deserializer during bootstrap. In addition, store the JSON string separately in NativeModuleLoader, and keep it separate from the map of the builtin source code, so we don't have to put it onto `NativeModule._source` and delete it later, though we still preserve it in `process.binding('natives')`, which we don't use anymore. This patch also makes the map of builtin source code and the config.gypi string available through side-effect-free getters in C++. PR-URL: https://github.com/nodejs/node/pull/24816 Reviewed-By: Gus Caplan --- lib/internal/bootstrap/cache.js | 4 +-- lib/internal/bootstrap/loaders.js | 10 ++++-- lib/internal/bootstrap/node.js | 2 +- lib/internal/process/per_thread.js | 14 ++------ src/env.h | 3 +- src/node_binding.cc | 7 ++++ src/node_native_module.cc | 48 +++++++++++++++++++++---- src/node_native_module.h | 18 ++++++++-- src/node_union_bytes.h | 10 ++++++ test/parallel/test-bootstrap-modules.js | 2 +- tools/js2c.py | 38 +++++++++++++------- 11 files changed, 114 insertions(+), 42 deletions(-) diff --git a/lib/internal/bootstrap/cache.js b/lib/internal/bootstrap/cache.js index 117babb4e416a7..8658f5b5d4ccec 100644 --- a/lib/internal/bootstrap/cache.js +++ b/lib/internal/bootstrap/cache.js @@ -6,10 +6,9 @@ // cannot be tampered with even with --expose-internals. const { NativeModule } = require('internal/bootstrap/loaders'); -const { getSource, compileCodeCache } = internalBinding('native_module'); +const { source, compileCodeCache } = internalBinding('native_module'); const { hasTracing } = process.binding('config'); -const source = getSource(); const depsModule = Object.keys(source).filter( (key) => NativeModule.isDepsModule(key) || key.startsWith('internal/deps') ); @@ -17,7 +16,6 @@ const depsModule = Object.keys(source).filter( // Modules with source code compiled in js2c that // cannot be compiled with the code cache. const cannotUseCache = [ - 'config', 'sys', // Deprecated. 'internal/v8_prof_polyfill', 'internal/v8_prof_processor', diff --git a/lib/internal/bootstrap/loaders.js b/lib/internal/bootstrap/loaders.js index cc3822995411f7..af4928cc6163d1 100644 --- a/lib/internal/bootstrap/loaders.js +++ b/lib/internal/bootstrap/loaders.js @@ -169,10 +169,15 @@ function NativeModule(id) { this.loading = false; } -NativeModule._source = getInternalBinding('natives'); +const { + source, + compileFunction +} = internalBinding('native_module'); + +NativeModule._source = source; NativeModule._cache = {}; -const config = getInternalBinding('config'); +const config = internalBinding('config'); // Think of this as module.exports in this file even though it is not // written in CommonJS style. @@ -323,7 +328,6 @@ NativeModule.prototype.proxifyExports = function() { this.exports = new Proxy(this.exports, handler); }; -const { compileFunction } = getInternalBinding('native_module'); NativeModule.prototype.compile = function() { const id = this.id; diff --git a/lib/internal/bootstrap/node.js b/lib/internal/bootstrap/node.js index 3f730f984ee4ae..29d4178dece48a 100644 --- a/lib/internal/bootstrap/node.js +++ b/lib/internal/bootstrap/node.js @@ -59,7 +59,7 @@ function startup() { } perThreadSetup.setupAssert(); - perThreadSetup.setupConfig(NativeModule._source); + perThreadSetup.setupConfig(); if (isMainThread) { mainThreadSetup.setupSignalHandlers(internalBinding); diff --git a/lib/internal/process/per_thread.js b/lib/internal/process/per_thread.js index ad634b757eca48..57cc9c38143f79 100644 --- a/lib/internal/process/per_thread.js +++ b/lib/internal/process/per_thread.js @@ -143,17 +143,9 @@ function setupMemoryUsage(_memoryUsage) { }; } -function setupConfig(_source) { - // NativeModule._source - // used for `process.config`, but not a real module - const config = _source.config; - delete _source.config; - - process.config = JSON.parse(config, function(key, value) { - if (value === 'true') return true; - if (value === 'false') return false; - return value; - }); +function setupConfig() { + // Serialized config.gypi + process.config = JSON.parse(internalBinding('native_module').config); } diff --git a/src/env.h b/src/env.h index afa5564647ec4c..8acaf8e6fd633d 100644 --- a/src/env.h +++ b/src/env.h @@ -140,6 +140,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2; V(channel_string, "channel") \ V(chunks_sent_since_last_write_string, "chunksSentSinceLastWrite") \ V(code_string, "code") \ + V(config_string, "config") \ V(constants_string, "constants") \ V(crypto_dsa_string, "dsa") \ V(crypto_ec_string, "ec") \ @@ -312,7 +313,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2; V(write_host_object_string, "_writeHostObject") \ V(write_queue_size_string, "writeQueueSize") \ V(x_forwarded_string, "x-forwarded-for") \ - V(zero_return_string, "ZERO_RETURN") \ + V(zero_return_string, "ZERO_RETURN") #define ENVIRONMENT_STRONG_PERSISTENT_PROPERTIES(V) \ V(as_external, v8::External) \ diff --git a/src/node_binding.cc b/src/node_binding.cc index 85d7f98452c132..cc422a39e08aee 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -390,6 +390,13 @@ void GetInternalBinding(const FunctionCallbackInfo& args) { DefineConstants(env->isolate(), exports); } else if (!strcmp(*module_v, "natives")) { exports = per_process_loader.GetSourceObject(env->context()); + // Legacy feature: process.binding('natives').config contains stringified + // config.gypi + CHECK(exports + ->Set(env->context(), + env->config_string(), + per_process_loader.GetConfigString(env->isolate())) + .FromJust()); } else { return ThrowIfNoSuchModule(env, *module_v); } diff --git a/src/node_native_module.cc b/src/node_native_module.cc index 6e5f08e35451f0..85f8d83d63a162 100644 --- a/src/node_native_module.cc +++ b/src/node_native_module.cc @@ -9,6 +9,7 @@ using v8::Array; using v8::ArrayBuffer; using v8::ArrayBufferCreationMode; using v8::Context; +using v8::DEFAULT; using v8::EscapableHandleScope; using v8::Function; using v8::FunctionCallbackInfo; @@ -19,11 +20,15 @@ using v8::Isolate; using v8::Local; using v8::Maybe; using v8::MaybeLocal; +using v8::Name; +using v8::None; using v8::Object; +using v8::PropertyCallbackInfo; using v8::Script; using v8::ScriptCompiler; using v8::ScriptOrigin; using v8::Set; +using v8::SideEffectType; using v8::String; using v8::Uint8Array; using v8::Value; @@ -70,10 +75,16 @@ void NativeModuleLoader::GetCacheUsage( args.GetReturnValue().Set(result); } -void NativeModuleLoader::GetSourceObject( - const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - args.GetReturnValue().Set(per_process_loader.GetSourceObject(env->context())); +void NativeModuleLoader::SourceObjectGetter( + Local property, const PropertyCallbackInfo& info) { + Local context = info.GetIsolate()->GetCurrentContext(); + info.GetReturnValue().Set(per_process_loader.GetSourceObject(context)); +} + +void NativeModuleLoader::ConfigStringGetter( + Local property, const PropertyCallbackInfo& info) { + info.GetReturnValue().Set( + per_process_loader.GetConfigString(info.GetIsolate())); } Local NativeModuleLoader::GetSourceObject( @@ -81,6 +92,10 @@ Local NativeModuleLoader::GetSourceObject( return MapToObject(context, source_); } +Local NativeModuleLoader::GetConfigString(Isolate* isolate) const { + return config_.ToStringChecked(isolate); +} + Local NativeModuleLoader::GetSource(Isolate* isolate, const char* id) const { const auto it = source_.find(id); @@ -88,7 +103,7 @@ Local NativeModuleLoader::GetSource(Isolate* isolate, return it->second.ToStringChecked(isolate); } -NativeModuleLoader::NativeModuleLoader() { +NativeModuleLoader::NativeModuleLoader() : config_(GetConfig()) { LoadJavaScriptSource(); LoadJavaScriptHash(); LoadCodeCache(); @@ -321,8 +336,27 @@ void NativeModuleLoader::Initialize(Local target, void* priv) { Environment* env = Environment::GetCurrent(context); - env->SetMethod( - target, "getSource", NativeModuleLoader::GetSourceObject); + CHECK(target + ->SetAccessor(env->context(), + env->config_string(), + ConfigStringGetter, + nullptr, + MaybeLocal(), + DEFAULT, + None, + SideEffectType::kHasNoSideEffect) + .FromJust()); + CHECK(target + ->SetAccessor(env->context(), + env->source_string(), + SourceObjectGetter, + nullptr, + MaybeLocal(), + DEFAULT, + None, + SideEffectType::kHasNoSideEffect) + .FromJust()); + env->SetMethod( target, "getCacheUsage", NativeModuleLoader::GetCacheUsage); env->SetMethod( diff --git a/src/node_native_module.h b/src/node_native_module.h index fc0e33c73500d1..02c769ef023907 100644 --- a/src/node_native_module.h +++ b/src/node_native_module.h @@ -40,6 +40,9 @@ class NativeModuleLoader { v8::Local context, void* priv); v8::Local GetSourceObject(v8::Local context) const; + // Returns config.gypi as a JSON string + v8::Local GetConfigString(v8::Isolate* isolate) const; + v8::Local GetSource(v8::Isolate* isolate, const char* id) const; // Run a script with JS source bundled inside the binary as if it's wrapped @@ -56,9 +59,15 @@ class NativeModuleLoader { private: static void GetCacheUsage(const v8::FunctionCallbackInfo& args); - // For legacy process.binding('natives') which is mutable, and for - // internalBinding('native_module').source for internal use - static void GetSourceObject(const v8::FunctionCallbackInfo& args); + // Passing map of builtin module source code into JS land as + // internalBinding('native_module').source + static void SourceObjectGetter( + v8::Local property, + const v8::PropertyCallbackInfo& info); + // Passing config.gypi into JS land as internalBinding('native_module').config + static void ConfigStringGetter( + v8::Local property, + const v8::PropertyCallbackInfo& info); // Compile code cache for a specific native module static void CompileCodeCache(const v8::FunctionCallbackInfo& args); // Compile a specific native module as a function @@ -67,6 +76,7 @@ class NativeModuleLoader { // Generated by tools/js2c.py as node_javascript.cc void LoadJavaScriptSource(); // Loads data into source_ void LoadJavaScriptHash(); // Loads data into source_hash_ + UnionBytes GetConfig(); // Return data for config.gypi // Generated by tools/generate_code_cache.js as node_code_cache.cc when // the build is configured with --code-cache-path=.... They are noops @@ -94,6 +104,8 @@ class NativeModuleLoader { bool has_code_cache_ = false; NativeModuleRecordMap source_; NativeModuleRecordMap code_cache_; + UnionBytes config_; + NativeModuleHashMap source_hash_; NativeModuleHashMap code_cache_hash_; }; diff --git a/src/node_union_bytes.h b/src/node_union_bytes.h index 128bb11ed0d0ab..66d8509beaf188 100644 --- a/src/node_union_bytes.h +++ b/src/node_union_bytes.h @@ -55,21 +55,31 @@ class UnionBytes { : is_one_byte_(false), two_bytes_(data), length_(length) {} UnionBytes(const uint8_t* data, size_t length) : is_one_byte_(true), one_bytes_(data), length_(length) {} + + UnionBytes(const UnionBytes&) = default; + UnionBytes& operator=(const UnionBytes&) = default; + UnionBytes(UnionBytes&&) = default; + UnionBytes& operator=(UnionBytes&&) = default; + bool is_one_byte() const { return is_one_byte_; } const uint16_t* two_bytes_data() const { CHECK(!is_one_byte_); + CHECK_NE(two_bytes_, nullptr); return two_bytes_; } const uint8_t* one_bytes_data() const { CHECK(is_one_byte_); + CHECK_NE(one_bytes_, nullptr); return one_bytes_; } v8::Local ToStringChecked(v8::Isolate* isolate) const { if (is_one_byte_) { + CHECK_NE(one_bytes_, nullptr); NonOwningExternalOneByteResource* source = new NonOwningExternalOneByteResource(one_bytes_, length_); return v8::String::NewExternalOneByte(isolate, source).ToLocalChecked(); } else { + CHECK_NE(two_bytes_, nullptr); NonOwningExternalTwoByteResource* source = new NonOwningExternalTwoByteResource(two_bytes_, length_); return v8::String::NewExternalTwoByte(isolate, source).ToLocalChecked(); diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index eb5afea961ba5d..9816aa05466ff3 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -9,7 +9,7 @@ const common = require('../common'); const assert = require('assert'); const isMainThread = common.isMainThread; -const kMaxModuleCount = isMainThread ? 59 : 80; +const kMaxModuleCount = isMainThread ? 61 : 82; assert(list.length <= kMaxModuleCount, `Total length: ${list.length}\n` + list.join('\n') diff --git a/tools/js2c.py b/tools/js2c.py index e9a52047739adf..d103a3d2364980 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -193,6 +193,10 @@ def ReadMacros(lines): {hash_initializers} }} +UnionBytes NativeModuleLoader::GetConfig() {{ + return UnionBytes(config_raw, arraysize(config_raw)); // config.gypi +}} + }} // namespace native_module }} // namespace node @@ -248,21 +252,23 @@ def JS2C(source, target): definitions = [] initializers = [] hash_initializers = [] + config_initializers = [] - def AddModule(module, source): - var = '%s_raw' % (module.replace('-', '_').replace('/', '_')) - source_hash = hashlib.sha256(source).hexdigest() - + def GetDefinition(var, source): # Treat non-ASCII as UTF-8 and convert it to UTF-16. if any(ord(c) > 127 for c in source): source = map(ord, source.decode('utf-8').encode('utf-16be')) source = [source[i] * 256 + source[i+1] for i in xrange(0, len(source), 2)] source = ToCArray(source) - definition = TWO_BYTE_STRING.format(var=var, data=source) + return TWO_BYTE_STRING.format(var=var, data=source) else: source = ToCArray(map(ord, source), step=20) - definition = ONE_BYTE_STRING.format(var=var, data=source) + return ONE_BYTE_STRING.format(var=var, data=source) + def AddModule(module, source): + var = '%s_raw' % (module.replace('-', '_').replace('/', '_')) + source_hash = hashlib.sha256(source).hexdigest() + definition = GetDefinition(var, source) initializer = INITIALIZER.format(module=module, var=var) hash_initializer = HASH_INITIALIZER.format(module=module, @@ -292,11 +298,17 @@ def AddModule(module, source): # if its a gypi file we're going to want it as json # later on anyway, so get it out of the way now - if name.endswith(".gypi"): + if name.endswith('.gypi'): + # Currently only config.gypi is allowed + assert name == 'config.gypi' + lines = re.sub(r'\'true\'', 'true', lines) + lines = re.sub(r'\'false\'', 'false', lines) lines = re.sub(r'#.*?\n', '', lines) lines = re.sub(r'\'', '"', lines) - - AddModule(name.split('.', 1)[0], lines) + definition = GetDefinition('config_raw', lines) + definitions.append(definition) + else: + AddModule(name.split('.', 1)[0], lines) # Add deprecated aliases for deps without 'deps/' if deprecated_deps is not None: @@ -306,9 +318,11 @@ def AddModule(module, source): # Emit result output = open(str(target[0]), "w") - output.write(TEMPLATE.format(definitions=''.join(definitions), - initializers=''.join(initializers), - hash_initializers=''.join(hash_initializers))) + output.write( + TEMPLATE.format(definitions=''.join(definitions), + initializers=''.join(initializers), + hash_initializers=''.join(hash_initializers), + config_initializers=''.join(config_initializers))) output.close() def main():