From de6e17071f55b67556f3393d71bc25278b2dc5f0 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 22 Nov 2022 09:43:54 +0200 Subject: [PATCH 1/6] js: refactor how modules are loaded This refactor tries to simplify the implementation of `require` and connected code by splitting it heavily into different parts. These changes are similar to what will be needed for native ESM support as shown in https://github.com/grafana/k6/pull/2563 , but without any of the native parts and without doing anything that isn't currently needed. This will hopefully make ESM PR much smaller and less intrusive. This includes still keeping the wrong relativity of `require` as explained in https://github.com/grafana/k6/issues/2674. It also tries to simplify connected code, but due to this being very sensitive code and the changes already being quite big, this is done only to an extent. The lack of new tests is mostly due to there not being really any new code and the tests that were created along these changes already being merged months ago with https://github.com/grafana/k6/pull/2782. Future changes will try to address the above as well as potentially moving the whole module types and logic in separate package to be reused in tests. --- js/bundle.go | 293 +++++++++++++++++++++------------------- js/bundle_test.go | 2 +- js/cjsmodule.go | 70 ++++++++++ js/gomodule.go | 93 +++++++++++++ js/initcontext.go | 295 +++++++++-------------------------------- js/initcontext_test.go | 25 ++-- js/modules.go | 122 +++++++++++++++++ js/runner.go | 4 +- 8 files changed, 517 insertions(+), 387 deletions(-) create mode 100644 js/cjsmodule.go create mode 100644 js/gomodule.go create mode 100644 js/modules.go diff --git a/js/bundle.go b/js/bundle.go index e97a4c5fccb..68c953b558f 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "net/url" + "path/filepath" "runtime" "github.com/dop251/goja" @@ -26,16 +27,18 @@ import ( // You can use this to produce identical BundleInstance objects. type Bundle struct { Filename *url.URL - Source string - Program *goja.Program + Source []byte Options lib.Options - - BaseInitContext *InitContext + logger *logrus.Logger CompatibilityMode lib.CompatibilityMode // parsed value preInitState *lib.TestPreInitState - exports map[string]goja.Callable + filesystems map[string]afero.Fs + pwd *url.URL + + callableExports map[string]struct{} + modResolution *modulesResolution } // A BundleInstance is a self-contained instance of a Bundle. @@ -45,17 +48,18 @@ type BundleInstance struct { // TODO: maybe just have a reference to the Bundle? or save and pass rtOpts? env map[string]string - exports map[string]goja.Callable - moduleVUImpl *moduleVUImpl - pgm programWithSource + mainModuleInstance moduleInstance + moduleVUImpl *moduleVUImpl } func (bi *BundleInstance) getCallableExport(name string) goja.Callable { - return bi.exports[name] + fn, ok := goja.AssertFunction(bi.mainModuleInstance.exports().Get(name)) + _ = ok // TODO maybe return it + return fn } func (bi *BundleInstance) getExported(name string) goja.Value { - return bi.pgm.exports.Get(name) + return bi.mainModuleInstance.exports().ToObject(bi.Runtime).Get(name) } // NewBundle creates a new bundle from a source file and a filesystem. @@ -74,35 +78,34 @@ func newBundle( return nil, err } - // Compile sources, both ES5 and ES6 are supported. - code := string(src.Data) - c := compiler.New(piState.Logger) - c.Options = compiler.Options{ - CompatibilityMode: compatMode, - Strict: true, - SourceMapLoader: generateSourceMapLoader(piState.Logger, filesystems), - } - pgm, _, err := c.Compile(code, src.URL.String(), false) - if err != nil { - return nil, err - } // Make a bundle, instantiate it into a throwaway VM to populate caches. - rt := goja.New() bundle := Bundle{ Filename: src.URL, - Source: code, - Program: pgm, - BaseInitContext: NewInitContext(piState.Logger, rt, c, compatMode, filesystems, loader.Dir(src.URL)), + Source: src.Data, Options: options, CompatibilityMode: compatMode, - exports: make(map[string]goja.Callable), + callableExports: make(map[string]struct{}), + modResolution: newModuleResolution(getJSModules()), + filesystems: filesystems, + pwd: loader.Dir(src.URL), + logger: piState.Logger, preInitState: piState, } - if err = bundle.instantiate(bundle.BaseInitContext, 0); err != nil { + c := bundle.newCompiler(piState.Logger) + if err = bundle.modResolution.setMain(src, c); err != nil { + return nil, err + } + // Instantiate the bundle into a new VM using a bound init context. This uses a context with a + // runtime, but no state, to allow module-provided types to function within the init context. + // TODO use a real context + vuImpl := &moduleVUImpl{ctx: context.Background(), runtime: goja.New()} + vuImpl.eventLoop = eventloop.New(vuImpl) + instance, err := bundle.instantiate(vuImpl, 0, c) + if err != nil { return nil, err } - err = bundle.getExports(piState.Logger, rt, updateOptions) + err = bundle.populateExports(piState.Logger, updateOptions, instance) if err != nil { return nil, err } @@ -140,11 +143,11 @@ func NewBundleFromArchive(piState *lib.TestPreInitState, arc *lib.Archive) (*Bun func (b *Bundle) makeArchive() *lib.Archive { arc := &lib.Archive{ Type: "js", - Filesystems: b.BaseInitContext.filesystems, + Filesystems: b.filesystems, Options: b.Options, FilenameURL: b.Filename, - Data: []byte(b.Source), - PwdURL: b.BaseInitContext.pwd, + Data: b.Source, + PwdURL: b.pwd, Env: make(map[string]string, len(b.preInitState.RuntimeOptions.Env)), CompatibilityMode: b.CompatibilityMode.String(), K6Version: consts.Version, @@ -158,19 +161,16 @@ func (b *Bundle) makeArchive() *lib.Archive { return arc } -// getExports validates and extracts exported objects -func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateOptions bool) error { - pgm := b.BaseInitContext.programs[b.Filename.String()] // this is the main script and it's always present - exportsV := pgm.module.Get("exports") - if goja.IsNull(exportsV) || goja.IsUndefined(exportsV) { +// populateExports validates and extracts exported objects +func (b *Bundle) populateExports(logger logrus.FieldLogger, updateOptions bool, instance moduleInstance) error { + exports := instance.exports() + if exports == nil { return errors.New("exports must be an object") } - exports := exportsV.ToObject(rt) - for _, k := range exports.Keys() { v := exports.Get(k) - if fn, ok := goja.AssertFunction(v); ok && k != consts.Options { - b.exports[k] = fn + if _, ok := goja.AssertFunction(v); ok && k != consts.Options { + b.callableExports[k] = struct{}{} continue } switch k { @@ -197,7 +197,7 @@ func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateO } } - if len(b.exports) == 0 { + if len(b.callableExports) == 0 { return errors.New("no exported functions in script") } @@ -208,43 +208,34 @@ func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateO func (b *Bundle) Instantiate(ctx context.Context, vuID uint64) (*BundleInstance, error) { // Instantiate the bundle into a new VM using a bound init context. This uses a context with a // runtime, but no state, to allow module-provided types to function within the init context. - rt := goja.New() - vuImpl := &moduleVUImpl{ - ctx: ctx, - runtime: rt, - } - init := newBoundInitContext(b.BaseInitContext, vuImpl) - if err := b.instantiate(init, vuID); err != nil { + vuImpl := &moduleVUImpl{ctx: ctx, runtime: goja.New()} + vuImpl.eventLoop = eventloop.New(vuImpl) + instance, err := b.instantiate(vuImpl, vuID, b.newCompiler(b.logger)) + if err != nil { return nil, err } - pgm := init.programs[b.Filename.String()] // this is the main script and it's always present bi := &BundleInstance{ - Runtime: rt, - exports: make(map[string]goja.Callable), - env: b.preInitState.RuntimeOptions.Env, - moduleVUImpl: vuImpl, - pgm: pgm, + Runtime: vuImpl.runtime, + env: b.preInitState.RuntimeOptions.Env, + moduleVUImpl: vuImpl, + mainModuleInstance: instance, } // Grab any exported functions that could be executed. These were // already pre-validated in cmd.validateScenarioConfig(), just get them here. - exports := pgm.module.Get("exports").ToObject(rt) - for k := range b.exports { - fn, _ := goja.AssertFunction(exports.Get(k)) - bi.exports[k] = fn - } + exports := instance.exports() jsOptions := exports.Get("options") var jsOptionsObj *goja.Object if jsOptions == nil || goja.IsNull(jsOptions) || goja.IsUndefined(jsOptions) { - jsOptionsObj = rt.NewObject() + jsOptionsObj = vuImpl.runtime.NewObject() err := exports.Set("options", jsOptionsObj) if err != nil { return nil, fmt.Errorf("couldn't set exported options with merged values: %w", err) } } else { - jsOptionsObj = jsOptions.ToObject(rt) + jsOptionsObj = jsOptions.ToObject(vuImpl.runtime) } var instErr error @@ -257,137 +248,161 @@ func (b *Bundle) Instantiate(ctx context.Context, vuID uint64) (*BundleInstance, return bi, instErr } -// Instantiates the bundle into an existing runtime. Not public because it also messes with a bunch -// of other things, will potentially thrash data and makes a mess in it if the operation fails. - -func (b *Bundle) initializeProgramObject(rt *goja.Runtime, init *InitContext) programWithSource { - pgm := programWithSource{ - pgm: b.Program, - src: b.Source, - exports: rt.NewObject(), - module: rt.NewObject(), +func (b *Bundle) newCompiler(logger logrus.FieldLogger) *compiler.Compiler { + c := compiler.New(logger) + c.Options = compiler.Options{ + CompatibilityMode: b.CompatibilityMode, + Strict: true, + SourceMapLoader: generateSourceMapLoader(logger, b.filesystems), } - _ = pgm.module.Set("exports", pgm.exports) - init.programs[b.Filename.String()] = pgm - return pgm + return c } -//nolint:funlen -func (b *Bundle) instantiate(init *InitContext, vuID uint64) (err error) { - rt := init.moduleVUImpl.runtime - logger := init.logger - rt.SetFieldNameMapper(common.FieldNameMapper{}) - rt.SetRandSource(common.NewRandSource()) - - env := make(map[string]string, len(b.preInitState.RuntimeOptions.Env)) - for key, value := range b.preInitState.RuntimeOptions.Env { - env[key] = value - } - rt.Set("__ENV", env) - rt.Set("__VU", vuID) - _ = rt.Set("console", newConsole(logger)) - - if init.compatibilityMode == lib.CompatibilityModeExtended { - rt.Set("global", rt.GlobalObject()) +func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64, c *compiler.Compiler) (moduleInstance, error) { + rt := vuImpl.runtime + err := b.setupJSRuntime(rt, int64(vuID), b.logger) + if err != nil { + return nil, err } initenv := &common.InitEnvironment{ - Logger: logger, - FileSystems: init.filesystems, - CWD: init.pwd, + Logger: b.logger, + FileSystems: b.filesystems, + CWD: b.pwd, Registry: b.preInitState.Registry, LookupEnv: b.preInitState.LookupEnv, } - unbindInit := b.setInitGlobals(rt, init) - init.moduleVUImpl.initEnv = initenv - init.moduleVUImpl.eventLoop = eventloop.New(init.moduleVUImpl) - pgm := b.initializeProgramObject(rt, init) + + cjsLoad := generateCJSLoad(b, c) + modSys := newModuleSystem(b.modResolution, vuImpl, cjsLoad) + unbindInit := b.setInitGlobals(rt, modSys) + vuImpl.initEnv = initenv + defer func() { + unbindInit() + vuImpl.initEnv = nil + }() // TODO: make something cleaner for interrupting scripts, and more unified // (e.g. as a part of the event loop or RunWithPanicCatching()? - initCtxDone := init.moduleVUImpl.ctx.Done() initDone := make(chan struct{}) - watchDone := make(chan struct{}) go func() { select { - case <-initCtxDone: - rt.Interrupt(init.moduleVUImpl.ctx.Err()) - case <-initDone: // do nothing + case <-vuImpl.ctx.Done(): + rt.Interrupt(vuImpl.ctx.Err()) + case initDone <- struct{}{}: // do nothing } - close(watchDone) + close(initDone) }() - err = common.RunWithPanicCatching(logger, rt, func() error { - return init.moduleVUImpl.eventLoop.Start(func() error { - f, errRun := rt.RunProgram(b.Program) - if errRun != nil { - return errRun - } - if call, ok := goja.AssertFunction(f); ok { - if _, errRun = call(pgm.exports, pgm.module, pgm.exports); errRun != nil { - return errRun - } - return nil + var instance moduleInstance + err = common.RunWithPanicCatching(b.logger, rt, func() error { + return vuImpl.eventLoop.Start(func() error { + //nolint:shadow,govet // here we shadow err on purpose + mod, err := b.modResolution.resolve(b.pwd, b.Filename.String(), cjsLoad) + if err != nil { + return err // TODO wrap as this should never happen } - panic("Somehow a commonjs main module is not wrapped in a function") + instance = mod.Instantiate(vuImpl) + return instance.execute() }) }) - close(initDone) - <-watchDone + + <-initDone if err != nil { var exception *goja.Exception if errors.As(err, &exception) { err = &scriptException{inner: exception} } - return err + return nil, err } - exportsV := pgm.module.Get("exports") - if goja.IsNull(exportsV) { - return errors.New("exports must be an object") + if exports := instance.exports(); exports == nil { + return nil, errors.New("exports must be an object") } - pgm.exports = exportsV.ToObject(rt) - init.programs[b.Filename.String()] = pgm - unbindInit() - init.moduleVUImpl.ctx = nil - init.moduleVUImpl.initEnv = nil // If we've already initialized the original VU init context, forbid // any subsequent VUs to open new files if vuID == 0 { - init.allowOnlyOpenedFiles() + allowOnlyOpenedFiles(b.filesystems["file"]) } rt.SetRandSource(common.NewRandSource()) + return instance, nil +} + +func (b *Bundle) setupJSRuntime(rt *goja.Runtime, vuID int64, logger logrus.FieldLogger) error { + rt.SetFieldNameMapper(common.FieldNameMapper{}) + rt.SetRandSource(common.NewRandSource()) + + env := make(map[string]string, len(b.preInitState.RuntimeOptions.Env)) + for key, value := range b.preInitState.RuntimeOptions.Env { + env[key] = value + } + err := rt.Set("__ENV", env) + if err != nil { + return err + } + err = rt.Set("__VU", vuID) + if err != nil { + return err + } + err = rt.Set("console", newConsole(logger)) + if err != nil { + return err + } + + if b.CompatibilityMode == lib.CompatibilityModeExtended { + err = rt.Set("global", rt.GlobalObject()) + if err != nil { + return err + } + } return nil } -func (b *Bundle) setInitGlobals(rt *goja.Runtime, init *InitContext) (unset func()) { +func (b *Bundle) setInitGlobals(rt *goja.Runtime, modSys *moduleSystem) (unset func()) { mustSet := func(k string, v interface{}) { if err := rt.Set(k, v); err != nil { panic(fmt.Errorf("failed to set '%s' global object: %w", k, err)) } } - mustSet("require", init.Require) - mustSet("open", init.Open) + r := requireImpl{ + vu: modSys.vu, + modules: modSys, + pwd: b.pwd, + } + mustSet("require", r.require) + + mustSet("open", func(filename string, args ...string) (goja.Value, error) { + if modSys.vu.State() != nil { // fix + return nil, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "open") + } + + if filename == "" { + return nil, errors.New("open() can't be used with an empty filename") + } + // This uses the pwd from the requireImpl + return openImpl(rt, b.filesystems["file"], r.pwd, filename, args...) + }) return func() { mustSet("require", goja.Undefined()) mustSet("open", goja.Undefined()) } } -func generateSourceMapLoader(logger logrus.FieldLogger, filesystems map[string]afero.Fs, -) func(path string) ([]byte, error) { - return func(path string) ([]byte, error) { - u, err := url.Parse(path) - if err != nil { - return nil, err +func generateCJSLoad(b *Bundle, c *compiler.Compiler) cjsModuleLoader { + return func(specifier *url.URL, name string) (*cjsModule, error) { + if filepath.IsAbs(name) && runtime.GOOS == "windows" { + b.logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and won't work if"+ + " you move the script between machines or run it with `k6 cloud`; if absolute paths are required,"+ + " import them with the `file://` schema for slightly better compatibility", + name) } - data, err := loader.Load(logger, filesystems, u, path) + d, err := loader.Load(b.logger, b.filesystems, specifier, name) if err != nil { return nil, err } - return data.Data, nil + return cjsmoduleFromString(specifier, d.Data, c) } } diff --git a/js/bundle_test.go b/js/bundle_test.go index ac3924f5b90..14b60688ba3 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -124,7 +124,7 @@ func TestNewBundle(t *testing.T) { b, err := getSimpleBundle(t, "-", `export default function() {};`) require.NoError(t, err) assert.Equal(t, "file://-", b.Filename.String()) - assert.Equal(t, "file:///", b.BaseInitContext.pwd.String()) + assert.Equal(t, "file:///", b.pwd.String()) }) t.Run("CompatibilityMode", func(t *testing.T) { t.Parallel() diff --git a/js/cjsmodule.go b/js/cjsmodule.go new file mode 100644 index 00000000000..21d6d7736b4 --- /dev/null +++ b/js/cjsmodule.go @@ -0,0 +1,70 @@ +package js + +import ( + "fmt" + "net/url" + + "github.com/dop251/goja" + "go.k6.io/k6/js/compiler" + "go.k6.io/k6/js/modules" +) + +// cjsModule represents a commonJS module +type cjsModule struct { + prg *goja.Program + url *url.URL +} + +var _ module = &cjsModule{} + +type cjsModuleInstance struct { + mod *cjsModule + moduleObj *goja.Object + vu modules.VU +} + +func (c *cjsModule) Instantiate(vu modules.VU) moduleInstance { + return &cjsModuleInstance{vu: vu, mod: c} +} + +func (c *cjsModuleInstance) execute() error { + rt := c.vu.Runtime() + exports := rt.NewObject() + c.moduleObj = rt.NewObject() + err := c.moduleObj.Set("exports", exports) + if err != nil { + return fmt.Errorf("error while getting ready to import commonJS, couldn't set exports property of module: %w", + err) + } + + // Run the program. + f, err := rt.RunProgram(c.mod.prg) + if err != nil { + return err + } + if call, ok := goja.AssertFunction(f); ok { + if _, err = call(exports, c.moduleObj, exports); err != nil { + return err + } + } + + return nil +} + +func (c *cjsModuleInstance) exports() *goja.Object { + exportsV := c.moduleObj.Get("exports") + if goja.IsNull(exportsV) || goja.IsUndefined(exportsV) { + return nil + } + return exportsV.ToObject(c.vu.Runtime()) +} + +type cjsModuleLoader func(specifier *url.URL, name string) (*cjsModule, error) + +func cjsmoduleFromString(fileURL *url.URL, data []byte, c *compiler.Compiler) (*cjsModule, error) { + pgm, _, err := c.Compile(string(data), fileURL.String(), false) + if err != nil { + return nil, err + } + return &cjsModule{prg: pgm, url: fileURL}, nil +} diff --git a/js/gomodule.go b/js/gomodule.go new file mode 100644 index 00000000000..1abe4742acc --- /dev/null +++ b/js/gomodule.go @@ -0,0 +1,93 @@ +package js + +import ( + "github.com/dop251/goja" + "go.k6.io/k6/js/modules" +) + +// baseGoModule is a go module that does not implement modules.Module interface +// TODO maybe depracate those in the future +type baseGoModule struct { + mod interface{} +} + +var _ module = &baseGoModule{} + +func (b *baseGoModule) Instantiate(vu modules.VU) moduleInstance { + return &baseGoModuleInstance{mod: b.mod, vu: vu} +} + +type baseGoModuleInstance struct { + mod interface{} + vu modules.VU + exportsO *goja.Object // this is so we only initialize the exports once per instance +} + +func (b *baseGoModuleInstance) execute() error { + return nil +} + +func (b *baseGoModuleInstance) exports() *goja.Object { + if b.exportsO == nil { + // TODO check this does not panic a lot + rt := b.vu.Runtime() + b.exportsO = rt.ToValue(b.mod).ToObject(rt) + } + return b.exportsO +} + +// goModule is a go module which implements modules.Module +type goModule struct { + modules.Module +} + +var _ module = &goModule{} + +func (g *goModule) Instantiate(vu modules.VU) moduleInstance { + return &goModuleInstance{vu: vu, module: g} +} + +type goModuleInstance struct { + modules.Instance + module *goModule + vu modules.VU + exportsO *goja.Object // this is so we only initialize the exports once per instance +} + +var _ moduleInstance = &goModuleInstance{} + +func (gi *goModuleInstance) execute() error { + gi.Instance = gi.module.NewModuleInstance(gi.vu) + return nil +} + +func (gi *goModuleInstance) exports() *goja.Object { + if gi.exportsO == nil { + rt := gi.vu.Runtime() + gi.exportsO = rt.ToValue(toESModuleExports(gi.Instance.Exports())).ToObject(rt) + } + return gi.exportsO +} + +func toESModuleExports(exp modules.Exports) interface{} { + if exp.Named == nil { + return exp.Default + } + if exp.Default == nil { + return exp.Named + } + + result := make(map[string]interface{}, len(exp.Named)+2) + + for k, v := range exp.Named { + result[k] = v + } + // Maybe check that those weren't set + result["default"] = exp.Default + // this so babel works with the `default` when it transpiles from ESM to commonjs. + // This should probably be removed once we have support for ESM directly. So that require doesn't get support for + // that while ESM has. + result["__esModule"] = true + + return result +} diff --git a/js/initcontext.go b/js/initcontext.go index 68792c67083..ca00a7fd9ae 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -1,255 +1,37 @@ package js import ( - "context" "errors" "fmt" "net/url" "path/filepath" - "runtime" "strings" "github.com/dop251/goja" "github.com/sirupsen/logrus" "github.com/spf13/afero" - "go.k6.io/k6/js/common" - "go.k6.io/k6/js/compiler" "go.k6.io/k6/js/modules" - "go.k6.io/k6/lib" "go.k6.io/k6/lib/fsext" "go.k6.io/k6/loader" ) -type programWithSource struct { - pgm *goja.Program - src string - module *goja.Object - exports *goja.Object -} - -const openCantBeUsedOutsideInitContextMsg = `The "open()" function is only available in the init stage ` + +const cantBeUsedOutsideInitContextMsg = `the "%s" function is only available in the init stage ` + `(i.e. the global scope), see https://k6.io/docs/using-k6/test-life-cycle for more information` -// InitContext provides APIs for use in the init context. -// -// TODO: refactor most/all of this state away, use common.InitEnvironment instead -type InitContext struct { - // Bound runtime; used to instantiate objects. - compiler *compiler.Compiler - - moduleVUImpl *moduleVUImpl - - // Filesystem to load files and scripts from with the map key being the scheme - filesystems map[string]afero.Fs - pwd *url.URL - - // Cache of loaded programs and files. - programs map[string]programWithSource - // merge this and the above - k6ModulesCache map[string]goja.Value - - compatibilityMode lib.CompatibilityMode - - logger logrus.FieldLogger - - moduleRegistry map[string]interface{} -} - -// NewInitContext creates a new initcontext with the provided arguments -func NewInitContext( - logger logrus.FieldLogger, rt *goja.Runtime, c *compiler.Compiler, compatMode lib.CompatibilityMode, - filesystems map[string]afero.Fs, pwd *url.URL, -) *InitContext { - return &InitContext{ - compiler: c, - filesystems: filesystems, - pwd: pwd, - programs: make(map[string]programWithSource), - compatibilityMode: compatMode, - logger: logger, - moduleRegistry: getJSModules(), - k6ModulesCache: make(map[string]goja.Value), - moduleVUImpl: &moduleVUImpl{ - // TODO: pass a real context as we did for https://github.com/grafana/k6/pull/2800, - // also see https://github.com/grafana/k6/issues/2804 - ctx: context.Background(), - runtime: rt, - }, - } -} - -func newBoundInitContext(base *InitContext, vuImpl *moduleVUImpl) *InitContext { - // we don't copy the exports as otherwise they will be shared and we don't want this. - // this means that all the files will be executed again but once again only once per compilation - // of the main file. - programs := make(map[string]programWithSource, len(base.programs)) - for key, program := range base.programs { - programs[key] = programWithSource{ - src: program.src, - pgm: program.pgm, - } - } - return &InitContext{ - filesystems: base.filesystems, - pwd: base.pwd, - compiler: base.compiler, - - programs: programs, - compatibilityMode: base.compatibilityMode, - k6ModulesCache: make(map[string]goja.Value), - logger: base.logger, - moduleRegistry: base.moduleRegistry, - moduleVUImpl: vuImpl, - } -} - -// Require is called when a module/file needs to be loaded by a script -func (i *InitContext) Require(arg string) (export goja.Value) { - switch { - case arg == "k6", strings.HasPrefix(arg, "k6/"): - var ok bool - if export, ok = i.k6ModulesCache[arg]; ok { - return export - } - defer func() { i.k6ModulesCache[arg] = export }() - // Builtin or external modules ("k6", "k6/*", or "k6/x/*") are handled - // specially, as they don't exist on the filesystem. This intentionally - // shadows attempts to name your own modules this. - v, err := i.requireModule(arg) - if err != nil { - common.Throw(i.moduleVUImpl.runtime, err) - } - return v - default: - // Fall back to loading from the filesystem. - v, err := i.requireFile(arg) - if err != nil { - common.Throw(i.moduleVUImpl.runtime, err) - } - return v - } -} - -func toESModuleExports(exp modules.Exports) interface{} { - if exp.Named == nil { - return exp.Default - } - if exp.Default == nil { - return exp.Named - } - - result := make(map[string]interface{}, len(exp.Named)+2) - - for k, v := range exp.Named { - result[k] = v - } - // Maybe check that those weren't set - result["default"] = exp.Default - // this so babel works with the `default` when it transpiles from ESM to commonjs. - // This should probably be removed once we have support for ESM directly. So that require doesn't get support for - // that while ESM has. - result["__esModule"] = true - - return result -} - -func (i *InitContext) requireModule(name string) (goja.Value, error) { - mod, ok := i.moduleRegistry[name] - if !ok { - return nil, fmt.Errorf("unknown module: %s", name) - } - if m, ok := mod.(modules.Module); ok { - instance := m.NewModuleInstance(i.moduleVUImpl) - return i.moduleVUImpl.runtime.ToValue(toESModuleExports(instance.Exports())), nil - } - - return i.moduleVUImpl.runtime.ToValue(mod), nil -} - -func (i *InitContext) requireFile(name string) (goja.Value, error) { - // Resolve the file path, push the target directory as pwd to make relative imports work. - pwd := i.pwd - fileURL, err := loader.Resolve(pwd, name) - if err != nil { - return nil, err - } - - // First, check if we have a cached program already. - pgm, ok := i.programs[fileURL.String()] - if !ok || pgm.module == nil { - if filepath.IsAbs(name) && runtime.GOOS == "windows" { - i.logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and won't work if"+ - " you move the script between machines or run it with `k6 cloud`; if absolute paths are required,"+ - " import them with the `file://` schema for slightly better compatibility", - name) - } - i.pwd = loader.Dir(fileURL) - defer func() { i.pwd = pwd }() - exports := i.moduleVUImpl.runtime.NewObject() - pgm.module = i.moduleVUImpl.runtime.NewObject() - _ = pgm.module.Set("exports", exports) - - if pgm.pgm == nil { - // Load the sources; the loader takes care of remote loading, etc. - data, err := loader.Load(i.logger, i.filesystems, fileURL, name) - if err != nil { - return goja.Undefined(), err - } - - pgm.src = string(data.Data) - - // Compile the sources; this handles ES5 vs ES6 automatically. - pgm.pgm, err = i.compileImport(pgm.src, data.URL.String()) - if err != nil { - return goja.Undefined(), err - } - } - - i.programs[fileURL.String()] = pgm - - // Run the program. - f, err := i.moduleVUImpl.runtime.RunProgram(pgm.pgm) - if err != nil { - delete(i.programs, fileURL.String()) - return goja.Undefined(), err - } - if call, ok := goja.AssertFunction(f); ok { - if _, err = call(exports, pgm.module, exports); err != nil { - return nil, err - } - } - } - - return pgm.module.Get("exports"), nil -} - -func (i *InitContext) compileImport(src, filename string) (*goja.Program, error) { - pgm, _, err := i.compiler.Compile(src, filename, false) - return pgm, err -} - -// Open implements open() in the init context and will read and return the +// openImpl implements openImpl() in the init context and will read and return the // contents of a file. If the second argument is "b" it returns an ArrayBuffer // instance, otherwise a string representation. -func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) { - if i.moduleVUImpl.State() != nil { - return nil, errors.New(openCantBeUsedOutsideInitContextMsg) - } - - if filename == "" { - return nil, errors.New("open() can't be used with an empty filename") - } - +func openImpl(rt *goja.Runtime, fs afero.Fs, basePWD *url.URL, filename string, args ...string) (goja.Value, error) { // Here IsAbs should be enough but unfortunately it doesn't handle absolute paths starting from // the current drive on windows like `\users\noname\...`. Also it makes it more easy to test and // will probably be need for archive execution under windows if always consider '/...' as an // absolute path. if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) { - filename = filepath.Join(i.pwd.Path, filename) + filename = filepath.Join(basePWD.Path, filename) } filename = filepath.Clean(filename) - fs := i.filesystems["file"] + if filename[0:1] != afero.FilePathSeparator { filename = afero.FilePathSeparator + filename } @@ -260,10 +42,10 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) } if len(args) > 0 && args[0] == "b" { - ab := i.moduleVUImpl.runtime.NewArrayBuffer(data) - return i.moduleVUImpl.runtime.ToValue(&ab), nil + ab := rt.NewArrayBuffer(data) + return rt.ToValue(&ab), nil } - return i.moduleVUImpl.runtime.ToValue(string(data)), nil + return rt.ToValue(string(data)), nil } func readFile(fileSystem afero.Fs, filename string) (data []byte, err error) { @@ -289,9 +71,7 @@ func readFile(fileSystem afero.Fs, filename string) (data []byte, err error) { } // allowOnlyOpenedFiles enables seen only files -func (i *InitContext) allowOnlyOpenedFiles() { - fs := i.filesystems["file"] - +func allowOnlyOpenedFiles(fs afero.Fs) { alreadyOpenedFS, ok := fs.(fsext.OnlyCachedEnabler) if !ok { return @@ -299,3 +79,60 @@ func (i *InitContext) allowOnlyOpenedFiles() { alreadyOpenedFS.AllowOnlyCached() } + +type requireImpl struct { + vu modules.VU + modules *moduleSystem + pwd *url.URL +} + +func (r *requireImpl) require(specifier string) (*goja.Object, error) { + // TODO remove this in the future when we address https://github.com/grafana/k6/issues/2674 + // This is currently needed as each time require is called we need to record it's new pwd + // to be used if a require *or* open is used within the file as they are relative to the + // latest call to require. + // This is *not* the actual require behaviour defined in commonJS as it is actually always relative + // to the file it is in. This is unlikely to be an issue but this code is here to keep backwards + // compatibility *for now*. + // With native ESM this won't even be possible as `require` might not be called - instead an import + // might be used in which case we won't be able to be doing this hack. In that case we either will + // need some goja specific helper or to use stack traces as goja_nodejs does. + currentPWD := r.pwd + if specifier != "k6" && !strings.HasPrefix(specifier, "k6/") { + defer func() { + r.pwd = currentPWD + }() + // In theory we can give that downwards, but this makes the code more tightly coupled + // plus as explained above this will be removed in the future so the code reflects more + // closely what will be needed then + fileURL, err := loader.Resolve(r.pwd, specifier) + if err != nil { + return nil, err + } + r.pwd = loader.Dir(fileURL) + } + + if r.vu.State() != nil { // fix + return nil, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "require") + } + if specifier == "" { + return nil, errors.New("require() can't be used with an empty specifier") + } + + return r.modules.Require(currentPWD, specifier) +} + +func generateSourceMapLoader(logger logrus.FieldLogger, filesystems map[string]afero.Fs, +) func(path string) ([]byte, error) { + return func(path string) ([]byte, error) { + u, err := url.Parse(path) + if err != nil { + return nil, err + } + data, err := loader.Load(logger, filesystems, u, path) + if err != nil { + return nil, err + } + return data.Data, nil + } +} diff --git a/js/initcontext_test.go b/js/initcontext_test.go index 688b2123835..9044ca1df43 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -25,7 +25,7 @@ import ( "go.k6.io/k6/metrics" ) -func TestInitContextRequire(t *testing.T) { +func TestRequire(t *testing.T) { t.Parallel() t.Run("Modules", func(t *testing.T) { t.Run("Nonexistent", func(t *testing.T) { @@ -48,13 +48,11 @@ func TestInitContextRequire(t *testing.T) { bi, err := b.Instantiate(context.Background(), 0) assert.NoError(t, err, "instance error") - exports := bi.pgm.exports - require.NotNil(t, exports) - _, defaultOk := goja.AssertFunction(exports.Get("default")) + _, defaultOk := goja.AssertFunction(bi.getExported("default")) assert.True(t, defaultOk, "default export is not a function") - assert.Equal(t, "abc123", exports.Get("dummy").String()) + assert.Equal(t, "abc123", bi.getExported("dummy").String()) - k6 := exports.Get("_k6").ToObject(bi.Runtime) + k6 := bi.getExported("_k6").ToObject(bi.Runtime) require.NotNil(t, k6) _, groupOk := goja.AssertFunction(k6.Get("group")) assert.True(t, groupOk, "k6.group is not a function") @@ -73,13 +71,11 @@ func TestInitContextRequire(t *testing.T) { bi, err := b.Instantiate(context.Background(), 0) require.NoError(t, err) - exports := bi.pgm.exports - require.NotNil(t, exports) - _, defaultOk := goja.AssertFunction(exports.Get("default")) + _, defaultOk := goja.AssertFunction(bi.getExported("default")) assert.True(t, defaultOk, "default export is not a function") - assert.Equal(t, "abc123", exports.Get("dummy").String()) + assert.Equal(t, "abc123", bi.getExported("dummy").String()) - _, groupOk := goja.AssertFunction(exports.Get("_group")) + _, groupOk := goja.AssertFunction(bi.getExported("_group")) assert.True(t, groupOk, "{ group } is not a function") }) }) @@ -107,7 +103,7 @@ func TestInitContextRequire(t *testing.T) { require.NoError(t, afero.WriteFile(fs, "/file.js", []byte(`throw new Error("aaaa")`), 0o755)) _, err := getSimpleBundle(t, "/script.js", `import "/file.js"; export default function() {}`, fs) assert.EqualError(t, err, - "Error: aaaa\n\tat file:///file.js:2:7(3)\n\tat go.k6.io/k6/js.(*InitContext).Require-fm (native)\n\tat file:///script.js:1:0(15)\n") + "Error: aaaa\n\tat file:///file.js:2:7(3)\n\tat go.k6.io/k6/js.(*requireImpl).require-fm (native)\n\tat file:///script.js:1:0(15)\n") }) imports := map[string]struct { @@ -175,9 +171,6 @@ func TestInitContextRequire(t *testing.T) { libName) b, err := getSimpleBundle(t, "/path/to/script.js", data, fs) require.NoError(t, err) - if constPath != "" { - assert.Contains(t, b.BaseInitContext.programs, "file://"+constPath) - } _, err = b.Instantiate(context.Background(), 0) require.NoError(t, err) @@ -538,7 +531,7 @@ func TestRequestWithMultipleBinaryFiles(t *testing.T) { <-ch } -func TestInitContextVU(t *testing.T) { +func Test__VU(t *testing.T) { t.Parallel() b, err := getSimpleBundle(t, "/script.js", ` let vu = __VU; diff --git a/js/modules.go b/js/modules.go new file mode 100644 index 00000000000..06e2c954e47 --- /dev/null +++ b/js/modules.go @@ -0,0 +1,122 @@ +package js + +import ( + "fmt" + "net/url" + "strings" + + "github.com/dop251/goja" + "go.k6.io/k6/js/compiler" + "go.k6.io/k6/js/modules" + "go.k6.io/k6/loader" +) + +type module interface { + Instantiate(vu modules.VU) moduleInstance +} + +type moduleInstance interface { + execute() error + exports() *goja.Object +} +type moduleCacheElement struct { + mod module + err error +} + +type modulesResolution struct { + cache map[string]moduleCacheElement + goModules map[string]interface{} +} + +func newModuleResolution(goModules map[string]interface{}) *modulesResolution { + return &modulesResolution{goModules: goModules, cache: make(map[string]moduleCacheElement)} +} + +func (mr *modulesResolution) setMain(main *loader.SourceData, c *compiler.Compiler) error { + mod, err := cjsmoduleFromString(main.URL, main.Data, c) + mr.cache[main.URL.String()] = moduleCacheElement{mod: mod, err: err} + return err +} + +func (mr *modulesResolution) resolveSpecifier(basePWD *url.URL, arg string) (*url.URL, error) { + specifier, err := loader.Resolve(basePWD, arg) + if err != nil { + return nil, err + } + return specifier, nil +} + +func (mr *modulesResolution) requireModule(name string) (module, error) { + mod, ok := mr.goModules[name] + if !ok { + return nil, fmt.Errorf("unknown module: %s", name) + } + if m, ok := mod.(modules.Module); ok { + return &goModule{Module: m}, nil + } + + return &baseGoModule{mod: mod}, nil +} + +func (mr *modulesResolution) resolve(basePWD *url.URL, arg string, loadCJS cjsModuleLoader) (module, error) { + if cached, ok := mr.cache[arg]; ok { + return cached.mod, cached.err + } + switch { + case arg == "k6", strings.HasPrefix(arg, "k6/"): + // Builtin or external modules ("k6", "k6/*", or "k6/x/*") are handled + // specially, as they don't exist on the filesystem. + mod, err := mr.requireModule(arg) + mr.cache[arg] = moduleCacheElement{mod: mod, err: err} + return mod, err + default: + specifier, err := mr.resolveSpecifier(basePWD, arg) + if err != nil { + return nil, err + } + // try cache with the final specifier + if cached, ok := mr.cache[specifier.String()]; ok { + return cached.mod, cached.err + } + // Fall back to loading from the filesystem. + mod, err := loadCJS(specifier, arg) + mr.cache[specifier.String()] = moduleCacheElement{mod: mod, err: err} + return mod, err + } +} + +type moduleSystem struct { + vu modules.VU + instanceCache map[module]moduleInstance + resolution *modulesResolution + cjsLoad cjsModuleLoader +} + +func newModuleSystem(resolution *modulesResolution, vu modules.VU, cjsLoad cjsModuleLoader) *moduleSystem { + return &moduleSystem{ + resolution: resolution, + instanceCache: make(map[module]moduleInstance), + vu: vu, + cjsLoad: cjsLoad, + } +} + +// Require is called when a module/file needs to be loaded by a script +func (ms *moduleSystem) Require(pwd *url.URL, arg string) (*goja.Object, error) { + mod, err := ms.resolution.resolve(pwd, arg, ms.cjsLoad) + if err != nil { + return nil, err + } + if instance, ok := ms.instanceCache[mod]; ok { + return instance.exports(), nil + } + + instance := mod.Instantiate(ms.vu) + ms.instanceCache[mod] = instance + if err = instance.execute(); err != nil { + return nil, err + } + + return instance.exports(), nil +} diff --git a/js/runner.go b/js/runner.go index 056bba77485..3569f8f5090 100644 --- a/js/runner.go +++ b/js/runner.go @@ -252,7 +252,7 @@ func (r *Runner) newVU( // instead of "Value is not an object: undefined ..." _ = vu.Runtime.GlobalObject().Set("open", func() { - common.Throw(vu.Runtime, errors.New(openCantBeUsedOutsideInitContextMsg)) + common.Throw(vu.Runtime, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "open")) }) return vu, nil @@ -355,7 +355,7 @@ func (r *Runner) GetOptions() lib.Options { // IsExecutable returns whether the given name is an exported and // executable function in the script. func (r *Runner) IsExecutable(name string) bool { - _, exists := r.Bundle.exports[name] + _, exists := r.Bundle.callableExports[name] return exists } From d8fc49bed400bfabed14307e83dc169f42fc9e45 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 9 Feb 2023 10:38:11 +0200 Subject: [PATCH 2/6] fixup! js: refactor how modules are loaded --- js/bundle.go | 10 +++++----- js/modules.go | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/js/bundle.go b/js/bundle.go index 68c953b558f..9a9f2104099 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -38,7 +38,7 @@ type Bundle struct { pwd *url.URL callableExports map[string]struct{} - modResolution *modulesResolution + moduleResolver *moduleResolver } // A BundleInstance is a self-contained instance of a Bundle. @@ -85,14 +85,14 @@ func newBundle( Options: options, CompatibilityMode: compatMode, callableExports: make(map[string]struct{}), - modResolution: newModuleResolution(getJSModules()), + moduleResolver: newModuleResolution(getJSModules()), filesystems: filesystems, pwd: loader.Dir(src.URL), logger: piState.Logger, preInitState: piState, } c := bundle.newCompiler(piState.Logger) - if err = bundle.modResolution.setMain(src, c); err != nil { + if err = bundle.moduleResolver.setMain(src, c); err != nil { return nil, err } // Instantiate the bundle into a new VM using a bound init context. This uses a context with a @@ -274,7 +274,7 @@ func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64, c *compiler.Comp } cjsLoad := generateCJSLoad(b, c) - modSys := newModuleSystem(b.modResolution, vuImpl, cjsLoad) + modSys := newModuleSystem(b.moduleResolver, vuImpl, cjsLoad) unbindInit := b.setInitGlobals(rt, modSys) vuImpl.initEnv = initenv defer func() { @@ -298,7 +298,7 @@ func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64, c *compiler.Comp err = common.RunWithPanicCatching(b.logger, rt, func() error { return vuImpl.eventLoop.Start(func() error { //nolint:shadow,govet // here we shadow err on purpose - mod, err := b.modResolution.resolve(b.pwd, b.Filename.String(), cjsLoad) + mod, err := b.moduleResolver.resolve(b.pwd, b.Filename.String(), cjsLoad) if err != nil { return err // TODO wrap as this should never happen } diff --git a/js/modules.go b/js/modules.go index 06e2c954e47..d694125ed39 100644 --- a/js/modules.go +++ b/js/modules.go @@ -24,22 +24,22 @@ type moduleCacheElement struct { err error } -type modulesResolution struct { +type moduleResolver struct { cache map[string]moduleCacheElement goModules map[string]interface{} } -func newModuleResolution(goModules map[string]interface{}) *modulesResolution { - return &modulesResolution{goModules: goModules, cache: make(map[string]moduleCacheElement)} +func newModuleResolution(goModules map[string]interface{}) *moduleResolver { + return &moduleResolver{goModules: goModules, cache: make(map[string]moduleCacheElement)} } -func (mr *modulesResolution) setMain(main *loader.SourceData, c *compiler.Compiler) error { +func (mr *moduleResolver) setMain(main *loader.SourceData, c *compiler.Compiler) error { mod, err := cjsmoduleFromString(main.URL, main.Data, c) mr.cache[main.URL.String()] = moduleCacheElement{mod: mod, err: err} return err } -func (mr *modulesResolution) resolveSpecifier(basePWD *url.URL, arg string) (*url.URL, error) { +func (mr *moduleResolver) resolveSpecifier(basePWD *url.URL, arg string) (*url.URL, error) { specifier, err := loader.Resolve(basePWD, arg) if err != nil { return nil, err @@ -47,7 +47,7 @@ func (mr *modulesResolution) resolveSpecifier(basePWD *url.URL, arg string) (*ur return specifier, nil } -func (mr *modulesResolution) requireModule(name string) (module, error) { +func (mr *moduleResolver) requireModule(name string) (module, error) { mod, ok := mr.goModules[name] if !ok { return nil, fmt.Errorf("unknown module: %s", name) @@ -59,7 +59,7 @@ func (mr *modulesResolution) requireModule(name string) (module, error) { return &baseGoModule{mod: mod}, nil } -func (mr *modulesResolution) resolve(basePWD *url.URL, arg string, loadCJS cjsModuleLoader) (module, error) { +func (mr *moduleResolver) resolve(basePWD *url.URL, arg string, loadCJS cjsModuleLoader) (module, error) { if cached, ok := mr.cache[arg]; ok { return cached.mod, cached.err } @@ -89,13 +89,13 @@ func (mr *modulesResolution) resolve(basePWD *url.URL, arg string, loadCJS cjsMo type moduleSystem struct { vu modules.VU instanceCache map[module]moduleInstance - resolution *modulesResolution + resolver *moduleResolver cjsLoad cjsModuleLoader } -func newModuleSystem(resolution *modulesResolution, vu modules.VU, cjsLoad cjsModuleLoader) *moduleSystem { +func newModuleSystem(resolution *moduleResolver, vu modules.VU, cjsLoad cjsModuleLoader) *moduleSystem { return &moduleSystem{ - resolution: resolution, + resolver: resolution, instanceCache: make(map[module]moduleInstance), vu: vu, cjsLoad: cjsLoad, @@ -104,7 +104,7 @@ func newModuleSystem(resolution *modulesResolution, vu modules.VU, cjsLoad cjsMo // Require is called when a module/file needs to be loaded by a script func (ms *moduleSystem) Require(pwd *url.URL, arg string) (*goja.Object, error) { - mod, err := ms.resolution.resolve(pwd, arg, ms.cjsLoad) + mod, err := ms.resolver.resolve(pwd, arg, ms.cjsLoad) if err != nil { return nil, err } From f0205307e0135b48e84bcb08b497776c3fbd738a Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 28 Feb 2023 16:53:04 +0200 Subject: [PATCH 3/6] fixup! js: refactor how modules are loaded --- js/bundle.go | 12 ++++++------ js/modules.go | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/js/bundle.go b/js/bundle.go index 9a9f2104099..c14a69dcc03 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -79,19 +79,20 @@ func newBundle( } // Make a bundle, instantiate it into a throwaway VM to populate caches. - bundle := Bundle{ + bundle := &Bundle{ Filename: src.URL, Source: src.Data, Options: options, CompatibilityMode: compatMode, callableExports: make(map[string]struct{}), - moduleResolver: newModuleResolution(getJSModules()), filesystems: filesystems, pwd: loader.Dir(src.URL), logger: piState.Logger, preInitState: piState, } c := bundle.newCompiler(piState.Logger) + bundle.moduleResolver = newModuleResolution(getJSModules(), generateCJSLoad(bundle, c)) + if err = bundle.moduleResolver.setMain(src, c); err != nil { return nil, err } @@ -110,7 +111,7 @@ func newBundle( return nil, err } - return &bundle, nil + return bundle, nil } // NewBundleFromArchive creates a new bundle from an lib.Archive. @@ -273,8 +274,7 @@ func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64, c *compiler.Comp LookupEnv: b.preInitState.LookupEnv, } - cjsLoad := generateCJSLoad(b, c) - modSys := newModuleSystem(b.moduleResolver, vuImpl, cjsLoad) + modSys := newModuleSystem(b.moduleResolver, vuImpl) unbindInit := b.setInitGlobals(rt, modSys) vuImpl.initEnv = initenv defer func() { @@ -298,7 +298,7 @@ func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64, c *compiler.Comp err = common.RunWithPanicCatching(b.logger, rt, func() error { return vuImpl.eventLoop.Start(func() error { //nolint:shadow,govet // here we shadow err on purpose - mod, err := b.moduleResolver.resolve(b.pwd, b.Filename.String(), cjsLoad) + mod, err := b.moduleResolver.resolve(b.pwd, b.Filename.String()) if err != nil { return err // TODO wrap as this should never happen } diff --git a/js/modules.go b/js/modules.go index d694125ed39..7a32c3d6dcb 100644 --- a/js/modules.go +++ b/js/modules.go @@ -27,10 +27,11 @@ type moduleCacheElement struct { type moduleResolver struct { cache map[string]moduleCacheElement goModules map[string]interface{} + loadCJS cjsModuleLoader } -func newModuleResolution(goModules map[string]interface{}) *moduleResolver { - return &moduleResolver{goModules: goModules, cache: make(map[string]moduleCacheElement)} +func newModuleResolution(goModules map[string]interface{}, loadCJS cjsModuleLoader) *moduleResolver { + return &moduleResolver{goModules: goModules, cache: make(map[string]moduleCacheElement), loadCJS: loadCJS} } func (mr *moduleResolver) setMain(main *loader.SourceData, c *compiler.Compiler) error { @@ -59,7 +60,7 @@ func (mr *moduleResolver) requireModule(name string) (module, error) { return &baseGoModule{mod: mod}, nil } -func (mr *moduleResolver) resolve(basePWD *url.URL, arg string, loadCJS cjsModuleLoader) (module, error) { +func (mr *moduleResolver) resolve(basePWD *url.URL, arg string) (module, error) { if cached, ok := mr.cache[arg]; ok { return cached.mod, cached.err } @@ -80,7 +81,7 @@ func (mr *moduleResolver) resolve(basePWD *url.URL, arg string, loadCJS cjsModul return cached.mod, cached.err } // Fall back to loading from the filesystem. - mod, err := loadCJS(specifier, arg) + mod, err := mr.loadCJS(specifier, arg) mr.cache[specifier.String()] = moduleCacheElement{mod: mod, err: err} return mod, err } @@ -93,18 +94,17 @@ type moduleSystem struct { cjsLoad cjsModuleLoader } -func newModuleSystem(resolution *moduleResolver, vu modules.VU, cjsLoad cjsModuleLoader) *moduleSystem { +func newModuleSystem(resolution *moduleResolver, vu modules.VU) *moduleSystem { return &moduleSystem{ resolver: resolution, instanceCache: make(map[module]moduleInstance), vu: vu, - cjsLoad: cjsLoad, } } // Require is called when a module/file needs to be loaded by a script func (ms *moduleSystem) Require(pwd *url.URL, arg string) (*goja.Object, error) { - mod, err := ms.resolver.resolve(pwd, arg, ms.cjsLoad) + mod, err := ms.resolver.resolve(pwd, arg) if err != nil { return nil, err } From 92c9f5aa8973d30730d585d4a2a93ce68b493e22 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 28 Feb 2023 17:00:33 +0200 Subject: [PATCH 4/6] fixup! fixup! js: refactor how modules are loaded --- js/bundle.go | 6 +++--- js/modules.go | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/js/bundle.go b/js/bundle.go index c14a69dcc03..f5647fe703d 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -101,7 +101,7 @@ func newBundle( // TODO use a real context vuImpl := &moduleVUImpl{ctx: context.Background(), runtime: goja.New()} vuImpl.eventLoop = eventloop.New(vuImpl) - instance, err := bundle.instantiate(vuImpl, 0, c) + instance, err := bundle.instantiate(vuImpl, 0) if err != nil { return nil, err } @@ -211,7 +211,7 @@ func (b *Bundle) Instantiate(ctx context.Context, vuID uint64) (*BundleInstance, // runtime, but no state, to allow module-provided types to function within the init context. vuImpl := &moduleVUImpl{ctx: ctx, runtime: goja.New()} vuImpl.eventLoop = eventloop.New(vuImpl) - instance, err := b.instantiate(vuImpl, vuID, b.newCompiler(b.logger)) + instance, err := b.instantiate(vuImpl, vuID) if err != nil { return nil, err } @@ -259,7 +259,7 @@ func (b *Bundle) newCompiler(logger logrus.FieldLogger) *compiler.Compiler { return c } -func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64, c *compiler.Compiler) (moduleInstance, error) { +func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64) (moduleInstance, error) { rt := vuImpl.runtime err := b.setupJSRuntime(rt, int64(vuID), b.logger) if err != nil { diff --git a/js/modules.go b/js/modules.go index 7a32c3d6dcb..4671cdf71e6 100644 --- a/js/modules.go +++ b/js/modules.go @@ -91,7 +91,6 @@ type moduleSystem struct { vu modules.VU instanceCache map[module]moduleInstance resolver *moduleResolver - cjsLoad cjsModuleLoader } func newModuleSystem(resolution *moduleResolver, vu modules.VU) *moduleSystem { From 0c8c08cf48197b776867da35cf67523ee554f84b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 6 Mar 2023 12:31:19 +0200 Subject: [PATCH 5/6] Drop bundle.logger use TestPreInitState instead --- js/bundle.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/js/bundle.go b/js/bundle.go index f5647fe703d..9c27ad0a362 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -29,7 +29,6 @@ type Bundle struct { Filename *url.URL Source []byte Options lib.Options - logger *logrus.Logger CompatibilityMode lib.CompatibilityMode // parsed value preInitState *lib.TestPreInitState @@ -87,7 +86,6 @@ func newBundle( callableExports: make(map[string]struct{}), filesystems: filesystems, pwd: loader.Dir(src.URL), - logger: piState.Logger, preInitState: piState, } c := bundle.newCompiler(piState.Logger) @@ -261,13 +259,13 @@ func (b *Bundle) newCompiler(logger logrus.FieldLogger) *compiler.Compiler { func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64) (moduleInstance, error) { rt := vuImpl.runtime - err := b.setupJSRuntime(rt, int64(vuID), b.logger) + err := b.setupJSRuntime(rt, int64(vuID), b.preInitState.Logger) if err != nil { return nil, err } initenv := &common.InitEnvironment{ - Logger: b.logger, + Logger: b.preInitState.Logger, FileSystems: b.filesystems, CWD: b.pwd, Registry: b.preInitState.Registry, @@ -295,7 +293,7 @@ func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64) (moduleInstance, }() var instance moduleInstance - err = common.RunWithPanicCatching(b.logger, rt, func() error { + err = common.RunWithPanicCatching(b.preInitState.Logger, rt, func() error { return vuImpl.eventLoop.Start(func() error { //nolint:shadow,govet // here we shadow err on purpose mod, err := b.moduleResolver.resolve(b.pwd, b.Filename.String()) @@ -394,12 +392,12 @@ func (b *Bundle) setInitGlobals(rt *goja.Runtime, modSys *moduleSystem) (unset f func generateCJSLoad(b *Bundle, c *compiler.Compiler) cjsModuleLoader { return func(specifier *url.URL, name string) (*cjsModule, error) { if filepath.IsAbs(name) && runtime.GOOS == "windows" { - b.logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and won't work if"+ - " you move the script between machines or run it with `k6 cloud`; if absolute paths are required,"+ - " import them with the `file://` schema for slightly better compatibility", + b.preInitState.Logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and "+ + "won't work if you move the script between machines or run it with `k6 cloud`; if absolute paths are "+ + "required, import them with the `file://` schema for slightly better compatibility", name) } - d, err := loader.Load(b.logger, b.filesystems, specifier, name) + d, err := loader.Load(b.preInitState.Logger, b.filesystems, specifier, name) if err != nil { return nil, err } From aa208f8f92f3eedc0774ba0044c5074526bad510 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 6 Mar 2023 12:35:07 +0200 Subject: [PATCH 6/6] Use loader.SourceData directly in the bundle --- js/bundle.go | 14 ++++++-------- js/bundle_test.go | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/js/bundle.go b/js/bundle.go index 9c27ad0a362..0f5210a6a29 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -26,9 +26,8 @@ import ( // A Bundle is a self-contained bundle of scripts and resources. // You can use this to produce identical BundleInstance objects. type Bundle struct { - Filename *url.URL - Source []byte - Options lib.Options + sourceData *loader.SourceData + Options lib.Options CompatibilityMode lib.CompatibilityMode // parsed value preInitState *lib.TestPreInitState @@ -79,8 +78,7 @@ func newBundle( // Make a bundle, instantiate it into a throwaway VM to populate caches. bundle := &Bundle{ - Filename: src.URL, - Source: src.Data, + sourceData: src, Options: options, CompatibilityMode: compatMode, callableExports: make(map[string]struct{}), @@ -144,8 +142,8 @@ func (b *Bundle) makeArchive() *lib.Archive { Type: "js", Filesystems: b.filesystems, Options: b.Options, - FilenameURL: b.Filename, - Data: b.Source, + FilenameURL: b.sourceData.URL, + Data: b.sourceData.Data, PwdURL: b.pwd, Env: make(map[string]string, len(b.preInitState.RuntimeOptions.Env)), CompatibilityMode: b.CompatibilityMode.String(), @@ -296,7 +294,7 @@ func (b *Bundle) instantiate(vuImpl *moduleVUImpl, vuID uint64) (moduleInstance, err = common.RunWithPanicCatching(b.preInitState.Logger, rt, func() error { return vuImpl.eventLoop.Start(func() error { //nolint:shadow,govet // here we shadow err on purpose - mod, err := b.moduleResolver.resolve(b.pwd, b.Filename.String()) + mod, err := b.moduleResolver.resolve(b.pwd, b.sourceData.URL.String()) if err != nil { return err // TODO wrap as this should never happen } diff --git a/js/bundle_test.go b/js/bundle_test.go index 14b60688ba3..6323b648617 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -123,7 +123,7 @@ func TestNewBundle(t *testing.T) { t.Parallel() b, err := getSimpleBundle(t, "-", `export default function() {};`) require.NoError(t, err) - assert.Equal(t, "file://-", b.Filename.String()) + assert.Equal(t, "file://-", b.sourceData.URL.String()) assert.Equal(t, "file:///", b.pwd.String()) }) t.Run("CompatibilityMode", func(t *testing.T) {