diff --git a/js/bundle.go b/js/bundle.go index 4dbf251cf37..b0f191ac0fd 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "net/url" + "path/filepath" "runtime" "github.com/dop251/goja" @@ -27,17 +28,19 @@ import ( // You can use this to produce identical BundleInstance objects. type Bundle struct { Filename *url.URL - Source string - Program *goja.Program + Source []byte Options lib.Options - - BaseInitContext *InitContext + logger *logrus.Logger RuntimeOptions lib.RuntimeOptions CompatibilityMode lib.CompatibilityMode // parsed value registry *metrics.Registry - exports map[string]goja.Callable + filesystems map[string]afero.Fs + pwd *url.URL + + callableExports map[string]struct{} + modResolution *modulesResolution } // A BundleInstance is a self-contained instance of a Bundle. @@ -47,17 +50,18 @@ type BundleInstance struct { // TODO: maybe just have a reference to the Bundle? or save and pass rtOpts? env map[string]string - exports map[string]goja.Callable - moduleVUImpl *moduleVUImpl - pgm programWithSource + mainModuleInstance moduleInstance + moduleVUImpl *moduleVUImpl } func (bi *BundleInstance) getCallableExport(name string) goja.Callable { - return bi.exports[name] + fn, ok := goja.AssertFunction(bi.mainModuleInstance.exports().Get(name)) + _ = ok // TODO maybe return it + return fn } func (bi *BundleInstance) getExported(name string) goja.Value { - return bi.pgm.exports.Get(name) + return bi.mainModuleInstance.exports().ToObject(bi.Runtime).Get(name) } // NewBundle creates a new bundle from a source file and a filesystem. @@ -76,36 +80,35 @@ func newBundle( return nil, err } - // Compile sources, both ES5 and ES6 are supported. - code := string(src.Data) - c := compiler.New(piState.Logger) - c.Options = compiler.Options{ - CompatibilityMode: compatMode, - Strict: true, - SourceMapLoader: generateSourceMapLoader(piState.Logger, filesystems), - } - pgm, _, err := c.Compile(code, src.URL.String(), false) - if err != nil { - return nil, err - } // Make a bundle, instantiate it into a throwaway VM to populate caches. - rt := goja.New() bundle := Bundle{ Filename: src.URL, - Source: code, - Program: pgm, - BaseInitContext: NewInitContext(piState.Logger, rt, c, compatMode, filesystems, loader.Dir(src.URL)), + Source: src.Data, RuntimeOptions: piState.RuntimeOptions, Options: options, CompatibilityMode: compatMode, - exports: make(map[string]goja.Callable), + callableExports: make(map[string]struct{}), registry: piState.Registry, + modResolution: newModuleResolution(getJSModules()), + filesystems: filesystems, + pwd: loader.Dir(src.URL), + logger: piState.Logger, } - if err = bundle.instantiate(bundle.BaseInitContext, 0); err != nil { + c := bundle.newCompiler(piState.Logger) + if err = bundle.modResolution.setMain(src, c); err != nil { + return nil, err + } + // Instantiate the bundle into a new VM using a bound init context. This uses a context with a + // runtime, but no state, to allow module-provided types to function within the init context. + // TODO use a real context + vuImpl := &moduleVUImpl{ctx: context.Background(), runtime: goja.New()} + vuImpl.eventLoop = eventloop.New(vuImpl) + instance, err := bundle.instantiate(piState.Logger, vuImpl, 0, c) + if err != nil { return nil, err } - err = bundle.getExports(piState.Logger, rt, updateOptions) + err = bundle.populateExports(piState.Logger, updateOptions, instance) if err != nil { return nil, err } @@ -143,11 +146,11 @@ func NewBundleFromArchive(piState *lib.TestPreInitState, arc *lib.Archive) (*Bun func (b *Bundle) makeArchive() *lib.Archive { arc := &lib.Archive{ Type: "js", - Filesystems: b.BaseInitContext.filesystems, + Filesystems: b.filesystems, Options: b.Options, FilenameURL: b.Filename, - Data: []byte(b.Source), - PwdURL: b.BaseInitContext.pwd, + Data: b.Source, + PwdURL: b.pwd, Env: make(map[string]string, len(b.RuntimeOptions.Env)), CompatibilityMode: b.CompatibilityMode.String(), K6Version: consts.Version, @@ -161,19 +164,16 @@ func (b *Bundle) makeArchive() *lib.Archive { return arc } -// getExports validates and extracts exported objects -func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateOptions bool) error { - pgm := b.BaseInitContext.programs[b.Filename.String()] // this is the main script and it's always present - exportsV := pgm.module.Get("exports") - if goja.IsNull(exportsV) || goja.IsUndefined(exportsV) { +// populateExports validates and extracts exported objects +func (b *Bundle) populateExports(logger logrus.FieldLogger, updateOptions bool, instance moduleInstance) error { + exports := instance.exports() + if exports == nil { return errors.New("exports must be an object") } - exports := exportsV.ToObject(rt) - for _, k := range exports.Keys() { v := exports.Get(k) - if fn, ok := goja.AssertFunction(v); ok && k != consts.Options { - b.exports[k] = fn + if _, ok := goja.AssertFunction(v); ok && k != consts.Options { + b.callableExports[k] = struct{}{} continue } switch k { @@ -200,7 +200,7 @@ func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateO } } - if len(b.exports) == 0 { + if len(b.callableExports) == 0 { return errors.New("no exported functions in script") } @@ -211,43 +211,34 @@ func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateO func (b *Bundle) Instantiate(ctx context.Context, vuID uint64) (*BundleInstance, error) { // Instantiate the bundle into a new VM using a bound init context. This uses a context with a // runtime, but no state, to allow module-provided types to function within the init context. - rt := goja.New() - vuImpl := &moduleVUImpl{ - ctx: ctx, - runtime: rt, - } - init := newBoundInitContext(b.BaseInitContext, vuImpl) - if err := b.instantiate(init, vuID); err != nil { + vuImpl := &moduleVUImpl{ctx: ctx, runtime: goja.New()} + vuImpl.eventLoop = eventloop.New(vuImpl) + instance, err := b.instantiate(b.logger, vuImpl, vuID, b.newCompiler(b.logger)) + if err != nil { return nil, err } - pgm := init.programs[b.Filename.String()] // this is the main script and it's always present bi := &BundleInstance{ - Runtime: rt, - exports: make(map[string]goja.Callable), - env: b.RuntimeOptions.Env, - moduleVUImpl: vuImpl, - pgm: pgm, + Runtime: vuImpl.runtime, + env: b.RuntimeOptions.Env, + moduleVUImpl: vuImpl, + mainModuleInstance: instance, } // Grab any exported functions that could be executed. These were // already pre-validated in cmd.validateScenarioConfig(), just get them here. - exports := pgm.module.Get("exports").ToObject(rt) - for k := range b.exports { - fn, _ := goja.AssertFunction(exports.Get(k)) - bi.exports[k] = fn - } + exports := instance.exports() jsOptions := exports.Get("options") var jsOptionsObj *goja.Object if jsOptions == nil || goja.IsNull(jsOptions) || goja.IsUndefined(jsOptions) { - jsOptionsObj = rt.NewObject() + jsOptionsObj = vuImpl.runtime.NewObject() err := exports.Set("options", jsOptionsObj) if err != nil { return nil, fmt.Errorf("couldn't set exported options with merged values: %w", err) } } else { - jsOptionsObj = jsOptions.ToObject(rt) + jsOptionsObj = jsOptions.ToObject(vuImpl.runtime) } var instErr error @@ -260,78 +251,72 @@ func (b *Bundle) Instantiate(ctx context.Context, vuID uint64) (*BundleInstance, return bi, instErr } -// Instantiates the bundle into an existing runtime. Not public because it also messes with a bunch -// of other things, will potentially thrash data and makes a mess in it if the operation fails. - -func (b *Bundle) initializeProgramObject(rt *goja.Runtime, init *InitContext) programWithSource { - pgm := programWithSource{ - pgm: b.Program, - src: b.Source, - exports: rt.NewObject(), - module: rt.NewObject(), +func (b *Bundle) newCompiler(logger logrus.FieldLogger) *compiler.Compiler { + c := compiler.New(logger) + c.Options = compiler.Options{ + CompatibilityMode: b.CompatibilityMode, + Strict: true, + SourceMapLoader: generateSourceMapLoader(logger, b.filesystems), } - _ = pgm.module.Set("exports", pgm.exports) - init.programs[b.Filename.String()] = pgm - return pgm + return c } -//nolint:funlen -func (b *Bundle) instantiate(init *InitContext, vuID uint64) (err error) { - rt := init.moduleVUImpl.runtime - logger := init.logger - rt.SetFieldNameMapper(common.FieldNameMapper{}) - rt.SetRandSource(common.NewRandSource()) - - env := make(map[string]string, len(b.RuntimeOptions.Env)) - for key, value := range b.RuntimeOptions.Env { - env[key] = value +func (b *Bundle) instantiate( + logger logrus.FieldLogger, vuImpl *moduleVUImpl, vuID uint64, c *compiler.Compiler, +) (instance moduleInstance, err error) { + rt := vuImpl.runtime + err = b.setupJSRuntime(rt, int64(vuID), logger) + if err != nil { + return nil, err } - rt.Set("__ENV", env) - rt.Set("__VU", vuID) - _ = rt.Set("console", newConsole(logger)) - if init.compatibilityMode == lib.CompatibilityModeExtended { - rt.Set("global", rt.GlobalObject()) + initenv := &common.InitEnvironment{Logger: logger, FileSystems: b.filesystems, CWD: b.pwd, Registry: b.registry} + cjsLoad := func(specifier *url.URL, name string) (*cjsModule, error) { + if filepath.IsAbs(name) && runtime.GOOS == "windows" { + logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and won't work if"+ + " you move the script between machines or run it with `k6 cloud`; if absolute paths are required,"+ + " import them with the `file://` schema for slightly better compatibility", + name) + } + //nolint:shadow,govet // here we shadow err on purpose + d, err := loader.Load(logger, b.filesystems, specifier, name) + if err != nil { + return nil, err + } + return cjsmoduleFromString(specifier, d.Data, c) } - initenv := &common.InitEnvironment{ - Logger: logger, - FileSystems: init.filesystems, - CWD: init.pwd, - Registry: b.registry, - } - unbindInit := b.setInitGlobals(rt, init) - init.moduleVUImpl.initEnv = initenv - init.moduleVUImpl.eventLoop = eventloop.New(init.moduleVUImpl) - pgm := b.initializeProgramObject(rt, init) + modSys := newModuleSystem(b.modResolution, vuImpl, cjsLoad) + unbindInit := b.setInitGlobals(rt, modSys) + vuImpl.initEnv = initenv + defer func() { + unbindInit() + vuImpl.initEnv = nil + }() // TODO: make something cleaner for interrupting scripts, and more unified // (e.g. as a part of the event loop or RunWithPanicCatching()? - initCtxDone := init.moduleVUImpl.ctx.Done() + initCtxDone := vuImpl.ctx.Done() initDone := make(chan struct{}) watchDone := make(chan struct{}) go func() { select { case <-initCtxDone: - rt.Interrupt(init.moduleVUImpl.ctx.Err()) + rt.Interrupt(vuImpl.ctx.Err()) case <-initDone: // do nothing } close(watchDone) }() err = common.RunWithPanicCatching(logger, rt, func() error { - return init.moduleVUImpl.eventLoop.Start(func() error { - f, errRun := rt.RunProgram(b.Program) - if errRun != nil { - return errRun - } - if call, ok := goja.AssertFunction(f); ok { - if _, errRun = call(pgm.exports, pgm.module, pgm.exports); errRun != nil { - return errRun - } - return nil + return vuImpl.eventLoop.Start(func() error { + //nolint:shadow,govet // here we shadow err on purpose + mod, err := b.modResolution.resolve(b.pwd, b.Filename.String(), cjsLoad) + if err != nil { + return err // TODO wrap as this should never happen } - panic("Somehow a commonjs main module is not wrapped in a function") + instance = mod.Instantiate(vuImpl) + return instance.execute() }) }) close(initDone) @@ -342,54 +327,79 @@ func (b *Bundle) instantiate(init *InitContext, vuID uint64) (err error) { if errors.As(err, &exception) { err = &scriptException{inner: exception} } - return err + return nil, err } - exportsV := pgm.module.Get("exports") - if goja.IsNull(exportsV) { - return errors.New("exports must be an object") + if exports := instance.exports(); exports == nil { + return nil, errors.New("exports must be an object") } - pgm.exports = exportsV.ToObject(rt) - init.programs[b.Filename.String()] = pgm - unbindInit() - init.moduleVUImpl.ctx = nil - init.moduleVUImpl.initEnv = nil // If we've already initialized the original VU init context, forbid // any subsequent VUs to open new files if vuID == 0 { - init.allowOnlyOpenedFiles() + allowOnlyOpenedFiles(b.filesystems["file"]) } rt.SetRandSource(common.NewRandSource()) + return instance, nil +} + +func (b *Bundle) setupJSRuntime(rt *goja.Runtime, vuID int64, logger logrus.FieldLogger) error { + rt.SetFieldNameMapper(common.FieldNameMapper{}) + rt.SetRandSource(common.NewRandSource()) + + env := make(map[string]string, len(b.RuntimeOptions.Env)) + for key, value := range b.RuntimeOptions.Env { + env[key] = value + } + err := rt.Set("__ENV", env) + if err != nil { + return err + } + err = rt.Set("__VU", vuID) + if err != nil { + return err + } + err = rt.Set("console", newConsole(logger)) + if err != nil { + return err + } + + if b.CompatibilityMode == lib.CompatibilityModeExtended { + err = rt.Set("global", rt.GlobalObject()) + if err != nil { + return err + } + } return nil } -func (b *Bundle) setInitGlobals(rt *goja.Runtime, init *InitContext) (unset func()) { +func (b *Bundle) setInitGlobals(rt *goja.Runtime, modSys *moduleSystem) (unset func()) { mustSet := func(k string, v interface{}) { if err := rt.Set(k, v); err != nil { panic(fmt.Errorf("failed to set '%s' global object: %w", k, err)) } } - mustSet("require", init.Require) - mustSet("open", init.Open) - return func() { - mustSet("require", goja.Undefined()) - mustSet("open", goja.Undefined()) + r := requireImpl{ + vu: modSys.vu, + modules: modSys, + pwd: b.pwd, } -} + mustSet("require", r.require) -func generateSourceMapLoader(logger logrus.FieldLogger, filesystems map[string]afero.Fs, -) func(path string) ([]byte, error) { - return func(path string) ([]byte, error) { - u, err := url.Parse(path) - if err != nil { - return nil, err + mustSet("open", func(filename string, args ...string) (goja.Value, error) { + if modSys.vu.State() != nil { // fix + return nil, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "open") } - data, err := loader.Load(logger, filesystems, u, path) - if err != nil { - return nil, err + + if filename == "" { + return nil, errors.New("open() can't be used with an empty filename") } - return data.Data, nil + // This uses the pwd from the requireImpl + return openImpl(rt, b.filesystems["file"], r.pwd, filename, args...) + }) + return func() { + mustSet("require", goja.Undefined()) + mustSet("open", goja.Undefined()) } } diff --git a/js/bundle_test.go b/js/bundle_test.go index 658523bb7b5..377009d6433 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -124,7 +124,7 @@ func TestNewBundle(t *testing.T) { b, err := getSimpleBundle(t, "-", `export default function() {};`) require.NoError(t, err) assert.Equal(t, "file://-", b.Filename.String()) - assert.Equal(t, "file:///", b.BaseInitContext.pwd.String()) + assert.Equal(t, "file:///", b.pwd.String()) }) t.Run("CompatibilityMode", func(t *testing.T) { t.Parallel() diff --git a/js/cjsmodule.go b/js/cjsmodule.go new file mode 100644 index 00000000000..21d6d7736b4 --- /dev/null +++ b/js/cjsmodule.go @@ -0,0 +1,70 @@ +package js + +import ( + "fmt" + "net/url" + + "github.com/dop251/goja" + "go.k6.io/k6/js/compiler" + "go.k6.io/k6/js/modules" +) + +// cjsModule represents a commonJS module +type cjsModule struct { + prg *goja.Program + url *url.URL +} + +var _ module = &cjsModule{} + +type cjsModuleInstance struct { + mod *cjsModule + moduleObj *goja.Object + vu modules.VU +} + +func (c *cjsModule) Instantiate(vu modules.VU) moduleInstance { + return &cjsModuleInstance{vu: vu, mod: c} +} + +func (c *cjsModuleInstance) execute() error { + rt := c.vu.Runtime() + exports := rt.NewObject() + c.moduleObj = rt.NewObject() + err := c.moduleObj.Set("exports", exports) + if err != nil { + return fmt.Errorf("error while getting ready to import commonJS, couldn't set exports property of module: %w", + err) + } + + // Run the program. + f, err := rt.RunProgram(c.mod.prg) + if err != nil { + return err + } + if call, ok := goja.AssertFunction(f); ok { + if _, err = call(exports, c.moduleObj, exports); err != nil { + return err + } + } + + return nil +} + +func (c *cjsModuleInstance) exports() *goja.Object { + exportsV := c.moduleObj.Get("exports") + if goja.IsNull(exportsV) || goja.IsUndefined(exportsV) { + return nil + } + return exportsV.ToObject(c.vu.Runtime()) +} + +type cjsModuleLoader func(specifier *url.URL, name string) (*cjsModule, error) + +func cjsmoduleFromString(fileURL *url.URL, data []byte, c *compiler.Compiler) (*cjsModule, error) { + pgm, _, err := c.Compile(string(data), fileURL.String(), false) + if err != nil { + return nil, err + } + return &cjsModule{prg: pgm, url: fileURL}, nil +} diff --git a/js/gomodule.go b/js/gomodule.go new file mode 100644 index 00000000000..1abe4742acc --- /dev/null +++ b/js/gomodule.go @@ -0,0 +1,93 @@ +package js + +import ( + "github.com/dop251/goja" + "go.k6.io/k6/js/modules" +) + +// baseGoModule is a go module that does not implement modules.Module interface +// TODO maybe depracate those in the future +type baseGoModule struct { + mod interface{} +} + +var _ module = &baseGoModule{} + +func (b *baseGoModule) Instantiate(vu modules.VU) moduleInstance { + return &baseGoModuleInstance{mod: b.mod, vu: vu} +} + +type baseGoModuleInstance struct { + mod interface{} + vu modules.VU + exportsO *goja.Object // this is so we only initialize the exports once per instance +} + +func (b *baseGoModuleInstance) execute() error { + return nil +} + +func (b *baseGoModuleInstance) exports() *goja.Object { + if b.exportsO == nil { + // TODO check this does not panic a lot + rt := b.vu.Runtime() + b.exportsO = rt.ToValue(b.mod).ToObject(rt) + } + return b.exportsO +} + +// goModule is a go module which implements modules.Module +type goModule struct { + modules.Module +} + +var _ module = &goModule{} + +func (g *goModule) Instantiate(vu modules.VU) moduleInstance { + return &goModuleInstance{vu: vu, module: g} +} + +type goModuleInstance struct { + modules.Instance + module *goModule + vu modules.VU + exportsO *goja.Object // this is so we only initialize the exports once per instance +} + +var _ moduleInstance = &goModuleInstance{} + +func (gi *goModuleInstance) execute() error { + gi.Instance = gi.module.NewModuleInstance(gi.vu) + return nil +} + +func (gi *goModuleInstance) exports() *goja.Object { + if gi.exportsO == nil { + rt := gi.vu.Runtime() + gi.exportsO = rt.ToValue(toESModuleExports(gi.Instance.Exports())).ToObject(rt) + } + return gi.exportsO +} + +func toESModuleExports(exp modules.Exports) interface{} { + if exp.Named == nil { + return exp.Default + } + if exp.Default == nil { + return exp.Named + } + + result := make(map[string]interface{}, len(exp.Named)+2) + + for k, v := range exp.Named { + result[k] = v + } + // Maybe check that those weren't set + result["default"] = exp.Default + // this so babel works with the `default` when it transpiles from ESM to commonjs. + // This should probably be removed once we have support for ESM directly. So that require doesn't get support for + // that while ESM has. + result["__esModule"] = true + + return result +} diff --git a/js/initcontext.go b/js/initcontext.go index 8b8083ea2c3..ca00a7fd9ae 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -1,254 +1,37 @@ package js import ( - "context" "errors" "fmt" "net/url" "path/filepath" - "runtime" "strings" "github.com/dop251/goja" "github.com/sirupsen/logrus" "github.com/spf13/afero" - "go.k6.io/k6/js/common" - "go.k6.io/k6/js/compiler" "go.k6.io/k6/js/modules" - "go.k6.io/k6/lib" "go.k6.io/k6/lib/fsext" "go.k6.io/k6/loader" ) -type programWithSource struct { - pgm *goja.Program - src string - module *goja.Object - exports *goja.Object -} - -const openCantBeUsedOutsideInitContextMsg = `The "open()" function is only available in the init stage ` + +const cantBeUsedOutsideInitContextMsg = `the "%s" function is only available in the init stage ` + `(i.e. the global scope), see https://k6.io/docs/using-k6/test-life-cycle for more information` -// InitContext provides APIs for use in the init context. -// -// TODO: refactor most/all of this state away, use common.InitEnvironment instead -type InitContext struct { - // Bound runtime; used to instantiate objects. - compiler *compiler.Compiler - - moduleVUImpl *moduleVUImpl - - // Filesystem to load files and scripts from with the map key being the scheme - filesystems map[string]afero.Fs - pwd *url.URL - - // Cache of loaded programs and files. - programs map[string]programWithSource - exportsCache map[string]goja.Value - - compatibilityMode lib.CompatibilityMode - - logger logrus.FieldLogger - - moduleRegistry map[string]interface{} -} - -// NewInitContext creates a new initcontext with the provided arguments -func NewInitContext( - logger logrus.FieldLogger, rt *goja.Runtime, c *compiler.Compiler, compatMode lib.CompatibilityMode, - filesystems map[string]afero.Fs, pwd *url.URL, -) *InitContext { - return &InitContext{ - compiler: c, - filesystems: filesystems, - pwd: pwd, - programs: make(map[string]programWithSource), - compatibilityMode: compatMode, - logger: logger, - moduleRegistry: getJSModules(), - exportsCache: make(map[string]goja.Value), - moduleVUImpl: &moduleVUImpl{ - // TODO: pass a real context as we did for https://github.com/grafana/k6/pull/2800, - // also see https://github.com/grafana/k6/issues/2804 - ctx: context.Background(), - runtime: rt, - }, - } -} - -func newBoundInitContext(base *InitContext, vuImpl *moduleVUImpl) *InitContext { - // we don't copy the exports as otherwise they will be shared and we don't want this. - // this means that all the files will be executed again but once again only once per compilation - // of the main file. - programs := make(map[string]programWithSource, len(base.programs)) - for key, program := range base.programs { - programs[key] = programWithSource{ - src: program.src, - pgm: program.pgm, - } - } - return &InitContext{ - filesystems: base.filesystems, - pwd: base.pwd, - compiler: base.compiler, - - programs: programs, - compatibilityMode: base.compatibilityMode, - exportsCache: make(map[string]goja.Value), - logger: base.logger, - moduleRegistry: base.moduleRegistry, - moduleVUImpl: vuImpl, - } -} - -// Require is called when a module/file needs to be loaded by a script -func (i *InitContext) Require(arg string) (export goja.Value) { - var ok bool - if export, ok = i.exportsCache[arg]; ok { - return export - } - defer func() { i.exportsCache[arg] = export }() - switch { - case arg == "k6", strings.HasPrefix(arg, "k6/"): - // Builtin or external modules ("k6", "k6/*", or "k6/x/*") are handled - // specially, as they don't exist on the filesystem. This intentionally - // shadows attempts to name your own modules this. - v, err := i.requireModule(arg) - if err != nil { - common.Throw(i.moduleVUImpl.runtime, err) - } - return v - default: - // Fall back to loading from the filesystem. - v, err := i.requireFile(arg) - if err != nil { - common.Throw(i.moduleVUImpl.runtime, err) - } - return v - } -} - -func toESModuleExports(exp modules.Exports) interface{} { - if exp.Named == nil { - return exp.Default - } - if exp.Default == nil { - return exp.Named - } - - result := make(map[string]interface{}, len(exp.Named)+2) - - for k, v := range exp.Named { - result[k] = v - } - // Maybe check that those weren't set - result["default"] = exp.Default - // this so babel works with the `default` when it transpiles from ESM to commonjs. - // This should probably be removed once we have support for ESM directly. So that require doesn't get support for - // that while ESM has. - result["__esModule"] = true - - return result -} - -func (i *InitContext) requireModule(name string) (goja.Value, error) { - mod, ok := i.moduleRegistry[name] - if !ok { - return nil, fmt.Errorf("unknown module: %s", name) - } - if m, ok := mod.(modules.Module); ok { - instance := m.NewModuleInstance(i.moduleVUImpl) - return i.moduleVUImpl.runtime.ToValue(toESModuleExports(instance.Exports())), nil - } - - return i.moduleVUImpl.runtime.ToValue(mod), nil -} - -func (i *InitContext) requireFile(name string) (goja.Value, error) { - // Resolve the file path, push the target directory as pwd to make relative imports work. - pwd := i.pwd - fileURL, err := loader.Resolve(pwd, name) - if err != nil { - return nil, err - } - - // First, check if we have a cached program already. - pgm, ok := i.programs[fileURL.String()] - if !ok || pgm.module == nil { - if filepath.IsAbs(name) && runtime.GOOS == "windows" { - i.logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and won't work if"+ - " you move the script between machines or run it with `k6 cloud`; if absolute paths are required,"+ - " import them with the `file://` schema for slightly better compatibility", - name) - } - i.pwd = loader.Dir(fileURL) - defer func() { i.pwd = pwd }() - exports := i.moduleVUImpl.runtime.NewObject() - pgm.module = i.moduleVUImpl.runtime.NewObject() - _ = pgm.module.Set("exports", exports) - - if pgm.pgm == nil { - // Load the sources; the loader takes care of remote loading, etc. - data, err := loader.Load(i.logger, i.filesystems, fileURL, name) - if err != nil { - return goja.Undefined(), err - } - - pgm.src = string(data.Data) - - // Compile the sources; this handles ES5 vs ES6 automatically. - pgm.pgm, err = i.compileImport(pgm.src, data.URL.String()) - if err != nil { - return goja.Undefined(), err - } - } - - i.programs[fileURL.String()] = pgm - - // Run the program. - f, err := i.moduleVUImpl.runtime.RunProgram(pgm.pgm) - if err != nil { - delete(i.programs, fileURL.String()) - return goja.Undefined(), err - } - if call, ok := goja.AssertFunction(f); ok { - if _, err = call(exports, pgm.module, exports); err != nil { - return nil, err - } - } - } - - return pgm.module.Get("exports"), nil -} - -func (i *InitContext) compileImport(src, filename string) (*goja.Program, error) { - pgm, _, err := i.compiler.Compile(src, filename, false) - return pgm, err -} - -// Open implements open() in the init context and will read and return the +// openImpl implements openImpl() in the init context and will read and return the // contents of a file. If the second argument is "b" it returns an ArrayBuffer // instance, otherwise a string representation. -func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) { - if i.moduleVUImpl.State() != nil { - return nil, errors.New(openCantBeUsedOutsideInitContextMsg) - } - - if filename == "" { - return nil, errors.New("open() can't be used with an empty filename") - } - +func openImpl(rt *goja.Runtime, fs afero.Fs, basePWD *url.URL, filename string, args ...string) (goja.Value, error) { // Here IsAbs should be enough but unfortunately it doesn't handle absolute paths starting from // the current drive on windows like `\users\noname\...`. Also it makes it more easy to test and // will probably be need for archive execution under windows if always consider '/...' as an // absolute path. if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) { - filename = filepath.Join(i.pwd.Path, filename) + filename = filepath.Join(basePWD.Path, filename) } filename = filepath.Clean(filename) - fs := i.filesystems["file"] + if filename[0:1] != afero.FilePathSeparator { filename = afero.FilePathSeparator + filename } @@ -259,10 +42,10 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) } if len(args) > 0 && args[0] == "b" { - ab := i.moduleVUImpl.runtime.NewArrayBuffer(data) - return i.moduleVUImpl.runtime.ToValue(&ab), nil + ab := rt.NewArrayBuffer(data) + return rt.ToValue(&ab), nil } - return i.moduleVUImpl.runtime.ToValue(string(data)), nil + return rt.ToValue(string(data)), nil } func readFile(fileSystem afero.Fs, filename string) (data []byte, err error) { @@ -288,9 +71,7 @@ func readFile(fileSystem afero.Fs, filename string) (data []byte, err error) { } // allowOnlyOpenedFiles enables seen only files -func (i *InitContext) allowOnlyOpenedFiles() { - fs := i.filesystems["file"] - +func allowOnlyOpenedFiles(fs afero.Fs) { alreadyOpenedFS, ok := fs.(fsext.OnlyCachedEnabler) if !ok { return @@ -298,3 +79,60 @@ func (i *InitContext) allowOnlyOpenedFiles() { alreadyOpenedFS.AllowOnlyCached() } + +type requireImpl struct { + vu modules.VU + modules *moduleSystem + pwd *url.URL +} + +func (r *requireImpl) require(specifier string) (*goja.Object, error) { + // TODO remove this in the future when we address https://github.com/grafana/k6/issues/2674 + // This is currently needed as each time require is called we need to record it's new pwd + // to be used if a require *or* open is used within the file as they are relative to the + // latest call to require. + // This is *not* the actual require behaviour defined in commonJS as it is actually always relative + // to the file it is in. This is unlikely to be an issue but this code is here to keep backwards + // compatibility *for now*. + // With native ESM this won't even be possible as `require` might not be called - instead an import + // might be used in which case we won't be able to be doing this hack. In that case we either will + // need some goja specific helper or to use stack traces as goja_nodejs does. + currentPWD := r.pwd + if specifier != "k6" && !strings.HasPrefix(specifier, "k6/") { + defer func() { + r.pwd = currentPWD + }() + // In theory we can give that downwards, but this makes the code more tightly coupled + // plus as explained above this will be removed in the future so the code reflects more + // closely what will be needed then + fileURL, err := loader.Resolve(r.pwd, specifier) + if err != nil { + return nil, err + } + r.pwd = loader.Dir(fileURL) + } + + if r.vu.State() != nil { // fix + return nil, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "require") + } + if specifier == "" { + return nil, errors.New("require() can't be used with an empty specifier") + } + + return r.modules.Require(currentPWD, specifier) +} + +func generateSourceMapLoader(logger logrus.FieldLogger, filesystems map[string]afero.Fs, +) func(path string) ([]byte, error) { + return func(path string) ([]byte, error) { + u, err := url.Parse(path) + if err != nil { + return nil, err + } + data, err := loader.Load(logger, filesystems, u, path) + if err != nil { + return nil, err + } + return data.Data, nil + } +} diff --git a/js/initcontext_test.go b/js/initcontext_test.go index 00ef84ec387..180d62dd3a6 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -25,7 +25,7 @@ import ( "go.k6.io/k6/metrics" ) -func TestInitContextRequire(t *testing.T) { +func TestRequire(t *testing.T) { t.Parallel() t.Run("Modules", func(t *testing.T) { t.Run("Nonexistent", func(t *testing.T) { @@ -48,13 +48,11 @@ func TestInitContextRequire(t *testing.T) { bi, err := b.Instantiate(context.Background(), 0) assert.NoError(t, err, "instance error") - exports := bi.pgm.exports - require.NotNil(t, exports) - _, defaultOk := goja.AssertFunction(exports.Get("default")) + _, defaultOk := goja.AssertFunction(bi.getExported("default")) assert.True(t, defaultOk, "default export is not a function") - assert.Equal(t, "abc123", exports.Get("dummy").String()) + assert.Equal(t, "abc123", bi.getExported("dummy").String()) - k6 := exports.Get("_k6").ToObject(bi.Runtime) + k6 := bi.getExported("_k6").ToObject(bi.Runtime) require.NotNil(t, k6) _, groupOk := goja.AssertFunction(k6.Get("group")) assert.True(t, groupOk, "k6.group is not a function") @@ -73,13 +71,11 @@ func TestInitContextRequire(t *testing.T) { bi, err := b.Instantiate(context.Background(), 0) require.NoError(t, err) - exports := bi.pgm.exports - require.NotNil(t, exports) - _, defaultOk := goja.AssertFunction(exports.Get("default")) + _, defaultOk := goja.AssertFunction(bi.getExported("default")) assert.True(t, defaultOk, "default export is not a function") - assert.Equal(t, "abc123", exports.Get("dummy").String()) + assert.Equal(t, "abc123", bi.getExported("dummy").String()) - _, groupOk := goja.AssertFunction(exports.Get("_group")) + _, groupOk := goja.AssertFunction(bi.getExported("_group")) assert.True(t, groupOk, "{ group } is not a function") }) }) @@ -107,7 +103,7 @@ func TestInitContextRequire(t *testing.T) { require.NoError(t, afero.WriteFile(fs, "/file.js", []byte(`throw new Error("aaaa")`), 0o755)) _, err := getSimpleBundle(t, "/script.js", `import "/file.js"; export default function() {}`, fs) assert.EqualError(t, err, - "Error: aaaa\n\tat file:///file.js:2:7(3)\n\tat go.k6.io/k6/js.(*InitContext).Require-fm (native)\n\tat file:///script.js:1:0(15)\n") + "Error: aaaa\n\tat file:///file.js:2:7(3)\n\tat go.k6.io/k6/js.(*requireImpl).require-fm (native)\n\tat file:///script.js:1:0(15)\n") }) imports := map[string]struct { @@ -175,9 +171,6 @@ func TestInitContextRequire(t *testing.T) { libName) b, err := getSimpleBundle(t, "/path/to/script.js", data, fs) require.NoError(t, err) - if constPath != "" { - assert.Contains(t, b.BaseInitContext.programs, "file://"+constPath) - } _, err = b.Instantiate(context.Background(), 0) require.NoError(t, err) @@ -538,7 +531,7 @@ func TestRequestWithMultipleBinaryFiles(t *testing.T) { <-ch } -func TestInitContextVU(t *testing.T) { +func Test__VU(t *testing.T) { t.Parallel() b, err := getSimpleBundle(t, "/script.js", ` let vu = __VU; diff --git a/js/modules.go b/js/modules.go new file mode 100644 index 00000000000..06e2c954e47 --- /dev/null +++ b/js/modules.go @@ -0,0 +1,122 @@ +package js + +import ( + "fmt" + "net/url" + "strings" + + "github.com/dop251/goja" + "go.k6.io/k6/js/compiler" + "go.k6.io/k6/js/modules" + "go.k6.io/k6/loader" +) + +type module interface { + Instantiate(vu modules.VU) moduleInstance +} + +type moduleInstance interface { + execute() error + exports() *goja.Object +} +type moduleCacheElement struct { + mod module + err error +} + +type modulesResolution struct { + cache map[string]moduleCacheElement + goModules map[string]interface{} +} + +func newModuleResolution(goModules map[string]interface{}) *modulesResolution { + return &modulesResolution{goModules: goModules, cache: make(map[string]moduleCacheElement)} +} + +func (mr *modulesResolution) setMain(main *loader.SourceData, c *compiler.Compiler) error { + mod, err := cjsmoduleFromString(main.URL, main.Data, c) + mr.cache[main.URL.String()] = moduleCacheElement{mod: mod, err: err} + return err +} + +func (mr *modulesResolution) resolveSpecifier(basePWD *url.URL, arg string) (*url.URL, error) { + specifier, err := loader.Resolve(basePWD, arg) + if err != nil { + return nil, err + } + return specifier, nil +} + +func (mr *modulesResolution) requireModule(name string) (module, error) { + mod, ok := mr.goModules[name] + if !ok { + return nil, fmt.Errorf("unknown module: %s", name) + } + if m, ok := mod.(modules.Module); ok { + return &goModule{Module: m}, nil + } + + return &baseGoModule{mod: mod}, nil +} + +func (mr *modulesResolution) resolve(basePWD *url.URL, arg string, loadCJS cjsModuleLoader) (module, error) { + if cached, ok := mr.cache[arg]; ok { + return cached.mod, cached.err + } + switch { + case arg == "k6", strings.HasPrefix(arg, "k6/"): + // Builtin or external modules ("k6", "k6/*", or "k6/x/*") are handled + // specially, as they don't exist on the filesystem. + mod, err := mr.requireModule(arg) + mr.cache[arg] = moduleCacheElement{mod: mod, err: err} + return mod, err + default: + specifier, err := mr.resolveSpecifier(basePWD, arg) + if err != nil { + return nil, err + } + // try cache with the final specifier + if cached, ok := mr.cache[specifier.String()]; ok { + return cached.mod, cached.err + } + // Fall back to loading from the filesystem. + mod, err := loadCJS(specifier, arg) + mr.cache[specifier.String()] = moduleCacheElement{mod: mod, err: err} + return mod, err + } +} + +type moduleSystem struct { + vu modules.VU + instanceCache map[module]moduleInstance + resolution *modulesResolution + cjsLoad cjsModuleLoader +} + +func newModuleSystem(resolution *modulesResolution, vu modules.VU, cjsLoad cjsModuleLoader) *moduleSystem { + return &moduleSystem{ + resolution: resolution, + instanceCache: make(map[module]moduleInstance), + vu: vu, + cjsLoad: cjsLoad, + } +} + +// Require is called when a module/file needs to be loaded by a script +func (ms *moduleSystem) Require(pwd *url.URL, arg string) (*goja.Object, error) { + mod, err := ms.resolution.resolve(pwd, arg, ms.cjsLoad) + if err != nil { + return nil, err + } + if instance, ok := ms.instanceCache[mod]; ok { + return instance.exports(), nil + } + + instance := mod.Instantiate(ms.vu) + ms.instanceCache[mod] = instance + if err = instance.execute(); err != nil { + return nil, err + } + + return instance.exports(), nil +} diff --git a/js/runner.go b/js/runner.go index edb1d43c648..c925782a8c0 100644 --- a/js/runner.go +++ b/js/runner.go @@ -253,7 +253,7 @@ func (r *Runner) newVU( // instead of "Value is not an object: undefined ..." _ = vu.Runtime.GlobalObject().Set("open", func() { - common.Throw(vu.Runtime, errors.New(openCantBeUsedOutsideInitContextMsg)) + common.Throw(vu.Runtime, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "open")) }) return vu, nil @@ -353,7 +353,7 @@ func (r *Runner) GetOptions() lib.Options { // IsExecutable returns whether the given name is an exported and // executable function in the script. func (r *Runner) IsExecutable(name string) bool { - _, exists := r.Bundle.exports[name] + _, exists := r.Bundle.callableExports[name] return exists }