refactor(js): make Compiler invoke a preprocessor, pre-compile core.js

This attempts to further separate the concept of compiler.Compiler and Babel being
the same thing.

A compiler.Compiler does two things:

1. Parses source code into a goja.Program, which in the Goja domain
means *compiling*.
2. Optionally runs the code through preprocessors and pre-compiles shims
such as core.js to be used in the runtime VU Goja VM.

I think introducing the concept of a preprocessor makes sense, as that's
what Babel is (besides being a compiler, which some may disagree with).
This design is more cohesive (all shims are now in js/compiler/lib/, the
js/lib/ package is gone) and gives us a place to expand by further
adding shims of our own.

js/bundle.go

@@ -31,7 +31,6 @@ import (
 	"github.com/dop251/goja"
 	"github.com/loadimpact/k6/js/common"
 	"github.com/loadimpact/k6/js/compiler"
-	jslib "github.com/loadimpact/k6/js/lib"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/loader"
 	"github.com/pkg/errors"
@@ -41,10 +40,11 @@ import (
 // A Bundle is a self-contained bundle of scripts and resources.
 // You can use this to produce identical BundleInstance objects.
 type Bundle struct {
-	Filename *url.URL
-	Source   string
-	Program  *goja.Program
-	Options  lib.Options
+	Filename   *url.URL
+	Source     string
+	PreProgram *goja.Program // runs preprocessor code, applies shims, etc.
+	Program    *goja.Program
+	Options    lib.Options
 
 	BaseInitContext *InitContext
 
@@ -72,16 +72,17 @@ func NewBundle(src *loader.SourceData, filesystems map[string]afero.Fs, rtOpts l
 	// Compile sources, both ES5 and ES6 are supported.
 	code := string(src.Data)
 	c := compiler.New()
-	pgm, _, err := c.Compile(code, src.URL.String(), "", "", true, compatMode)
+	pgm, prePgm, _, err := c.Compile(code, src.URL.String(), "", "", true, compatMode)
 	if err != nil {
 		return nil, err
 	}
 	// Make a bundle, instantiate it into a throwaway VM to populate caches.
 	rt := goja.New()
 	bundle := Bundle{
-		Filename: src.URL,
-		Source:   code,
-		Program:  pgm,
+		Filename:   src.URL,
+		Source:     code,
+		Program:    pgm,
+		PreProgram: prePgm,
 		BaseInitContext: NewInitContext(rt, c, compatMode, new(context.Context),
 			filesystems, loader.Dir(src.URL)),
 		Env: rtOpts.Env,
@@ -149,7 +150,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 	}
 
 	c := compiler.New()
-	pgm, _, err := c.Compile(string(arc.Data), arc.FilenameURL.String(), "", "", true, compatMode)
+	pgm, prePgm, _, err := c.Compile(string(arc.Data), arc.FilenameURL.String(), "", "", true, compatMode)
 	if err != nil {
 		return nil, err
 	}
@@ -170,6 +171,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 		Filename:        arc.FilenameURL,
 		Source:          string(arc.Data),
 		Program:         pgm,
+		PreProgram:      prePgm,
 		Options:         arc.Options,
 		BaseInitContext: initctx,
 		Env:             env,
@@ -247,8 +249,8 @@ func (b *Bundle) instantiate(rt *goja.Runtime, init *InitContext) error {
 	rt.SetFieldNameMapper(common.FieldNameMapper{})
 	rt.SetRandSource(common.NewRandSource())
 
-	if init.compatibilityMode == compiler.CompatibilityModeES6 {
-		if _, err := rt.RunProgram(jslib.GetCoreJS()); err != nil {
+	if b.PreProgram != nil {
+		if _, err := rt.RunProgram(b.PreProgram); err != nil {
 			return err
 		}
 	}

js/common/util.go

@@ -29,7 +29,7 @@ import (
 func RunString(rt *goja.Runtime, src string) (goja.Value, error) {
 	var err error
 	c := compiler.New()
-	src, _, err = c.Transform(src, "__string__")
+	src, _, err = c.Preprocess(src, "__string__")
 	if err != nil {
 		return goja.Undefined(), err
 	}

js/compiler/compiler.go

@@ -44,8 +44,8 @@ var (
 		"highlightCode": false,
 	}
 
-	once sync.Once // nolint:gochecknoglobals
-	babl *babel    // nolint:gochecknoglobals
+	once     sync.Once     // nolint:gochecknoglobals
+	preprocs *preprocessor // nolint:gochecknoglobals
 )
 
 // CompatibilityMode specifies the JS compatibility mode
@@ -68,91 +68,101 @@ func New() *Compiler {
 	return &Compiler{}
 }
 
-// Transform the given code into ES5, while synchronizing to ensure only a single
-// babel instance is in use.
-func (c *Compiler) Transform(src, filename string) (code string, srcmap SourceMap, err error) {
-	var b *babel
-	if b, err = newBabel(); err != nil {
+// Preprocess the given code by compiling it to ES5 with Babel and pre-compiling
+// core.js, while synchronizing to ensure only a single preprocessor instance
+// is in use.
+func (c *Compiler) Preprocess(src, filename string) (code string, srcmap SourceMap, err error) {
+	var pp *preprocessor
+	if pp, err = newPreprocessor(); err != nil {
 		return
 	}
 
-	b.mutex.Lock()
-	defer b.mutex.Unlock()
-	return b.Transform(src, filename)
+	pp.mutex.Lock()
+	defer pp.mutex.Unlock()
+	return pp.babelTransform(src, filename)
 }
 
 // Compile the program
 func (c *Compiler) Compile(src, filename string, pre, post string,
-	strict bool, compatMode CompatibilityMode) (*goja.Program, string, error) {
-	return c.compile(src, filename, pre, post, strict, compatMode)
-}
-
-func (c *Compiler) compile(src, filename string, pre, post string,
-	strict bool, compatMode CompatibilityMode) (*goja.Program, string, error) {
+	strict bool, compatMode CompatibilityMode) (*goja.Program, *goja.Program, string, error) {
+	var preProgram *goja.Program
 	code := pre + src + post
 	ast, err := parser.ParseFile(nil, filename, code, 0)
 	if err != nil {
 		if compatMode == CompatibilityModeES6 {
-			code, _, err := c.Transform(src, filename)
-			if err != nil {
-				return nil, code, err
+			codeProcessed, _, errP := c.Preprocess(src, filename)
+			if errP != nil {
+				return nil, nil, code, errP
 			}
-			return c.compile(code, filename, pre, post, strict, compatMode)
+			return c.Compile(codeProcessed, filename, pre, post, strict, compatMode)
 		}
-		return nil, src, err
+		return nil, nil, src, err
+	}
+	if compatMode == CompatibilityModeES6 && preprocs != nil {
+		preProgram = preprocs.corejs
 	}
 	pgm, err := goja.CompileAST(ast, strict)
-	return pgm, code, err
+	return pgm, preProgram, code, err
 }
 
-type babel struct {
+type preprocessor struct {
 	vm        *goja.Runtime
-	this      goja.Value
+	babel     goja.Value
+	corejs    *goja.Program
 	transform goja.Callable
 	mutex     sync.Mutex //TODO: cache goja.CompileAST() in an init() function?
 }
 
-func newBabel() (*babel, error) {
+func newPreprocessor() (*preprocessor, error) {
 	var err error
 
 	once.Do(func() {
 		conf := rice.Config{
 			LocateOrder: []rice.LocateMethod{rice.LocateEmbedded},
 		}
+		// Compile Babel
 		babelSrc := conf.MustFindBox("lib").MustString("babel.min.js")
 		vm := goja.New()
 		if _, err = vm.RunString(babelSrc); err != nil {
 			return
 		}
 
-		this := vm.Get("Babel")
-		bObj := this.ToObject(vm)
-		babl = &babel{vm: vm, this: this}
-		if err = vm.ExportTo(bObj.Get("transform"), &babl.transform); err != nil {
+		// Compile core.js for use in VU Goja runtimes
+		corejs := goja.MustCompile(
+			"corejs.min.js",
+			conf.MustFindBox("lib").MustString("corejs.min.js"),
+			true,
+		)
+
+		babel := vm.Get("Babel")
+		bObj := babel.ToObject(vm)
+		preprocs = &preprocessor{vm: vm, babel: babel, corejs: corejs}
+		if err = vm.ExportTo(bObj.Get("transform"), &preprocs.transform); err != nil {
 			return
 		}
 	})
 
-	return babl, err
+	return preprocs, err
 }
 
-func (b *babel) Transform(src, filename string) (code string, srcmap SourceMap, err error) {
+// Transform the given code into ES5 with Babel
+func (pp *preprocessor) babelTransform(src, filename string) (code string, srcmap SourceMap, err error) {
 	opts := DefaultOpts
 	opts["filename"] = filename
 
 	startTime := time.Now()
-	v, err := b.transform(b.this, b.vm.ToValue(src), b.vm.ToValue(opts))
+	v, err := pp.transform(pp.babel, pp.vm.ToValue(src), pp.vm.ToValue(opts))
 	if err != nil {
 		return
 	}
 	logrus.WithField("t", time.Since(startTime)).Debug("Babel: Transformed")
 
-	vO := v.ToObject(b.vm)
-	if err = b.vm.ExportTo(vO.Get("code"), &code); err != nil {
+	vO := v.ToObject(pp.vm)
+	if err = pp.vm.ExportTo(vO.Get("code"), &code); err != nil {
 		return
 	}
 	var rawmap map[string]interface{}
-	if err = b.vm.ExportTo(vO.Get("map"), &rawmap); err != nil {
+	if err = pp.vm.ExportTo(vO.Get("map"), &rawmap); err != nil {
 		return
 	}
 	if err = mapstructure.Decode(rawmap, &srcmap); err != nil {

js/compiler/compiler_test.go

@@ -30,23 +30,23 @@ import (
 func TestTransform(t *testing.T) {
 	c := New()
 	t.Run("blank", func(t *testing.T) {
-		src, _, err := c.Transform("", "test.js")
+		src, _, err := c.Preprocess("", "test.js")
 		assert.NoError(t, err)
 		assert.Equal(t, `"use strict";`, src)
 		// assert.Equal(t, 3, srcmap.Version)
 		// assert.Equal(t, "test.js", srcmap.File)
 		// assert.Equal(t, "", srcmap.Mappings)
 	})
 	t.Run("double-arrow", func(t *testing.T) {
-		src, _, err := c.Transform("()=> true", "test.js")
+		src, _, err := c.Preprocess("()=> true", "test.js")
 		assert.NoError(t, err)
 		assert.Equal(t, `"use strict";(function () {return true;});`, src)
 		// assert.Equal(t, 3, srcmap.Version)
 		// assert.Equal(t, "test.js", srcmap.File)
 		// assert.Equal(t, "aAAA,qBAAK,IAAL", srcmap.Mappings)
 	})
 	t.Run("longer", func(t *testing.T) {
-		src, _, err := c.Transform(strings.Join([]string{
+		src, _, err := c.Preprocess(strings.Join([]string{
 			`function add(a, b) {`,
 			`    return a + b;`,
 			`};`,
@@ -71,24 +71,28 @@ func TestCompile(t *testing.T) {
 	c := New()
 	t.Run("ES5", func(t *testing.T) {
 		src := `1+(function() { return 2; })()`
-		pgm, code, err := c.Compile(src, "script.js", "", "", true, CompatibilityModeES51)
+		pgm, prePgm, code, err := c.Compile(src, "script.js", "", "", true, CompatibilityModeES51)
 		if !assert.NoError(t, err) {
 			return
 		}
+		// Running in ES5 mode, so nothing to preprocess
+		assert.Nil(t, prePgm)
 		assert.Equal(t, src, code)
 		v, err := goja.New().RunProgram(pgm)
 		if assert.NoError(t, err) {
 			assert.Equal(t, int64(3), v.Export())
 		}
 
 		t.Run("Wrap", func(t *testing.T) {
-			pgm, code, err := c.Compile(src, "script.js",
+			pgm, prePgm, code, err := c.Compile(src, "script.js",
 				"(function(){return ", "})", true, CompatibilityModeES51)
 			if !assert.NoError(t, err) {
 				return
 			}
 			assert.Equal(t, `(function(){return 1+(function() { return 2; })()})`, code)
-			v, err := goja.New().RunProgram(pgm)
+
+			assert.Nil(t, prePgm)
+			v, err = goja.New().RunProgram(pgm)
 			if assert.NoError(t, err) {
 				fn, ok := goja.AssertFunction(v)
 				if assert.True(t, ok, "not a function") {
@@ -102,30 +106,35 @@ func TestCompile(t *testing.T) {
 
 		t.Run("Invalid", func(t *testing.T) {
 			src := `1+(function() { return 2; )()`
-			_, _, err := c.Compile(src, "script.js", "", "", true, CompatibilityModeES6)
+			_, _, _, err := c.Compile(src, "script.js", "", "", true, CompatibilityModeES6)
 			assert.IsType(t, &goja.Exception{}, err)
 			assert.Contains(t, err.Error(), `SyntaxError: script.js: Unexpected token (1:26)
 > 1 | 1+(function() { return 2; )()`)
 		})
 	})
 	t.Run("ES6", func(t *testing.T) {
-		pgm, code, err := c.Compile(`1+(()=>2)()`, "script.js", "", "", true, CompatibilityModeES6)
+		pgm, prePgm, code, err := c.Compile(`1+(()=>2)()`, "script.js", "", "", true, CompatibilityModeES6)
 		if !assert.NoError(t, err) {
 			return
 		}
 		assert.Equal(t, `"use strict";1 + function () {return 2;}();`, code)
-		v, err := goja.New().RunProgram(pgm)
+		vm := goja.New()
+		_, err = vm.RunProgram(prePgm)
+		assert.NoError(t, err)
+		v, err := vm.RunProgram(pgm)
 		if assert.NoError(t, err) {
 			assert.Equal(t, int64(3), v.Export())
 		}
 
 		t.Run("Wrap", func(t *testing.T) {
-			pgm, code, err := c.Compile(`fn(1+(()=>2)())`, "script.js", "(function(fn){", "})", true, CompatibilityModeES6)
+			pgm, prePgm, code, err := c.Compile(`fn(1+(()=>2)())`, "script.js", "(function(fn){", "})", true, CompatibilityModeES6)
 			if !assert.NoError(t, err) {
 				return
 			}
 			assert.Equal(t, `(function(fn){"use strict";fn(1 + function () {return 2;}());})`, code)
 			rt := goja.New()
+			_, err = rt.RunProgram(prePgm)
+			assert.NoError(t, err)
 			v, err := rt.RunProgram(pgm)
 			if assert.NoError(t, err) {
 				fn, ok := goja.AssertFunction(v)
@@ -141,7 +150,7 @@ func TestCompile(t *testing.T) {
 		})
 
 		t.Run("Invalid", func(t *testing.T) {
-			_, _, err := c.Compile(`1+(=>2)()`, "script.js", "", "", true, CompatibilityModeES6)
+			_, _, _, err := c.Compile(`1+(=>2)()`, "script.js", "", "", true, CompatibilityModeES6)
 			assert.IsType(t, &goja.Exception{}, err)
 			assert.Contains(t, err.Error(), `SyntaxError: script.js: Unexpected token (1:3)
 > 1 | 1+(=>2)()`)

js/initcontext.go

@@ -181,7 +181,7 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) {
 }
 
 func (i *InitContext) compileImport(src, filename string) (*goja.Program, error) {
-	pgm, _, err := i.compiler.Compile(src, filename,
+	pgm, _, _, err := i.compiler.Compile(src, filename,
 		"(function(module, exports){\n", "\n})\n", true, i.compatibilityMode)
 	return pgm, err
 }