Embedding JS dependencies as Go code (#658)

closes #545

.circleci/config.yml

@@ -37,7 +37,7 @@ jobs:
             export PATH=$GOPATH/bin:$PATH
             echo "mode: set" > coverage.txt
             for pkg in $(go list ./... | grep -v vendor); do
-                go test -race -timeout 180s -coverprofile=$(echo $pkg | tr / -).coverage $pkg
+                go test -race -timeout 210s -coverprofile=$(echo $pkg | tr / -).coverage $pkg
             done
             grep -h -v "^mode:" *.coverage >> coverage.txt
             rm -f *.coverage

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "js2/compiler/lib/babel-standalone-bower"]
-	path = js/compiler/lib/babel-standalone-bower
-	url = https://github.com/Daniel15/babel-standalone-bower
-[submodule "js/lib/core-js"]
-	path = js/lib/core-js
-	url = https://github.com/zloirock/core-js

Makefile

@@ -13,7 +13,7 @@ format:
 .PHONY: check
 check:
 	gometalinter --deadline 10m --config gometalinter.json --vendor ./...
-	go test -race -timeout 180s ./...
+	go test -race -timeout 210s ./...
 
 .PHONY: docs
 docs:

build-release.sh

@@ -6,9 +6,6 @@ eval $(go env)
 # To override the latest git tag as the version, pass something else as the first arg.
 VERSION=${1:-$(git describe --tags --abbrev=0)}
 
-# Fail early if external dependencies aren't installed.
-rice --help > /dev/null || (echo "ERROR: rice is not installed, run: go get github.com/GeertJohan/go.rice/rice"; exit 1)
-
 make_archive() {
 	FMT=$1
 	DIR=$2
@@ -40,7 +37,6 @@ build_dist() {
 
 	# Build a binary, embed what we can by means of static assets inside it.
 	GOARCH=$GOARCH GOOS=$GOOS go build -o dist/$DIR/$BIN
-	rice append --exec=dist/$DIR/$BIN -i ./js/compiler -i ./js/lib
 
 	# Archive it all, native format depends on the platform. Subshell to not mess with $PWD.
 	( cd dist && make_archive $FMT $DIR )
@@ -60,7 +56,7 @@ checksum() {
 		echo "ERROR: unable to find a command to compute sha-256 hash"
 		return 1
 	fi
-	
+
 	rm -f dist/$CHECKSUM_FILE
 	( cd dist && for x in $(ls -1); do $CHECKSUM_CMD $x >> $CHECKSUM_FILE; done )
 }

gometalinter.json

@@ -7,5 +7,5 @@
 		"goconst", "gosimple", "staticcheck",
 		"misspell"
 	],
-	"Exclude": ["/usr/local/go/src", "/go/src"]
+	"Exclude": ["/usr/local/go/src", "/go/src", "rice-box.go"]
 }

js/bundle.go

@@ -56,6 +56,11 @@ type BundleInstance struct {
 
 // Creates a new bundle from a source file and a filesystem.
 func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) {
+	compiler, err := compiler.New()
+	if err != nil {
+		return nil, err
+	}
+
 	// Compile sources, both ES5 and ES6 are supported.
 	code := string(src.Data)
 	pgm, _, err := compiler.Compile(code, src.Filename, "", "", true)
@@ -75,7 +80,7 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu
 		Filename:        src.Filename,
 		Source:          code,
 		Program:         pgm,
-		BaseInitContext: NewInitContext(rt, new(context.Context), cachedFS, loader.Dir(src.Filename)),
+		BaseInitContext: NewInitContext(rt, compiler, new(context.Context), cachedFS, loader.Dir(src.Filename)),
 		Env:             rtOpts.Env,
 	}
 	if err := bundle.instantiate(rt, bundle.BaseInitContext); err != nil {
@@ -126,6 +131,11 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu
 }
 
 func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, error) {
+	compiler, err := compiler.New()
+	if err != nil {
+		return nil, err
+	}
+
 	if arc.Type != "js" {
 		return nil, errors.Errorf("expected bundle type 'js', got '%s'", arc.Type)
 	}
@@ -135,7 +145,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 		return nil, err
 	}
 
-	initctx := NewInitContext(goja.New(), new(context.Context), nil, arc.Pwd)
+	initctx := NewInitContext(goja.New(), compiler, new(context.Context), nil, arc.Pwd)
 	for filename, data := range arc.Scripts {
 		src := string(data)
 		pgm, err := initctx.compileImport(src, filename)
@@ -217,7 +227,7 @@ func (b *Bundle) instantiate(rt *goja.Runtime, init *InitContext) error {
 	rt.SetFieldNameMapper(common.FieldNameMapper{})
 	rt.SetRandSource(common.NewRandSource())
 
-	if _, err := rt.RunProgram(jslib.CoreJS); err != nil {
+	if _, err := rt.RunProgram(jslib.GetCoreJS()); err != nil {
 		return err
 	}
 

js/bundle_test.go

@@ -52,7 +52,7 @@ func TestNewBundle(t *testing.T) {
 	})
 	t.Run("Invalid", func(t *testing.T) {
 		_, err := getSimpleBundle("/script.js", "\x00")
-		assert.EqualError(t, err, "SyntaxError: /script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n    | ^ at <eval>:2:26853(114)")
+		assert.Contains(t, err.Error(), "SyntaxError: /script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n")
 	})
 	t.Run("Error", func(t *testing.T) {
 		_, err := getSimpleBundle("/script.js", `throw new Error("aaaa");`)

js/common/util.go

@@ -27,7 +27,12 @@ import (
 
 // Runs an ES6 string in the given runtime. Use this rather than writing ES5 in tests.
 func RunString(rt *goja.Runtime, src string) (goja.Value, error) {
-	src, _, err := compiler.Transform(src, "__string__")
+	compiler, err := compiler.New()
+	if err != nil {
+		panic(err)
+	}
+
+	src, _, err = compiler.Transform(src, "__string__")
 	if err != nil {
 		return goja.Undefined(), err
 	}

js/common/util_test.go

@@ -35,7 +35,7 @@ func TestRunString(t *testing.T) {
 	})
 	t.Run("Invalid", func(t *testing.T) {
 		_, err := RunString(goja.New(), `let a = #;`)
-		assert.EqualError(t, err, "SyntaxError: __string__: Unexpected character '#' (1:8)\n> 1 | let a = #;\n    |         ^ at <eval>:2:26853(114)")
+		assert.Contains(t, err.Error(), "SyntaxError: __string__: Unexpected character '#' (1:8)\n> 1 | let a = #;\n")
 	})
 }
 

js/compiler/compiler.go

@@ -18,6 +18,8 @@
  *
  */
 
+//go:generate rice embed-go
+
 package compiler
 
 import (
@@ -32,9 +34,6 @@ import (
 )
 
 var (
-	lib      = rice.MustFindBox("lib")
-	babelSrc = lib.MustString("babel-standalone-bower/babel.min.js")
-
 	DefaultOpts = map[string]interface{}{
 		"presets":       []string{"latest"},
 		"ast":           false,
@@ -44,6 +43,9 @@ var (
 		"retainLines":   true,
 		"highlightCode": false,
 	}
+
+	compilerInstance *Compiler
+	once             sync.Once
 )
 
 // A Compiler uses Babel to compile ES6 code into something ES5-compatible.
@@ -58,6 +60,21 @@ type Compiler struct {
 
 // Constructs a new compiler.
 func New() (*Compiler, error) {
+	var err error
+	once.Do(func() {
+		compilerInstance, err = new()
+	})
+
+	return compilerInstance, err
+}
+
+func new() (*Compiler, error) {
+	conf := rice.Config{
+		LocateOrder: []rice.LocateMethod{rice.LocateEmbedded},
+	}
+
+	babelSrc := conf.MustFindBox("lib").MustString("babel.min.js")
+
 	c := &Compiler{vm: goja.New()}
 	if _, err := c.vm.RunString(babelSrc); err != nil {
 		return nil, err

js/compiler/compiler_test.go

@@ -116,9 +116,8 @@ func TestCompile(t *testing.T) {
 			src := `1+(function() { return 2; )()`
 			_, _, err := c.Compile(src, "script.js", "", "", true)
 			assert.IsType(t, &goja.Exception{}, err)
-			assert.EqualError(t, err, `SyntaxError: script.js: Unexpected token (1:26)
-> 1 | 1+(function() { return 2; )()
-    |                           ^ at <eval>:2:26853(114)`)
+			assert.Contains(t, err.Error(), `SyntaxError: script.js: Unexpected token (1:26)
+> 1 | 1+(function() { return 2; )()`)
 		})
 	})
 	t.Run("ES6", func(t *testing.T) {
@@ -156,9 +155,8 @@ func TestCompile(t *testing.T) {
 		t.Run("Invalid", func(t *testing.T) {
 			_, _, err := c.Compile(`1+(=>2)()`, "script.js", "", "", true)
 			assert.IsType(t, &goja.Exception{}, err)
-			assert.EqualError(t, err, `SyntaxError: script.js: Unexpected token (1:3)
-> 1 | 1+(=>2)()
-    |    ^ at <eval>:2:26853(114)`)
+			assert.Contains(t, err.Error(), `SyntaxError: script.js: Unexpected token (1:3)
+> 1 | 1+(=>2)()`)
 		})
 	})
 }

js/initcontext.go

@@ -41,7 +41,8 @@ type programWithSource struct {
 // Provides APIs for use in the init context.
 type InitContext struct {
 	// Bound runtime; used to instantiate objects.
-	runtime *goja.Runtime
+	runtime  *goja.Runtime
+	compiler *compiler.Compiler
 
 	// Pointer to a context that bridged modules are invoked with.
 	ctxPtr *context.Context
@@ -55,12 +56,13 @@ type InitContext struct {
 	files    map[string][]byte
 }
 
-func NewInitContext(rt *goja.Runtime, ctxPtr *context.Context, fs afero.Fs, pwd string) *InitContext {
+func NewInitContext(rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string) *InitContext {
 	return &InitContext{
-		runtime: rt,
-		ctxPtr:  ctxPtr,
-		fs:      fs,
-		pwd:     pwd,
+		runtime:  rt,
+		compiler: compiler,
+		ctxPtr:   ctxPtr,
+		fs:       fs,
+		pwd:      pwd,
 
 		programs: make(map[string]programWithSource),
 		files:    make(map[string][]byte),
@@ -156,7 +158,7 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) {
 }
 
 func (i *InitContext) compileImport(src, filename string) (*goja.Program, error) {
-	pgm, _, err := compiler.Compile(src, filename, "(function(){", "})()", true)
+	pgm, _, err := i.compiler.Compile(src, filename, "(function(){", "})()", true)
 	return pgm, err
 }
 

js/initcontext_test.go

@@ -118,7 +118,7 @@ func TestInitContextRequire(t *testing.T) {
 				Filename: "/script.js",
 				Data:     []byte(`import "/file.js"; export default function() {}`),
 			}, fs, lib.RuntimeOptions{})
-			assert.EqualError(t, err, "SyntaxError: /file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n    | ^ at <eval>:2:26853(114)")
+			assert.Contains(t, err.Error(), "SyntaxError: /file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n")
 		})
 		t.Run("Error", func(t *testing.T) {
 			fs := afero.NewMemMapFs()

js/lib/lib.go

@@ -18,15 +18,19 @@
  *
  */
 
+//go:generate rice embed-go
+
 package lib
 
 import (
 	"github.com/GeertJohan/go.rice"
 	"github.com/dop251/goja"
 )
 
-var CoreJS *goja.Program = goja.MustCompile(
-	"core-js/shim.js",
-	rice.MustFindBox("core-js").MustString("client/shim.js"),
-	true,
-)
+func GetCoreJS() *goja.Program {
+	return goja.MustCompile(
+		"core-js/shim.min.js",
+		rice.MustFindBox("core-js").MustString("shim.min.js"),
+		true,
+	)
+}