Rewrite script/files loading to be be url based

api/v1/setup_teardown_routes_test.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 	"time"
 
@@ -130,9 +131,10 @@ func TestSetupData(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
+		testCase := testCase
 		t.Run(testCase.name, func(t *testing.T) {
 			runner, err := js.New(
-				&lib.SourceData{Filename: "/script.js", Data: testCase.script},
+				&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
 				afero.NewMemMapFs(),
 				lib.RuntimeOptions{},
 			)

cmd/run.go

@@ -29,6 +29,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -508,13 +509,14 @@ func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData,
 		if err != nil {
 			return nil, err
 		}
-		return &lib.SourceData{Filename: "-", Data: data}, nil
+		return &lib.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil
 	}
-	abspath := filepath.Join(pwd, src)
-	if ok, _ := afero.Exists(fs, abspath); ok {
-		src = abspath
+	pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd))}
+	srcURL, err := loader.Resolve(pwdURL, src)
+	if err != nil {
+		return nil, err
 	}
-	return loader.Load(fs, pwd, src)
+	return loader.Load(map[string]afero.Fs{"file": fs, "https": afero.NewMemMapFs()}, srcURL, src)
 }
 
 // Creates a new runner.

cmd/runtime_options_test.go

@@ -23,6 +23,7 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
@@ -220,8 +221,8 @@ func TestEnvVars(t *testing.T) {
 
 			runner, err := newRunner(
 				&lib.SourceData{
-					Data:     []byte(jsCode),
-					Filename: "/script.js",
+					Data: []byte(jsCode),
+					URL:  &url.URL{Path: "/script.js"},
 				},
 				typeJS,
 				afero.NewOsFs(),
@@ -234,16 +235,15 @@ func TestEnvVars(t *testing.T) {
 			assert.NoError(t, archive.Write(archiveBuf))
 
 			getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) {
-				r, err := newRunner(
+				return newRunner(
 					&lib.SourceData{
-						Data:     []byte(archiveBuf.Bytes()),
-						Filename: "/script.tar",
+						Data: archiveBuf.Bytes(),
+						URL:  &url.URL{Path: "/script.js"},
 					},
 					typeArchive,
 					afero.NewOsFs(),
 					rtOpts,
 				)
-				return r, err
 			}
 
 			_, err = getRunnerErr(lib.RuntimeOptions{})

converter/har/converter_test.go

@@ -57,8 +57,8 @@ func TestBuildK6RequestObject(t *testing.T) {
 	v, err := buildK6RequestObject(req)
 	assert.NoError(t, err)
 	_, err = js.New(&lib.SourceData{
-		Filename: "/script.js",
-		Data:     []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
+		URL:  &url.URL{Path: "/script.js"},
+		Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
 	}, afero.NewMemMapFs(), lib.RuntimeOptions{})
 	assert.NoError(t, err)
 }

core/engine_test.go

@@ -23,6 +23,7 @@ package core
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"testing"
 	"time"
 
@@ -556,7 +557,7 @@ func TestSentReceivedMetrics(t *testing.T) {
 
 	runTest := func(t *testing.T, ts testScript, tc testCase, noConnReuse bool) (float64, float64) {
 		r, err := js.New(
-			&lib.SourceData{Filename: "/script.js", Data: []byte(ts.Code)},
+			&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)},
 			afero.NewMemMapFs(),
 			lib.RuntimeOptions{},
 		)
@@ -697,7 +698,7 @@ func TestRunTags(t *testing.T) {
 	`))
 
 	r, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)
@@ -797,7 +798,7 @@ func TestSetupTeardownThresholds(t *testing.T) {
 	`))
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)
@@ -860,7 +861,7 @@ func TestEmittedMetricsWhenScalingDown(t *testing.T) {
 	`))
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)
@@ -920,7 +921,7 @@ func TestMinIterationDuration(t *testing.T) {
 	t.Parallel()
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: []byte(`
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(`
 		import { Counter } from "k6/metrics";
 
 		let testCounter = new Counter("testcounter");

core/local/local_test.go

@@ -23,6 +23,7 @@ package local
 import (
 	"context"
 	"net"
+	"net/url"
 	"runtime"
 	"sync/atomic"
 	"testing"
@@ -481,7 +482,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) {
 	}`)
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)

js/bundle.go

@@ -23,7 +23,8 @@ package js
 import (
 	"context"
 	"encoding/json"
-	"os"
+	"net/url"
+	"time"
 
 	"github.com/dop251/goja"
 	"github.com/loadimpact/k6/js/common"
@@ -38,7 +39,7 @@ import (
 // A Bundle is a self-contained bundle of scripts and resources.
 // You can use this to produce identical BundleInstance objects.
 type Bundle struct {
-	Filename string
+	Filename *url.URL
 	Source   string
 	Program  *goja.Program
 	Options  lib.Options
@@ -55,16 +56,32 @@ type BundleInstance struct {
 	Default goja.Callable
 }
 
+type cacheOnReadFs struct {
+	afero.Fs
+	cache afero.Fs
+}
+
+func newCacheOnReadFs(base, layer afero.Fs, cacheTime time.Duration) afero.Fs {
+	return cacheOnReadFs{
+		Fs:    afero.NewCacheOnReadFs(base, layer, cacheTime),
+		cache: layer,
+	}
+}
+
+func (c cacheOnReadFs) GetCachedFs() afero.Fs {
+	return c.cache
+}
+
 // NewBundle creates a new bundle from a source file and a filesystem.
-func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) {
+func NewBundle(src *lib.SourceData, fileFS afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) {
 	compiler, err := compiler.New()
 	if err != nil {
 		return nil, err
 	}
 
 	// Compile sources, both ES5 and ES6 are supported.
 	code := string(src.Data)
-	pgm, _, err := compiler.Compile(code, src.Filename, "", "", true)
+	pgm, _, err := compiler.Compile(code, src.URL.String(), "", "", true)
 	if err != nil {
 		return nil, err
 	}
@@ -73,15 +90,19 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu
 	// written every time something is read from the real filesystem. This cache is then used for
 	// successive spawns to read from (they have no access to the real disk).
 	mirrorFS := afero.NewMemMapFs()
-	cachedFS := afero.NewCacheOnReadFs(fs, mirrorFS, 0)
+	cachedFS := newCacheOnReadFs(fileFS, mirrorFS, 0)
+	fses := map[string]afero.Fs{
+		"file":  cachedFS,
+		"https": afero.NewMemMapFs(),
+	}
 
 	// Make a bundle, instantiate it into a throwaway VM to populate caches.
 	rt := goja.New()
 	bundle := Bundle{
-		Filename:        src.Filename,
+		Filename:        src.URL,
 		Source:          code,
 		Program:         pgm,
-		BaseInitContext: NewInitContext(rt, compiler, new(context.Context), cachedFS, loader.Dir(src.Filename)),
+		BaseInitContext: NewInitContext(rt, compiler, new(context.Context), fses, loader.Dir(src.URL)),
 		Env:             rtOpts.Env,
 	}
 	if err := bundle.instantiate(rt, bundle.BaseInitContext); err != nil {
@@ -146,8 +167,17 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 	if err != nil {
 		return nil, err
 	}
-	initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FS, arc.Pwd)
-	initctx.files = arc.Files
+	pwdURL, err := loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd)
+	if err != nil {
+		return nil, err
+	}
+
+	filenameURL, err := loader.Resolve(pwdURL, arc.Filename)
+	if err != nil {
+		return nil, err
+	}
+
+	initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FSes, pwdURL)
 
 	env := arc.Env
 	if env == nil {
@@ -159,7 +189,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 	}
 
 	bundle := &Bundle{
-		Filename:        arc.Filename,
+		Filename:        filenameURL,
 		Source:          string(arc.Data),
 		Program:         pgm,
 		Options:         arc.Options,
@@ -175,28 +205,18 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 func (b *Bundle) makeArchive() *lib.Archive {
 	arc := &lib.Archive{
 		Type:     "js",
-		FS:       afero.NewMemMapFs(),
+		FSes:     b.BaseInitContext.fses,
 		Options:  b.Options,
-		Filename: b.Filename,
+		Filename: b.Filename.String(),
 		Data:     []byte(b.Source),
-		Pwd:      b.BaseInitContext.pwd,
+		Pwd:      b.BaseInitContext.pwd.String(),
 		Env:      make(map[string]string, len(b.Env)),
 	}
 	// Copy env so changes in the archive are not reflected in the source Bundle
 	for k, v := range b.Env {
 		arc.Env[k] = v
 	}
 
-	arc.Scripts = make(map[string][]byte, len(b.BaseInitContext.programs))
-	for name, pgm := range b.BaseInitContext.programs {
-		arc.Scripts[name] = []byte(pgm.src)
-		err := afero.WriteFile(arc.FS, name, []byte(pgm.src), os.ModePerm)
-		if err != nil {
-			return nil
-		}
-	}
-	arc.Files = b.BaseInitContext.files
-
 	return arc
 }