Rewrite script/files loading to be be url based

This includes also change to the archive file structure. Now instead of
separating files by whether they are scripts or no, they are separated
based on whether their URI scheme.

Through the (majority) of k6 now instead of simple strings a url.URL is
used to idenitify files(scripts or otherwise).

This also means that all imports or `open` can have schemes.
Previously remote modules were supported specifically without scheme.
With this change we specifically prefer if they are with a scheme. The
old variant is supported but logs a warning.
Additionally if remote module requires a relative/absolute path that doesn't have a
scheme it is relative/absolute given the remote module url.

Because of some of the changes, now caching is done through a afero.Fs
instead of additional map. This also fixes not laoding remotely imported
files from an archive, but instead requesting them again.

fixes #1037, closes #838, fixes #887 and fixes #1051

api/v1/setup_teardown_routes_test.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 	"time"
 
@@ -130,9 +131,10 @@ func TestSetupData(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
+		testCase := testCase
 		t.Run(testCase.name, func(t *testing.T) {
 			runner, err := js.New(
-				&lib.SourceData{Filename: "/script.js", Data: testCase.script},
+				&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
 				afero.NewMemMapFs(),
 				lib.RuntimeOptions{},
 			)

cmd/run.go

@@ -29,9 +29,9 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
@@ -508,13 +508,14 @@ func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData,
 		if err != nil {
 			return nil, err
 		}
-		return &lib.SourceData{Filename: "-", Data: data}, nil
+		return &lib.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil
 	}
-	abspath := filepath.Join(pwd, src)
-	if ok, _ := afero.Exists(fs, abspath); ok {
-		src = abspath
+	pwdURL := &url.URL{Scheme: "file", Path: pwd}
+	srcURL, err := loader.Resolve(pwdURL, src)
+	if err != nil {
+		return nil, err
 	}
-	return loader.Load(fs, pwd, src)
+	return loader.Load(map[string]afero.Fs{"file": fs, "https": afero.NewMemMapFs()}, srcURL, src)
 }
 
 // Creates a new runner.

cmd/runtime_options_test.go

@@ -23,6 +23,7 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
@@ -220,8 +221,8 @@ func TestEnvVars(t *testing.T) {
 
 			runner, err := newRunner(
 				&lib.SourceData{
-					Data:     []byte(jsCode),
-					Filename: "/script.js",
+					Data: []byte(jsCode),
+					URL:  &url.URL{Path: "/script.js"},
 				},
 				typeJS,
 				afero.NewOsFs(),
@@ -234,16 +235,15 @@ func TestEnvVars(t *testing.T) {
 			assert.NoError(t, archive.Write(archiveBuf))
 
 			getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) {
-				r, err := newRunner(
+				return newRunner(
 					&lib.SourceData{
-						Data:     []byte(archiveBuf.Bytes()),
-						Filename: "/script.tar",
+						Data: archiveBuf.Bytes(),
+						URL:  &url.URL{Path: "/script.js"},
 					},
 					typeArchive,
 					afero.NewOsFs(),
 					rtOpts,
 				)
-				return r, err
 			}
 
 			_, err = getRunnerErr(lib.RuntimeOptions{})

converter/har/converter_test.go

@@ -57,8 +57,8 @@ func TestBuildK6RequestObject(t *testing.T) {
 	v, err := buildK6RequestObject(req)
 	assert.NoError(t, err)
 	_, err = js.New(&lib.SourceData{
-		Filename: "/script.js",
-		Data:     []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
+		URL:  &url.URL{Path: "/script.js"},
+		Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
 	}, afero.NewMemMapFs(), lib.RuntimeOptions{})
 	assert.NoError(t, err)
 }

core/engine_test.go

@@ -23,6 +23,7 @@ package core
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"testing"
 	"time"
 
@@ -556,7 +557,7 @@ func TestSentReceivedMetrics(t *testing.T) {
 
 	runTest := func(t *testing.T, ts testScript, tc testCase, noConnReuse bool) (float64, float64) {
 		r, err := js.New(
-			&lib.SourceData{Filename: "/script.js", Data: []byte(ts.Code)},
+			&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)},
 			afero.NewMemMapFs(),
 			lib.RuntimeOptions{},
 		)
@@ -697,7 +698,7 @@ func TestRunTags(t *testing.T) {
 	`))
 
 	r, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)
@@ -797,7 +798,7 @@ func TestSetupTeardownThresholds(t *testing.T) {
 	`))
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)
@@ -860,7 +861,7 @@ func TestEmittedMetricsWhenScalingDown(t *testing.T) {
 	`))
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)
@@ -920,7 +921,7 @@ func TestMinIterationDuration(t *testing.T) {
 	t.Parallel()
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: []byte(`
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(`
 		import { Counter } from "k6/metrics";
 
 		let testCounter = new Counter("testcounter");

core/local/local_test.go

@@ -23,6 +23,7 @@ package local
 import (
 	"context"
 	"net"
+	"net/url"
 	"runtime"
 	"sync/atomic"
 	"testing"
@@ -481,7 +482,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) {
 	}`)
 
 	runner, err := js.New(
-		&lib.SourceData{Filename: "/script.js", Data: script},
+		&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
 		afero.NewMemMapFs(),
 		lib.RuntimeOptions{},
 	)

js/bundle.go

@@ -23,7 +23,8 @@ package js
 import (
 	"context"
 	"encoding/json"
-	"os"
+	"net/url"
+	"time"
 
 	"github.com/dop251/goja"
 	"github.com/loadimpact/k6/js/common"
@@ -38,7 +39,7 @@ import (
 // A Bundle is a self-contained bundle of scripts and resources.
 // You can use this to produce identical BundleInstance objects.
 type Bundle struct {
-	Filename string
+	Filename *url.URL
 	Source   string
 	Program  *goja.Program
 	Options  lib.Options
@@ -55,16 +56,32 @@ type BundleInstance struct {
 	Default goja.Callable
 }
 
+type cacheOnReadFs struct {
+	afero.Fs
+	cache afero.Fs
+}
+
+func newCacheOnReadFs(base, layer afero.Fs, cacheTime time.Duration) afero.Fs {
+	return cacheOnReadFs{
+		Fs:    afero.NewCacheOnReadFs(base, layer, cacheTime),
+		cache: layer,
+	}
+}
+
+func (c cacheOnReadFs) GetCachedFs() afero.Fs {
+	return c.cache
+}
+
 // NewBundle creates a new bundle from a source file and a filesystem.
-func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) {
+func NewBundle(src *lib.SourceData, fileFS afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) {
 	compiler, err := compiler.New()
 	if err != nil {
 		return nil, err
 	}
 
 	// Compile sources, both ES5 and ES6 are supported.
 	code := string(src.Data)
-	pgm, _, err := compiler.Compile(code, src.Filename, "", "", true)
+	pgm, _, err := compiler.Compile(code, src.URL.String(), "", "", true)
 	if err != nil {
 		return nil, err
 	}
@@ -73,15 +90,19 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu
 	// written every time something is read from the real filesystem. This cache is then used for
 	// successive spawns to read from (they have no access to the real disk).
 	mirrorFS := afero.NewMemMapFs()
-	cachedFS := afero.NewCacheOnReadFs(fs, mirrorFS, 0)
+	cachedFS := newCacheOnReadFs(fileFS, mirrorFS, 0)
+	fses := map[string]afero.Fs{
+		"file":  cachedFS,
+		"https": afero.NewMemMapFs(),
+	}
 
 	// Make a bundle, instantiate it into a throwaway VM to populate caches.
 	rt := goja.New()
 	bundle := Bundle{
-		Filename:        src.Filename,
+		Filename:        src.URL,
 		Source:          code,
 		Program:         pgm,
-		BaseInitContext: NewInitContext(rt, compiler, new(context.Context), cachedFS, loader.Dir(src.Filename)),
+		BaseInitContext: NewInitContext(rt, compiler, new(context.Context), fses, loader.Dir(src.URL)),
 		Env:             rtOpts.Env,
 	}
 	if err := bundle.instantiate(rt, bundle.BaseInitContext); err != nil {
@@ -146,8 +167,17 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 	if err != nil {
 		return nil, err
 	}
-	initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FS, arc.Pwd)
-	initctx.files = arc.Files
+	pwdURL, err := loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd)
+	if err != nil {
+		return nil, err
+	}
+
+	filenameURL, err := loader.Resolve(pwdURL, arc.Filename)
+	if err != nil {
+		return nil, err
+	}
+
+	initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FSes, pwdURL)
 
 	env := arc.Env
 	if env == nil {
@@ -159,7 +189,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 	}
 
 	bundle := &Bundle{
-		Filename:        arc.Filename,
+		Filename:        filenameURL,
 		Source:          string(arc.Data),
 		Program:         pgm,
 		Options:         arc.Options,
@@ -175,28 +205,18 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
 func (b *Bundle) makeArchive() *lib.Archive {
 	arc := &lib.Archive{
 		Type:     "js",
-		FS:       afero.NewMemMapFs(),
+		FSes:     b.BaseInitContext.fses,
 		Options:  b.Options,
-		Filename: b.Filename,
+		Filename: b.Filename.String(),
 		Data:     []byte(b.Source),
-		Pwd:      b.BaseInitContext.pwd,
+		Pwd:      b.BaseInitContext.pwd.String(),
 		Env:      make(map[string]string, len(b.Env)),
 	}
 	// Copy env so changes in the archive are not reflected in the source Bundle
 	for k, v := range b.Env {
 		arc.Env[k] = v
 	}
 
-	arc.Scripts = make(map[string][]byte, len(b.BaseInitContext.programs))
-	for name, pgm := range b.BaseInitContext.programs {
-		arc.Scripts[name] = []byte(pgm.src)
-		err := afero.WriteFile(arc.FS, name, []byte(pgm.src), os.ModePerm)
-		if err != nil {
-			return nil
-		}
-	}
-	arc.Files = b.BaseInitContext.files
-
 	return arc
 }