Rewrite script/files loading to be be url based

api/v1/setup_teardown_routes_test.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 	"time"
 
@@ -34,8 +35,8 @@ import (
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/types"
+	"github.com/loadimpact/k6/loader"
 	"github.com/manyminds/api2go/jsonapi"
-	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	null "gopkg.in/guregu/null.v3"
@@ -130,10 +131,11 @@ func TestSetupData(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
+		testCase := testCase
 		t.Run(testCase.name, func(t *testing.T) {
 			runner, err := js.New(
-				&lib.SourceData{Filename: "/script.js", Data: testCase.script},
-				afero.NewMemMapFs(),
+				&loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
+				nil,
 				lib.RuntimeOptions{},
 			)
 			require.NoError(t, err)

cmd/archive.go

@@ -51,8 +51,8 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -62,7 +62,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -71,7 +71,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/cloud.go

@@ -71,8 +71,8 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		}
 
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -82,7 +82,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -91,7 +91,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/collectors.go

@@ -29,6 +29,7 @@ import (
 	"github.com/kelseyhightower/envconfig"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/consts"
+	"github.com/loadimpact/k6/loader"
 	"github.com/loadimpact/k6/stats/cloud"
 	"github.com/loadimpact/k6/stats/datadog"
 	"github.com/loadimpact/k6/stats/influxdb"
@@ -61,7 +62,7 @@ func parseCollector(s string) (t, arg string) {
 	}
 }
 
-func newCollector(collectorName, arg string, src *lib.SourceData, conf Config) (lib.Collector, error) {
+func newCollector(collectorName, arg string, src *loader.SourceData, conf Config) (lib.Collector, error) {
 	getCollector := func() (lib.Collector, error) {
 		switch collectorName {
 		case collectorJSON:

cmd/inspect.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 )
 
@@ -43,8 +42,8 @@ var inspectCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		fs := afero.NewOsFs()
-		src, err := readSource(args[0], pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(args[0], pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -59,20 +58,24 @@ var inspectCmd = &cobra.Command{
 			return err
 		}
 
-		var opts lib.Options
+		var (
+			opts lib.Options
+			b    *js.Bundle
+		)
 		switch typ {
 		case typeArchive:
-			arc, err := lib.ReadArchive(bytes.NewBuffer(src.Data))
+			var arc *lib.Archive
+			arc, err = lib.ReadArchive(bytes.NewBuffer(src.Data))
 			if err != nil {
 				return err
 			}
-			b, err := js.NewBundleFromArchive(arc, runtimeOptions)
+			b, err = js.NewBundleFromArchive(arc, runtimeOptions)
 			if err != nil {
 				return err
 			}
 			opts = b.Options
 		case typeJS:
-			b, err := js.NewBundle(src, fs, runtimeOptions)
+			b, err = js.NewBundle(src, filesystems, runtimeOptions)
 			if err != nil {
 				return err
 			}

cmd/run.go

@@ -29,6 +29,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -43,6 +44,7 @@ import (
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/consts"
+	"github.com/loadimpact/k6/lib/fsext"
 	"github.com/loadimpact/k6/lib/types"
 	"github.com/loadimpact/k6/loader"
 	"github.com/loadimpact/k6/ui"
@@ -116,8 +118,8 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -127,7 +129,7 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -138,7 +140,7 @@ a commandline interface for interacting with it.`,
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, cliConf, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), cliConf, r)
 		if err != nil {
 			return err
 		}
@@ -503,29 +505,55 @@ func init() {
 	runCmd.Flags().AddFlagSet(runCmdFlagSet())
 }
 
+func createFilesystems() map[string]afero.Fs {
+	// We want to eliminate disk access at runtime, so we set up a memory mapped cache that's
+	// written every time something is read from the real filesystem. This cache is then used for
+	// successive spawns to read from (they have no access to the real disk).
+	// Also initialize the same for `https` but the caching is handled manually in the loader package
+	osfs := afero.NewOsFs()
+	if runtime.GOOS == "windows" {
+		// This is done so that we can continue to use paths with /|"\" through the code but also to
+		// be easier to travers the cachedFs later as it doesn't work very well if you have windows
+		// volumes
+		osfs = fsext.NewTrimFilePathSeparatorFs(osfs)
+	}
+	return map[string]afero.Fs{
+		"file":  fsext.NewCacheOnReadFs(osfs, afero.NewMemMapFs(), 0),
+		"https": afero.NewMemMapFs(),
+	}
+}
+
 // Reads a source file from any supported destination.
-func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, error) {
+func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*loader.SourceData, error) {
 	if src == "-" {
 		data, err := ioutil.ReadAll(stdin)
 		if err != nil {
 			return nil, err
 		}
-		return &lib.SourceData{Filename: "-", Data: data}, nil
+		return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil
+	}
+	pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"}
+	srcLocalPath := filepath.Join(pwd, src)
+	if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok {
+		// there is file on the local disk ... lets use it :)
+		return loader.Load(filesystems, &url.URL{Scheme: "file", Path: srcLocalPath}, src)
 	}
-	abspath := filepath.Join(pwd, src)
-	if ok, _ := afero.Exists(fs, abspath); ok {
-		src = abspath
+	srcURL, err := loader.Resolve(pwdURL, filepath.ToSlash(src))
+	if err != nil {
+		return nil, err
 	}
-	return loader.Load(fs, pwd, src)
+	return loader.Load(filesystems, srcURL, src)
 }
 
 // Creates a new runner.
-func newRunner(src *lib.SourceData, typ string, fs afero.Fs, rtOpts lib.RuntimeOptions) (lib.Runner, error) {
+func newRunner(
+	src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions,
+) (lib.Runner, error) {
 	switch typ {
 	case "":
-		return newRunner(src, detectType(src.Data), fs, rtOpts)
+		return newRunner(src, detectType(src.Data), filesystems, rtOpts)
 	case typeJS:
-		return js.New(src, fs, rtOpts)
+		return js.New(src, filesystems, rtOpts)
 	case typeArchive:
 		arc, err := lib.ReadArchive(bytes.NewReader(src.Data))
 		if err != nil {

cmd/runtime_options_test.go

@@ -23,12 +23,14 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
 	"testing"
 
 	"github.com/loadimpact/k6/lib"
+	"github.com/loadimpact/k6/loader"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -226,13 +228,15 @@ func TestEnvVars(t *testing.T) {
 				}
 			}
 
+			fs := afero.NewMemMapFs()
+			require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(jsCode), 0644))
 			runner, err := newRunner(
-				&lib.SourceData{
-					Data:     []byte(jsCode),
-					Filename: "/script.js",
+				&loader.SourceData{
+					Data: []byte(jsCode),
+					URL:  &url.URL{Path: "/script.js", Scheme: "file"},
 				},
 				typeJS,
-				afero.NewOsFs(),
+				map[string]afero.Fs{"file": fs},
 				rtOpts,
 			)
 			require.NoError(t, err)
@@ -242,16 +246,15 @@ func TestEnvVars(t *testing.T) {
 			assert.NoError(t, archive.Write(archiveBuf))
 
 			getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) {
-				r, err := newRunner(
-					&lib.SourceData{
-						Data:     []byte(archiveBuf.Bytes()),
-						Filename: "/script.tar",
+				return newRunner(
+					&loader.SourceData{
+						Data: archiveBuf.Bytes(),
+						URL:  &url.URL{Path: "/script.js"},
 					},
 					typeArchive,
-					afero.NewOsFs(),
+					nil,
 					rtOpts,
 				)
-				return r, err
 			}
 
 			_, err = getRunnerErr(lib.RuntimeOptions{})

converter/har/converter_test.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
+	"github.com/loadimpact/k6/loader"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -56,10 +56,10 @@ func TestBuildK6RequestObject(t *testing.T) {
 	}
 	v, err := buildK6RequestObject(req)
 	assert.NoError(t, err)
-	_, err = js.New(&lib.SourceData{
-		Filename: "/script.js",
-		Data:     []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
-	}, afero.NewMemMapFs(), lib.RuntimeOptions{})
+	_, err = js.New(&loader.SourceData{
+		URL:  &url.URL{Path: "/script.js"},
+		Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
+	}, nil, lib.RuntimeOptions{})
 	assert.NoError(t, err)
 }