Rewrite script/files loading to be be url based

api/v1/setup_teardown_routes_test.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 	"time"
 
@@ -35,7 +36,6 @@ import (
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/types"
 	"github.com/manyminds/api2go/jsonapi"
-	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	null "gopkg.in/guregu/null.v3"
@@ -130,10 +130,11 @@ func TestSetupData(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
+		testCase := testCase
 		t.Run(testCase.name, func(t *testing.T) {
 			runner, err := js.New(
-				&lib.SourceData{Filename: "/script.js", Data: testCase.script},
-				afero.NewMemMapFs(),
+				&lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
+				nil,
 				lib.RuntimeOptions{},
 			)
 			require.NoError(t, err)

cmd/archive.go

@@ -23,7 +23,6 @@ package cmd
 import (
 	"os"
 
-	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -51,8 +50,8 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -62,7 +61,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -71,7 +70,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(filesystems["file"], Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/cloud.go

@@ -35,7 +35,6 @@ import (
 	"github.com/loadimpact/k6/stats/cloud"
 	"github.com/loadimpact/k6/ui"
 	"github.com/pkg/errors"
-	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
@@ -71,8 +70,8 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		}
 
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -82,7 +81,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -91,7 +90,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(filesystems["file"], Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/inspect.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 )
 
@@ -43,8 +42,8 @@ var inspectCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		fs := afero.NewOsFs()
-		src, err := readSource(args[0], pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(args[0], pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -59,20 +58,24 @@ var inspectCmd = &cobra.Command{
 			return err
 		}
 
-		var opts lib.Options
+		var (
+			opts lib.Options
+			b    *js.Bundle
+		)
 		switch typ {
 		case typeArchive:
-			arc, err := lib.ReadArchive(bytes.NewBuffer(src.Data))
+			var arc *lib.Archive
+			arc, err = lib.ReadArchive(bytes.NewBuffer(src.Data))
 			if err != nil {
 				return err
 			}
-			b, err := js.NewBundleFromArchive(arc, runtimeOptions)
+			b, err = js.NewBundleFromArchive(arc, runtimeOptions)
 			if err != nil {
 				return err
 			}
 			opts = b.Options
 		case typeJS:
-			b, err := js.NewBundle(src, fs, runtimeOptions)
+			b, err = js.NewBundle(src, filesystems, runtimeOptions)
 			if err != nil {
 				return err
 			}

cmd/run.go

@@ -29,6 +29,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -43,6 +44,7 @@ import (
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/consts"
+	"github.com/loadimpact/k6/lib/fsext"
 	"github.com/loadimpact/k6/lib/types"
 	"github.com/loadimpact/k6/loader"
 	"github.com/loadimpact/k6/ui"
@@ -116,8 +118,8 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := createFilesystems()
+		src, err := readSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -127,7 +129,7 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -138,7 +140,7 @@ a commandline interface for interacting with it.`,
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, cliConf, r)
+		conf, err := getConsolidatedConfig(filesystems["file"], cliConf, r)
 		if err != nil {
 			return err
 		}
@@ -503,29 +505,45 @@ func init() {
 	runCmd.Flags().AddFlagSet(runCmdFlagSet())
 }
 
+func createFilesystems() map[string]afero.Fs {
+	// We want to eliminate disk access at runtime, so we set up a memory mapped cache that's
+	// written every time something is read from the real filesystem. This cache is then used for
+	// successive spawns to read from (they have no access to the real disk).
+	// Also initialize the same for `https` but the caching is handled manually in the loader package
+	return map[string]afero.Fs{
+		"file": fsext.NewCacheOnReadFs(
+			fsext.NewUnprependPathFs(afero.NewOsFs(), afero.FilePathSeparator),
+			afero.NewMemMapFs(), 0),
+		"https": afero.NewMemMapFs(),
+	}
+}
+
 // Reads a source file from any supported destination.
-func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, error) {
+func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*lib.SourceData, error) {
 	if src == "-" {
 		data, err := ioutil.ReadAll(stdin)
 		if err != nil {
 			return nil, err
 		}
-		return &lib.SourceData{Filename: "-", Data: data}, nil
+		return &lib.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil
 	}
-	abspath := filepath.Join(pwd, src)
-	if ok, _ := afero.Exists(fs, abspath); ok {
-		src = abspath
+	pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"}
+	srcURL, err := loader.Resolve(pwdURL, filepath.ToSlash(src))
+	if err != nil {
+		return nil, err
 	}
-	return loader.Load(fs, pwd, src)
+	return loader.Load(filesystems, srcURL, src)
 }
 
 // Creates a new runner.
-func newRunner(src *lib.SourceData, typ string, fs afero.Fs, rtOpts lib.RuntimeOptions) (lib.Runner, error) {
+func newRunner(
+	src *lib.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions,
+) (lib.Runner, error) {
 	switch typ {
 	case "":
-		return newRunner(src, detectType(src.Data), fs, rtOpts)
+		return newRunner(src, detectType(src.Data), filesystems, rtOpts)
 	case typeJS:
-		return js.New(src, fs, rtOpts)
+		return js.New(src, filesystems, rtOpts)
 	case typeArchive:
 		arc, err := lib.ReadArchive(bytes.NewReader(src.Data))
 		if err != nil {

cmd/runtime_options_test.go

@@ -23,13 +23,13 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
 	"testing"
 
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -220,11 +220,11 @@ func TestEnvVars(t *testing.T) {
 
 			runner, err := newRunner(
 				&lib.SourceData{
-					Data:     []byte(jsCode),
-					Filename: "/script.js",
+					Data: []byte(jsCode),
+					URL:  &url.URL{Path: "/script.js"},
 				},
 				typeJS,
-				afero.NewOsFs(),
+				nil,
 				rtOpts,
 			)
 			require.NoError(t, err)
@@ -234,16 +234,15 @@ func TestEnvVars(t *testing.T) {
 			assert.NoError(t, archive.Write(archiveBuf))
 
 			getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) {
-				r, err := newRunner(
+				return newRunner(
 					&lib.SourceData{
-						Data:     []byte(archiveBuf.Bytes()),
-						Filename: "/script.tar",
+						Data: archiveBuf.Bytes(),
+						URL:  &url.URL{Path: "/script.js"},
 					},
 					typeArchive,
-					afero.NewOsFs(),
+					nil,
 					rtOpts,
 				)
-				return r, err
 			}
 
 			_, err = getRunnerErr(lib.RuntimeOptions{})

converter/har/converter_test.go

@@ -27,7 +27,6 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -57,9 +56,9 @@ func TestBuildK6RequestObject(t *testing.T) {
 	v, err := buildK6RequestObject(req)
 	assert.NoError(t, err)
 	_, err = js.New(&lib.SourceData{
-		Filename: "/script.js",
-		Data:     []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
-	}, afero.NewMemMapFs(), lib.RuntimeOptions{})
+		URL:  &url.URL{Path: "/script.js"},
+		Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
+	}, nil, lib.RuntimeOptions{})
 	assert.NoError(t, err)
 }