Rewrite script/files loading to be be url based

.golangci.yml

@@ -38,7 +38,7 @@ linters-settings:
   dupl:
     threshold: 150
   goconst:
-    min-len: 5
+    min-len: 10
     min-occurrences: 4
 
 linters:

api/v1/setup_teardown_routes_test.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 	"time"
 
@@ -34,8 +35,8 @@ import (
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/types"
+	"github.com/loadimpact/k6/loader"
 	"github.com/manyminds/api2go/jsonapi"
-	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	null "gopkg.in/guregu/null.v3"
@@ -130,10 +131,11 @@ func TestSetupData(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
+		testCase := testCase
 		t.Run(testCase.name, func(t *testing.T) {
 			runner, err := js.New(
-				&lib.SourceData{Filename: "/script.js", Data: testCase.script},
-				afero.NewMemMapFs(),
+				&loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
+				nil,
 				lib.RuntimeOptions{},
 			)
 			require.NoError(t, err)

cmd/archive.go

@@ -23,6 +23,7 @@ package cmd
 import (
 	"os"
 
+	"github.com/loadimpact/k6/loader"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -51,8 +52,8 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -62,7 +63,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -71,7 +72,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/cloud.go

@@ -32,6 +32,7 @@ import (
 	"github.com/kelseyhightower/envconfig"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/consts"
+	"github.com/loadimpact/k6/loader"
 	"github.com/loadimpact/k6/stats/cloud"
 	"github.com/loadimpact/k6/ui"
 	"github.com/pkg/errors"
@@ -71,8 +72,8 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		}
 
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -82,7 +83,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -91,7 +92,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/collectors.go

@@ -29,6 +29,7 @@ import (
 	"github.com/kelseyhightower/envconfig"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/consts"
+	"github.com/loadimpact/k6/loader"
 	"github.com/loadimpact/k6/stats/cloud"
 	"github.com/loadimpact/k6/stats/datadog"
 	"github.com/loadimpact/k6/stats/influxdb"
@@ -61,7 +62,7 @@ func parseCollector(s string) (t, arg string) {
 	}
 }
 
-func newCollector(collectorName, arg string, src *lib.SourceData, conf Config) (lib.Collector, error) {
+func newCollector(collectorName, arg string, src *loader.SourceData, conf Config) (lib.Collector, error) {
 	getCollector := func() (lib.Collector, error) {
 		switch collectorName {
 		case collectorJSON:

cmd/inspect.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
+	"github.com/loadimpact/k6/loader"
 	"github.com/spf13/cobra"
 )
 
@@ -43,8 +43,8 @@ var inspectCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		fs := afero.NewOsFs()
-		src, err := readSource(args[0], pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(args[0], pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -59,20 +59,24 @@ var inspectCmd = &cobra.Command{
 			return err
 		}
 
-		var opts lib.Options
+		var (
+			opts lib.Options
+			b    *js.Bundle
+		)
 		switch typ {
 		case typeArchive:
-			arc, err := lib.ReadArchive(bytes.NewBuffer(src.Data))
+			var arc *lib.Archive
+			arc, err = lib.ReadArchive(bytes.NewBuffer(src.Data))
 			if err != nil {
 				return err
 			}
-			b, err := js.NewBundleFromArchive(arc, runtimeOptions)
+			b, err = js.NewBundleFromArchive(arc, runtimeOptions)
 			if err != nil {
 				return err
 			}
 			opts = b.Options
 		case typeJS:
-			b, err := js.NewBundle(src, fs, runtimeOptions)
+			b, err = js.NewBundle(src, filesystems, runtimeOptions)
 			if err != nil {
 				return err
 			}

cmd/run.go

@@ -26,12 +26,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
@@ -116,8 +113,8 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -127,7 +124,7 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -138,7 +135,7 @@ a commandline interface for interacting with it.`,
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, cliConf, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), cliConf, r)
 		if err != nil {
 			return err
 		}
@@ -503,29 +500,15 @@ func init() {
 	runCmd.Flags().AddFlagSet(runCmdFlagSet())
 }
 
-// Reads a source file from any supported destination.
-func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, error) {
-	if src == "-" {
-		data, err := ioutil.ReadAll(stdin)
-		if err != nil {
-			return nil, err
-		}
-		return &lib.SourceData{Filename: "-", Data: data}, nil
-	}
-	abspath := filepath.Join(pwd, src)
-	if ok, _ := afero.Exists(fs, abspath); ok {
-		src = abspath
-	}
-	return loader.Load(fs, pwd, src)
-}
-
 // Creates a new runner.
-func newRunner(src *lib.SourceData, typ string, fs afero.Fs, rtOpts lib.RuntimeOptions) (lib.Runner, error) {
+func newRunner(
+	src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions,
+) (lib.Runner, error) {
 	switch typ {
 	case "":
-		return newRunner(src, detectType(src.Data), fs, rtOpts)
+		return newRunner(src, detectType(src.Data), filesystems, rtOpts)
 	case typeJS:
-		return js.New(src, fs, rtOpts)
+		return js.New(src, filesystems, rtOpts)
 	case typeArchive:
 		arc, err := lib.ReadArchive(bytes.NewReader(src.Data))
 		if err != nil {

cmd/runtime_options_test.go

@@ -23,12 +23,14 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
 	"testing"
 
 	"github.com/loadimpact/k6/lib"
+	"github.com/loadimpact/k6/loader"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -226,13 +228,15 @@ func TestEnvVars(t *testing.T) {
 				}
 			}
 
+			fs := afero.NewMemMapFs()
+			require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(jsCode), 0644))
 			runner, err := newRunner(
-				&lib.SourceData{
-					Data:     []byte(jsCode),
-					Filename: "/script.js",
+				&loader.SourceData{
+					Data: []byte(jsCode),
+					URL:  &url.URL{Path: "/script.js", Scheme: "file"},
 				},
 				typeJS,
-				afero.NewOsFs(),
+				map[string]afero.Fs{"file": fs},
 				rtOpts,
 			)
 			require.NoError(t, err)
@@ -242,16 +246,15 @@ func TestEnvVars(t *testing.T) {
 			assert.NoError(t, archive.Write(archiveBuf))
 
 			getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) {
-				r, err := newRunner(
-					&lib.SourceData{
-						Data:     []byte(archiveBuf.Bytes()),
-						Filename: "/script.tar",
+				return newRunner(
+					&loader.SourceData{
+						Data: archiveBuf.Bytes(),
+						URL:  &url.URL{Path: "/script.js"},
 					},
 					typeArchive,
-					afero.NewOsFs(),
+					nil,
 					rtOpts,
 				)
-				return r, err
 			}
 
 			_, err = getRunnerErr(lib.RuntimeOptions{})

converter/har/converter_test.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
+	"github.com/loadimpact/k6/loader"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -56,10 +56,10 @@ func TestBuildK6RequestObject(t *testing.T) {
 	}
 	v, err := buildK6RequestObject(req)
 	assert.NoError(t, err)
-	_, err = js.New(&lib.SourceData{
-		Filename: "/script.js",
-		Data:     []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
-	}, afero.NewMemMapFs(), lib.RuntimeOptions{})
+	_, err = js.New(&loader.SourceData{
+		URL:  &url.URL{Path: "/script.js"},
+		Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
+	}, nil, lib.RuntimeOptions{})
 	assert.NoError(t, err)
 }