Rewrite script/files loading to be be url based (#1059)

This also includes a change to the archive file structure. Now instead of
separating files by whether they are scripts or not, they are separated
based on their URI scheme.

Throughout the (majority) of k6 now, instead of simple strings, a url.URL is
used to identify files (scripts or otherwise).
This also means that all imports can have schemes. Previously remote modules
were supported specifically without a scheme. With this change k6 prefers if
they are with a scheme. The old variant is supported, but logs a warning.
Additionally if remote module requires a relative/absolute path that doesn't
have a scheme, it is relative/absolute given the remote module url.

Because of some of the changes, now caching is done through a afero.Fs
instead of additional maps. This also fixes the remotely imported files not 
properly loading from an archive, but instead requesting them again. This is
supported with old archives and for github/cdnjs's shortcut loaders. 

For future-proof reasons the archive now also records the GOOS value of the 
k6 that generated it. 

Anonymize case insensitively as windows paths are (usually) case insensitive. 

fixes #1037, closes #838, fixes #887 and fixes #1051

.golangci.yml

@@ -38,7 +38,7 @@ linters-settings:
   dupl:
     threshold: 150
   goconst:
-    min-len: 5
+    min-len: 10
     min-occurrences: 4
 
 linters:

api/v1/setup_teardown_routes_test.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 	"time"
 
@@ -34,8 +35,8 @@ import (
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/types"
+	"github.com/loadimpact/k6/loader"
 	"github.com/manyminds/api2go/jsonapi"
-	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	null "gopkg.in/guregu/null.v3"
@@ -130,10 +131,11 @@ func TestSetupData(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
+		testCase := testCase
 		t.Run(testCase.name, func(t *testing.T) {
 			runner, err := js.New(
-				&lib.SourceData{Filename: "/script.js", Data: testCase.script},
-				afero.NewMemMapFs(),
+				&loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
+				nil,
 				lib.RuntimeOptions{},
 			)
 			require.NoError(t, err)

cmd/archive.go

@@ -23,6 +23,7 @@ package cmd
 import (
 	"os"
 
+	"github.com/loadimpact/k6/loader"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -51,8 +52,8 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -62,7 +63,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -71,7 +72,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/cloud.go

@@ -32,6 +32,7 @@ import (
 	"github.com/kelseyhightower/envconfig"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/consts"
+	"github.com/loadimpact/k6/loader"
 	"github.com/loadimpact/k6/stats/cloud"
 	"github.com/loadimpact/k6/ui"
 	"github.com/pkg/errors"
@@ -71,8 +72,8 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		}
 
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -82,7 +83,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -91,7 +92,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
 		if err != nil {
 			return err
 		}

cmd/collectors.go

@@ -29,6 +29,7 @@ import (
 	"github.com/kelseyhightower/envconfig"
 	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/consts"
+	"github.com/loadimpact/k6/loader"
 	"github.com/loadimpact/k6/stats/cloud"
 	"github.com/loadimpact/k6/stats/datadog"
 	"github.com/loadimpact/k6/stats/influxdb"
@@ -61,7 +62,7 @@ func parseCollector(s string) (t, arg string) {
 	}
 }
 
-func newCollector(collectorName, arg string, src *lib.SourceData, conf Config) (lib.Collector, error) {
+func newCollector(collectorName, arg string, src *loader.SourceData, conf Config) (lib.Collector, error) {
 	getCollector := func() (lib.Collector, error) {
 		switch collectorName {
 		case collectorJSON:

cmd/inspect.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
+	"github.com/loadimpact/k6/loader"
 	"github.com/spf13/cobra"
 )
 
@@ -43,8 +43,8 @@ var inspectCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		fs := afero.NewOsFs()
-		src, err := readSource(args[0], pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(args[0], pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -59,20 +59,24 @@ var inspectCmd = &cobra.Command{
 			return err
 		}
 
-		var opts lib.Options
+		var (
+			opts lib.Options
+			b    *js.Bundle
+		)
 		switch typ {
 		case typeArchive:
-			arc, err := lib.ReadArchive(bytes.NewBuffer(src.Data))
+			var arc *lib.Archive
+			arc, err = lib.ReadArchive(bytes.NewBuffer(src.Data))
 			if err != nil {
 				return err
 			}
-			b, err := js.NewBundleFromArchive(arc, runtimeOptions)
+			b, err = js.NewBundleFromArchive(arc, runtimeOptions)
 			if err != nil {
 				return err
 			}
 			opts = b.Options
 		case typeJS:
-			b, err := js.NewBundle(src, fs, runtimeOptions)
+			b, err = js.NewBundle(src, filesystems, runtimeOptions)
 			if err != nil {
 				return err
 			}

cmd/run.go

@@ -26,12 +26,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
@@ -116,8 +113,8 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 		filename := args[0]
-		fs := afero.NewOsFs()
-		src, err := readSource(filename, pwd, fs, os.Stdin)
+		filesystems := loader.CreateFilesystems()
+		src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
 		if err != nil {
 			return err
 		}
@@ -127,7 +124,7 @@ a commandline interface for interacting with it.`,
 			return err
 		}
 
-		r, err := newRunner(src, runType, fs, runtimeOptions)
+		r, err := newRunner(src, runType, filesystems, runtimeOptions)
 		if err != nil {
 			return err
 		}
@@ -138,7 +135,7 @@ a commandline interface for interacting with it.`,
 		if err != nil {
 			return err
 		}
-		conf, err := getConsolidatedConfig(fs, cliConf, r)
+		conf, err := getConsolidatedConfig(afero.NewOsFs(), cliConf, r)
 		if err != nil {
 			return err
 		}
@@ -503,29 +500,15 @@ func init() {
 	runCmd.Flags().AddFlagSet(runCmdFlagSet())
 }
 
-// Reads a source file from any supported destination.
-func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, error) {
-	if src == "-" {
-		data, err := ioutil.ReadAll(stdin)
-		if err != nil {
-			return nil, err
-		}
-		return &lib.SourceData{Filename: "-", Data: data}, nil
-	}
-	abspath := filepath.Join(pwd, src)
-	if ok, _ := afero.Exists(fs, abspath); ok {
-		src = abspath
-	}
-	return loader.Load(fs, pwd, src)
-}
-
 // Creates a new runner.
-func newRunner(src *lib.SourceData, typ string, fs afero.Fs, rtOpts lib.RuntimeOptions) (lib.Runner, error) {
+func newRunner(
+	src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions,
+) (lib.Runner, error) {
 	switch typ {
 	case "":
-		return newRunner(src, detectType(src.Data), fs, rtOpts)
+		return newRunner(src, detectType(src.Data), filesystems, rtOpts)
 	case typeJS:
-		return js.New(src, fs, rtOpts)
+		return js.New(src, filesystems, rtOpts)
 	case typeArchive:
 		arc, err := lib.ReadArchive(bytes.NewReader(src.Data))
 		if err != nil {

cmd/runtime_options_test.go

@@ -23,12 +23,14 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
 	"testing"
 
 	"github.com/loadimpact/k6/lib"
+	"github.com/loadimpact/k6/loader"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -226,13 +228,15 @@ func TestEnvVars(t *testing.T) {
 				}
 			}
 
+			fs := afero.NewMemMapFs()
+			require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(jsCode), 0644))
 			runner, err := newRunner(
-				&lib.SourceData{
-					Data:     []byte(jsCode),
-					Filename: "/script.js",
+				&loader.SourceData{
+					Data: []byte(jsCode),
+					URL:  &url.URL{Path: "/script.js", Scheme: "file"},
 				},
 				typeJS,
-				afero.NewOsFs(),
+				map[string]afero.Fs{"file": fs},
 				rtOpts,
 			)
 			require.NoError(t, err)
@@ -242,16 +246,15 @@ func TestEnvVars(t *testing.T) {
 			assert.NoError(t, archive.Write(archiveBuf))
 
 			getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) {
-				r, err := newRunner(
-					&lib.SourceData{
-						Data:     []byte(archiveBuf.Bytes()),
-						Filename: "/script.tar",
+				return newRunner(
+					&loader.SourceData{
+						Data: archiveBuf.Bytes(),
+						URL:  &url.URL{Path: "/script.js"},
 					},
 					typeArchive,
-					afero.NewOsFs(),
+					nil,
 					rtOpts,
 				)
-				return r, err
 			}
 
 			_, err = getRunnerErr(lib.RuntimeOptions{})

converter/har/converter_test.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/loadimpact/k6/js"
 	"github.com/loadimpact/k6/lib"
-	"github.com/spf13/afero"
+	"github.com/loadimpact/k6/loader"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -56,10 +56,10 @@ func TestBuildK6RequestObject(t *testing.T) {
 	}
 	v, err := buildK6RequestObject(req)
 	assert.NoError(t, err)
-	_, err = js.New(&lib.SourceData{
-		Filename: "/script.js",
-		Data:     []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
-	}, afero.NewMemMapFs(), lib.RuntimeOptions{})
+	_, err = js.New(&loader.SourceData{
+		URL:  &url.URL{Path: "/script.js"},
+		Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
+	}, nil, lib.RuntimeOptions{})
 	assert.NoError(t, err)
 }