From ea8384fd9d356ae1dc3a05de8ffb34d73e312519 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 31 Jul 2019 13:05:20 +0300 Subject: [PATCH] Rewrite script/files loading to be be url based (#1059) This also includes a change to the archive file structure. Now instead of separating files by whether they are scripts or not, they are separated based on their URI scheme. Throughout the (majority) of k6 now, instead of simple strings, a url.URL is used to identify files (scripts or otherwise). This also means that all imports can have schemes. Previously remote modules were supported specifically without a scheme. With this change k6 prefers if they are with a scheme. The old variant is supported, but logs a warning. Additionally if remote module requires a relative/absolute path that doesn't have a scheme, it is relative/absolute given the remote module url. Because of some of the changes, now caching is done through a afero.Fs instead of additional maps. This also fixes the remotely imported files not properly loading from an archive, but instead requesting them again. This is supported with old archives and for github/cdnjs's shortcut loaders. For future-proof reasons the archive now also records the GOOS value of the k6 that generated it. Anonymize case insensitively as windows paths are (usually) case insensitive. fixes #1037, closes #838, fixes #887 and fixes #1051 --- .golangci.yml | 2 +- api/v1/setup_teardown_routes_test.go | 8 +- cmd/archive.go | 9 +- cmd/cloud.go | 9 +- cmd/collectors.go | 3 +- cmd/inspect.go | 18 +- cmd/run.go | 35 +-- cmd/runtime_options_test.go | 23 +- converter/har/converter_test.go | 10 +- core/engine_test.go | 23 +- core/local/local_test.go | 7 +- js/bundle.go | 55 ++-- js/bundle_test.go | 109 ++++---- js/console_test.go | 45 +++- js/http_bench_test.go | 8 +- js/initcontext.go | 80 +++--- js/initcontext_test.go | 83 +++--- js/module_loading_test.go | 46 +--- js/modules/k6/marshalling_test.go | 7 +- js/runner.go | 6 +- js/runner_test.go | 383 +++++++++------------------ lib/archive.go | 297 +++++++++++++-------- lib/archive_test.go | 326 ++++++++++++++++++++--- lib/fsext/cacheonread.go | 27 ++ lib/fsext/changepathfs.go | 190 +++++++++++++ lib/fsext/changepathfs_test.go | 167 ++++++++++++ lib/fsext/trimpathseparator_test.go | 30 +++ lib/fsext/walk.go | 84 ++++++ lib/models.go | 6 - lib/old_archive_test.go | 208 +++++++++++++++ loader/cdnjs_test.go | 34 ++- loader/filesystems.go | 27 ++ loader/github_test.go | 51 +++- loader/loader.go | 231 +++++++++++----- loader/loader_test.go | 225 +++++++++++----- loader/readsource.go | 48 ++++ loader/readsource_test.go | 88 ++++++ stats/cloud/collector.go | 5 +- stats/cloud/collector_test.go | 14 +- 39 files changed, 2159 insertions(+), 868 deletions(-) create mode 100644 lib/fsext/cacheonread.go create mode 100644 lib/fsext/changepathfs.go create mode 100644 lib/fsext/changepathfs_test.go create mode 100644 lib/fsext/trimpathseparator_test.go create mode 100644 lib/fsext/walk.go create mode 100644 lib/old_archive_test.go create mode 100644 loader/filesystems.go create mode 100644 loader/readsource.go create mode 100644 loader/readsource_test.go diff --git a/.golangci.yml b/.golangci.yml index ed38e6f2a27..361c5f6977f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -38,7 +38,7 @@ linters-settings: dupl: threshold: 150 goconst: - min-len: 5 + min-len: 10 min-occurrences: 4 linters: diff --git a/api/v1/setup_teardown_routes_test.go b/api/v1/setup_teardown_routes_test.go index ef352d6a260..14476da7122 100644 --- a/api/v1/setup_teardown_routes_test.go +++ b/api/v1/setup_teardown_routes_test.go @@ -26,6 +26,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "net/url" "testing" "time" @@ -34,8 +35,8 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/manyminds/api2go/jsonapi" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" @@ -130,10 +131,11 @@ func TestSetupData(t *testing.T) { }, } for _, testCase := range testCases { + testCase := testCase t.Run(testCase.name, func(t *testing.T) { runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: testCase.script}, - afero.NewMemMapFs(), + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script}, + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) diff --git a/cmd/archive.go b/cmd/archive.go index edb4b65e6d3..adb8ad6cb7e 100644 --- a/cmd/archive.go +++ b/cmd/archive.go @@ -23,6 +23,7 @@ package cmd import ( "os" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -51,8 +52,8 @@ An archive is a fully self-contained test run, and can be executed identically e return err } filename := args[0] - fs := afero.NewOsFs() - src, err := readSource(filename, pwd, fs, os.Stdin) + filesystems := loader.CreateFilesystems() + src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } @@ -62,7 +63,7 @@ An archive is a fully self-contained test run, and can be executed identically e return err } - r, err := newRunner(src, runType, fs, runtimeOptions) + r, err := newRunner(src, runType, filesystems, runtimeOptions) if err != nil { return err } @@ -71,7 +72,7 @@ An archive is a fully self-contained test run, and can be executed identically e if err != nil { return err } - conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r) + conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r) if err != nil { return err } diff --git a/cmd/cloud.go b/cmd/cloud.go index 3e913994ddd..7fdcd48686c 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -32,6 +32,7 @@ import ( "github.com/kelseyhightower/envconfig" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/consts" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats/cloud" "github.com/loadimpact/k6/ui" "github.com/pkg/errors" @@ -71,8 +72,8 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud } filename := args[0] - fs := afero.NewOsFs() - src, err := readSource(filename, pwd, fs, os.Stdin) + filesystems := loader.CreateFilesystems() + src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } @@ -82,7 +83,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud return err } - r, err := newRunner(src, runType, fs, runtimeOptions) + r, err := newRunner(src, runType, filesystems, runtimeOptions) if err != nil { return err } @@ -91,7 +92,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud if err != nil { return err } - conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r) + conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r) if err != nil { return err } diff --git a/cmd/collectors.go b/cmd/collectors.go index 93979b9bb64..f92dc7dcb6f 100644 --- a/cmd/collectors.go +++ b/cmd/collectors.go @@ -29,6 +29,7 @@ import ( "github.com/kelseyhightower/envconfig" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/consts" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats/cloud" "github.com/loadimpact/k6/stats/datadog" "github.com/loadimpact/k6/stats/influxdb" @@ -61,7 +62,7 @@ func parseCollector(s string) (t, arg string) { } } -func newCollector(collectorName, arg string, src *lib.SourceData, conf Config) (lib.Collector, error) { +func newCollector(collectorName, arg string, src *loader.SourceData, conf Config) (lib.Collector, error) { getCollector := func() (lib.Collector, error) { switch collectorName { case collectorJSON: diff --git a/cmd/inspect.go b/cmd/inspect.go index efab0afbeb7..c48221d5cf1 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -28,7 +28,7 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" - "github.com/spf13/afero" + "github.com/loadimpact/k6/loader" "github.com/spf13/cobra" ) @@ -43,8 +43,8 @@ var inspectCmd = &cobra.Command{ if err != nil { return err } - fs := afero.NewOsFs() - src, err := readSource(args[0], pwd, fs, os.Stdin) + filesystems := loader.CreateFilesystems() + src, err := loader.ReadSource(args[0], pwd, filesystems, os.Stdin) if err != nil { return err } @@ -59,20 +59,24 @@ var inspectCmd = &cobra.Command{ return err } - var opts lib.Options + var ( + opts lib.Options + b *js.Bundle + ) switch typ { case typeArchive: - arc, err := lib.ReadArchive(bytes.NewBuffer(src.Data)) + var arc *lib.Archive + arc, err = lib.ReadArchive(bytes.NewBuffer(src.Data)) if err != nil { return err } - b, err := js.NewBundleFromArchive(arc, runtimeOptions) + b, err = js.NewBundleFromArchive(arc, runtimeOptions) if err != nil { return err } opts = b.Options case typeJS: - b, err := js.NewBundle(src, fs, runtimeOptions) + b, err = js.NewBundle(src, filesystems, runtimeOptions) if err != nil { return err } diff --git a/cmd/run.go b/cmd/run.go index 9c2bd2c94ee..caf9df0e3dc 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -26,12 +26,9 @@ import ( "context" "encoding/json" "fmt" - "io" - "io/ioutil" "net/http" "os" "os/signal" - "path/filepath" "runtime" "strings" "syscall" @@ -116,8 +113,8 @@ a commandline interface for interacting with it.`, return err } filename := args[0] - fs := afero.NewOsFs() - src, err := readSource(filename, pwd, fs, os.Stdin) + filesystems := loader.CreateFilesystems() + src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } @@ -127,7 +124,7 @@ a commandline interface for interacting with it.`, return err } - r, err := newRunner(src, runType, fs, runtimeOptions) + r, err := newRunner(src, runType, filesystems, runtimeOptions) if err != nil { return err } @@ -138,7 +135,7 @@ a commandline interface for interacting with it.`, if err != nil { return err } - conf, err := getConsolidatedConfig(fs, cliConf, r) + conf, err := getConsolidatedConfig(afero.NewOsFs(), cliConf, r) if err != nil { return err } @@ -503,29 +500,15 @@ func init() { runCmd.Flags().AddFlagSet(runCmdFlagSet()) } -// Reads a source file from any supported destination. -func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, error) { - if src == "-" { - data, err := ioutil.ReadAll(stdin) - if err != nil { - return nil, err - } - return &lib.SourceData{Filename: "-", Data: data}, nil - } - abspath := filepath.Join(pwd, src) - if ok, _ := afero.Exists(fs, abspath); ok { - src = abspath - } - return loader.Load(fs, pwd, src) -} - // Creates a new runner. -func newRunner(src *lib.SourceData, typ string, fs afero.Fs, rtOpts lib.RuntimeOptions) (lib.Runner, error) { +func newRunner( + src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions, +) (lib.Runner, error) { switch typ { case "": - return newRunner(src, detectType(src.Data), fs, rtOpts) + return newRunner(src, detectType(src.Data), filesystems, rtOpts) case typeJS: - return js.New(src, fs, rtOpts) + return js.New(src, filesystems, rtOpts) case typeArchive: arc, err := lib.ReadArchive(bytes.NewReader(src.Data)) if err != nil { diff --git a/cmd/runtime_options_test.go b/cmd/runtime_options_test.go index 7fbf43b443a..2321f691cb6 100644 --- a/cmd/runtime_options_test.go +++ b/cmd/runtime_options_test.go @@ -23,12 +23,14 @@ package cmd import ( "bytes" "fmt" + "net/url" "os" "runtime" "strings" "testing" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -226,13 +228,15 @@ func TestEnvVars(t *testing.T) { } } + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(jsCode), 0644)) runner, err := newRunner( - &lib.SourceData{ - Data: []byte(jsCode), - Filename: "/script.js", + &loader.SourceData{ + Data: []byte(jsCode), + URL: &url.URL{Path: "/script.js", Scheme: "file"}, }, typeJS, - afero.NewOsFs(), + map[string]afero.Fs{"file": fs}, rtOpts, ) require.NoError(t, err) @@ -242,16 +246,15 @@ func TestEnvVars(t *testing.T) { assert.NoError(t, archive.Write(archiveBuf)) getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) { - r, err := newRunner( - &lib.SourceData{ - Data: []byte(archiveBuf.Bytes()), - Filename: "/script.tar", + return newRunner( + &loader.SourceData{ + Data: archiveBuf.Bytes(), + URL: &url.URL{Path: "/script.js"}, }, typeArchive, - afero.NewOsFs(), + nil, rtOpts, ) - return r, err } _, err = getRunnerErr(lib.RuntimeOptions{}) diff --git a/converter/har/converter_test.go b/converter/har/converter_test.go index 575044f9b4d..fdf92cce8cc 100644 --- a/converter/har/converter_test.go +++ b/converter/har/converter_test.go @@ -27,7 +27,7 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" - "github.com/spf13/afero" + "github.com/loadimpact/k6/loader" "github.com/stretchr/testify/assert" ) @@ -56,10 +56,10 @@ func TestBuildK6RequestObject(t *testing.T) { } v, err := buildK6RequestObject(req) assert.NoError(t, err) - _, err = js.New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + _, err = js.New(&loader.SourceData{ + URL: &url.URL{Path: "/script.js"}, + Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)), + }, nil, lib.RuntimeOptions{}) assert.NoError(t, err) } diff --git a/core/engine_test.go b/core/engine_test.go index fe81be047a1..784915594ad 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -23,6 +23,7 @@ package core import ( "context" "fmt" + "net/url" "testing" "time" @@ -32,11 +33,11 @@ import ( "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" "github.com/loadimpact/k6/stats/dummy" log "github.com/sirupsen/logrus" logtest "github.com/sirupsen/logrus/hooks/test" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" @@ -556,8 +557,8 @@ func TestSentReceivedMetrics(t *testing.T) { runTest := func(t *testing.T, ts testScript, tc testCase, noConnReuse bool) (float64, float64) { r, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: []byte(ts.Code)}, - afero.NewMemMapFs(), + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)}, + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -697,8 +698,8 @@ func TestRunTags(t *testing.T) { `)) r, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, - afero.NewMemMapFs(), + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -797,8 +798,8 @@ func TestSetupTeardownThresholds(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, - afero.NewMemMapFs(), + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -860,8 +861,8 @@ func TestEmittedMetricsWhenScalingDown(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, - afero.NewMemMapFs(), + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -920,7 +921,7 @@ func TestMinIterationDuration(t *testing.T) { t.Parallel() runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: []byte(` + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(` import { Counter } from "k6/metrics"; let testCounter = new Counter("testcounter"); @@ -935,7 +936,7 @@ func TestMinIterationDuration(t *testing.T) { export default function () { testCounter.add(1); };`)}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) diff --git a/core/local/local_test.go b/core/local/local_test.go index 23c27d88179..4af9be82b0c 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -23,12 +23,14 @@ package local import ( "context" "net" + "net/url" "runtime" "sync/atomic" "testing" "time" "github.com/loadimpact/k6/lib/netext" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" @@ -37,7 +39,6 @@ import ( "github.com/loadimpact/k6/stats" "github.com/pkg/errors" logtest "github.com/sirupsen/logrus/hooks/test" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" @@ -481,8 +482,8 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { }`) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, - afero.NewMemMapFs(), + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) diff --git a/js/bundle.go b/js/bundle.go index a2e9d47e47d..9214d6a1bec 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -23,7 +23,8 @@ package js import ( "context" "encoding/json" - "os" + "net/url" + "runtime" "github.com/loadimpact/k6/lib/consts" @@ -40,7 +41,7 @@ import ( // A Bundle is a self-contained bundle of scripts and resources. // You can use this to produce identical BundleInstance objects. type Bundle struct { - Filename string + Filename *url.URL Source string Program *goja.Program Options lib.Options @@ -58,7 +59,7 @@ type BundleInstance struct { } // NewBundle creates a new bundle from a source file and a filesystem. -func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { +func NewBundle(src *loader.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { compiler, err := compiler.New() if err != nil { return nil, err @@ -66,24 +67,17 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu // Compile sources, both ES5 and ES6 are supported. code := string(src.Data) - pgm, _, err := compiler.Compile(code, src.Filename, "", "", true) + pgm, _, err := compiler.Compile(code, src.URL.String(), "", "", true) if err != nil { return nil, err } - - // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's - // written every time something is read from the real filesystem. This cache is then used for - // successive spawns to read from (they have no access to the real disk). - mirrorFS := afero.NewMemMapFs() - cachedFS := afero.NewCacheOnReadFs(fs, mirrorFS, 0) - // Make a bundle, instantiate it into a throwaway VM to populate caches. rt := goja.New() bundle := Bundle{ - Filename: src.Filename, + Filename: src.URL, Source: code, Program: pgm, - BaseInitContext: NewInitContext(rt, compiler, new(context.Context), cachedFS, loader.Dir(src.Filename)), + BaseInitContext: NewInitContext(rt, compiler, new(context.Context), filesystems, loader.Dir(src.URL)), Env: rtOpts.Env, } if err := bundle.instantiate(rt, bundle.BaseInitContext); err != nil { @@ -144,12 +138,12 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, return nil, errors.Errorf("expected bundle type 'js', got '%s'", arc.Type) } - pgm, _, err := compiler.Compile(string(arc.Data), arc.Filename, "", "", true) + pgm, _, err := compiler.Compile(string(arc.Data), arc.FilenameURL.String(), "", "", true) if err != nil { return nil, err } - initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FS, arc.Pwd) - initctx.files = arc.Files + + initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.Filesystems, arc.PwdURL) env := arc.Env if env == nil { @@ -161,7 +155,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, } bundle := &Bundle{ - Filename: arc.Filename, + Filename: arc.FilenameURL, Source: string(arc.Data), Program: pgm, Options: arc.Options, @@ -176,30 +170,21 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, func (b *Bundle) makeArchive() *lib.Archive { arc := &lib.Archive{ - Type: "js", - FS: afero.NewMemMapFs(), - Options: b.Options, - Filename: b.Filename, - Data: []byte(b.Source), - Pwd: b.BaseInitContext.pwd, - Env: make(map[string]string, len(b.Env)), - K6Version: consts.Version, + Type: "js", + Filesystems: b.BaseInitContext.filesystems, + Options: b.Options, + FilenameURL: b.Filename, + Data: []byte(b.Source), + PwdURL: b.BaseInitContext.pwd, + Env: make(map[string]string, len(b.Env)), + K6Version: consts.Version, + Goos: runtime.GOOS, } // Copy env so changes in the archive are not reflected in the source Bundle for k, v := range b.Env { arc.Env[k] = v } - arc.Scripts = make(map[string][]byte, len(b.BaseInitContext.programs)) - for name, pgm := range b.BaseInitContext.programs { - arc.Scripts[name] = []byte(pgm.src) - err := afero.WriteFile(arc.FS, name, []byte(pgm.src), os.ModePerm) - if err != nil { - return nil - } - } - arc.Files = b.BaseInitContext.files - return arc } diff --git a/js/bundle_test.go b/js/bundle_test.go index 745c27a452d..971922f9b28 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -24,6 +24,7 @@ import ( "crypto/tls" "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "runtime" @@ -31,24 +32,42 @@ import ( "testing" "time" - "github.com/loadimpact/k6/lib/consts" - "github.com/dop251/goja" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/consts" + "github.com/loadimpact/k6/lib/fsext" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/guregu/null.v3" ) +const isWindows = runtime.GOOS == "windows" + func getSimpleBundle(filename, data string) (*Bundle, error) { + return getSimpleBundleWithFs(filename, data, afero.NewMemMapFs()) +} + +func getSimpleBundleWithOptions(filename, data string, options lib.RuntimeOptions) (*Bundle, error) { + return NewBundle( + &loader.SourceData{ + URL: &url.URL{Path: filename, Scheme: "file"}, + Data: []byte(data), + }, + map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": afero.NewMemMapFs()}, + options, + ) +} + +func getSimpleBundleWithFs(filename, data string, fs afero.Fs) (*Bundle, error) { return NewBundle( - &lib.SourceData{ - Filename: filename, - Data: []byte(data), + &loader.SourceData{ + URL: &url.URL{Path: filename, Scheme: "file"}, + Data: []byte(data), }, - afero.NewMemMapFs(), + map[string]afero.Fs{"file": fs, "https": afero.NewMemMapFs()}, lib.RuntimeOptions{}, ) } @@ -60,11 +79,11 @@ func TestNewBundle(t *testing.T) { }) t.Run("Invalid", func(t *testing.T) { _, err := getSimpleBundle("/script.js", "\x00") - assert.Contains(t, err.Error(), "SyntaxError: /script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") + assert.Contains(t, err.Error(), "SyntaxError: file:///script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") }) t.Run("Error", func(t *testing.T) { _, err := getSimpleBundle("/script.js", `throw new Error("aaaa");`) - assert.EqualError(t, err, "Error: aaaa at /script.js:1:7(3)") + assert.EqualError(t, err, "Error: aaaa at file:///script.js:1:7(3)") }) t.Run("InvalidExports", func(t *testing.T) { _, err := getSimpleBundle("/script.js", `exports = null`) @@ -89,8 +108,8 @@ func TestNewBundle(t *testing.T) { t.Run("stdin", func(t *testing.T) { b, err := getSimpleBundle("-", `export default function() {};`) if assert.NoError(t, err) { - assert.Equal(t, "-", b.Filename) - assert.Equal(t, "/", b.BaseInitContext.pwd) + assert.Equal(t, "file://-", b.Filename.String()) + assert.Equal(t, "file:///", b.BaseInitContext.pwd.String()) } }) t.Run("Options", func(t *testing.T) { @@ -359,16 +378,13 @@ func TestNewBundleFromArchive(t *testing.T) { assert.NoError(t, afero.WriteFile(fs, "/path/to/file.txt", []byte(`hi`), 0644)) assert.NoError(t, afero.WriteFile(fs, "/path/to/exclaim.js", []byte(`export default function(s) { return s + "!" };`), 0644)) - src := &lib.SourceData{ - Filename: "/path/to/script.js", - Data: []byte(` + data := ` import exclaim from "./exclaim.js"; export let options = { vus: 12345 }; export let file = open("./file.txt"); export default function() { return exclaim(file); }; - `), - } - b, err := NewBundle(src, fs, lib.RuntimeOptions{}) + ` + b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs) if !assert.NoError(t, err) { return } @@ -387,13 +403,17 @@ func TestNewBundleFromArchive(t *testing.T) { arc := b.makeArchive() assert.Equal(t, "js", arc.Type) assert.Equal(t, lib.Options{VUs: null.IntFrom(12345)}, arc.Options) - assert.Equal(t, "/path/to/script.js", arc.Filename) - assert.Equal(t, string(src.Data), string(arc.Data)) - assert.Equal(t, "/path/to", arc.Pwd) - assert.Len(t, arc.Scripts, 1) - assert.Equal(t, `export default function(s) { return s + "!" };`, string(arc.Scripts["/path/to/exclaim.js"])) - assert.Len(t, arc.Files, 1) - assert.Equal(t, `hi`, string(arc.Files["/path/to/file.txt"])) + assert.Equal(t, "file:///path/to/script.js", arc.FilenameURL.String()) + assert.Equal(t, data, string(arc.Data)) + assert.Equal(t, "file:///path/to/", arc.PwdURL.String()) + + exclaimData, err := afero.ReadFile(arc.Filesystems["file"], "/path/to/exclaim.js") + assert.NoError(t, err) + assert.Equal(t, `export default function(s) { return s + "!" };`, string(exclaimData)) + + fileData, err := afero.ReadFile(arc.Filesystems["file"], "/path/to/file.txt") + assert.NoError(t, err) + assert.Equal(t, `hi`, string(fileData)) assert.Equal(t, consts.Version, arc.K6Version) b2, err := NewBundleFromArchive(arc, lib.RuntimeOptions{}) @@ -496,8 +516,12 @@ func TestOpen(t *testing.T) { prefix, err := ioutil.TempDir("", "k6_open_test") require.NoError(t, err) fs := afero.NewOsFs() + filePath := filepath.Join(prefix, "/path/to/file.txt") require.NoError(t, fs.MkdirAll(filepath.Join(prefix, "/path/to"), 0755)) - require.NoError(t, afero.WriteFile(fs, filepath.Join(prefix, "/path/to/file.txt"), []byte(`hi`), 0644)) + require.NoError(t, afero.WriteFile(fs, filePath, []byte(`hi`), 0644)) + if isWindows { + fs = fsext.NewTrimFilePathSeparatorFs(fs) + } return fs, prefix, func() { require.NoError(t, os.RemoveAll(prefix)) } }, } @@ -516,21 +540,18 @@ func TestOpen(t *testing.T) { if openPath != "" && (openPath[0] == '/' || openPath[0] == '\\') { openPath = filepath.Join(prefix, openPath) } - if runtime.GOOS == "windows" { + if isWindows { openPath = strings.Replace(openPath, `\`, `\\`, -1) } var pwd = tCase.pwd if pwd == "" { pwd = "/path/to/" } - src := &lib.SourceData{ - Filename: filepath.Join(prefix, filepath.Join(pwd, "script.js")), - Data: []byte(` - export let file = open("` + openPath + `"); - export default function() { return file }; - `), - } - sourceBundle, err := NewBundle(src, fs, lib.RuntimeOptions{}) + data := ` + export let file = open("` + openPath + `"); + export default function() { return file };` + + sourceBundle, err := getSimpleBundleWithFs(filepath.ToSlash(filepath.Join(prefix, pwd, "script.js")), data, fs) if tCase.isError { assert.Error(t, err) return @@ -554,7 +575,7 @@ func TestOpen(t *testing.T) { } t.Run(tCase.name, testFunc) - if runtime.GOOS == "windows" { + if isWindows { // windowsify the testcase tCase.openPath = strings.Replace(tCase.openPath, `/`, `\`, -1) tCase.pwd = strings.Replace(tCase.pwd, `/`, `\`, -1) @@ -600,19 +621,13 @@ func TestBundleEnv(t *testing.T) { "TEST_A": "1", "TEST_B": "", }} - - b1, err := NewBundle( - &lib.SourceData{ - Filename: "/script.js", - Data: []byte(` - export default function() { - if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); } - if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); } - } - `), - }, - afero.NewMemMapFs(), rtOpts, - ) + data := ` + export default function() { + if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); } + if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); } + } + ` + b1, err := getSimpleBundleWithOptions("/script.js", data, rtOpts) if !assert.NoError(t, err) { return } diff --git a/js/console_test.go b/js/console_test.go index b85bbbdacd2..8b0448d5fd1 100644 --- a/js/console_test.go +++ b/js/console_test.go @@ -24,6 +24,7 @@ import ( "context" "fmt" "io/ioutil" + "net/url" "os" "testing" @@ -32,6 +33,7 @@ import ( "github.com/dop251/goja" "github.com/loadimpact/k6/js/common" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" log "github.com/sirupsen/logrus" logtest "github.com/sirupsen/logrus/hooks/test" @@ -68,7 +70,29 @@ func TestConsoleContext(t *testing.T) { assert.Equal(t, "b", entry.Message) } } +func getSimpleRunner(path, data string) (*Runner, error) { + return getSimpleRunnerWithFileFs(path, data, afero.NewMemMapFs()) +} + +func getSimpleRunnerWithOptions(path, data string, options lib.RuntimeOptions) (*Runner, error) { + return New(&loader.SourceData{ + URL: &url.URL{Path: path, Scheme: "file"}, + Data: []byte(data), + }, map[string]afero.Fs{ + "file": afero.NewMemMapFs(), + "https": afero.NewMemMapFs()}, + options) +} +func getSimpleRunnerWithFileFs(path, data string, fileFs afero.Fs) (*Runner, error) { + return New(&loader.SourceData{ + URL: &url.URL{Path: path, Scheme: "file"}, + Data: []byte(data), + }, map[string]afero.Fs{ + "file": fileFs, + "https": afero.NewMemMapFs()}, + lib.RuntimeOptions{}) +} func TestConsole(t *testing.T) { levels := map[string]log.Level{ "log": log.InfoLevel, @@ -87,16 +111,15 @@ func TestConsole(t *testing.T) { `{}`: {Message: "[object Object]"}, } for name, level := range levels { + name, level := name, level t.Run(name, func(t *testing.T) { for args, result := range argsets { + args, result := args, result t.Run(args, func(t *testing.T) { - r, err := New(&lib.SourceData{ - Filename: "/script", - Data: []byte(fmt.Sprintf( - `export default function() { console.%s(%s); }`, - name, args, - )), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r, err := getSimpleRunner("/script.js", fmt.Sprintf( + `export default function() { console.%s(%s); }`, + name, args, + )) assert.NoError(t, err) samples := make(chan stats.SampleContainer, 100) @@ -179,13 +202,11 @@ func TestFileConsole(t *testing.T) { } } - r, err := New(&lib.SourceData{ - Filename: "/script", - Data: []byte(fmt.Sprintf( + r, err := getSimpleRunner("/script", + fmt.Sprintf( `export default function() { console.%s(%s); }`, name, args, - )), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + )) assert.NoError(t, err) err = r.SetOptions(lib.Options{ diff --git a/js/http_bench_test.go b/js/http_bench_test.go index 17f45fb0914..4fdfdd10f6a 100644 --- a/js/http_bench_test.go +++ b/js/http_bench_test.go @@ -7,7 +7,6 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/stats" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "gopkg.in/guregu/null.v3" ) @@ -17,17 +16,14 @@ func BenchmarkHTTPRequests(b *testing.B) { tb := testutils.NewHTTPMultiBin(b) defer tb.Cleanup() - r, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + r, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; export default function() { let url = "HTTPBIN_URL"; let res = http.get(url + "/cookies/set?k2=v2&k1=v1"); if (res.status != 200) { throw new Error("wrong status: " + res.status) } } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(b, err) { return } diff --git a/js/initcontext.go b/js/initcontext.go index 9842a0f2396..f43c3b14cf6 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -22,6 +22,7 @@ package js import ( "context" + "net/url" "path/filepath" "strings" @@ -49,28 +50,26 @@ type InitContext struct { // Pointer to a context that bridged modules are invoked with. ctxPtr *context.Context - // Filesystem to load files and scripts from. - fs afero.Fs - pwd string + // Filesystem to load files and scripts from with the map key being the scheme + filesystems map[string]afero.Fs + pwd *url.URL // Cache of loaded programs and files. programs map[string]programWithSource - files map[string][]byte } // NewInitContext creates a new initcontext with the provided arguments func NewInitContext( - rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string, + rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, filesystems map[string]afero.Fs, pwd *url.URL, ) *InitContext { return &InitContext{ - runtime: rt, - compiler: compiler, - ctxPtr: ctxPtr, - fs: fs, - pwd: filepath.ToSlash(pwd), + runtime: rt, + compiler: compiler, + ctxPtr: ctxPtr, + filesystems: filesystems, + pwd: pwd, programs: make(map[string]programWithSource), - files: make(map[string][]byte), } } @@ -89,12 +88,11 @@ func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Ru runtime: rt, ctxPtr: ctxPtr, - fs: base.fs, - pwd: base.pwd, - compiler: base.compiler, + filesystems: base.filesystems, + pwd: base.pwd, + compiler: base.compiler, programs: programs, - files: base.files, } } @@ -130,12 +128,15 @@ func (i *InitContext) requireModule(name string) (goja.Value, error) { func (i *InitContext) requireFile(name string) (goja.Value, error) { // Resolve the file path, push the target directory as pwd to make relative imports work. pwd := i.pwd - filename := loader.Resolve(pwd, name) + fileURL, err := loader.Resolve(pwd, name) + if err != nil { + return nil, err + } // First, check if we have a cached program already. - pgm, ok := i.programs[filename] + pgm, ok := i.programs[fileURL.String()] if !ok || pgm.exports == nil { - i.pwd = loader.Dir(filename) + i.pwd = loader.Dir(fileURL) defer func() { i.pwd = pwd }() // Swap the importing scope's exports out, then put it back again. @@ -150,21 +151,22 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) { i.runtime.Set("module", module) if pgm.pgm == nil { // Load the sources; the loader takes care of remote loading, etc. - data, err := loader.Load(i.fs, pwd, name) + data, err := loader.Load(i.filesystems, fileURL, name) if err != nil { return goja.Undefined(), err } + pgm.src = string(data.Data) // Compile the sources; this handles ES5 vs ES6 automatically. - pgm.pgm, err = i.compileImport(pgm.src, data.Filename) + pgm.pgm, err = i.compileImport(pgm.src, data.URL.String()) if err != nil { return goja.Undefined(), err } } pgm.exports = module.Get("exports") - i.programs[filename] = pgm + i.programs[fileURL.String()] = pgm // Run the program. if _, err := i.runtime.RunProgram(pgm.pgm); err != nil { @@ -193,28 +195,22 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) // will probably be need for archive execution under windows if always consider '/...' as an // absolute path. if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) { - filename = filepath.Join(i.pwd, filename) + filename = filepath.Join(i.pwd.Path, filename) } - filename = filepath.ToSlash(filename) - - data, ok := i.files[filename] - if !ok { - var ( - err error - isDir bool - ) - - // Workaround for https://github.com/spf13/afero/issues/201 - if isDir, err = afero.IsDir(i.fs, filename); err != nil { - return nil, err - } else if isDir { - return nil, errors.New("open() can't be used with directories") - } - data, err = afero.ReadFile(i.fs, filename) - if err != nil { - return nil, err - } - i.files[filename] = data + filename = filepath.Clean(filename) + fs := i.filesystems["file"] + if filename[0:1] != afero.FilePathSeparator { + filename = afero.FilePathSeparator + filename + } + // Workaround for https://github.com/spf13/afero/issues/201 + if isDir, err := afero.IsDir(fs, filename); err != nil { + return nil, err + } else if isDir { + return nil, errors.New("open() can't be used with directories") + } + data, err := afero.ReadFile(fs, filename) + if err != nil { + return nil, err } if len(args) > 0 && args[0] == "b" { diff --git a/js/initcontext_test.go b/js/initcontext_test.go index f127e3b0194..de5a7012dc5 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -40,6 +40,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestInitContextRequire(t *testing.T) { @@ -111,25 +112,20 @@ func TestInitContextRequire(t *testing.T) { t.Run("Nonexistent", func(t *testing.T) { path := filepath.FromSlash("/nonexistent.js") _, err := getSimpleBundle("/script.js", `import "/nonexistent.js"; export default function() {}`) - assert.EqualError(t, err, fmt.Sprintf("GoError: open %s: file does not exist", path)) + assert.Contains(t, err.Error(), fmt.Sprintf(`"file://%s" couldn't be found on local disk`, filepath.ToSlash(path))) }) t.Run("Invalid", func(t *testing.T) { fs := afero.NewMemMapFs() assert.NoError(t, afero.WriteFile(fs, "/file.js", []byte{0x00}, 0755)) - _, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`import "/file.js"; export default function() {}`), - }, fs, lib.RuntimeOptions{}) - assert.Contains(t, err.Error(), "SyntaxError: /file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") + _, err := getSimpleBundleWithFs("/script.js", `import "/file.js"; export default function() {}`, fs) + require.Error(t, err) + assert.Contains(t, err.Error(), "SyntaxError: file:///file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") }) t.Run("Error", func(t *testing.T) { fs := afero.NewMemMapFs() assert.NoError(t, afero.WriteFile(fs, "/file.js", []byte(`throw new Error("aaaa")`), 0755)) - _, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`import "/file.js"; export default function() {}`), - }, fs, lib.RuntimeOptions{}) - assert.EqualError(t, err, "Error: aaaa at /file.js:2:7(4)") + _, err := getSimpleBundleWithFs("/script.js", `import "/file.js"; export default function() {}`, fs) + assert.EqualError(t, err, "Error: aaaa at file:///file.js:2:7(4)") }) imports := map[string]struct { @@ -162,23 +158,16 @@ func TestInitContextRequire(t *testing.T) { }}, } for libName, data := range imports { + libName, data := libName, data t.Run("lib=\""+libName+"\"", func(t *testing.T) { for constName, constPath := range data.ConstPaths { + constName, constPath := constName, constPath name := "inline" if constName != "" { name = "const=\"" + constName + "\"" } t.Run(name, func(t *testing.T) { fs := afero.NewMemMapFs() - src := &lib.SourceData{ - Filename: `/path/to/script.js`, - Data: []byte(fmt.Sprintf(` - import fn from "%s"; - let v = fn(); - export default function() { - }; - `, libName)), - } jsLib := `export default function() { return 12345; }` if constName != "" { @@ -195,12 +184,17 @@ func TestInitContextRequire(t *testing.T) { assert.NoError(t, fs.MkdirAll(filepath.Dir(data.LibPath), 0755)) assert.NoError(t, afero.WriteFile(fs, data.LibPath, []byte(jsLib), 0644)) - b, err := NewBundle(src, fs, lib.RuntimeOptions{}) + data := fmt.Sprintf(` + import fn from "%s"; + let v = fn(); + export default function() {};`, + libName) + b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs) if !assert.NoError(t, err) { return } if constPath != "" { - assert.Contains(t, b.BaseInitContext.programs, constPath) + assert.Contains(t, b.BaseInitContext.programs, "file://"+constPath) } _, err = b.Instantiate() @@ -216,18 +210,15 @@ func TestInitContextRequire(t *testing.T) { fs := afero.NewMemMapFs() assert.NoError(t, afero.WriteFile(fs, "/a.js", []byte(`const myvar = "a";`), 0644)) assert.NoError(t, afero.WriteFile(fs, "/b.js", []byte(`const myvar = "b";`), 0644)) - b, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + data := ` import "./a.js"; import "./b.js"; export default function() { if (typeof myvar != "undefined") { throw new Error("myvar is set in global scope"); } - }; - `), - }, fs, lib.RuntimeOptions{}) + };` + b, err := getSimpleBundleWithFs("/script.js", data, fs) if !assert.NoError(t, err) { return } @@ -253,17 +244,15 @@ func createAndReadFile(t *testing.T, file string, content []byte, expectedLength binaryArg = ",\"b\"" } - b, err := NewBundle(&lib.SourceData{ - Filename: "/path/to/script.js", - Data: []byte(fmt.Sprintf(` - export let data = open("/path/to/%s"%s); - var expectedLength = %d; - if (data.length != expectedLength) { - throw new Error("Length not equal, expected: " + expectedLength + ", actual: " + data.length); - } - export default function() {} - `, file, binaryArg, expectedLength)), - }, fs, lib.RuntimeOptions{}) + data := fmt.Sprintf(` + export let data = open("/path/to/%s"%s); + var expectedLength = %d; + if (data.length != expectedLength) { + throw new Error("Length not equal, expected: " + expectedLength + ", actual: " + data.length); + } + export default function() {} + `, file, binaryArg, expectedLength) + b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs) if !assert.NoError(t, err) { return nil, err @@ -322,12 +311,8 @@ func TestInitContextOpen(t *testing.T) { } t.Run("Nonexistent", func(t *testing.T) { - fs := afero.NewMemMapFs() path := filepath.FromSlash("/nonexistent.txt") - _, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`open("/nonexistent.txt"); export default function() {}`), - }, fs, lib.RuntimeOptions{}) + _, err := getSimpleBundle("/script.js", `open("/nonexistent.txt"); export default function() {}`) assert.EqualError(t, err, fmt.Sprintf("GoError: open %s: file does not exist", path)) }) @@ -363,9 +348,8 @@ func TestRequestWithBinaryFile(t *testing.T) { assert.NoError(t, fs.MkdirAll("/path/to", 0755)) assert.NoError(t, afero.WriteFile(fs, "/path/to/file.bin", []byte("hi!"), 0644)) - b, err := NewBundle(&lib.SourceData{ - Filename: "/path/to/script.js", - Data: []byte(fmt.Sprintf(` + b, err := getSimpleBundleWithFs("/path/to/script.js", + fmt.Sprintf(` import http from "k6/http"; let binFile = open("/path/to/file.bin", "b"); export default function() { @@ -376,9 +360,8 @@ func TestRequestWithBinaryFile(t *testing.T) { var res = http.post("%s", data); return true; } - `, srv.URL)), - }, fs, lib.RuntimeOptions{}) - assert.NoError(t, err) + `, srv.URL), fs) + require.NoError(t, err) bi, err := b.Instantiate() assert.NoError(t, err) diff --git a/js/module_loading_test.go b/js/module_loading_test.go index df94a096ca9..c59426e4ed4 100644 --- a/js/module_loading_test.go +++ b/js/module_loading_test.go @@ -65,9 +65,7 @@ func TestLoadOnceGlobalVars(t *testing.T) { return C(); } `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import { A } from "./A.js"; import { B } from "./B.js"; @@ -79,12 +77,10 @@ func TestLoadOnceGlobalVars(t *testing.T) { throw new Error("A() != B() (" + A() + ") != (" + B() + ")"); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -115,9 +111,7 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) { return http.get("HTTPBIN_URL/get"); } `)), os.ModePerm)) - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import { A } from "./A.js"; export default function(data) { @@ -126,13 +120,11 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) { throw new Error("wrong status "+ resp.status); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) require.NoError(t, r1.SetOptions(lib.Options{Hosts: tb.Dialer.Hosts})) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -159,9 +151,7 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) { return globalVar; } `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import { A } from "./A.js"; export default function(data) { @@ -172,12 +162,10 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) { throw new Error("wrong value of a " + a); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -231,14 +219,10 @@ func TestLoadCycle(t *testing.T) { `), os.ModePerm)) data, err := afero.ReadFile(fs, "/main.js") require.NoError(t, err) - r1, err := New(&lib.SourceData{ - Filename: "/main.js", - Data: data, - }, fs, lib.RuntimeOptions{}) + r1, err := getSimpleRunnerWithFileFs("/main.js", string(data), fs) require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -281,9 +265,7 @@ func TestLoadCycleBinding(t *testing.T) { } `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - Filename: "/main.js", - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/main.js", ` import {foo} from './a.js'; import {bar} from './b.js'; export default function() { @@ -296,12 +278,10 @@ func TestLoadCycleBinding(t *testing.T) { throw new Error("Wrong value of bar() "+ barMessage); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -341,9 +321,7 @@ func TestBrowserified(t *testing.T) { }); `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import {alpha, bravo } from "./browserified.js"; export default function(data) { @@ -361,12 +339,10 @@ func TestBrowserified(t *testing.T) { throw new Error("bravo.B() != 'b' (" + bravo.B() + ") != 'b'"); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) diff --git a/js/modules/k6/marshalling_test.go b/js/modules/k6/marshalling_test.go index 22e552a34f8..8c4ca3ae3a8 100644 --- a/js/modules/k6/marshalling_test.go +++ b/js/modules/k6/marshalling_test.go @@ -22,6 +22,7 @@ package k6_test import ( "context" + "net/url" "testing" "time" @@ -29,8 +30,8 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -114,8 +115,8 @@ func TestSetupDataMarshalling(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, - afero.NewMemMapFs(), + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + nil, lib.RuntimeOptions{}, ) diff --git a/js/runner.go b/js/runner.go index 603161369b4..4d3f97635bc 100644 --- a/js/runner.go +++ b/js/runner.go @@ -34,6 +34,7 @@ import ( "github.com/loadimpact/k6/js/common" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/netext" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" "github.com/oxtoacart/bpool" "github.com/pkg/errors" @@ -62,8 +63,9 @@ type Runner struct { setupData []byte } -func New(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) { - bundle, err := NewBundle(src, fs, rtOpts) +// New returns a new Runner for the provide source +func New(src *loader.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) { + bundle, err := NewBundle(src, filesystems, rtOpts) if err != nil { return nil, err } diff --git a/js/runner_test.go b/js/runner_test.go index cbd77ce187c..ddcb80fc2f9 100644 --- a/js/runner_test.go +++ b/js/runner_test.go @@ -32,7 +32,6 @@ import ( "net" "net/http" "os" - "runtime" "strings" "testing" "time" @@ -59,13 +58,10 @@ import ( func TestRunnerNew(t *testing.T) { t.Run("Valid", func(t *testing.T) { - r, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r, err := getSimpleRunner("/script.js", ` let counter = 0; export default function() { counter++; } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) assert.NoError(t, err) t.Run("NewVU", func(t *testing.T) { @@ -84,19 +80,13 @@ func TestRunnerNew(t *testing.T) { }) t.Run("Invalid", func(t *testing.T) { - _, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`blarg`), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - assert.EqualError(t, err, "ReferenceError: blarg is not defined at /script.js:1:1(0)") + _, err := getSimpleRunner("/script.js", `blarg`) + assert.EqualError(t, err, "ReferenceError: blarg is not defined at file:///script.js:1:1(0)") }) } func TestRunnerGetDefaultGroup(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`export default function() {};`), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r1, err := getSimpleRunner("/script.js", `export default function() {};`) if assert.NoError(t, err) { assert.NotNil(t, r1.GetDefaultGroup()) } @@ -108,10 +98,7 @@ func TestRunnerGetDefaultGroup(t *testing.T) { } func TestRunnerOptions(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`export default function() {};`), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r1, err := getSimpleRunner("/script.js", `export default function() {};`) if !assert.NoError(t, err) { return } @@ -151,9 +138,7 @@ func TestOptionsSettingToScript(t *testing.T) { variant := variant t.Run(fmt.Sprintf("Variant#%d", i), func(t *testing.T) { t.Parallel() - src := &lib.SourceData{ - Filename: "/script.js", - Data: []byte(variant + ` + data := variant + ` export default function() { if (!options) { throw new Error("Expected options to be defined!"); @@ -161,10 +146,9 @@ func TestOptionsSettingToScript(t *testing.T) { if (options.teardownTimeout != __ENV.expectedTeardownTimeout) { throw new Error("expected teardownTimeout to be " + __ENV.expectedTeardownTimeout + " but it was " + options.teardownTimeout); } - }; - `), - } - r, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{Env: map[string]string{"expectedTeardownTimeout": "4s"}}) + };` + r, err := getSimpleRunnerWithOptions("/script.js", data, + lib.RuntimeOptions{Env: map[string]string{"expectedTeardownTimeout": "4s"}}) require.NoError(t, err) newOptions := lib.Options{TeardownTimeout: types.NullDurationFrom(4 * time.Second)} @@ -183,9 +167,7 @@ func TestOptionsSettingToScript(t *testing.T) { func TestOptionsPropagationToScript(t *testing.T) { t.Parallel() - src := &lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + data := ` export let options = { setupTimeout: "1s", myOption: "test" }; export default function() { if (options.external) { @@ -197,12 +179,11 @@ func TestOptionsPropagationToScript(t *testing.T) { if (options.setupTimeout != __ENV.expectedSetupTimeout) { throw new Error("expected setupTimeout to be " + __ENV.expectedSetupTimeout + " but it was " + options.setupTimeout); } - }; - `), - } + };` expScriptOptions := lib.Options{SetupTimeout: types.NullDurationFrom(1 * time.Second)} - r1, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{Env: map[string]string{"expectedSetupTimeout": "1s"}}) + r1, err := getSimpleRunnerWithOptions("/script.js", data, + lib.RuntimeOptions{Env: map[string]string{"expectedSetupTimeout": "1s"}}) require.NoError(t, err) require.Equal(t, expScriptOptions, r1.GetOptions()) @@ -233,7 +214,7 @@ func TestMetricName(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - script := []byte(tb.Replacer.Replace(` + script := tb.Replacer.Replace(` import { Counter } from "k6/metrics"; let myCounter = new Counter("not ok name @"); @@ -241,13 +222,9 @@ func TestMetricName(t *testing.T) { export default function(data) { myCounter.add(1); } - `)) + `) - _, err := New( - &lib.SourceData{Filename: "/script.js", Data: script}, - afero.NewMemMapFs(), - lib.RuntimeOptions{}, - ) + _, err := getSimpleRunner("/script.js", script) require.Error(t, err) } @@ -255,7 +232,7 @@ func TestSetupDataIsolation(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - script := []byte(tb.Replacer.Replace(` + script := tb.Replacer.Replace(` import { Counter } from "k6/metrics"; export let options = { @@ -285,13 +262,9 @@ func TestSetupDataIsolation(t *testing.T) { } myCounter.add(1); } - `)) + `) - runner, err := New( - &lib.SourceData{Filename: "/script.js", Data: script}, - afero.NewMemMapFs(), - lib.RuntimeOptions{}, - ) + runner, err := getSimpleRunner("/script.js", script) require.NoError(t, err) engine, err := core.NewEngine(local.New(runner), runner.GetOptions()) @@ -322,13 +295,13 @@ func TestSetupDataIsolation(t *testing.T) { require.Equal(t, 501, count, "mycounter should be the number of iterations + 1 for the teardown") } -func testSetupDataHelper(t *testing.T, src *lib.SourceData) { +func testSetupDataHelper(t *testing.T, data string) { t.Helper() expScriptOptions := lib.Options{ SetupTimeout: types.NullDurationFrom(1 * time.Second), TeardownTimeout: types.NullDurationFrom(1 * time.Second), } - r1, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r1, err := getSimpleRunner("/script.js", data) // TODO fix this require.NoError(t, err) require.Equal(t, expScriptOptions, r1.GetOptions()) @@ -349,71 +322,56 @@ func testSetupDataHelper(t *testing.T, src *lib.SourceData) { } } func TestSetupDataReturnValue(t *testing.T) { - src := &lib.SourceData{ - Filename: "/script.js", - Data: []byte(` - export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; - export function setup() { - return 42; - } - export default function(data) { - if (data != 42) { - throw new Error("default: wrong data: " + JSON.stringify(data)) - } - }; - - export function teardown(data) { - if (data != 42) { - throw new Error("teardown: wrong data: " + JSON.stringify(data)) - } - }; - `), + testSetupDataHelper(t, ` + export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; + export function setup() { + return 42; } - testSetupDataHelper(t, src) + export default function(data) { + if (data != 42) { + throw new Error("default: wrong data: " + JSON.stringify(data)) + } + }; + + export function teardown(data) { + if (data != 42) { + throw new Error("teardown: wrong data: " + JSON.stringify(data)) + } + };`) } func TestSetupDataNoSetup(t *testing.T) { - src := &lib.SourceData{ - Filename: "/script.js", - Data: []byte(` - export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; - export default function(data) { - if (data !== undefined) { - throw new Error("default: wrong data: " + JSON.stringify(data)) - } - }; + testSetupDataHelper(t, ` + export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; + export default function(data) { + if (data !== undefined) { + throw new Error("default: wrong data: " + JSON.stringify(data)) + } + }; - export function teardown(data) { - if (data !== undefined) { - console.log(data); - throw new Error("teardown: wrong data: " + JSON.stringify(data)) - } - }; - `), - } - testSetupDataHelper(t, src) + export function teardown(data) { + if (data !== undefined) { + console.log(data); + throw new Error("teardown: wrong data: " + JSON.stringify(data)) + } + };`) } func TestSetupDataNoReturn(t *testing.T) { - src := &lib.SourceData{ - Filename: "/script.js", - Data: []byte(` - export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; - export function setup() { } - export default function(data) { - if (data !== undefined) { - throw new Error("default: wrong data: " + JSON.stringify(data)) - } - }; + testSetupDataHelper(t, ` + export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; + export function setup() { } + export default function(data) { + if (data !== undefined) { + throw new Error("default: wrong data: " + JSON.stringify(data)) + } + }; - export function teardown(data) { - if (data !== undefined) { - throw new Error("teardown: wrong data: " + JSON.stringify(data)) - } - }; - `), - } - testSetupDataHelper(t, src) + export function teardown(data) { + if (data !== undefined) { + throw new Error("teardown: wrong data: " + JSON.stringify(data)) + } + };`) } func TestRunnerIntegrationImports(t *testing.T) { t.Run("Modules", func(t *testing.T) { @@ -424,12 +382,10 @@ func TestRunnerIntegrationImports(t *testing.T) { "k6/html", } for _, mod := range modules { + mod := mod t.Run(mod, func(t *testing.T) { t.Run("Source", func(t *testing.T) { - _, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(fmt.Sprintf(`import "%s"; export default function() {}`, mod)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + _, err := getSimpleRunner("/script.js", fmt.Sprintf(`import "%s"; export default function() {}`, mod)) assert.NoError(t, err) }) }) @@ -438,8 +394,8 @@ func TestRunnerIntegrationImports(t *testing.T) { t.Run("Files", func(t *testing.T) { fs := afero.NewMemMapFs() - assert.NoError(t, fs.MkdirAll("/path/to", 0755)) - assert.NoError(t, afero.WriteFile(fs, "/path/to/lib.js", []byte(`export default "hi!";`), 0644)) + require.NoError(t, fs.MkdirAll("/path/to", 0755)) + require.NoError(t, afero.WriteFile(fs, "/path/to/lib.js", []byte(`export default "hi!";`), 0644)) testdata := map[string]struct{ filename, path string }{ "Absolute": {"/path/script.js", "/path/to/lib.js"}, @@ -449,33 +405,26 @@ func TestRunnerIntegrationImports(t *testing.T) { "STDIN-Relative": {"-", "./path/to/lib.js"}, } for name, data := range testdata { + name, data := name, data t.Run(name, func(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: data.filename, - Data: []byte(fmt.Sprintf(` + r1, err := getSimpleRunnerWithFileFs(data.filename, fmt.Sprintf(` import hi from "%s"; export default function() { if (hi != "hi!") { throw new Error("incorrect value"); } - }`, data.path)), - }, fs, lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + }`, data.path), fs) + require.NoError(t, err) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) testdata := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range testdata { + r := r t.Run(name, func(t *testing.T) { vu, err := r.NewVU(make(chan stats.SampleContainer, 100)) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) err = vu.RunOnce(context.Background()) - assert.NoError(t, err) + require.NoError(t, err) }) } }) @@ -484,16 +433,11 @@ func TestRunnerIntegrationImports(t *testing.T) { } func TestVURunContext(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` export let options = { vus: 10 }; export default function() { fn(); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r1.SetOptions(r1.GetOptions().Apply(lib.Options{Throw: null.BoolFrom(true)})) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) @@ -533,16 +477,13 @@ func TestVURunContext(t *testing.T) { func TestVURunInterrupt(t *testing.T) { //TODO: figure out why interrupt sometimes fails... data race in goja? - if runtime.GOOS == "windows" { + if isWindows { t.Skip() } - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` export default function() { while(true) {} } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) require.NoError(t, err) require.NoError(t, r1.SetOptions(lib.Options{Throw: null.BoolFrom(true)})) @@ -572,9 +513,7 @@ func TestVURunInterrupt(t *testing.T) { } func TestVUIntegrationGroups(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import { group } from "k6"; export default function() { fnOuter(); @@ -585,16 +524,11 @@ func TestVUIntegrationGroups(t *testing.T) { }) }); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) testdata := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range testdata { @@ -635,23 +569,16 @@ func TestVUIntegrationGroups(t *testing.T) { } func TestVUIntegrationMetrics(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import { group } from "k6"; import { Trend } from "k6/metrics"; let myMetric = new Trend("my_metric"); export default function() { myMetric.add(5); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) testdata := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range testdata { @@ -709,23 +636,15 @@ func TestVUIntegrationInsecureRequests(t *testing.T) { } for name, data := range testdata { t.Run(name, func(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { http.get("https://expired.badssl.com/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r1.SetOptions(lib.Options{Throw: null.BoolFrom(true)}.Apply(data.opts)) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } - + require.NoError(t, err) runners := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range runners { t.Run(name, func(t *testing.T) { @@ -748,18 +667,14 @@ func TestVUIntegrationInsecureRequests(t *testing.T) { } func TestVUIntegrationBlacklistOption(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { http.get("http://10.1.2.3/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) cidr, err := lib.ParseCIDR("10.0.0.0/8") + if !assert.NoError(t, err) { return } @@ -787,9 +702,7 @@ func TestVUIntegrationBlacklistOption(t *testing.T) { } func TestVUIntegrationBlacklistScript(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export let options = { @@ -798,8 +711,7 @@ func TestVUIntegrationBlacklistScript(t *testing.T) { }; export default function() { http.get("http://10.1.2.3/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) if !assert.NoError(t, err) { return } @@ -828,9 +740,8 @@ func TestVUIntegrationHosts(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunner("/script.js", + tb.Replacer.Replace(` import { check, fail } from "k6"; import http from "k6/http"; export default function() { @@ -839,8 +750,7 @@ func TestVUIntegrationHosts(t *testing.T) { "is correct IP": (r) => r.remote_ip === "127.0.0.1" }) || fail("failed to override dns"); } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(t, err) { return } @@ -912,13 +822,10 @@ func TestVUIntegrationTLSConfig(t *testing.T) { } for name, data := range testdata { t.Run(name, func(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { http.get("https://sha256.badssl.com/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) if !assert.NoError(t, err) { return } @@ -951,17 +858,14 @@ func TestVUIntegrationTLSConfig(t *testing.T) { } func TestVUIntegrationHTTP2(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { let res = http.request("GET", "https://http2.akamai.com/demo"); if (res.status != 200) { throw new Error("wrong status: " + res.status) } if (res.proto != "HTTP/2.0") { throw new Error("wrong proto: " + res.proto) } } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) if !assert.NoError(t, err) { return } @@ -1001,12 +905,9 @@ func TestVUIntegrationHTTP2(t *testing.T) { } func TestVUIntegrationOpenFunctionError(t *testing.T) { - r, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r, err := getSimpleRunner("/script.js", ` export default function() { open("/tmp/foo") } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) assert.NoError(t, err) vu, err := r.NewVU(make(chan stats.SampleContainer, 100)) @@ -1020,9 +921,7 @@ func TestVUIntegrationCookiesReset(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; export default function() { let url = "HTTPBIN_URL"; @@ -1038,8 +937,7 @@ func TestVUIntegrationCookiesReset(t *testing.T) { throw new Error("wrong cookies: " + res.body); } } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(t, err) { return } @@ -1073,9 +971,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; export default function() { let url = "HTTPBIN_URL"; @@ -1095,8 +991,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { } } } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(t, err) { return } @@ -1130,14 +1025,11 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { } func TestVUIntegrationVUID(t *testing.T) { - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` export default function() { if (__VU != 1234) { throw new Error("wrong __VU: " + __VU); } }`, - ), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + ) if !assert.NoError(t, err) { return } @@ -1226,13 +1118,10 @@ func TestVUIntegrationClientCerts(t *testing.T) { } go func() { _ = srv.Serve(listener) }() - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(fmt.Sprintf(` + r1, err := getSimpleRunner("/script.js", fmt.Sprintf(` import http from "k6/http"; export default function() { http.get("https://%s")} - `, listener.Addr().String())), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `, listener.Addr().String())) if !assert.NoError(t, err) { return } @@ -1309,17 +1198,14 @@ func TestHTTPRequestInInitContext(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - _, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + _, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import { check, fail } from "k6"; import http from "k6/http"; let res = http.get("HTTPBIN_URL/"); export default function() { console.log(test); } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if assert.Error(t, err) { assert.Equal( t, @@ -1392,11 +1278,9 @@ func TestInitContextForbidden(t *testing.T) { defer tb.Cleanup() for _, test := range table { + test := test t.Run(test[0], func(t *testing.T) { - _, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(test[1])), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + _, err := getSimpleRunner("/script.js", tb.Replacer.Replace(test[1])) if assert.Error(t, err) { assert.Equal( t, @@ -1412,10 +1296,7 @@ func TestArchiveRunningIntegraty(t *testing.T) { defer tb.Cleanup() fs := afero.NewMemMapFs() - require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm)) - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + data := tb.Replacer.Replace(` let fput = open("/home/somebody/test.json"); export let options = { setupTimeout: "10s", teardownTimeout: "10s" }; export function setup() { @@ -1426,8 +1307,10 @@ func TestArchiveRunningIntegraty(t *testing.T) { throw new Error("incorrect answer " + data); } } - `)), - }, fs, lib.RuntimeOptions{}) + `) + require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm)) + require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(data), os.ModePerm)) + r1, err := getSimpleRunnerWithFileFs("/script.js", data, fs) require.NoError(t, err) buf := bytes.NewBuffer(nil) @@ -1458,18 +1341,15 @@ func TestArchiveNotPanicking(t *testing.T) { fs := afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/non/existent", []byte(`42`), os.ModePerm)) - r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunnerWithFileFs("/script.js", tb.Replacer.Replace(` let fput = open("/non/existent"); export default function(data) { } - `)), - }, fs, lib.RuntimeOptions{}) + `), fs) require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) + arc.Filesystems = map[string]afero.Fs{"file": afero.NewMemMapFs()} r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) // we do want this to error here as this is where we find out that a given file is not in the // archive @@ -1481,9 +1361,7 @@ func TestStuffNotPanicking(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(` + r, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; import ws from "k6/ws"; import { group } from "k6"; @@ -1521,8 +1399,7 @@ func TestStuffNotPanicking(t *testing.T) { } }); } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) require.NoError(t, err) ch := make(chan stats.SampleContainer, 1000) diff --git a/lib/archive.go b/lib/archive.go index db9f0325593..f18db2d2c72 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -24,8 +24,10 @@ import ( "archive/tar" "bytes" "encoding/json" + "fmt" "io" "io/ioutil" + "net/url" "os" "path" "path/filepath" @@ -34,12 +36,17 @@ import ( "strings" "time" + "github.com/loadimpact/k6/lib/fsext" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" ) -var volumeRE = regexp.MustCompile(`^([a-zA-Z]):(.*)`) -var sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js -var homeDirRE = regexp.MustCompile(`^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`) +//nolint: gochecknoglobals, lll +var ( + volumeRE = regexp.MustCompile(`^[/\\]?([a-zA-Z]):(.*)`) + sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js + homeDirRE = regexp.MustCompile(`(?i)^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`) +) // NormalizeAndAnonymizePath Normalizes (to use a / path separator) and anonymizes a file path, by scrubbing usernames from home directories. func NormalizeAndAnonymizePath(path string) string { @@ -51,20 +58,10 @@ func NormalizeAndAnonymizePath(path string) string { return homeDirRE.ReplaceAllString(p, `$1/$2/nobody`) } -type normalizedFS struct { - afero.Fs -} - -func (m *normalizedFS) Open(name string) (afero.File, error) { - return m.Fs.Open(NormalizeAndAnonymizePath(name)) -} - -func (m *normalizedFS) OpenFile(name string, flag int, mode os.FileMode) (afero.File, error) { - return m.Fs.OpenFile(NormalizeAndAnonymizePath(name), flag, mode) -} - -func (m *normalizedFS) Stat(name string) (os.FileInfo, error) { - return m.Fs.Stat(NormalizeAndAnonymizePath(name)) +func newNormalizedFs(fs afero.Fs) afero.Fs { + return fsext.NewChangePathFs(fs, fsext.ChangePathFunc(func(name string) (string, error) { + return NormalizeAndAnonymizePath(name), nil + })) } // An Archive is a rollup of all resources and options needed to reproduce a test identically elsewhere. @@ -75,34 +72,43 @@ type Archive struct { // Options to use. Options Options `json:"options"` + // TODO: rewrite the encoding, decoding of json to use another type with only the fields it + // needs in order to remove Filename and Pwd from this // Filename and contents of the main file being executed. - Filename string `json:"filename"` - Data []byte `json:"-"` + Filename string `json:"filename"` // only for json + FilenameURL *url.URL `json:"-"` + Data []byte `json:"-"` // Working directory for resolving relative paths. - Pwd string `json:"pwd"` + Pwd string `json:"pwd"` // only for json + PwdURL *url.URL `json:"-"` - // Archived filesystem. - Scripts map[string][]byte `json:"-"` // included scripts - Files map[string][]byte `json:"-"` // non-script resources - - FS afero.Fs `json:"-"` + Filesystems map[string]afero.Fs `json:"-"` // Environment variables Env map[string]string `json:"env"` K6Version string `json:"k6version"` + Goos string `json:"goos"` } -// Reads an archive created by Archive.Write from a reader. -func ReadArchive(in io.Reader) (*Archive, error) { - r := tar.NewReader(in) - arc := &Archive{ - Scripts: make(map[string][]byte), - Files: make(map[string][]byte), - FS: &normalizedFS{Fs: afero.NewMemMapFs()}, +func (arc *Archive) getFs(name string) afero.Fs { + fs, ok := arc.Filesystems[name] + if !ok { + fs = afero.NewMemMapFs() + if name == "file" { + fs = newNormalizedFs(fs) + } + arc.Filesystems[name] = fs } + return fs +} + +// ReadArchive reads an archive created by Archive.Write from a reader. +func ReadArchive(in io.Reader) (*Archive, error) { + r := tar.NewReader(in) + arc := &Archive{Filesystems: make(map[string]afero.Fs, 2)} for { hdr, err := r.Next() if err != nil { @@ -122,15 +128,27 @@ func ReadArchive(in io.Reader) (*Archive, error) { switch hdr.Name { case "metadata.json": - if err := json.Unmarshal(data, &arc); err != nil { + if err = json.Unmarshal(data, &arc); err != nil { return nil, err } // Path separator normalization for older archives (<=0.20.0) - arc.Filename = NormalizeAndAnonymizePath(arc.Filename) - arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd) + if arc.K6Version == "" { + arc.Filename = NormalizeAndAnonymizePath(arc.Filename) + arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd) + } + arc.PwdURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd) + if err != nil { + return nil, err + } + arc.FilenameURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Filename) + if err != nil { + return nil, err + } + continue case "data": arc.Data = data + continue } // Path separator normalization for older archives (<=0.20.0) @@ -140,30 +158,70 @@ func ReadArchive(in io.Reader) (*Archive, error) { continue } pfx := normPath[:idx] - name := normPath[idx+1:] - if name != "" && name[0] == '_' { - name = name[1:] - } + name := normPath[idx:] - var dst map[string][]byte switch pfx { - case "files": - dst = arc.Files - case "scripts": - dst = arc.Scripts + case "files", "scripts": // old archives + // in old archives (pre 0.25.0) names without "_" at the beginning were https, the ones with "_" are local files + pfx = "https" + if len(name) >= 2 && name[0:2] == "/_" { + pfx = "file" + name = name[2:] + } + fallthrough + case "https", "file": + fs := arc.getFs(pfx) + name = filepath.FromSlash(name) + err = afero.WriteFile(fs, name, data, os.FileMode(hdr.Mode)) + if err != nil { + return nil, err + } + err = fs.Chtimes(name, hdr.AccessTime, hdr.ModTime) + if err != nil { + return nil, err + } default: - continue + return nil, fmt.Errorf("unknown file prefix `%s` for file `%s`", pfx, normPath) } + } + scheme, pathOnFs := getURLPathOnFs(arc.FilenameURL) + var err error + pathOnFs, err = url.PathUnescape(pathOnFs) + if err != nil { + return nil, err + } + err = afero.WriteFile(arc.getFs(scheme), pathOnFs, arc.Data, 0644) // TODO fix the mode ? + if err != nil { + return nil, err + } - dst[name] = data + return arc, nil +} - err = afero.WriteFile(arc.FS, name, data, os.ModePerm) - if err != nil { - return nil, err - } +func normalizeAndAnonymizeURL(u *url.URL) { + if u.Scheme == "file" { + u.Path = NormalizeAndAnonymizePath(u.Path) } +} - return arc, nil +func getURLPathOnFs(u *url.URL) (scheme string, pathOnFs string) { + scheme = "https" + switch { + case u.Opaque != "": + return scheme, "/" + u.Opaque + case u.Scheme == "": + return scheme, path.Clean(u.String()[len("//"):]) + default: + scheme = u.Scheme + } + return scheme, path.Clean(u.String()[len(u.Scheme)+len(":/"):]) +} + +func getURLtoString(u *url.URL) string { + if u.Opaque == "" && u.Scheme == "" { + return u.String()[len("//"):] // https url without a scheme + } + return u.String() } // Write serialises the archive to a writer. @@ -173,11 +231,18 @@ func ReadArchive(in io.Reader) (*Archive, error) { // the current one. func (arc *Archive) Write(out io.Writer) error { w := tar.NewWriter(out) - t := time.Now() + now := time.Now() metaArc := *arc - metaArc.Filename = NormalizeAndAnonymizePath(metaArc.Filename) - metaArc.Pwd = NormalizeAndAnonymizePath(metaArc.Pwd) + normalizeAndAnonymizeURL(metaArc.FilenameURL) + normalizeAndAnonymizeURL(metaArc.PwdURL) + metaArc.Filename = getURLtoString(metaArc.FilenameURL) + metaArc.Pwd = getURLtoString(metaArc.PwdURL) + var actualDataPath, err = url.PathUnescape(path.Join(getURLPathOnFs(metaArc.FilenameURL))) + if err != nil { + return err + } + var madeLinkToData bool metadata, err := metaArc.json() if err != nil { return err @@ -186,10 +251,10 @@ func (arc *Archive) Write(out io.Writer) error { Name: "metadata.json", Mode: 0644, Size: int64(len(metadata)), - ModTime: t, + ModTime: now, Typeflag: tar.TypeReg, }) - if _, err := w.Write(metadata); err != nil { + if _, err = w.Write(metadata); err != nil { return err } @@ -197,27 +262,20 @@ func (arc *Archive) Write(out io.Writer) error { Name: "data", Mode: 0644, Size: int64(len(arc.Data)), - ModTime: t, + ModTime: now, Typeflag: tar.TypeReg, }) - if _, err := w.Write(arc.Data); err != nil { + if _, err = w.Write(arc.Data); err != nil { return err } - - arcfs := []struct { - name string - files map[string][]byte - }{ - {"scripts", arc.Scripts}, - {"files", arc.Files}, - } - for _, entry := range arcfs { - _ = w.WriteHeader(&tar.Header{ - Name: entry.name, - Mode: 0755, - ModTime: t, - Typeflag: tar.TypeDir, - }) + for _, name := range [...]string{"file", "https"} { + filesystem, ok := arc.Filesystems[name] + if !ok { + continue + } + if cachedfs, ok := filesystem.(fsext.CacheOnReadFs); ok { + filesystem = cachedfs.GetCachingFs() + } // A couple of things going on here: // - You can't just create file entries, you need to create directory entries too. @@ -227,21 +285,32 @@ func (arc *Archive) Write(out io.Writer) error { // - We don't want to leak private information (eg. usernames) in archives, so make sure to // anonymize paths before stuffing them in a shareable archive. foundDirs := make(map[string]bool) - paths := make([]string, 0, len(entry.files)) - files := make(map[string][]byte, len(entry.files)) - for filePath, data := range entry.files { - filePath = NormalizeAndAnonymizePath(filePath) - files[filePath] = data - paths = append(paths, filePath) - dir := path.Dir(filePath) - for { - foundDirs[dir] = true - idx := strings.LastIndexByte(dir, os.PathSeparator) - if idx == -1 { - break - } - dir = dir[:idx] + paths := make([]string, 0, 10) + infos := make(map[string]os.FileInfo) // ... fix this ? + files := make(map[string][]byte) + + walkFunc := filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + normalizedPath := NormalizeAndAnonymizePath(filePath) + + infos[normalizedPath] = info + if info.IsDir() { + foundDirs[normalizedPath] = true + return nil } + + paths = append(paths, normalizedPath) + files[normalizedPath], err = afero.ReadFile(filesystem, filePath) + return err + }) + + if err = fsext.Walk(filesystem, afero.FilePathSeparator, walkFunc); err != nil { + return err + } + if len(files) == 0 { + continue // we don't need to write anything for this fs, if this is not done the root will be written } dirs := make([]string, 0, len(foundDirs)) for dirpath := range foundDirs { @@ -250,35 +319,51 @@ func (arc *Archive) Write(out io.Writer) error { sort.Strings(paths) sort.Strings(dirs) - for _, dirpath := range dirs { - if dirpath == "" || dirpath[0] == '/' { - dirpath = "_" + dirpath - } + for _, dirPath := range dirs { _ = w.WriteHeader(&tar.Header{ - Name: path.Clean(entry.name + "/" + dirpath), - Mode: 0755, - ModTime: t, - Typeflag: tar.TypeDir, + Name: path.Clean(path.Join(name, dirPath)), + Mode: 0755, // MemMapFs is buggy + AccessTime: now, // MemMapFs is buggy + ChangeTime: now, // MemMapFs is buggy + ModTime: now, // MemMapFs is buggy + Typeflag: tar.TypeDir, }) } for _, filePath := range paths { - data := files[filePath] - if filePath[0] == '/' { - filePath = "_" + filePath + var fullFilePath = path.Clean(path.Join(name, filePath)) + // we either have opaque + if fullFilePath == actualDataPath { + madeLinkToData = true + err = w.WriteHeader(&tar.Header{ + Name: fullFilePath, + Size: 0, + Typeflag: tar.TypeLink, + Linkname: "data", + }) + } else { + err = w.WriteHeader(&tar.Header{ + Name: fullFilePath, + Mode: 0644, // MemMapFs is buggy + Size: int64(len(files[filePath])), + AccessTime: infos[filePath].ModTime(), + ChangeTime: infos[filePath].ModTime(), + ModTime: infos[filePath].ModTime(), + Typeflag: tar.TypeReg, + }) + if err == nil { + _, err = w.Write(files[filePath]) + } } - _ = w.WriteHeader(&tar.Header{ - Name: path.Clean(entry.name + "/" + filePath), - Mode: 0644, - Size: int64(len(data)), - ModTime: t, - Typeflag: tar.TypeReg, - }) - if _, err := w.Write(data); err != nil { + if err != nil { return err } } } + if !madeLinkToData { + // This should never happen we should always link to `data` from inside the file/https directories + return fmt.Errorf("archive creation failed because the main script wasn't present in the cached filesystem") + } return w.Close() } diff --git a/lib/archive_test.go b/lib/archive_test.go index 48ea6588ba0..bc1ffbb043f 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -23,10 +23,18 @@ package lib import ( "bytes" "fmt" + "net/url" + "os" + "path" + "path/filepath" "runtime" "testing" + "github.com/loadimpact/k6/lib/consts" + "github.com/loadimpact/k6/lib/fsext" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" ) @@ -42,6 +50,8 @@ func TestNormalizeAndAnonymizePath(t *testing.T) { "\\NOTSHARED\\dir\\dir\\myfile.txt": "/NOTSHARED/dir/dir/myfile.txt", "C:\\Users\\myname\\dir\\myfile.txt": "/C/Users/nobody/dir/myfile.txt", "D:\\Documents and Settings\\myname\\dir\\myfile.txt": "/D/Documents and Settings/nobody/dir/myfile.txt", + "C:\\uSers\\myname\\dir\\myfile.txt": "/C/uSers/nobody/dir/myfile.txt", + "D:\\doCUMENts aND Settings\\myname\\dir\\myfile.txt": "/D/doCUMENts aND Settings/nobody/dir/myfile.txt", } // TODO: fix this - the issue is that filepath.Clean replaces `/` with whatever the path // separator is on the current OS and as such this gets confused for shared folder on @@ -50,6 +60,7 @@ func TestNormalizeAndAnonymizePath(t *testing.T) { testdata["//etc/hosts"] = "/etc/hosts" } for from, to := range testdata { + from, to := from, to t.Run("path="+from, func(t *testing.T) { res := NormalizeAndAnonymizePath(from) assert.Equal(t, to, res) @@ -58,36 +69,113 @@ func TestNormalizeAndAnonymizePath(t *testing.T) { } } +func makeMemMapFs(t *testing.T, input map[string][]byte) afero.Fs { + fs := afero.NewMemMapFs() + for path, data := range input { + require.NoError(t, afero.WriteFile(fs, path, data, 0644)) + } + return fs +} + +func getMapKeys(m map[string]afero.Fs) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + + return keys +} + +func diffMapFilesystems(t *testing.T, first, second map[string]afero.Fs) bool { + require.ElementsMatch(t, getMapKeys(first), getMapKeys(second), + "fs map keys don't match %s, %s", getMapKeys(first), getMapKeys(second)) + for key, fs := range first { + secondFs := second[key] + diffFilesystems(t, fs, secondFs) + } + + return true +} + +func diffFilesystems(t *testing.T, first, second afero.Fs) { + diffFilesystemsDir(t, first, second, "/") +} + +func getInfoNames(infos []os.FileInfo) []string { + var names = make([]string, len(infos)) + for i, info := range infos { + names[i] = info.Name() + } + return names +} + +func diffFilesystemsDir(t *testing.T, first, second afero.Fs, dirname string) { + firstInfos, err := afero.ReadDir(first, dirname) + require.NoError(t, err, dirname) + + secondInfos, err := afero.ReadDir(first, dirname) + require.NoError(t, err, dirname) + + require.ElementsMatch(t, getInfoNames(firstInfos), getInfoNames(secondInfos), "directory: "+dirname) + for _, info := range firstInfos { + path := filepath.Join(dirname, info.Name()) + if info.IsDir() { + diffFilesystemsDir(t, first, second, path) + continue + } + firstData, err := afero.ReadFile(first, path) + require.NoError(t, err, path) + + secondData, err := afero.ReadFile(second, path) + require.NoError(t, err, path) + + assert.Equal(t, firstData, secondData, path) + } +} + func TestArchiveReadWrite(t *testing.T) { t.Run("Roundtrip", func(t *testing.T) { arc1 := &Archive{ - Type: "js", + Type: "js", + K6Version: consts.Version, Options: Options{ VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - Filename: "/path/to/script.js", - Data: []byte(`// contents...`), - Pwd: "/path/to", - Scripts: map[string][]byte{ - "/path/to/a.js": []byte(`// a contents`), - "/path/to/b.js": []byte(`// b contents`), - "cdnjs.com/libraries/Faker": []byte(`// faker contents`), - }, - Files: map[string][]byte{ - "/path/to/file1.txt": []byte(`hi!`), - "/path/to/file2.txt": []byte(`bye!`), - "github.com/loadimpact/k6/README.md": []byte(`README`), + FilenameURL: &url.URL{Scheme: "file", Path: "/path/to/a.js"}, + Data: []byte(`// a contents`), + PwdURL: &url.URL{Scheme: "file", Path: "/path/to"}, + Filesystems: map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + "/path/to/a.js": []byte(`// a contents`), + "/path/to/b.js": []byte(`// b contents`), + "/path/to/file1.txt": []byte(`hi!`), + "/path/to/file2.txt": []byte(`bye!`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/cdnjs.com/libraries/Faker": []byte(`// faker contents`), + "/github.com/loadimpact/k6/README.md": []byte(`README`), + }), }, } buf := bytes.NewBuffer(nil) - assert.NoError(t, arc1.Write(buf)) + require.NoError(t, arc1.Write(buf)) + + arc1Filesystems := arc1.Filesystems + arc1.Filesystems = nil arc2, err := ReadArchive(buf) - arc2.FS = nil - assert.NoError(t, err) + require.NoError(t, err) + + arc2Filesystems := arc2.Filesystems + arc2.Filesystems = nil + arc2.Filename = "" + arc2.Pwd = "" + assert.Equal(t, arc1, arc2) + + diffMapFilesystems(t, arc1Filesystems, arc2Filesystems) }) t.Run("Anonymized", func(t *testing.T) { @@ -95,7 +183,7 @@ func TestArchiveReadWrite(t *testing.T) { Pwd, PwdNormAnon string }{ {"/home/myname", "/home/nobody"}, - {"C:\\Users\\Administrator", "/C/Users/nobody"}, + {filepath.FromSlash("/C:/Users/Administrator"), "/C/Users/nobody"}, } for _, entry := range testdata { arc1 := &Archive{ @@ -104,18 +192,21 @@ func TestArchiveReadWrite(t *testing.T) { VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - Filename: fmt.Sprintf("%s/script.js", entry.Pwd), - Data: []byte(`// contents...`), - Pwd: entry.Pwd, - Scripts: map[string][]byte{ - fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`), - fmt.Sprintf("%s/b.js", entry.Pwd): []byte(`// b contents`), - "cdnjs.com/libraries/Faker": []byte(`// faker contents`), - }, - Files: map[string][]byte{ - fmt.Sprintf("%s/file1.txt", entry.Pwd): []byte(`hi!`), - fmt.Sprintf("%s/file2.txt", entry.Pwd): []byte(`bye!`), - "github.com/loadimpact/k6/README.md": []byte(`README`), + FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/a.js", entry.Pwd)}, + K6Version: consts.Version, + Data: []byte(`// a contents`), + PwdURL: &url.URL{Scheme: "file", Path: entry.Pwd}, + Filesystems: map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`), + fmt.Sprintf("%s/b.js", entry.Pwd): []byte(`// b contents`), + fmt.Sprintf("%s/file1.txt", entry.Pwd): []byte(`hi!`), + fmt.Sprintf("%s/file2.txt", entry.Pwd): []byte(`bye!`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/cdnjs.com/libraries/Faker": []byte(`// faker contents`), + "/github.com/loadimpact/k6/README.md": []byte(`README`), + }), }, } arc1Anon := &Archive{ @@ -124,28 +215,41 @@ func TestArchiveReadWrite(t *testing.T) { VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - Filename: fmt.Sprintf("%s/script.js", entry.PwdNormAnon), - Data: []byte(`// contents...`), - Pwd: entry.PwdNormAnon, - Scripts: map[string][]byte{ - fmt.Sprintf("%s/a.js", entry.PwdNormAnon): []byte(`// a contents`), - fmt.Sprintf("%s/b.js", entry.PwdNormAnon): []byte(`// b contents`), - "cdnjs.com/libraries/Faker": []byte(`// faker contents`), - }, - Files: map[string][]byte{ - fmt.Sprintf("%s/file1.txt", entry.PwdNormAnon): []byte(`hi!`), - fmt.Sprintf("%s/file2.txt", entry.PwdNormAnon): []byte(`bye!`), - "github.com/loadimpact/k6/README.md": []byte(`README`), + FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/a.js", entry.PwdNormAnon)}, + K6Version: consts.Version, + Data: []byte(`// a contents`), + PwdURL: &url.URL{Scheme: "file", Path: entry.PwdNormAnon}, + + Filesystems: map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + fmt.Sprintf("%s/a.js", entry.PwdNormAnon): []byte(`// a contents`), + fmt.Sprintf("%s/b.js", entry.PwdNormAnon): []byte(`// b contents`), + fmt.Sprintf("%s/file1.txt", entry.PwdNormAnon): []byte(`hi!`), + fmt.Sprintf("%s/file2.txt", entry.PwdNormAnon): []byte(`bye!`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/cdnjs.com/libraries/Faker": []byte(`// faker contents`), + "/github.com/loadimpact/k6/README.md": []byte(`README`), + }), }, } buf := bytes.NewBuffer(nil) - assert.NoError(t, arc1.Write(buf)) + require.NoError(t, arc1.Write(buf)) + + arc1Filesystems := arc1Anon.Filesystems + arc1Anon.Filesystems = nil arc2, err := ReadArchive(buf) - arc2.FS = nil assert.NoError(t, err) + arc2.Filename = "" + arc2.Pwd = "" + + arc2Filesystems := arc2.Filesystems + arc2.Filesystems = nil + assert.Equal(t, arc1Anon, arc2) + diffMapFilesystems(t, arc1Filesystems, arc2Filesystems) } }) } @@ -159,3 +263,139 @@ func TestArchiveJSONEscape(t *testing.T) { assert.NoError(t, err) assert.Contains(t, string(b), "test<.js") } + +func TestUsingCacheFromCacheOnReadFs(t *testing.T) { + var base = afero.NewMemMapFs() + var cached = afero.NewMemMapFs() + // we specifically have different contents in both places + require.NoError(t, afero.WriteFile(base, "/wrong", []byte(`ooops`), 0644)) + require.NoError(t, afero.WriteFile(cached, "/correct", []byte(`test`), 0644)) + + arc := &Archive{ + Type: "js", + FilenameURL: &url.URL{Scheme: "file", Path: "/correct"}, + K6Version: consts.Version, + Data: []byte(`test`), + PwdURL: &url.URL{Scheme: "file", Path: "/"}, + Filesystems: map[string]afero.Fs{ + "file": fsext.NewCacheOnReadFs(base, cached, 0), + }, + } + + buf := bytes.NewBuffer(nil) + require.NoError(t, arc.Write(buf)) + + newArc, err := ReadArchive(buf) + require.NoError(t, err) + + data, err := afero.ReadFile(newArc.Filesystems["file"], "/correct") + require.NoError(t, err) + require.Equal(t, string(data), "test") + + data, err = afero.ReadFile(newArc.Filesystems["file"], "/wrong") + require.Error(t, err) + require.Nil(t, data) +} + +func TestArchiveWithDataNotInFS(t *testing.T) { + t.Parallel() + + arc := &Archive{ + Type: "js", + FilenameURL: &url.URL{Scheme: "file", Path: "/script"}, + K6Version: consts.Version, + Data: []byte(`test`), + PwdURL: &url.URL{Scheme: "file", Path: "/"}, + Filesystems: nil, + } + + buf := bytes.NewBuffer(nil) + err := arc.Write(buf) + require.Error(t, err) + require.Contains(t, err.Error(), "the main script wasn't present in the cached filesystem") +} + +func TestMalformedMetadata(t *testing.T) { + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/metadata.json", []byte("{,}"), 0644)) + var b, err = dumpMemMapFsToBuf(fs) + require.NoError(t, err) + _, err = ReadArchive(b) + require.Error(t, err) + require.Equal(t, err.Error(), `invalid character ',' looking for beginning of object key string`) +} + +func TestStrangePaths(t *testing.T) { + var pathsToChange = []string{ + `/path/with spaces/a.js`, + `/path/with spaces/a.js`, + `/path/with日本語/b.js`, + `/path/with spaces and 日本語/file1.txt`, + } + for _, pathToChange := range pathsToChange { + otherMap := make(map[string][]byte, len(pathsToChange)) + for _, other := range pathsToChange { + otherMap[other] = []byte(`// ` + other + ` contents`) + } + arc1 := &Archive{ + Type: "js", + K6Version: consts.Version, + Options: Options{ + VUs: null.IntFrom(12345), + SystemTags: GetTagSet(DefaultSystemTagList...), + }, + FilenameURL: &url.URL{Scheme: "file", Path: pathToChange}, + Data: []byte(`// ` + pathToChange + ` contents`), + PwdURL: &url.URL{Scheme: "file", Path: path.Dir(pathToChange)}, + Filesystems: map[string]afero.Fs{ + "file": makeMemMapFs(t, otherMap), + }, + } + + buf := bytes.NewBuffer(nil) + require.NoError(t, arc1.Write(buf), pathToChange) + + arc1Filesystems := arc1.Filesystems + arc1.Filesystems = nil + + arc2, err := ReadArchive(buf) + require.NoError(t, err, pathToChange) + + arc2Filesystems := arc2.Filesystems + arc2.Filesystems = nil + arc2.Filename = "" + arc2.Pwd = "" + + assert.Equal(t, arc1, arc2, pathToChange) + + diffMapFilesystems(t, arc1Filesystems, arc2Filesystems) + } +} + +func TestStdinArchive(t *testing.T) { + var fs = afero.NewMemMapFs() + // we specifically have different contents in both places + require.NoError(t, afero.WriteFile(fs, "/-", []byte(`test`), 0644)) + + arc := &Archive{ + Type: "js", + FilenameURL: &url.URL{Scheme: "file", Path: "/-"}, + K6Version: consts.Version, + Data: []byte(`test`), + PwdURL: &url.URL{Scheme: "file", Path: "/"}, + Filesystems: map[string]afero.Fs{ + "file": fs, + }, + } + + buf := bytes.NewBuffer(nil) + require.NoError(t, arc.Write(buf)) + + newArc, err := ReadArchive(buf) + require.NoError(t, err) + + data, err := afero.ReadFile(newArc.Filesystems["file"], "/-") + require.NoError(t, err) + require.Equal(t, string(data), "test") + +} diff --git a/lib/fsext/cacheonread.go b/lib/fsext/cacheonread.go new file mode 100644 index 00000000000..9a534f3f664 --- /dev/null +++ b/lib/fsext/cacheonread.go @@ -0,0 +1,27 @@ +package fsext + +import ( + "time" + + "github.com/spf13/afero" +) + +// CacheOnReadFs is wrapper around afero.CacheOnReadFs with the ability to return the filesystem +// that is used as cache +type CacheOnReadFs struct { + afero.Fs + cache afero.Fs +} + +// NewCacheOnReadFs returns a new CacheOnReadFs +func NewCacheOnReadFs(base, layer afero.Fs, cacheTime time.Duration) afero.Fs { + return CacheOnReadFs{ + Fs: afero.NewCacheOnReadFs(base, layer, cacheTime), + cache: layer, + } +} + +// GetCachingFs returns the afero.Fs being used for cache +func (c CacheOnReadFs) GetCachingFs() afero.Fs { + return c.cache +} diff --git a/lib/fsext/changepathfs.go b/lib/fsext/changepathfs.go new file mode 100644 index 00000000000..7f7a81d6f2f --- /dev/null +++ b/lib/fsext/changepathfs.go @@ -0,0 +1,190 @@ +package fsext + +import ( + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/afero" +) + +var _ afero.Lstater = (*ChangePathFs)(nil) + +// ChangePathFs is a filesystem that wraps another afero.Fs and changes all given paths from all +// file and directory names, with a function, before calling the same method on the wrapped afero.Fs. +// Heavily based on afero.BasePathFs +type ChangePathFs struct { + source afero.Fs + fn ChangePathFunc +} + +// ChangePathFile is a file from ChangePathFs +type ChangePathFile struct { + afero.File + originalName string +} + +// NewChangePathFs return a ChangePathFs where all paths will be change with the provided funcs +func NewChangePathFs(source afero.Fs, fn ChangePathFunc) *ChangePathFs { + return &ChangePathFs{source: source, fn: fn} +} + +// ChangePathFunc is the function that will be called by ChangePathFs to change the path +type ChangePathFunc func(name string) (path string, err error) + +// NewTrimFilePathSeparatorFs is ChangePathFs that trims a Afero.FilePathSeparator from all paths +// Heavily based on afero.BasePathFs +func NewTrimFilePathSeparatorFs(source afero.Fs) *ChangePathFs { + return &ChangePathFs{source: source, fn: ChangePathFunc(func(name string) (path string, err error) { + if !strings.HasPrefix(name, afero.FilePathSeparator) { + return name, os.ErrNotExist + } + + return filepath.Clean(strings.TrimPrefix(name, afero.FilePathSeparator)), nil + })} +} + +// Name Returns the name of the file +func (f *ChangePathFile) Name() string { + return f.originalName +} + +//Chtimes changes the access and modification times of the named file +func (b *ChangePathFs) Chtimes(name string, atime, mtime time.Time) (err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return &os.PathError{Op: "chtimes", Path: name, Err: err} + } + return b.source.Chtimes(newName, atime, mtime) +} + +// Chmod changes the mode of the named file to mode. +func (b *ChangePathFs) Chmod(name string, mode os.FileMode) (err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return &os.PathError{Op: "chmod", Path: name, Err: err} + } + return b.source.Chmod(newName, mode) +} + +// Name return the name of this FileSystem +func (b *ChangePathFs) Name() string { + return "ChangePathFs" +} + +// Stat returns a FileInfo describing the named file, or an error, if any +// happens. +func (b *ChangePathFs) Stat(name string) (fi os.FileInfo, err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return nil, &os.PathError{Op: "stat", Path: name, Err: err} + } + return b.source.Stat(newName) +} + +// Rename renames a file. +func (b *ChangePathFs) Rename(oldName, newName string) (err error) { + var newOldName, newNewName string + if newOldName, err = b.fn(oldName); err != nil { + return &os.PathError{Op: "rename", Path: oldName, Err: err} + } + if newNewName, err = b.fn(newName); err != nil { + return &os.PathError{Op: "rename", Path: newName, Err: err} + } + return b.source.Rename(newOldName, newNewName) +} + +// RemoveAll removes a directory path and any children it contains. It +// does not fail if the path does not exist (return nil). +func (b *ChangePathFs) RemoveAll(name string) (err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return &os.PathError{Op: "remove_all", Path: name, Err: err} + } + return b.source.RemoveAll(newName) +} + +// Remove removes a file identified by name, returning an error, if any +// happens. +func (b *ChangePathFs) Remove(name string) (err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + return b.source.Remove(newName) +} + +// OpenFile opens a file using the given flags and the given mode. +func (b *ChangePathFs) OpenFile(name string, flag int, mode os.FileMode) (f afero.File, err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return nil, &os.PathError{Op: "openfile", Path: name, Err: err} + } + sourcef, err := b.source.OpenFile(newName, flag, mode) + if err != nil { + return nil, err + } + return &ChangePathFile{File: sourcef, originalName: name}, nil +} + +// Open opens a file, returning it or an error, if any happens. +func (b *ChangePathFs) Open(name string) (f afero.File, err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + sourcef, err := b.source.Open(newName) + if err != nil { + return nil, err + } + return &ChangePathFile{File: sourcef, originalName: name}, nil +} + +// Mkdir creates a directory in the filesystem, return an error if any +// happens. +func (b *ChangePathFs) Mkdir(name string, mode os.FileMode) (err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.Mkdir(newName, mode) +} + +// MkdirAll creates a directory path and all parents that does not exist +// yet. +func (b *ChangePathFs) MkdirAll(name string, mode os.FileMode) (err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.MkdirAll(newName, mode) +} + +// Create creates a file in the filesystem, returning the file and an +// error, if any happens +func (b *ChangePathFs) Create(name string) (f afero.File, err error) { + var newName string + if newName, err = b.fn(name); err != nil { + return nil, &os.PathError{Op: "create", Path: name, Err: err} + } + sourcef, err := b.source.Create(newName) + if err != nil { + return nil, err + } + return &ChangePathFile{File: sourcef, originalName: name}, nil +} + +// LstatIfPossible implements the afero.Lstater interface +func (b *ChangePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + var newName string + newName, err := b.fn(name) + if err != nil { + return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} + } + if lstater, ok := b.source.(afero.Lstater); ok { + return lstater.LstatIfPossible(newName) + } + fi, err := b.source.Stat(newName) + return fi, false, err +} diff --git a/lib/fsext/changepathfs_test.go b/lib/fsext/changepathfs_test.go new file mode 100644 index 00000000000..5e91ea77f97 --- /dev/null +++ b/lib/fsext/changepathfs_test.go @@ -0,0 +1,167 @@ +package fsext + +import ( + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestChangePathFs(t *testing.T) { + var m = afero.NewMemMapFs() + var prefix = "/another/" + var c = NewChangePathFs(m, ChangePathFunc(func(name string) (string, error) { + if !strings.HasPrefix(name, prefix) { + return "", fmt.Errorf("path %s doesn't start with `%s`", name, prefix) + } + return name[len(prefix):], nil + })) + + var filePath = "/another/path/to/file.txt" + + require.Equal(t, c.Name(), "ChangePathFs") + t.Run("Create", func(t *testing.T) { + f, err := c.Create(filePath) + require.NoError(t, err) + require.Equal(t, filePath, f.Name()) + + /** TODO figure out if this is error in MemMapFs + _, err = c.Create(filePath) + require.Error(t, err) + require.True(t, os.IsExist(err)) + */ + + _, err = c.Create("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + }) + + t.Run("Mkdir", func(t *testing.T) { + require.NoError(t, c.Mkdir("/another/path/too", 0644)) + checkErrorPath(t, c.Mkdir("/notanother/path/too", 0644), "/notanother/path/too") + }) + + t.Run("MkdirAll", func(t *testing.T) { + require.NoError(t, c.MkdirAll("/another/pattth/too", 0644)) + checkErrorPath(t, c.MkdirAll("/notanother/pattth/too", 0644), "/notanother/pattth/too") + }) + + t.Run("Open", func(t *testing.T) { + f, err := c.Open(filePath) + require.NoError(t, err) + require.Equal(t, filePath, f.Name()) + + _, err = c.Open("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + }) + + t.Run("OpenFile", func(t *testing.T) { + f, err := c.OpenFile(filePath, os.O_RDWR, 0644) + require.NoError(t, err) + require.Equal(t, filePath, f.Name()) + + _, err = c.OpenFile("/notanother/path/to/file.txt", os.O_RDWR, 0644) + checkErrorPath(t, err, "/notanother/path/to/file.txt") + + _, err = c.OpenFile("/another/nonexistant", os.O_RDWR, 0644) + require.True(t, os.IsNotExist(err)) + }) + + t.Run("Stat Chmod Chtimes", func(t *testing.T) { + info, err := c.Stat(filePath) + require.NoError(t, err) + require.Equal(t, "file.txt", info.Name()) + + sometime := time.Unix(10000, 13) + require.NotEqual(t, sometime, info.ModTime()) + require.NoError(t, c.Chtimes(filePath, time.Now(), sometime)) + require.Equal(t, sometime, info.ModTime()) + + mode := os.FileMode(0007) + require.NotEqual(t, mode, info.Mode()) + require.NoError(t, c.Chmod(filePath, mode)) + require.Equal(t, mode, info.Mode()) + + _, err = c.Stat("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + + checkErrorPath(t, c.Chtimes("/notanother/path/to/file.txt", time.Now(), time.Now()), "/notanother/path/to/file.txt") + + checkErrorPath(t, c.Chmod("/notanother/path/to/file.txt", mode), "/notanother/path/to/file.txt") + }) + + t.Run("LstatIfPossible", func(t *testing.T) { + info, ok, err := c.LstatIfPossible(filePath) + require.NoError(t, err) + require.False(t, ok) + require.Equal(t, "file.txt", info.Name()) + + _, _, err = c.LstatIfPossible("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + }) + + t.Run("Rename", func(t *testing.T) { + info, err := c.Stat(filePath) + require.NoError(t, err) + require.False(t, info.IsDir()) + + require.NoError(t, c.Rename(filePath, "/another/path/to/file.doc")) + + _, err = c.Stat(filePath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + info, err = c.Stat("/another/path/to/file.doc") + require.NoError(t, err) + require.False(t, info.IsDir()) + + checkErrorPath(t, + c.Rename("/notanother/path/to/file.txt", "/another/path/to/file.doc"), + "/notanother/path/to/file.txt") + + checkErrorPath(t, + c.Rename(filePath, "/notanother/path/to/file.doc"), + "/notanother/path/to/file.doc") + }) + + t.Run("Remove", func(t *testing.T) { + var removeFilePath = "/another/file/to/remove.txt" + _, err := c.Create(removeFilePath) + require.NoError(t, err) + + require.NoError(t, c.Remove(removeFilePath)) + + _, err = c.Stat(removeFilePath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = c.Create(removeFilePath) + require.NoError(t, err) + + require.NoError(t, c.RemoveAll(path.Dir(removeFilePath))) + + _, err = c.Stat(removeFilePath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + checkErrorPath(t, + c.Remove("/notanother/path/to/file.txt"), + "/notanother/path/to/file.txt") + + checkErrorPath(t, + c.RemoveAll("/notanother/path/to"), + "/notanother/path/to") + }) +} + +func checkErrorPath(t *testing.T, err error, path string) { + require.Error(t, err) + p, ok := err.(*os.PathError) + require.True(t, ok) + require.Equal(t, p.Path, path) + +} diff --git a/lib/fsext/trimpathseparator_test.go b/lib/fsext/trimpathseparator_test.go new file mode 100644 index 00000000000..678b0c1b758 --- /dev/null +++ b/lib/fsext/trimpathseparator_test.go @@ -0,0 +1,30 @@ +package fsext + +import ( + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestTrimAferoPathSeparatorFs(t *testing.T) { + m := afero.NewMemMapFs() + fs := NewTrimFilePathSeparatorFs(m) + expecteData := []byte("something") + err := afero.WriteFile(fs, filepath.FromSlash("/path/to/somewhere"), expecteData, 0644) + require.NoError(t, err) + data, err := afero.ReadFile(m, "/path/to/somewhere") + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + require.Nil(t, data) + + data, err = afero.ReadFile(m, "path/to/somewhere") + require.NoError(t, err) + require.Equal(t, expecteData, data) + + err = afero.WriteFile(fs, filepath.FromSlash("path/without/separtor"), expecteData, 0644) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) +} diff --git a/lib/fsext/walk.go b/lib/fsext/walk.go new file mode 100644 index 00000000000..2869f022c82 --- /dev/null +++ b/lib/fsext/walk.go @@ -0,0 +1,84 @@ +package fsext + +import ( + "os" + "path/filepath" + "sort" + + "github.com/spf13/afero" +) + +// Walk implements afero.Walk, but in a way that it doesn't loop to infinity and doesn't have +// problems if a given path part looks like a windows volume name +func Walk(fs afero.Fs, root string, walkFn filepath.WalkFunc) error { + info, err := fs.Stat(root) + if err != nil { + return walkFn(root, nil, err) + } + return walk(fs, root, info, walkFn) +} + +// readDirNames reads the directory named by dirname and returns +// a sorted list of directory entries. +// adapted from https://github.com/spf13/afero/blob/master/path.go#L27 +func readDirNames(fs afero.Fs, dirname string) ([]string, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + infos, err := f.Readdir(-1) + if err != nil { + return nil, err + } + err = f.Close() + + if err != nil { + return nil, err + } + + var names = make([]string, len(infos)) + for i, info := range infos { + names[i] = info.Name() + } + sort.Strings(names) + return names, nil +} + +// walk recursively descends path, calling walkFn +// adapted from https://github.com/spf13/afero/blob/master/path.go#L27 +func walk(fs afero.Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + + if !info.IsDir() { + return nil + } + + names, err := readDirNames(fs, path) + if err != nil { + return walkFn(path, info, err) + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := fs.Stat(filename) + if err != nil { + if err = walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(fs, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} diff --git a/lib/models.go b/lib/models.go index c187b5b2382..3944eeecba6 100644 --- a/lib/models.go +++ b/lib/models.go @@ -40,12 +40,6 @@ const GroupSeparator = "::" // Error emitted if you attempt to instantiate a Group or Check that contains the separator. var ErrNameContainsGroupSeparator = errors.New("group and check names may not contain '::'") -// Wraps a source file; data and filename. -type SourceData struct { - Data []byte - Filename string -} - // StageFields defines the fields used for a Stage; this is a dumb hack to make the JSON code // cleaner. pls fix. type StageFields struct { diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go new file mode 100644 index 00000000000..a72b1d975ce --- /dev/null +++ b/lib/old_archive_test.go @@ -0,0 +1,208 @@ +package lib + +import ( + "archive/tar" + "bytes" + "net/url" + "os" + "path" + "path/filepath" + "testing" + + "github.com/loadimpact/k6/lib/fsext" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func dumpMemMapFsToBuf(fs afero.Fs) (*bytes.Buffer, error) { + var b = bytes.NewBuffer(nil) + var w = tar.NewWriter(b) + err := fsext.Walk(fs, afero.FilePathSeparator, + filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error { + if filePath == afero.FilePathSeparator { + return nil // skip the root + } + if err != nil { + return err + } + if info.IsDir() { + return w.WriteHeader(&tar.Header{ + Name: path.Clean(filepath.ToSlash(filePath)[1:]), + Mode: 0555, + Typeflag: tar.TypeDir, + }) + } + var data []byte + data, err = afero.ReadFile(fs, filePath) + if err != nil { + return err + } + err = w.WriteHeader(&tar.Header{ + Name: path.Clean(filepath.ToSlash(filePath)[1:]), + Mode: 0644, + Size: int64(len(data)), + Typeflag: tar.TypeReg, + }) + if err != nil { + return err + } + _, err = w.Write(data) + if err != nil { + return err + } + return nil + })) + if err != nil { + return nil, err + } + return b, w.Close() +} + +func TestOldArchive(t *testing.T) { + var testCases = map[string]string{ + // map of filename to data for each main file tested + "github.com/loadimpact/k6/samples/example.js": `github file`, + "cdnjs.com/packages/Faker": `faker file`, + "C:/something/path2": `windows script`, + "/absolulte/path2": `unix script`, + } + for filename, data := range testCases { + filename, data := filename, data + t.Run(filename, func(t *testing.T) { + metadata := `{"filename": "` + filename + `"}` + fs := makeMemMapFs(t, map[string][]byte{ + // files + "/files/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), + "/files/cdnjs.com/packages/Faker": []byte(`faker file`), + "/files/example.com/path/to.js": []byte(`example.com file`), + "/files/_/C/something/path": []byte(`windows file`), + "/files/_/absolulte/path": []byte(`unix file`), + + // scripts + "/scripts/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), + "/scripts/cdnjs.com/packages/Faker2": []byte(`faker script`), + "/scripts/example.com/path/too.js": []byte(`example.com script`), + "/scripts/_/C/something/path2": []byte(`windows script`), + "/scripts/_/absolulte/path2": []byte(`unix script`), + "/data": []byte(data), + "/metadata.json": []byte(metadata), + }) + + buf, err := dumpMemMapFsToBuf(fs) + require.NoError(t, err) + + var ( + expectedFilesystems = map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + "/C:/something/path": []byte(`windows file`), + "/absolulte/path": []byte(`unix file`), + "/C:/something/path2": []byte(`windows script`), + "/absolulte/path2": []byte(`unix script`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/example.com/path/to.js": []byte(`example.com file`), + "/example.com/path/too.js": []byte(`example.com script`), + "/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), + "/cdnjs.com/packages/Faker": []byte(`faker file`), + "/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), + "/cdnjs.com/packages/Faker2": []byte(`faker script`), + }), + } + ) + + arc, err := ReadArchive(buf) + require.NoError(t, err) + + diffMapFilesystems(t, expectedFilesystems, arc.Filesystems) + }) + } +} + +func TestUnknownPrefix(t *testing.T) { + fs := makeMemMapFs(t, map[string][]byte{ + "/strange/something": []byte(`github file`), + }) + buf, err := dumpMemMapFsToBuf(fs) + require.NoError(t, err) + + _, err = ReadArchive(buf) + require.Error(t, err) + require.Equal(t, err.Error(), + "unknown file prefix `strange` for file `strange/something`") +} + +func TestFilenamePwdResolve(t *testing.T) { + var tests = []struct { + Filename, Pwd, version string + expectedFilenameURL, expectedPwdURL *url.URL + expectedError string + }{ + { + Filename: "/home/nobody/something.js", + Pwd: "/home/nobody", + expectedFilenameURL: &url.URL{Scheme: "file", Path: "/home/nobody/something.js"}, + expectedPwdURL: &url.URL{Scheme: "file", Path: "/home/nobody"}, + }, + { + Filename: "github.com/loadimpact/k6/samples/http2.js", + Pwd: "github.com/loadimpact/k6/samples", + expectedFilenameURL: &url.URL{Opaque: "github.com/loadimpact/k6/samples/http2.js"}, + expectedPwdURL: &url.URL{Opaque: "github.com/loadimpact/k6/samples"}, + }, + { + Filename: "cdnjs.com/libraries/Faker", + Pwd: "/home/nobody", + expectedFilenameURL: &url.URL{Opaque: "cdnjs.com/libraries/Faker"}, + expectedPwdURL: &url.URL{Scheme: "file", Path: "/home/nobody"}, + }, + { + Filename: "example.com/something/dot.js", + Pwd: "example.com/something/", + expectedFilenameURL: &url.URL{Host: "example.com", Scheme: "", Path: "/something/dot.js"}, + expectedPwdURL: &url.URL{Host: "example.com", Scheme: "", Path: "/something"}, + }, + { + Filename: "https://example.com/something/dot.js", + Pwd: "https://example.com/something", + expectedFilenameURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something/dot.js"}, + expectedPwdURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something"}, + version: "0.25.0", + }, + { + Filename: "ftps://example.com/something/dot.js", + Pwd: "https://example.com/something", + expectedError: "only supported schemes for imports are file and https", + version: "0.25.0", + }, + + { + Filename: "https://example.com/something/dot.js", + Pwd: "ftps://example.com/something", + expectedError: "only supported schemes for imports are file and https", + version: "0.25.0", + }, + } + + for _, test := range tests { + metadata := `{ + "filename": "` + test.Filename + `", + "pwd": "` + test.Pwd + `", + "k6version": "` + test.version + `" + }` + + buf, err := dumpMemMapFsToBuf(makeMemMapFs(t, map[string][]byte{ + "/metadata.json": []byte(metadata), + })) + require.NoError(t, err) + + arc, err := ReadArchive(buf) + if test.expectedError != "" { + require.Error(t, err) + require.Contains(t, err.Error(), test.expectedError) + } else { + require.NoError(t, err) + require.Equal(t, test.expectedFilenameURL, arc.FilenameURL) + require.Equal(t, test.expectedPwdURL, arc.PwdURL) + } + } +} diff --git a/loader/cdnjs_test.go b/loader/cdnjs_test.go index 78f3c69ad73..55de95eadba 100644 --- a/loader/cdnjs_test.go +++ b/loader/cdnjs_test.go @@ -21,10 +21,12 @@ package loader import ( + "net/url" "testing" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCDNJS(t *testing.T) { @@ -61,19 +63,28 @@ func TestCDNJS(t *testing.T) { `^https://cdnjs.cloudflare.com/ajax/libs/Faker/0.7.2/MinFaker.js$`, }, } + + var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"} for path, expected := range paths { + path, expected := path, expected t.Run(path, func(t *testing.T) { name, loader, parts := pickLoader(path) assert.Equal(t, "cdnjs", name) assert.Equal(t, expected.parts, parts) + src, err := loader(path, parts) - assert.NoError(t, err) + require.NoError(t, err) assert.Regexp(t, expected.src, src) - data, err := Load(afero.NewMemMapFs(), "/", path) - if assert.NoError(t, err) { - assert.Equal(t, path, data.Filename) - assert.NotEmpty(t, data.Data) - } + + resolvedURL, err := Resolve(root, path) + require.NoError(t, err) + require.Empty(t, resolvedURL.Scheme) + require.Equal(t, path, resolvedURL.Opaque) + + data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path) + require.NoError(t, err) + assert.Equal(t, resolvedURL, data.URL) + assert.NotEmpty(t, data.Data) }) } @@ -92,9 +103,14 @@ func TestCDNJS(t *testing.T) { assert.Equal(t, "cdnjs", name) assert.Equal(t, []string{"Faker", "3.1.0", "nonexistent.js"}, parts) src, err := loader(path, parts) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js", src) - _, err = Load(afero.NewMemMapFs(), "/", path) - assert.EqualError(t, err, "cdnjs: not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js") + + pathURL, err := url.Parse(src) + require.NoError(t, err) + + _, err = Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path) + require.Error(t, err) + assert.Contains(t, err.Error(), "not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js") }) } diff --git a/loader/filesystems.go b/loader/filesystems.go new file mode 100644 index 00000000000..2bfde5c1205 --- /dev/null +++ b/loader/filesystems.go @@ -0,0 +1,27 @@ +package loader + +import ( + "runtime" + + "github.com/loadimpact/k6/lib/fsext" + "github.com/spf13/afero" +) + +// CreateFilesystems creates the correct filesystem map for the current OS +func CreateFilesystems() map[string]afero.Fs { + // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's + // written every time something is read from the real filesystem. This cache is then used for + // successive spawns to read from (they have no access to the real disk). + // Also initialize the same for `https` but the caching is handled manually in the loader package + osfs := afero.NewOsFs() + if runtime.GOOS == "windows" { + // This is done so that we can continue to use paths with /|"\" through the code but also to + // be easier to traverse the cachedFs later as it doesn't work very well if you have windows + // volumes + osfs = fsext.NewTrimFilePathSeparatorFs(osfs) + } + return map[string]afero.Fs{ + "file": fsext.NewCacheOnReadFs(osfs, afero.NewMemMapFs(), 0), + "https": afero.NewMemMapFs(), + } +} diff --git a/loader/github_test.go b/loader/github_test.go index 2d6d2daa2f4..bd110185b44 100644 --- a/loader/github_test.go +++ b/loader/github_test.go @@ -21,23 +21,64 @@ package loader import ( + "net/url" "testing" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGithub(t *testing.T) { path := "github.com/github/gitignore/Go.gitignore" + expectedEndSrc := "https://raw.githubusercontent.com/github/gitignore/master/Go.gitignore" name, loader, parts := pickLoader(path) assert.Equal(t, "github", name) assert.Equal(t, []string{"github", "gitignore", "Go.gitignore"}, parts) src, err := loader(path, parts) assert.NoError(t, err) - assert.Equal(t, "https://raw.githubusercontent.com/github/gitignore/master/Go.gitignore", src) - data, err := Load(afero.NewMemMapFs(), "/", path) - if assert.NoError(t, err) { - assert.Equal(t, path, data.Filename) + assert.Equal(t, expectedEndSrc, src) + + var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"} + resolvedURL, err := Resolve(root, path) + require.NoError(t, err) + require.Empty(t, resolvedURL.Scheme) + require.Equal(t, path, resolvedURL.Opaque) + t.Run("not cached", func(t *testing.T) { + data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path) + require.NoError(t, err) + assert.Equal(t, data.URL, resolvedURL) + assert.Equal(t, path, data.URL.String()) assert.NotEmpty(t, data.Data) - } + }) + + t.Run("cached", func(t *testing.T) { + fs := afero.NewMemMapFs() + testData := []byte("test data") + + err := afero.WriteFile(fs, "/github.com/github/gitignore/Go.gitignore", testData, 0644) + require.NoError(t, err) + + data, err := Load(map[string]afero.Fs{"https": fs}, resolvedURL, path) + require.NoError(t, err) + assert.Equal(t, path, data.URL.String()) + assert.Equal(t, data.Data, testData) + }) + + t.Run("relative", func(t *testing.T) { + var tests = map[string]string{ + "./something.else": "github.com/github/gitignore/something.else", + "../something.else": "github.com/github/something.else", + "/something.else": "github.com/something.else", + } + for relative, expected := range tests { + relativeURL, err := Resolve(Dir(resolvedURL), relative) + require.NoError(t, err) + assert.Equal(t, expected, relativeURL.String()) + } + }) + + t.Run("dir", func(t *testing.T) { + require.Equal(t, &url.URL{Opaque: "github.com/github/gitignore"}, Dir(resolvedURL)) + }) } diff --git a/loader/loader.go b/loader/loader.go index 083ba8739c9..eb94568a24f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -1,7 +1,7 @@ /* * * k6 - a next-generation load testing tool - * Copyright (C) 2016 Load Impact + * Copyright (C) 2019 Load Impact * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -22,22 +22,29 @@ package loader import ( "io/ioutil" - "net" "net/http" "net/url" + "os" + "path" "path/filepath" "regexp" "strings" "time" - "github.com/loadimpact/k6/lib" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/afero" ) +// SourceData wraps a source file; data and filename. +type SourceData struct { + Data []byte + URL *url.URL +} + type loaderFunc func(path string, parts []string) (string, error) +//nolint: gochecknoglobals var ( loaders = []struct { name string @@ -47,114 +54,200 @@ var ( {"cdnjs", cdnjs, regexp.MustCompile(`^cdnjs.com/libraries/([^/]+)(?:/([(\d\.)]+-?[^/]*))?(?:/(.*))?$`)}, {"github", github, regexp.MustCompile(`^github.com/([^/]+)/([^/]+)/(.*)$`)}, } - invalidScriptErrMsg = `The file "%[1]s" couldn't be found on local disk, ` + - `and trying to retrieve it from https://%[1]s failed as well. Make ` + - `sure that you've specified the right path to the file. If you're ` + + httpsSchemeCouldntBeLoadedMsg = `The moduleSpecifier "%s" couldn't be retrieved from` + + ` the resolved url "%s". Error : "%s"` + fileSchemeCouldntBeLoadedMsg = `The moduleSpecifier "%s" couldn't be found on ` + + `local disk. Make sure that you've specified the right path to the file. If you're ` + `running k6 using the Docker image make sure you have mounted the ` + `local directory (-v /local/path/:/inside/docker/path) containing ` + `your script and modules so that they're accessible by k6 from ` + `inside of the container, see ` + `https://docs.k6.io/v1.0/docs/modules#section-using-local-modules-with-docker.` + errNoLoaderMatched = errors.New("no loader matched") ) -// Resolves a relative path to an absolute one. -func Resolve(pwd, name string) string { - if name[0] == '.' { - return filepath.ToSlash(filepath.Join(pwd, name)) +// Resolve a relative path to an absolute one. +func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { + if moduleSpecifier == "" { + return nil, errors.New("local or remote path required") + } + + if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' || filepath.IsAbs(moduleSpecifier) { + if pwd.Opaque != "" { // this is a loader reference + parts := strings.SplitN(pwd.Opaque, "/", 2) + if moduleSpecifier[0] == '/' { + return &url.URL{Opaque: path.Join(parts[0], moduleSpecifier)}, nil + } + return &url.URL{Opaque: path.Join(parts[0], path.Join(path.Dir(parts[1]+"/"), moduleSpecifier))}, nil + } + + // The file is in format like C:/something/path.js. But this will be decoded as scheme `C` + // ... which is not what we want we want it to be decode as file:///C:/something/path.js + if filepath.VolumeName(moduleSpecifier) != "" { + moduleSpecifier = "/" + moduleSpecifier + } + + // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we read + // it if it's missing + var finalPwd = pwd + if pwd.Opaque != "" { + if !strings.HasSuffix(pwd.Opaque, "/") { + finalPwd = &url.URL{Opaque: pwd.Opaque + "/"} + } + } else if !strings.HasSuffix(pwd.Path, "/") { + finalPwd = &url.URL{} + *finalPwd = *pwd + finalPwd.Path += "/" + } + return finalPwd.Parse(moduleSpecifier) + } + + if strings.Contains(moduleSpecifier, "://") { + u, err := url.Parse(moduleSpecifier) + if err != nil { + return nil, err + } + if u.Scheme != "file" && u.Scheme != "https" { + return nil, + errors.Errorf("only supported schemes for imports are file and https, %s has `%s`", + moduleSpecifier, u.Scheme) + } + if u.Scheme == "file" && pwd.Scheme == "https" { + return nil, errors.Errorf("origin (%s) not allowed to load local file: %s", pwd, moduleSpecifier) + } + return u, err + } + // here we only care if a loader is pickable, if it is and later there is an error in the loading + // from it we don't want to try another resolve + _, loader, _ := pickLoader(moduleSpecifier) + if loader == nil { + u, err := url.Parse("https://" + moduleSpecifier) + if err != nil { + return nil, err + } + u.Scheme = "" + return u, nil } - return name + return &url.URL{Opaque: moduleSpecifier}, nil } -// Returns the directory for the path. -func Dir(name string) string { - if name == "-" { - return "/" +// Dir returns the directory for the path. +func Dir(old *url.URL) *url.URL { + if old.Opaque != "" { // loader + return &url.URL{Opaque: path.Join(old.Opaque, "../")} } - return filepath.Dir(name) + return old.ResolveReference(&url.URL{Path: "./"}) } -func Load(fs afero.Fs, pwd, name string) (*lib.SourceData, error) { - log.WithFields(log.Fields{"pwd": pwd, "name": name}).Debug("Loading...") +// Load loads the provided moduleSpecifier from the given filesystems which are map of afero.Fs +// for a given scheme which is they key of the map. If the scheme is https then a request will +// be made if the files is not found in the map and written to the map. +func Load( + filesystems map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string, +) (*SourceData, error) { + log.WithFields( + log.Fields{ + "moduleSpecifier": moduleSpecifier, + "original moduleSpecifier": originalModuleSpecifier, + }).Debug("Loading...") - // We just need to make sure `import ""` doesn't crash the loader. - if name == "" { - return nil, errors.New("local or remote path required") + var pathOnFs string + switch { + case moduleSpecifier.Opaque != "": // This is loader + pathOnFs = filepath.Join(afero.FilePathSeparator, moduleSpecifier.Opaque) + case moduleSpecifier.Scheme == "": + pathOnFs = path.Clean(moduleSpecifier.String()) + default: + pathOnFs = path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):]) } - - // Do not allow the protocol to be specified, it messes everything up. - if strings.Contains(name, "://") { - return nil, errors.New("imports should not contain a protocol") + scheme := moduleSpecifier.Scheme + if scheme == "" { + scheme = "https" } - // Do not allow remote-loaded scripts to lift arbitrary files off the user's machine. - if (name[0] == '/' && pwd[0] != '/') || (filepath.VolumeName(name) != "" && filepath.VolumeName(pwd) == "") { - return nil, errors.Errorf("origin (%s) not allowed to load local file: %s", pwd, name) + pathOnFs, err := url.PathUnescape(filepath.FromSlash(pathOnFs)) + if err != nil { + return nil, err } - // If the file starts with ".", resolve it as a relative path. - name = Resolve(pwd, name) - log.WithField("name", name).Debug("Resolved...") + data, err := afero.ReadFile(filesystems[scheme], pathOnFs) - // If the resolved path starts with a "/" or has a volume, it's a local file. - if name[0] == '/' || filepath.VolumeName(name) != "" { - data, err := afero.ReadFile(fs, name) - if err != nil { - return nil, err + if err != nil { + if os.IsNotExist(err) { + if scheme == "https" { + var finalModuleSpecifierURL = &url.URL{} + + switch { + case moduleSpecifier.Opaque != "": // This is loader + finalModuleSpecifierURL, err = resolveUsingLoaders(moduleSpecifier.Opaque) + if err != nil { + return nil, err + } + case moduleSpecifier.Scheme == "": + log.WithField("url", moduleSpecifier).Warning( + "A url was resolved but it didn't have scheme. " + + "This will be deprecated in the future and all remote modules will " + + "need to explicitly use `https` as scheme") + *finalModuleSpecifierURL = *moduleSpecifier + finalModuleSpecifierURL.Scheme = scheme + default: + finalModuleSpecifierURL = moduleSpecifier + } + var result *SourceData + result, err = loadRemoteURL(finalModuleSpecifierURL) + if err != nil { + return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, finalModuleSpecifierURL, err) + } + result.URL = moduleSpecifier + // TODO maybe make an afero.Fs which makes request directly and than use CacheOnReadFs + // on top of as with the `file` scheme fs + _ = afero.WriteFile(filesystems[scheme], pathOnFs, result.Data, 0644) + return result, nil + } + return nil, errors.Errorf(fileSchemeCouldntBeLoadedMsg, moduleSpecifier) } - return &lib.SourceData{Filename: name, Data: data}, nil + return nil, err } - // If the file is from a known service, try loading from there. - loaderName, loader, loaderArgs := pickLoader(name) + return &SourceData{URL: moduleSpecifier, Data: data}, nil +} + +func resolveUsingLoaders(name string) (*url.URL, error) { + _, loader, loaderArgs := pickLoader(name) if loader != nil { - u, err := loader(name, loaderArgs) + urlString, err := loader(name, loaderArgs) if err != nil { return nil, err } - data, err := fetch(u) - if err != nil { - return nil, errors.Wrap(err, loaderName) - } - return &lib.SourceData{Filename: name, Data: data}, nil + return url.Parse(urlString) } - // If it's not a file, check is it a remote location. HTTPS is enforced, because it's 2017, HTTPS is easy, - // running arbitrary, trivially MitM'd code (even sandboxed) is very, very bad. - origURL := "https://" + name - parsedURL, err := url.Parse(origURL) - - if err != nil { - return nil, errors.Errorf(invalidScriptErrMsg, name) - } + return nil, errNoLoaderMatched +} - if _, err = net.LookupHost(parsedURL.Hostname()); err != nil { - return nil, errors.Errorf(invalidScriptErrMsg, name) +func loadRemoteURL(u *url.URL) (*SourceData, error) { + var oldQuery = u.RawQuery + if u.RawQuery != "" { + u.RawQuery += "&" } + u.RawQuery += "_k6=1" - // Load it and have a look. - url := origURL - if !strings.ContainsRune(url, '?') { - url += "?" - } else { - url += "&" - } - url += "_k6=1" - data, err := fetch(url) + data, err := fetch(u.String()) + u.RawQuery = oldQuery // If this fails, try to fetch without ?_k6=1 - some sources act weird around unknown GET args. if err != nil { - data2, err2 := fetch(origURL) - if err2 != nil { - return nil, errors.Errorf(invalidScriptErrMsg, name) + data, err = fetch(u.String()) + if err != nil { + return nil, err } - data = data2 } // TODO: Parse the HTML, look for meta tags!! // // - return &lib.SourceData{Filename: name, Data: data}, nil + return &SourceData{URL: u, Data: data}, nil } func pickLoader(path string) (string, loaderFunc, []string) { diff --git a/loader/loader_test.go b/loader/loader_test.go index 33e0f8c640c..745d6007509 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -18,31 +18,87 @@ * */ -package loader +package loader_test import ( "fmt" "net/http" + "net/url" "path/filepath" "testing" "github.com/loadimpact/k6/lib/testutils" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDir(t *testing.T) { testdata := map[string]string{ - "/path/to/file.txt": filepath.FromSlash("/path/to"), + "/path/to/file.txt": filepath.FromSlash("/path/to/"), "-": "/", } for name, dir := range testdata { + nameURL := &url.URL{Scheme: "file", Path: name} + dirURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(dir)} t.Run("path="+name, func(t *testing.T) { - assert.Equal(t, dir, Dir(name)) + assert.Equal(t, dirURL, loader.Dir(nameURL)) }) } } +func TestResolve(t *testing.T) { + + t.Run("Blank", func(t *testing.T) { + _, err := loader.Resolve(nil, "") + assert.EqualError(t, err, "local or remote path required") + }) + + t.Run("Protocol", func(t *testing.T) { + root, err := url.Parse("file:///") + require.NoError(t, err) + + t.Run("Missing", func(t *testing.T) { + u, err := loader.Resolve(root, "example.com/html") + require.NoError(t, err) + assert.Equal(t, u.String(), "//example.com/html") + // TODO: check that warning will be emitted if Loaded + }) + t.Run("WS", func(t *testing.T) { + moduleSpecifier := "ws://example.com/html" + _, err := loader.Resolve(root, moduleSpecifier) + assert.EqualError(t, err, + "only supported schemes for imports are file and https, "+moduleSpecifier+" has `ws`") + }) + + t.Run("HTTP", func(t *testing.T) { + moduleSpecifier := "http://example.com/html" + _, err := loader.Resolve(root, moduleSpecifier) + assert.EqualError(t, err, + "only supported schemes for imports are file and https, "+moduleSpecifier+" has `http`") + }) + }) + + t.Run("Remote Lifting Denied", func(t *testing.T) { + pwdURL, err := url.Parse("https://example.com/") + require.NoError(t, err) + + _, err = loader.Resolve(pwdURL, "file:///etc/shadow") + assert.EqualError(t, err, "origin (https://example.com/) not allowed to load local file: file:///etc/shadow") + }) + + t.Run("Fixes missing slash in pwd", func(t *testing.T) { + pwdURL, err := url.Parse("https://example.com/path/to") + require.NoError(t, err) + + moduleURL, err := loader.Resolve(pwdURL, "./something") + require.NoError(t, err) + require.Equal(t, "https://example.com/path/to/something", moduleURL.String()) + require.Equal(t, "https://example.com/path/to", pwdURL.String()) + }) + +} func TestLoad(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) sr := tb.Replacer.Replace @@ -55,69 +111,93 @@ func TestLoad(t *testing.T) { http.DefaultTransport = oldHTTPTransport }() - t.Run("Blank", func(t *testing.T) { - _, err := Load(nil, "/", "") - assert.EqualError(t, err, "local or remote path required") - }) - - t.Run("Protocol", func(t *testing.T) { - _, err := Load(nil, "/", sr("HTTPSBIN_URL/html")) - assert.EqualError(t, err, "imports should not contain a protocol") - }) - t.Run("Local", func(t *testing.T) { - fs := afero.NewMemMapFs() - assert.NoError(t, fs.MkdirAll("/path/to", 0755)) - assert.NoError(t, afero.WriteFile(fs, "/path/to/file.txt", []byte("hi"), 0644)) + filesystems := make(map[string]afero.Fs) + filesystems["file"] = afero.NewMemMapFs() + assert.NoError(t, filesystems["file"].MkdirAll("/path/to", 0755)) + assert.NoError(t, afero.WriteFile(filesystems["file"], "/path/to/file.txt", []byte("hi"), 0644)) testdata := map[string]struct{ pwd, path string }{ - "Absolute": {"/path", "/path/to/file.txt"}, - "Relative": {"/path", "./to/file.txt"}, - "Adjacent": {"/path/to", "./file.txt"}, + "Absolute": {"/path/", "/path/to/file.txt"}, + "Relative": {"/path/", "./to/file.txt"}, + "Adjacent": {"/path/to/", "./file.txt"}, } for name, data := range testdata { + data := data t.Run(name, func(t *testing.T) { - src, err := Load(fs, data.pwd, data.path) - if assert.NoError(t, err) { - assert.Equal(t, "/path/to/file.txt", src.Filename) - assert.Equal(t, "hi", string(src.Data)) - } + pwdURL, err := url.Parse("file://" + data.pwd) + require.NoError(t, err) + + moduleURL, err := loader.Resolve(pwdURL, data.path) + require.NoError(t, err) + + src, err := loader.Load(filesystems, moduleURL, data.path) + require.NoError(t, err) + + assert.Equal(t, "file:///path/to/file.txt", src.URL.String()) + assert.Equal(t, "hi", string(src.Data)) }) } t.Run("Nonexistent", func(t *testing.T) { + root, err := url.Parse("file:///") + require.NoError(t, err) + path := filepath.FromSlash("/nonexistent") - _, err := Load(fs, "/", "/nonexistent") - assert.EqualError(t, err, fmt.Sprintf("open %s: file does not exist", path)) - }) + pathURL, err := loader.Resolve(root, "/nonexistent") + require.NoError(t, err) - t.Run("Remote Lifting Denied", func(t *testing.T) { - _, err := Load(fs, "example.com", "/etc/shadow") - assert.EqualError(t, err, "origin (example.com) not allowed to load local file: /etc/shadow") + _, err = loader.Load(filesystems, pathURL, path) + require.Error(t, err) + assert.Contains(t, err.Error(), + fmt.Sprintf(`The moduleSpecifier "file://%s" couldn't be found on local disk. `, + filepath.ToSlash(path))) }) + }) t.Run("Remote", func(t *testing.T) { - src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/html")) - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/html")) + filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()} + t.Run("From local", func(t *testing.T) { + root, err := url.Parse("file:///") + require.NoError(t, err) + + moduleSpecifier := sr("HTTPSBIN_URL/html") + moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier) + require.NoError(t, err) + + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) + require.NoError(t, err) + assert.Equal(t, src.URL, moduleSpecifierURL) assert.Contains(t, string(src.Data), "Herman Melville - Moby-Dick") - } + }) t.Run("Absolute", func(t *testing.T) { - src, err := Load(nil, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT"), sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt")) - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt")) - assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") - } + pwdURL, err := url.Parse(sr("HTTPSBIN_URL")) + require.NoError(t, err) + + moduleSpecifier := sr("HTTPSBIN_URL/robots.txt") + moduleSpecifierURL, err := loader.Resolve(pwdURL, moduleSpecifier) + require.NoError(t, err) + + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) + require.NoError(t, err) + assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/robots.txt")) + assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") }) t.Run("Relative", func(t *testing.T) { - src, err := Load(nil, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT"), "./robots.txt") - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt")) - assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") - } + pwdURL, err := url.Parse(sr("HTTPSBIN_URL")) + require.NoError(t, err) + + moduleSpecifier := ("./robots.txt") + moduleSpecifierURL, err := loader.Resolve(pwdURL, moduleSpecifier) + require.NoError(t, err) + + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) + require.NoError(t, err) + assert.Equal(t, sr("HTTPSBIN_URL/robots.txt"), src.URL.String()) + assert.Equal(t, "User-agent: *\nDisallow: /deny\n", string(src.Data)) }) }) @@ -132,11 +212,19 @@ func TestLoad(t *testing.T) { }) t.Run("No _k6=1 Fallback", func(t *testing.T) { - src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/raw/something")) - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/raw/something")) - assert.Equal(t, responseStr, string(src.Data)) - } + root, err := url.Parse("file:///") + require.NoError(t, err) + + moduleSpecifier := sr("HTTPSBIN_URL/raw/something") + moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier) + require.NoError(t, err) + + filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()} + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) + + require.NoError(t, err) + assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/raw/something")) + assert.Equal(t, responseStr, string(src.Data)) }) tb.Mux.HandleFunc("/invalid", func(w http.ResponseWriter, r *http.Request) { @@ -144,19 +232,32 @@ func TestLoad(t *testing.T) { }) t.Run("Invalid", func(t *testing.T) { - src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/invalid")) - assert.Nil(t, src) - assert.Error(t, err) - - t.Run("Host", func(t *testing.T) { - src, err := Load(nil, "/", "some-path-that-doesnt-exist.js") - assert.Nil(t, src) - assert.Error(t, err) - }) - t.Run("URL", func(t *testing.T) { - src, err := Load(nil, "/", "192.168.0.%31") - assert.Nil(t, src) - assert.Error(t, err) + root, err := url.Parse("file:///") + require.NoError(t, err) + + t.Run("IP URL", func(t *testing.T) { + _, err := loader.Resolve(root, "192.168.0.%31") + require.Error(t, err) + require.Contains(t, err.Error(), `invalid URL escape "%31"`) }) + + var testData = [...]struct { + name, moduleSpecifier string + }{ + {"URL", sr("HTTPSBIN_URL/invalid")}, + {"HOST", "some-path-that-doesnt-exist.js"}, + } + + filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()} + for _, data := range testData { + moduleSpecifier := data.moduleSpecifier + t.Run(data.name, func(t *testing.T) { + moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier) + require.NoError(t, err) + + _, err = loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) + require.Error(t, err) + }) + } }) } diff --git a/loader/readsource.go b/loader/readsource.go new file mode 100644 index 00000000000..3efd4ce85a5 --- /dev/null +++ b/loader/readsource.go @@ -0,0 +1,48 @@ +package loader + +import ( + "io" + "io/ioutil" + "net/url" + "path/filepath" + + "github.com/loadimpact/k6/lib/fsext" + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +// ReadSource Reads a source file from any supported destination. +func ReadSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*SourceData, error) { + if src == "-" { + data, err := ioutil.ReadAll(stdin) + if err != nil { + return nil, err + } + // TODO: don't do it in this way ... + err = afero.WriteFile(filesystems["file"].(fsext.CacheOnReadFs).GetCachingFs(), "/-", data, 0644) + if err != nil { + return nil, errors.Wrap(err, "caching data read from -") + } + return &SourceData{URL: &url.URL{Path: "/-", Scheme: "file"}, Data: data}, err + } + var srcLocalPath string + if filepath.IsAbs(src) { + srcLocalPath = src + } else { + srcLocalPath = filepath.Join(pwd, src) + } + // All paths should start with a / in all fses. This is mostly for windows where it will start + // with a volume name : C:\something.js + srcLocalPath = filepath.Clean(afero.FilePathSeparator + srcLocalPath) + if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok { + // there is file on the local disk ... lets use it :) + return Load(filesystems, &url.URL{Scheme: "file", Path: filepath.ToSlash(srcLocalPath)}, src) + } + + pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} + srcURL, err := Resolve(pwdURL, filepath.ToSlash(src)) + if err != nil { + return nil, err + } + return Load(filesystems, srcURL, src) +} diff --git a/loader/readsource_test.go b/loader/readsource_test.go new file mode 100644 index 00000000000..e962f304f87 --- /dev/null +++ b/loader/readsource_test.go @@ -0,0 +1,88 @@ +package loader + +import ( + "bytes" + "io" + "net/url" + "testing" + + "github.com/loadimpact/k6/lib/fsext" + "github.com/pkg/errors" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +type errorReader string + +func (e errorReader) Read(_ []byte) (int, error) { + return 0, errors.New((string)(e)) +} + +var _ io.Reader = errorReader("") + +func TestReadSourceSTDINError(t *testing.T) { + _, err := ReadSource("-", "", nil, errorReader("1234")) + require.Error(t, err) + require.Equal(t, "1234", err.Error()) +} + +func TestReadSourceSTDINCache(t *testing.T) { + var data = []byte(`test contents`) + var r = bytes.NewReader(data) + var fs = afero.NewMemMapFs() + sourceData, err := ReadSource("-", "/path/to/pwd", + map[string]afero.Fs{"file": fsext.NewCacheOnReadFs(nil, fs, 0)}, r) + require.NoError(t, err) + require.Equal(t, &SourceData{ + URL: &url.URL{Scheme: "file", Path: "/-"}, + Data: data}, sourceData) + fileData, err := afero.ReadFile(fs, "/-") + require.NoError(t, err) + require.Equal(t, data, fileData) +} + +func TestReadSourceRelative(t *testing.T) { + var data = []byte(`test contents`) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/path/to/somewhere/script.js", data, 0644)) + sourceData, err := ReadSource("../somewhere/script.js", "/path/to/pwd", map[string]afero.Fs{"file": fs}, nil) + require.NoError(t, err) + require.Equal(t, &SourceData{ + URL: &url.URL{Scheme: "file", Path: "/path/to/somewhere/script.js"}, + Data: data}, sourceData) +} + +func TestReadSourceAbsolute(t *testing.T) { + var data = []byte(`test contents`) + var r = bytes.NewReader(data) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/a/b", data, 0644)) + require.NoError(t, afero.WriteFile(fs, "/c/a/b", []byte("wrong"), 0644)) + sourceData, err := ReadSource("/a/b", "/c", map[string]afero.Fs{"file": fs}, r) + require.NoError(t, err) + require.Equal(t, &SourceData{ + URL: &url.URL{Scheme: "file", Path: "/a/b"}, + Data: data}, sourceData) +} + +func TestReadSourceHttps(t *testing.T) { + var data = []byte(`test contents`) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644)) + sourceData, err := ReadSource("https://github.com/something", "/c", + map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil) + require.NoError(t, err) + require.Equal(t, &SourceData{ + URL: &url.URL{Scheme: "https", Host: "github.com", Path: "/something"}, + Data: data}, sourceData) +} + +func TestReadSourceHttpError(t *testing.T) { + var data = []byte(`test contents`) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644)) + _, err := ReadSource("http://github.com/something", "/c", + map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil) + require.Error(t, err) + require.Contains(t, err.Error(), `only supported schemes for imports are file and https`) +} diff --git a/stats/cloud/collector.go b/stats/cloud/collector.go index d5108a11f40..8fa8ec07453 100644 --- a/stats/cloud/collector.go +++ b/stats/cloud/collector.go @@ -30,6 +30,7 @@ import ( "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext" "github.com/loadimpact/k6/lib/netext/httpext" + "github.com/loadimpact/k6/loader" "github.com/pkg/errors" "gopkg.in/guregu/null.v3" @@ -97,7 +98,7 @@ func MergeFromExternal(external map[string]json.RawMessage, conf *Config) error } // New creates a new cloud collector -func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*Collector, error) { +func New(conf Config, src *loader.SourceData, opts lib.Options, version string) (*Collector, error) { if err := MergeFromExternal(opts.External, &conf); err != nil { return nil, err } @@ -107,7 +108,7 @@ func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*C } if !conf.Name.Valid || conf.Name.String == "" { - conf.Name = null.StringFrom(filepath.Base(src.Filename)) + conf.Name = null.StringFrom(filepath.Base(src.URL.Path)) } if conf.Name.String == "-" { conf.Name = null.StringFrom(TestName) diff --git a/stats/cloud/collector_test.go b/stats/cloud/collector_test.go index ccdec01b936..a0f926271c0 100644 --- a/stats/cloud/collector_test.go +++ b/stats/cloud/collector_test.go @@ -27,6 +27,7 @@ import ( "io/ioutil" "math/rand" "net/http" + "net/url" "sync" "testing" "time" @@ -42,6 +43,7 @@ import ( "github.com/loadimpact/k6/lib/netext/httpext" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" ) @@ -134,9 +136,9 @@ func TestCloudCollector(t *testing.T) { })) defer tb.Cleanup() - script := &lib.SourceData{ - Data: []byte(""), - Filename: "/script.js", + script := &loader.SourceData{ + Data: []byte(""), + URL: &url.URL{Path: "/script.js"}, } options := lib.Options{ @@ -280,9 +282,9 @@ func TestCloudCollectorMaxPerPacket(t *testing.T) { })) defer tb.Cleanup() - script := &lib.SourceData{ - Data: []byte(""), - Filename: "/script.js", + script := &loader.SourceData{ + Data: []byte(""), + URL: &url.URL{Path: "/script.js"}, } options := lib.Options{