From 4d362417fd14b0b8349150cb28c3e8f2e756932e Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 17 Sep 2019 20:05:10 +0000 Subject: [PATCH] sandbox: add gvisor runsc-based sandbox This creates a VM (running Container-Optimized OS) with configuration such that it boots up and downloads/configures the runsc Docker runtime, reloading the existing Docker daemon on the VM, and then creates a new privileged Docker container with the host's /var/run/docker.sock available to the container. From within that container it's then possible for the new sandbox HTTP server to create its own Docker containers running under gvisor (using docker run --runtime=runsc). This then adds a regional us-central1 load balancer and instance group manager & instane template to run these VMs automatically across us-central1. Then the play.golang.org frontend can hit that URL (http://sandbox.play-sandbox-fwd.il4.us-central1.lb.golang-org.internal) Fixes golang/go#25224 Updates golang/go#30439 (remove nacl) Updates golang/go#33629 (this CL makes the playground support 2 versions) Change-Id: I56c8a86875abcde9d29fa7592b23c0ecd3861458 Reviewed-on: https://go-review.googlesource.com/c/playground/+/195983 Run-TryBot: Brad Fitzpatrick Reviewed-by: Alexander Rakoczy Reviewed-by: Emmanuel Odeke --- .gitignore | 1 + Dockerfile | 25 +- Makefile | 23 +- go.sum | 8 - main.go | 4 + sandbox.go | 122 ++++++-- sandbox/.gitignore | 1 + sandbox/Dockerfile | 34 +++ sandbox/Dockerfile.gvisor | 33 +++ sandbox/Makefile | 52 ++++ sandbox/cloud-init.yaml | 17 ++ sandbox/konlet.yaml | 16 ++ sandbox/sandbox.go | 521 ++++++++++++++++++++++++++++++++++ sandbox/sandbox.tf | 143 ++++++++++ sandbox/sandboxtypes/types.go | 22 ++ server_test.go | 2 +- tests.go | 97 ++++++- vet.go | 3 +- 18 files changed, 1066 insertions(+), 58 deletions(-) create mode 100644 .gitignore create mode 100644 sandbox/.gitignore create mode 100644 sandbox/Dockerfile create mode 100644 sandbox/Dockerfile.gvisor create mode 100644 sandbox/Makefile create mode 100644 sandbox/cloud-init.yaml create mode 100644 sandbox/konlet.yaml create mode 100644 sandbox/sandbox.go create mode 100644 sandbox/sandbox.tf create mode 100644 sandbox/sandboxtypes/types.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..3fa8c86b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.terraform diff --git a/Dockerfile b/Dockerfile index 2e84a6cf..063f9a52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,14 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +############################################################################ FROM debian:stretch AS nacl RUN apt-get update && apt-get install -y --no-install-recommends curl bzip2 ca-certificates RUN curl -s https://storage.googleapis.com/nativeclient-mirror/nacl/nacl_sdk/trunk.544461/naclsdk_linux.tar.bz2 | tar -xj -C /tmp --strip-components=2 pepper_67/tools/sel_ldr_x86_64 +############################################################################ FROM debian:stretch AS build LABEL maintainer="golang-dev@googlegroups.com" @@ -64,12 +66,32 @@ COPY . /go/src/playground/ WORKDIR /go/src/playground RUN go install +############################################################################ +# Temporary Docker stage to add a pre-Go1.14 $GOROOT into our +# container for early linux/amd64 testing. +FROM golang:1.13 AS temp_pre_go14 + +ENV BUILD_DEPS 'curl git gcc patch libc6-dev ca-certificates' +RUN apt-get update && apt-get install -y --no-install-recommends ${BUILD_DEPS} + +# go1.14beta1: +ENV GO_REV a5bfd9da1d1b24f326399b6b75558ded14514f23 + +RUN cd /usr/local && git clone https://go.googlesource.com/go go1.14 && cd go1.14 && git reset --hard ${GO_REV} +WORKDIR /usr/local/go1.14/src +RUN ./make.bash +ENV GOROOT /usr/local/go1.14 +RUN ../bin/go install --tags=faketime std + +############################################################################ +# Final stage. FROM debian:stretch RUN apt-get update && apt-get install -y git ca-certificates --no-install-recommends COPY --from=build /usr/local/go /usr/local/go COPY --from=nacl /tmp/sel_ldr_x86_64 /usr/local/bin +COPY --from=temp_pre_go14 /usr/local/go1.14 /usr/local/go1.14 ENV GOPATH /go ENV PATH /usr/local/go/bin:$GOPATH/bin:$PATH @@ -101,9 +123,6 @@ COPY edit.html /app COPY static /app/static WORKDIR /app -# Run tests -RUN /app/playground test - # Whether we allow third-party imports via proxy.golang.org: ENV ALLOW_PLAY_MODULE_DOWNLOADS true diff --git a/Makefile b/Makefile index 2cf568d5..bac164f3 100644 --- a/Makefile +++ b/Makefile @@ -9,12 +9,27 @@ GCLOUD_ACCESS_TOKEN := $(shell gcloud auth print-access-token) docker: docker build -t golang/playground . -test: +runlocal: + docker network create sandnet || true + docker kill play_dev || true + docker run --name=play_dev --rm --network=sandnet -ti -p 127.0.0.1:8081:8080/tcp golang/playground + +test_go: # Run fast tests first: (and tests whether, say, things compile) GO111MODULE=on go test -v - # Then run the slower tests, which happen as one of the - # Dockerfile RUN steps: - docker build -t golang/playground . + +test_nacl: docker + docker kill sandbox_front_test || true + docker run --rm --name=sandbox_front_test --network=sandnet -t golang/playground testnacl + +test_gvisor: docker + docker kill sandbox_front_test || true + docker run --rm --name=sandbox_front_test --network=sandnet -t golang/playground test + +# Note: test_gvisor is not included in "test" yet, because it requires +# running a separate server first ("make runlocal" in the sandbox +# directory) +test: test_go test_nacl update-cloudbuild-trigger: # The gcloud CLI doesn't yet support updating a trigger. diff --git a/go.sum b/go.sum index cc443522..0c2bd239 100644 --- a/go.sum +++ b/go.sum @@ -59,14 +59,12 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -75,16 +73,13 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -103,17 +98,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 h1:nfPFGzJkUDX6uBmpN/pSw7MbOAWegH5QDQuoXFHedLg= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/main.go b/main.go index 94a96bfb..3bba9aba 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,10 @@ func main() { s.test() return } + if len(os.Args) > 1 && os.Args[1] == "testnacl" { + s.testNacl() + return + } port := os.Getenv("PORT") if port == "" { diff --git a/sandbox.go b/sandbox.go index 7b6909f7..e559725b 100644 --- a/sandbox.go +++ b/sandbox.go @@ -12,6 +12,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "go/ast" "go/doc" @@ -26,11 +27,12 @@ import ( "runtime" "strconv" "strings" - "syscall" "text/template" "time" + "cloud.google.com/go/compute/metadata" "github.com/bradfitz/gomemcache/memcache" + "golang.org/x/playground/sandbox/sandboxtypes" ) const ( @@ -79,7 +81,7 @@ type response struct { // If there is no cached *response for the combination of cachePrefix and request.Body, // handler calls cmdFunc and in case of a nil error, stores the value of *response in the cache. // The handler returned supports Cross-Origin Resource Sharing (CORS) from any domain. -func (s *server) commandHandler(cachePrefix string, cmdFunc func(*request) (*response, error)) http.HandlerFunc { +func (s *server) commandHandler(cachePrefix string, cmdFunc func(context.Context, *request) (*response, error)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { cachePrefix := cachePrefix // so we can modify it below w.Header().Set("Access-Control-Allow-Origin", "*") @@ -110,7 +112,7 @@ func (s *server) commandHandler(cachePrefix string, cmdFunc func(*request) (*res if err != memcache.ErrCacheMiss { s.log.Errorf("s.cache.Get(%q, &response): %v", key, err) } - resp, err = cmdFunc(&req) + resp, err = cmdFunc(r.Context(), &req) if err != nil { s.log.Errorf("cmdFunc error: %v", err) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) @@ -315,7 +317,7 @@ var failedTestPattern = "--- FAIL" // The output of successfully ran program is returned in *response.Events. // If a program cannot be built or has timed out, // *response.Errors contains an explanation for a user. -func compileAndRun(req *request) (*response, error) { +func compileAndRun(ctx context.Context, req *request) (*response, error) { // TODO(andybons): Add semaphore to limit number of running programs at once. tmpDir, err := ioutil.TempDir("", "sandbox") if err != nil { @@ -368,15 +370,34 @@ func compileAndRun(req *request) (*response, error) { } } + // TODO: remove all this once Go 1.14 is out. This is a transitional/debug step + // to support both nacl & gvisor temporarily. + useGvisor := os.Getenv("GO_VERSION") >= "go1.14" || + os.Getenv("DEBUG_FORCE_GVISOR") == "1" || + strings.Contains(req.Body, "//play:gvisor\n") + exe := filepath.Join(tmpDir, "a.out") goCache := filepath.Join(tmpDir, "gocache") - ctx, cancel := context.WithTimeout(context.Background(), maxCompileTime) + buildCtx, cancel := context.WithTimeout(ctx, maxCompileTime) defer cancel() - cmd := exec.CommandContext(ctx, "go", "build", "-o", exe, buildPkgArg) + goBin := "go" + if useGvisor { + goBin = "/usr/local/go1.14/bin/go" + } + cmd := exec.CommandContext(buildCtx, goBin, + "build", + "-o", exe, + "-tags=faketime", // required for Go 1.14+, no-op before + buildPkgArg) cmd.Dir = tmpDir var goPath string - cmd.Env = []string{"GOOS=nacl", "GOARCH=amd64p32", "GOCACHE=" + goCache} + if useGvisor { + cmd.Env = []string{"GOOS=linux", "GOARCH=amd64", "GOROOT=/usr/local/go1.14"} + } else { + cmd.Env = []string{"GOOS=nacl", "GOARCH=amd64p32"} + } + cmd.Env = append(cmd.Env, "GOCACHE="+goCache) if useModules { // Create a GOPATH just for modules to be downloaded // into GOPATH/pkg/mod. @@ -411,28 +432,62 @@ func compileAndRun(req *request) (*response, error) { } return nil, fmt.Errorf("error building go source: %v", err) } - ctx, cancel = context.WithTimeout(context.Background(), maxRunTime) + runCtx, cancel := context.WithTimeout(ctx, maxRunTime) defer cancel() - cmd = exec.CommandContext(ctx, "sel_ldr_x86_64", "-l", "/dev/null", "-S", "-e", exe, testParam) rec := new(Recorder) - cmd.Stdout = rec.Stdout() - cmd.Stderr = rec.Stderr() - var status int - if err := cmd.Run(); err != nil { - if ctx.Err() == context.DeadlineExceeded { - // Send what was captured before the timeout. - events, err := rec.Events() - if err != nil { - return nil, fmt.Errorf("error decoding events: %v", err) - } - return &response{Errors: "process took too long", Events: events}, nil + var exitCode int + if useGvisor { + f, err := os.Open(exe) + if err != nil { + return nil, err } - exitErr, ok := err.(*exec.ExitError) - if !ok { - return nil, fmt.Errorf("error running sandbox: %v", err) + defer f.Close() + req, err := http.NewRequestWithContext(ctx, "POST", sandboxBackendURL(), f) + if err != nil { + return nil, err + } + req.Header.Add("Idempotency-Key", "1") // lets Transport do retries with a POST + if testParam != "" { + req.Header.Add("X-Argument", testParam) + } + req.GetBody = func() (io.ReadCloser, error) { return os.Open(exe) } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err } - if ws, ok := exitErr.Sys().(syscall.WaitStatus); ok { - status = ws.ExitStatus() + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected response from backend: %v", res.Status) + } + var execRes sandboxtypes.Response + if err := json.NewDecoder(res.Body).Decode(&execRes); err != nil { + log.Printf("JSON decode error from backend: %v", err) + return nil, errors.New("error parsing JSON from backend") + } + if execRes.Error != "" { + return &response{Errors: execRes.Error}, nil + } + exitCode = execRes.ExitCode + rec.Stdout().Write(execRes.Stdout) + rec.Stderr().Write(execRes.Stderr) + } else { + cmd := exec.CommandContext(runCtx, "sel_ldr_x86_64", "-l", "/dev/null", "-S", "-e", exe, testParam) + cmd.Stdout = rec.Stdout() + cmd.Stderr = rec.Stderr() + if err := cmd.Run(); err != nil { + if ctx.Err() == context.DeadlineExceeded { + // Send what was captured before the timeout. + events, err := rec.Events() + if err != nil { + return nil, fmt.Errorf("error decoding events: %v", err) + } + return &response{Errors: "process took too long", Events: events}, nil + } + exitErr, ok := err.(*exec.ExitError) + if !ok { + return nil, fmt.Errorf("error running sandbox: %v", err) + } + exitCode = exitErr.ExitCode() } } events, err := rec.Events() @@ -455,7 +510,7 @@ func compileAndRun(req *request) (*response, error) { } return &response{ Events: events, - Status: status, + Status: exitCode, IsTest: testParam != "", TestsFailed: fails, VetErrors: vetOut, @@ -490,7 +545,8 @@ func playgroundGoproxy() string { } func (s *server) healthCheck() error { - resp, err := compileAndRun(&request{Body: healthProg}) + ctx := context.Background() // TODO: cap it to some reasonable timeout + resp, err := compileAndRun(ctx, &request{Body: healthProg}) if err != nil { return err } @@ -503,6 +559,18 @@ func (s *server) healthCheck() error { return nil } +func sandboxBackendURL() string { + if v := os.Getenv("SANDBOX_BACKEND_URL"); v != "" { + return v + } + id, _ := metadata.ProjectID() + switch id { + case "golang-org": + return "http://sandbox.play-sandbox-fwd.il4.us-central1.lb.golang-org.internal/run" + } + panic(fmt.Sprintf("no SANDBOX_BACKEND_URL environment and no default defined for project %q", id)) +} + const healthProg = ` package main diff --git a/sandbox/.gitignore b/sandbox/.gitignore new file mode 100644 index 00000000..0f2bb690 --- /dev/null +++ b/sandbox/.gitignore @@ -0,0 +1 @@ +*.yaml.expanded diff --git a/sandbox/Dockerfile b/sandbox/Dockerfile new file mode 100644 index 00000000..ceb5401a --- /dev/null +++ b/sandbox/Dockerfile @@ -0,0 +1,34 @@ +# This is the sandbox backend server. +# +# When it's run, the host maps in /var/run/docker.sock to this +# environment so the play-sandbox server can connect to the host's +# docker daemon, which has the gvisor "runsc" runtime available. + +FROM golang:1.13 AS build + +COPY . /go/src/playground +WORKDIR /go/src/playground/sandbox +RUN go install + +FROM debian:buster + +RUN apt-get update + +# Extra stuff for occasional debugging: +RUN apt-get install --yes strace lsof emacs25-nox net-tools tcpdump procps + +# Install Docker CLI: +RUN apt-get install --yes \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common +RUN bash -c "curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -" +RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian buster stable" +RUN apt-get update +RUN apt-get install --yes docker-ce-cli + +COPY --from=build /go/bin/sandbox /usr/local/bin/play-sandbox + +ENTRYPOINT ["/usr/local/bin/play-sandbox"] diff --git a/sandbox/Dockerfile.gvisor b/sandbox/Dockerfile.gvisor new file mode 100644 index 00000000..42188871 --- /dev/null +++ b/sandbox/Dockerfile.gvisor @@ -0,0 +1,33 @@ +# This is the environment that the untrusted playground programs run within +# under gvisor. + +############################################################################ +# Import the sandbox server's container (which is assumed to be +# already built, as enforced by the Makefile), just so we can copy its +# binary out of it. The same binary is used as both as the server and the +# gvisor-contained helper. +FROM golang/playground-sandbox AS server + +############################################################################ +# Temporary nacl compatibility for development & incremental +# deployment purposes, so we can run the new server architecture in +# nacl mode for a bit, then opt-in linux/amd64 gvisor mode, and +# then once Go 1.14 is out for real we remove the nacl option and +# delete all the nacl code. +FROM debian:buster AS nacl +RUN apt-get update && apt-get install -y --no-install-recommends curl bzip2 ca-certificates +RUN curl -s https://storage.googleapis.com/nativeclient-mirror/nacl/nacl_sdk/trunk.544461/naclsdk_linux.tar.bz2 | tar -xj -C /tmp --strip-components=2 pepper_67/tools/sel_ldr_x86_64 + + +############################################################################ +# This is the actual environment things run in: a minimal busybox with glibc +# binaries so we can use cgo. +FROM busybox:glibc + +COPY --from=server /usr/local/bin/play-sandbox /usr/local/bin/play-sandbox + +# And this, temporarily, for being able to test the old nacl binaries +# with the new sandbox: +COPY --from=nacl /tmp/sel_ldr_x86_64 /usr/local/bin + +ENTRYPOINT ["/usr/local/bin/play-sandbox"] diff --git a/sandbox/Makefile b/sandbox/Makefile new file mode 100644 index 00000000..06c74964 --- /dev/null +++ b/sandbox/Makefile @@ -0,0 +1,52 @@ +ZONE := us-central1-f +TEST_VM := gvisor-cos-test-vm +PROJ := golang-org +NETWORK := golang + +# Docker environment for the sandbox server itself (containing docker CLI, etc), running +# in a privileged container. +docker: + docker build -f Dockerfile --tag=golang/playground-sandbox .. + docker tag golang/playground-sandbox gcr.io/$(PROJ)/playground-sandbox:latest + +# dockergvisor builds the golang/playground-sandbox-gvisor docker +# image, which is the environment that the untrusted programs run in +# (a busybox:glibc world with this directory's sandbox binary which +# runs in --mode=contained) +dockergvisor: + docker build -f Dockerfile.gvisor --tag=golang/playground-sandbox-gvisor .. + docker tag golang/playground-sandbox-gvisor gcr.io/$(PROJ)/playground-sandbox-gvisor:latest + +push: docker dockergvisor + docker push gcr.io/$(PROJ)/playground-sandbox:latest + docker push gcr.io/$(PROJ)/playground-sandbox-gvisor:latest + +# runlocal runs the sandbox server locally, for use with the frontend +# parent directory's "test_nacl" or "test_gvisor" test targets. +runlocal: docker dockergvisor + docker network create sandnet || true + docker kill sandbox_dev || true + docker run --name=sandbox_dev --rm --network=sandnet -ti -p 127.0.0.1:8080:80/tcp -v /var/run/docker.sock:/var/run/docker.sock golang/playground-sandbox:latest --dev + +konlet.yaml.expanded: konlet.yaml + sed "s/PROJECT_NAME/$(PROJ)/" konlet.yaml > konlet.yaml.expanded + +# create_test_vm creates a test VM for interactive debugging. +create_test_vm: konlet.yaml.expanded + gcloud --project=$(PROJ) compute instances create $(TEST_VM) \ + --zone $(ZONE) \ + --network $(NETWORK) \ + --no-address \ + --image-project cos-cloud \ + --image cos-stable-76-12239-60-0 \ + --metadata-from-file gce-container-declaration=konlet.yaml.expanded,user-data=cloud-init.yaml + +# delete_test_vm deletes the test VM from create_test_vm. +delete_test_vm: + gcloud --project=$(PROJ) compute instances delete $(TEST_VM) --quiet --zone $(ZONE) + +# ssh connects to the create_test_vm VM. It must be run from the same network. +ssh: + gcloud --project=$(PROJ) compute ssh $(TEST_VM) --internal-ip --zone $(ZONE) + + diff --git a/sandbox/cloud-init.yaml b/sandbox/cloud-init.yaml new file mode 100644 index 00000000..41195a4e --- /dev/null +++ b/sandbox/cloud-init.yaml @@ -0,0 +1,17 @@ +#cloud-config + +write_files: +- path: /etc/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "runtimes": { "runsc": { "path": "/var/lib/docker/runsc", "runtimeArgs": [] } } + } + +runcmd: +- curl -L -o /var/lib/docker/runsc https://storage.googleapis.com/go-builder-data/runsc-1.0.1 +- chmod +x /var/lib/docker/runsc +- systemctl reload docker.service diff --git a/sandbox/konlet.yaml b/sandbox/konlet.yaml new file mode 100644 index 00000000..2850af34 --- /dev/null +++ b/sandbox/konlet.yaml @@ -0,0 +1,16 @@ +spec: + containers: + - name: playground + image: 'gcr.io/PROJECT_NAME/gvisor-playground-sandbox:latest' + volumeMounts: + - name: dockersock + mountPath: /var/run/docker.sock + securityContext: + privileged: true + stdin: false + tty: true + restartPolicy: Always + volumes: + - name: dockersock + hostPath: + path: /var/run/docker.sock diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go new file mode 100644 index 00000000..be7ab6d5 --- /dev/null +++ b/sandbox/sandbox.go @@ -0,0 +1,521 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The sandbox program is an HTTP server that receives untrusted +// linux/amd64 in a POST request and then executes them in a gvisor +// sandbox using Docker, returning the output as a response to the +// POST. +// +// It's part of the Go playground (https://play.golang.org/). +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "os/signal" + "runtime" + "sync" + "syscall" + "time" + + "golang.org/x/playground/sandbox/sandboxtypes" +) + +var ( + listenAddr = flag.String("listen", ":80", "HTTP server listen address. Only applicable when --mode=server") + mode = flag.String("mode", "server", "Whether to run in \"server\" mode or \"contained\" mode. The contained mode is used internally by the server mode.") + dev = flag.Bool("dev", false, "run in dev mode (show help messages)") + numWorkers = flag.Int("workers", runtime.NumCPU(), "number of parallel gvisor containers to pre-spin up & let run concurrently") +) + +const ( + maxBinarySize = 100 << 20 + runTimeout = 5 * time.Second + maxOutputSize = 100 << 20 + memoryLimitBytes = 100 << 20 +) + +var errTooMuchOutput = errors.New("Output too large") + +// containedStartMessage is the first thing written to stdout by the +// contained process when it starts up. This lets the parent HTTP +// server know that a particular container is ready to run a binary. +const containedStartMessage = "started\n" + +var ( + readyContainer chan *Container + runSem chan struct{} +) + +type Container struct { + name string + stdin io.WriteCloser + stdout io.ReadCloser + stderr io.ReadCloser + cmd *exec.Cmd + + waitOnce sync.Once + waitVal error +} + +func (c *Container) Close() { + setContainerWanted(c.name, false) + c.stdin.Close() + c.stdout.Close() + c.stderr.Close() + if c.cmd.Process != nil { + c.cmd.Process.Kill() + c.Wait() // just in case + } +} + +func (c *Container) Wait() error { + c.waitOnce.Do(c.wait) + return c.waitVal +} + +func (c *Container) wait() { + c.waitVal = c.cmd.Wait() +} + +func main() { + flag.Parse() + if *mode == "contained" { + runInGvisor() + panic("runInGvisor didn't exit") + } + if flag.NArg() != 0 { + flag.Usage() + os.Exit(1) + } + log.Printf("Go playground sandbox starting.") + + readyContainer = make(chan *Container, *numWorkers) + runSem = make(chan struct{}, *numWorkers) + go makeWorkers() + go handleSignals() + + if out, err := exec.Command("docker", "version").CombinedOutput(); err != nil { + log.Fatalf("failed to connect to docker: %v, %s", err, out) + } + if *dev { + log.Printf("Running in dev mode; container published to host at: http://localhost:8080/") + // TODO: XXXX FIXME: this is no longer the protocol since the addition of the processMeta JSON header, + // so write a client program to do this instead? + log.Printf("Run a binary with: curl -v --data-binary @/home/bradfitz/hello http://localhost:8080/run\n") + } else { + log.Printf("Listening on %s", *listenAddr) + } + + http.HandleFunc("/health", healthHandler) + http.HandleFunc("/healthz", healthHandler) + http.HandleFunc("/", rootHandler) + http.HandleFunc("/run", runHandler) + log.Fatal(http.ListenAndServe(*listenAddr, nil)) +} + +func handleSignals() { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGINT) + s := <-c + log.Fatalf("closing on signal %d: %v", s, s) +} + +func healthHandler(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "OK\n") +} + +func rootHandler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + io.WriteString(w, "Hi from sandbox\n") +} + +// processMeta is the JSON sent to the gvisor container before the untrusted binary. +// It currently contains only the arguments to pass to the binary. +// It might contain environment or other things later. +type processMeta struct { + Args []string `json:"args"` +} + +// runInGvisor is run when we're now inside gvisor. We have no network +// at this point. We can read our binary in from stdin and then run +// it. +func runInGvisor() { + const binPath = "/tmpfs/play" + if _, err := io.WriteString(os.Stdout, containedStartMessage); err != nil { + log.Fatalf("writing to stdout: %v", err) + } + slurp, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Fatalf("reading stdin in contained mode: %v", err) + } + nl := bytes.IndexByte(slurp, '\n') + if nl == -1 { + log.Fatalf("no newline found in input") + } + metaJSON, bin := slurp[:nl], slurp[nl+1:] + + if err := ioutil.WriteFile(binPath, bin, 0755); err != nil { + log.Fatalf("writing contained binary: %v", err) + } + + var meta processMeta + if err := json.NewDecoder(bytes.NewReader(metaJSON)).Decode(&meta); err != nil { + log.Fatalf("error decoding JSON meta: %v", err) + } + + // As part of a temporary transition plan, we also support + // running nacl binaries in this sandbox. The point isn't to + // double sandbox things as much as it is to let us transition + // things in steps: first to split the sandbox into two parts + // (frontend & backend), and then to change the type of binary + // (nacl to linux/amd64). This means we can do step 1 of the + // migration during the Go 1.13 dev cycle and have less + // risk/rush during the Go 1.14 release, which should just be + // a flag flip. + // This isn't a perfect heuristic, but it works and it's cheap: + isNacl := bytes.Contains(slurp, []byte("_rt0_amd64p32_nacl")) + + cmd := exec.Command(binPath) + if isNacl { + cmd = exec.Command("/usr/local/bin/sel_ldr_x86_64", "-l", "/dev/null", "-S", "-e", binPath) + } + cmd.Args = append(cmd.Args, meta.Args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + err = cmd.Wait() + os.Remove(binPath) // not that it matters much, this container will be nuked + os.Exit(errExitCode(err)) + return + +} + +func makeWorkers() { + for { + c, err := startContainer(context.Background()) + if err != nil { + log.Printf("error starting container: %v", err) + time.Sleep(5 * time.Second) + continue + } + readyContainer <- c + } +} + +func randHex(n int) string { + b := make([]byte, n/2) + _, err := rand.Read(b) + if err != nil { + panic(err) + } + return fmt.Sprintf("%x", b) +} + +var ( + wantedMu sync.Mutex + containerWanted = map[string]bool{} +) + +// setContainerWanted records whether a named container is wanted or +// not. Any unwanted containers are cleaned up asynchronously as a +// sanity check against leaks. +// +// TODO(bradfitz): add leak checker (background docker ps loop) +func setContainerWanted(name string, wanted bool) { + wantedMu.Lock() + defer wantedMu.Unlock() + if wanted { + containerWanted[name] = true + } else { + delete(containerWanted, name) + } +} + +func getContainer(ctx context.Context) (*Container, error) { + select { + case c := <-readyContainer: + return c, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func startContainer(ctx context.Context) (c *Container, err error) { + name := "play_run_" + randHex(8) + setContainerWanted(name, true) + var stdin io.WriteCloser + var stdout io.ReadCloser + var stderr io.ReadCloser + defer func() { + if err == nil { + return + } + setContainerWanted(name, false) + if stdin != nil { + stdin.Close() + } + if stdout != nil { + stdout.Close() + } + if stderr != nil { + stderr.Close() + } + }() + + cmd := exec.Command("docker", "run", + "--name="+name, + "--rm", + "--tmpfs=/tmpfs", + "-i", // read stdin + + "--runtime=runsc", + "--network=none", + "--memory="+fmt.Sprint(memoryLimitBytes), + + "gcr.io/golang-org/playground-sandbox-gvisor:latest", + "--mode=contained") + stdin, err = cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err = cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err = cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + errc := make(chan error, 1) + go func() { + buf := make([]byte, len(containedStartMessage)) + if _, err := io.ReadFull(stdout, buf); err != nil { + errc <- fmt.Errorf("error reading header from sandbox container: %v", err) + return + } + if string(buf) != containedStartMessage { + errc <- fmt.Errorf("sandbox container sent wrong header %q; want %q", buf, containedStartMessage) + return + } + errc <- nil + }() + select { + case <-ctx.Done(): + log.Printf("timeout starting container") + cmd.Process.Kill() + return nil, ctx.Err() + case err := <-errc: + if err != nil { + log.Printf("error starting container: %v", err) + return nil, err + } + } + return &Container{ + name: name, + stdin: stdin, + stdout: stdout, + stderr: stderr, + cmd: cmd, + }, nil +} + +func runHandler(w http.ResponseWriter, r *http.Request) { + t0 := time.Now() + tlast := t0 + var logmu sync.Mutex + logf := func(format string, args ...interface{}) { + if !*dev { + return + } + logmu.Lock() + defer logmu.Unlock() + t := time.Now() + d := t.Sub(tlast) + d0 := t.Sub(t0) + tlast = t + log.Print(fmt.Sprintf("+%10v +%10v ", d0, d) + fmt.Sprintf(format, args...)) + } + logf("/run") + + if r.Method != "POST" { + http.Error(w, "expected a POST", http.StatusBadRequest) + return + } + + // Bound the number of requests being processed at once. + // (Before we slurp the binary into memory) + select { + case runSem <- struct{}{}: + case <-r.Context().Done(): + return + } + defer func() { <-runSem }() + + bin, err := ioutil.ReadAll(http.MaxBytesReader(w, r.Body, maxBinarySize)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + logf("read %d bytes", len(bin)) + + c, err := getContainer(r.Context()) + if err != nil { + if cerr := r.Context().Err(); cerr != nil { + log.Printf("getContainer, client side cancellation: %v", cerr) + return + } + http.Error(w, "failed to get container", http.StatusInternalServerError) + log.Printf("failed to get container: %v", err) + return + } + logf("got container %s", c.name) + defer c.Close() + defer logf("leaving handler; about to close container") + + runTimer := time.NewTimer(runTimeout) + defer runTimer.Stop() + + errc := make(chan error, 2) // user-visible error + waitc := make(chan error, 1) + + copyOut := func(which string, dst *[]byte, r io.Reader) { + buf := make([]byte, 4<<10) + for { + n, err := r.Read(buf) + logf("%s: Read = %v, %v", which, n, err) + *dst = append(*dst, buf[:n]...) + if err == io.EOF { + return + } + if len(*dst) > maxOutputSize { + errc <- errTooMuchOutput + return + } + if err != nil { + log.Printf("reading %s: %v", which, err) + errc <- fmt.Errorf("error reading %v", which) + return + } + } + } + + res := &sandboxtypes.Response{} + go func() { + var meta processMeta + meta.Args = r.Header["X-Argument"] + metaJSON, _ := json.Marshal(&meta) + metaJSON = append(metaJSON, '\n') + if _, err := c.stdin.Write(metaJSON); err != nil { + log.Printf("stdin write meta: %v", err) + errc <- errors.New("failed to write meta to child") + return + } + if _, err := c.stdin.Write(bin); err != nil { + log.Printf("stdin write: %v", err) + errc <- errors.New("failed to write binary to child") + return + } + c.stdin.Close() + logf("wrote+closed") + go copyOut("stdout", &res.Stdout, c.stdout) + go copyOut("stderr", &res.Stderr, c.stderr) + waitc <- c.Wait() + }() + var waitErr error + select { + case waitErr = <-waitc: + logf("waited: %v", waitErr) + case err := <-errc: + logf("got error: %v", err) + if err == errTooMuchOutput { + sendError(w, err.Error()) + return + } + if err != nil { + http.Error(w, "failed to read stdout from docker run", http.StatusInternalServerError) + return + } + case <-runTimer.C: + logf("timeout") + sendError(w, "timeout running program") + return + } + + res.ExitCode = errExitCode(waitErr) + res.Stderr = cleanStderr(res.Stderr) + sendResponse(w, res) +} + +func errExitCode(err error) int { + if err == nil { + return 0 + } + if ee, ok := err.(*exec.ExitError); ok { + return ee.ExitCode() + } + return 1 +} + +func sendError(w http.ResponseWriter, errMsg string) { + sendResponse(w, &sandboxtypes.Response{Error: errMsg}) +} + +func sendResponse(w http.ResponseWriter, r *sandboxtypes.Response) { + jres, err := json.MarshalIndent(r, "", " ") + if err != nil { + http.Error(w, "error encoding JSON", http.StatusInternalServerError) + log.Printf("json marshal: %v", err) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", fmt.Sprint(len(jres))) + w.Write(jres) +} + +// cleanStderr removes spam stderr lines from the beginning of x +// and returns a slice of x. +func cleanStderr(x []byte) []byte { + for { + nl := bytes.IndexByte(x, '\n') + if nl == -1 || !isSpamStderrLine(x[:nl+1]) { + return x + } + x = x[nl+1:] + } +} + +var warningPrefix = []byte("WARNING: ") + +// isSpamStderrLine reports whether line is a spammy line of stderr +// output from Docker. Currently it only matches things starting with +// "WARNING: " like: +// WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap. +// +// TODO: remove this and instead just make the child process start by +// writing a known header to stderr, then have parent skip everything +// before that unique header. +func isSpamStderrLine(line []byte) bool { + return bytes.HasPrefix(line, warningPrefix) +} diff --git a/sandbox/sandbox.tf b/sandbox/sandbox.tf new file mode 100644 index 00000000..61029c12 --- /dev/null +++ b/sandbox/sandbox.tf @@ -0,0 +1,143 @@ +# TODO: move the network configuration into terraform too? It was created by hand with: +# gcloud compute networks subnets update golang --region=us-central1 --enable-private-ip-google-access + +terraform { + backend "gcs" { + bucket = "tf-state-prod-golang-org" + prefix = "terraform/state" + } +} + +provider "google-beta" { + project = "golang-org" + region = "us-central1" + zone = "us-central1-f" +} + +provider "google" { + project = "golang-org" + region = "us-central1" + zone = "us-central1-f" +} + +data "local_file" "cloud_init" { + filename = "${path.module}/cloud-init.yaml" +} + +data "local_file" "konlet" { + filename = "${path.module}/konlet.yaml.expanded" +} + +data "google_compute_image" "cos" { + family = "cos-stable" + project = "cos-cloud" +} + +resource "google_compute_instance_template" "inst_tmpl" { + name = "play-sandbox-tmpl" + machine_type = "n1-standard-1" + metadata = { + "ssh-keys" = "bradfitz:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtL4Di+zypBIRmvohzfC4mOzETaz/DtzANPKir/mUE1QiC8HuL8BDpIu3rZZY1gAnQf3IZMpgQYgx90TZskgWfi7xLN2mDh2iBZB0KUDlpNpwn1SzSUTQU10XV5mOLm1B7L+w3QXA1wz7kMjmztxZNli/tvZI5BtAX6c58c4Rn0pjQTPeEVPMMvEj6zPUy4QInWTgtrbTj3On/e95F3ZmUjEKFfjeFxVPh7fF6mSygzcxEyGYxTbMdUod/dNbZD/7HY5eTNQPSvjp+GaoodbFEdYWnk9vtoX/2VWYo0J/+w6oHprDzj7dPzKeJeCjpnKra7DyCS/RxIIPt5Giwdj2F bradfitz@bradfitz-glinux-desktop.sea.corp.google.com" + "gce-container-declaration" = data.local_file.konlet.content + "user-data" = data.local_file.cloud_init.content + } + network_interface { + network = "golang" + } + service_account { + scopes = ["storage-ro"] + } + disk { + source_image = data.google_compute_image.cos.self_link + auto_delete = true + boot = true + } + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + } + lifecycle { + create_before_destroy = true + } +} + +resource "google_compute_region_autoscaler" "default" { + provider = "google-beta" + + name = "play-sandbox-autoscaler" + region = "us-central1" + target = "${google_compute_region_instance_group_manager.rigm.self_link}" + + autoscaling_policy { + max_replicas = 10 + min_replicas = 2 + cooldown_period = 60 + + cpu_utilization { + target = 0.5 + } + } +} + +resource "google_compute_region_instance_group_manager" "rigm" { + provider = "google-beta" + name = "play-sandbox-rigm" + + base_instance_name = "playsandbox" + region = "us-central1" + + version { + name = "primary" + instance_template = "${google_compute_instance_template.inst_tmpl.self_link}" + } + + named_port { + name = "http" + port = 80 + } + + auto_healing_policies { + health_check = "${google_compute_health_check.default.self_link}" + initial_delay_sec = 30 + } +} + +data "google_compute_region_instance_group" "rig" { + provider = "google-beta" + self_link = "${google_compute_region_instance_group_manager.rigm.instance_group}" +} + +resource "google_compute_health_check" "default" { + name = "play-sandbox-rigm-health-check" + check_interval_sec = 5 + timeout_sec = 5 + healthy_threshold = 2 + unhealthy_threshold = 10 # 50 seconds + http_health_check { + request_path = "/healthz" + port = 80 + } +} + +resource "google_compute_region_backend_service" "default" { + name = "play-sandbox-backend-service" + region = "us-central1" + health_checks = ["${google_compute_health_check.default.self_link}"] + backend { + group = "${data.google_compute_region_instance_group.rig.self_link}" + } +} + +resource "google_compute_forwarding_rule" "default" { + name = "play-sandbox-fwd" + region = "us-central1" + network = "golang" + ports = ["80"] + load_balancing_scheme = "INTERNAL" + ip_protocol = "TCP" + backend_service = "${google_compute_region_backend_service.default.self_link}" + + # Adding a service label gives us a DNS name: + # sandbox.play-sandbox-fwd.il4.us-central1.lb.golang-org.internal + service_label = "sandbox" +} diff --git a/sandbox/sandboxtypes/types.go b/sandbox/sandboxtypes/types.go new file mode 100644 index 00000000..3ebfb46e --- /dev/null +++ b/sandbox/sandboxtypes/types.go @@ -0,0 +1,22 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The sandboxtypes package contains the shared types +// to communicate between the different sandbox components. +package sandboxtypes + +// Response is the response from the x/playground/sandbox backend to +// the x/playground frontend. +// +// The stdout/stderr are base64 encoded which isn't ideal but is good +// enough for now. Maybe we'll move to protobufs later. +type Response struct { + // Error, if non-empty, means we failed to run the binary. + // It's meant to be user-visible. + Error string `json:"error,omitempty"` + + ExitCode int `json:"exitCode"` + Stdout []byte `json:"stdout"` + Stderr []byte `json:"stderr"` +} diff --git a/server_test.go b/server_test.go index 773a0934..40ba0654 100644 --- a/server_test.go +++ b/server_test.go @@ -161,7 +161,7 @@ func TestCommandHandler(t *testing.T) { if err != nil { t.Fatalf("newServer(testingOptions(t)): %v", err) } - testHandler := s.commandHandler("test", func(r *request) (*response, error) { + testHandler := s.commandHandler("test", func(_ context.Context, r *request) (*response, error) { if r.Body == "fail" { return nil, fmt.Errorf("non recoverable") } diff --git a/tests.go b/tests.go index 18f6ea0a..4022b6a3 100644 --- a/tests.go +++ b/tests.go @@ -8,8 +8,10 @@ package main import ( + "context" "fmt" stdlog "log" + "net" "os" "reflect" "strings" @@ -19,12 +21,27 @@ import ( type compileTest struct { name string // test name prog, want, errors string + wantFunc func(got string) error // alternative to want withVet bool wantEvents []Event wantVetErrors string } +func (s *server) testNacl() { + log.Printf("testing nacl mode") + s.runTests() +} + func (s *server) test() { + if _, err := net.ResolveIPAddr("ip", "sandbox_dev.sandnet."); err != nil { + log.Fatalf("sandbox_dev.sandnet not available") + } + os.Setenv("DEBUG_FORCE_GVISOR", "1") + os.Setenv("SANDBOX_BACKEND_URL", "http://sandbox_dev.sandnet/run") + s.runTests() +} + +func (s *server) runTests() { if err := s.healthCheck(); err != nil { stdlog.Fatal(err) } @@ -33,44 +50,66 @@ func (s *server) test() { defer func(old string) { os.Setenv("ALLOW_PLAY_MODULE_DOWNLOADS", old) }(os.Getenv("ALLOW_PLAY_MODULE_DOWNLOADS")) os.Setenv("ALLOW_PLAY_MODULE_DOWNLOADS", "true") + failed := false for i, t := range tests { - fmt.Printf("testing case %d (%q)...\n", i, t.name) - resp, err := compileAndRun(&request{Body: t.prog, WithVet: t.withVet}) + stdlog.Printf("testing case %d (%q)...\n", i, t.name) + resp, err := compileAndRun(context.Background(), &request{Body: t.prog, WithVet: t.withVet}) if err != nil { stdlog.Fatal(err) } if t.wantEvents != nil { if !reflect.DeepEqual(resp.Events, t.wantEvents) { - stdlog.Fatalf("resp.Events = %q, want %q", resp.Events, t.wantEvents) + stdlog.Printf("resp.Events = %q, want %q", resp.Events, t.wantEvents) + failed = true } continue } if t.errors != "" { if resp.Errors != t.errors { - stdlog.Fatalf("resp.Errors = %q, want %q", resp.Errors, t.errors) + stdlog.Printf("resp.Errors = %q, want %q", resp.Errors, t.errors) + failed = true } continue } if resp.Errors != "" { - stdlog.Fatal(resp.Errors) + stdlog.Print(resp.Errors) + failed = true + continue } if resp.VetErrors != t.wantVetErrors { - stdlog.Fatalf("resp.VetErrs = %q, want %q", resp.VetErrors, t.wantVetErrors) + stdlog.Printf("resp.VetErrs = %q, want %q", resp.VetErrors, t.wantVetErrors) + failed = true + continue } if t.withVet && (resp.VetErrors != "") == resp.VetOK { - stdlog.Fatalf("resp.VetErrs & VetOK inconsistent; VetErrs = %q; VetOK = %v", resp.VetErrors, resp.VetOK) + stdlog.Printf("resp.VetErrs & VetOK inconsistent; VetErrs = %q; VetOK = %v", resp.VetErrors, resp.VetOK) + failed = true + continue } if len(resp.Events) == 0 { - stdlog.Fatalf("unexpected output: %q, want %q", "", t.want) + stdlog.Printf("unexpected output: %q, want %q", "", t.want) + failed = true + continue } var b strings.Builder for _, e := range resp.Events { b.WriteString(e.Message) } - if !strings.Contains(b.String(), t.want) { - stdlog.Fatalf("unexpected output: %q, want %q", b.String(), t.want) + if t.wantFunc != nil { + if err := t.wantFunc(b.String()); err != nil { + stdlog.Printf("%v\n", err) + failed = true + } + } else { + if !strings.Contains(b.String(), t.want) { + stdlog.Printf("unexpected output: %q, want %q", b.String(), t.want) + failed = true + } } } + if failed { + stdlog.Fatalf("FAILED") + } fmt.Println("OK") } @@ -190,11 +229,16 @@ import ( func main() { filepath.Walk("/", func(path string, info os.FileInfo, err error) error { + if path == "/proc" || path == "/sys" { + return filepath.SkipDir + } fmt.Println(path) return nil }) } -`, want: `/ +`, wantFunc: func(got string) error { + // The environment for the old nacl sandbox: + if strings.TrimSpace(got) == `/ /dev /dev/null /dev/random @@ -211,8 +255,31 @@ func main() { /usr/local/go /usr/local/go/lib /usr/local/go/lib/time -/usr/local/go/lib/time/zoneinfo.zip`}, - +/usr/local/go/lib/time/zoneinfo.zip` { + return nil + } + have := map[string]bool{} + for _, f := range strings.Split(got, "\n") { + have[f] = true + } + for _, expect := range []string{ + "/.dockerenv", + "/__runsc_containers__", + "/etc/hostname", + "/dev/zero", + "/lib/ld-linux-x86-64.so.2", + "/lib/libc.so.6", + "/etc/nsswitch.conf", + "/bin/env", + "/tmpfs", + } { + if !have[expect] { + return fmt.Errorf("missing expected sandbox file %q; got:\n%s", expect, got) + } + } + return nil + }, + }, { name: "test_passes", prog: ` @@ -404,7 +471,9 @@ func main() { package main import ("fmt"; "github.com/bradfitz/iter") func main() { for i := range iter.N(5) { fmt.Println(i) } } -`, want: "0\n1\n2\n3\n4\n"}, +`, + want: "0\n1\n2\n3\n4\n", + }, { name: "compile_with_vet", diff --git a/vet.go b/vet.go index a9cae12a..032a5aa2 100644 --- a/vet.go +++ b/vet.go @@ -5,6 +5,7 @@ package main import ( + "context" "fmt" "io/ioutil" "os" @@ -21,7 +22,7 @@ import ( // the /compile (compileAndRun) handler instead with the WithVet // boolean set. This code path doesn't support modules and only exists // as a temporary compatiblity bridge to older javascript clients. -func vetCheck(req *request) (*response, error) { +func vetCheck(ctx context.Context, req *request) (*response, error) { tmpDir, err := ioutil.TempDir("", "vet") if err != nil { return nil, fmt.Errorf("error creating temp directory: %v", err)