diff --git a/runner.arch/Dockerfile b/runner/Dockerfile.arch
similarity index 91%
rename from runner.arch/Dockerfile
rename to runner/Dockerfile.arch
index fe91b4639..5e9ad2996 100644
--- a/runner.arch/Dockerfile
+++ b/runner/Dockerfile.arch
@@ -20,3 +20,6 @@ RUN mkdir /storage /cache && \
 
 WORKDIR /home/pkgeval
 USER pkgeval
+
+COPY ./entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/runner.ubuntu/Dockerfile b/runner/Dockerfile.ubuntu
similarity index 91%
rename from runner.ubuntu/Dockerfile
rename to runner/Dockerfile.ubuntu
index e9e2e1cd0..d37ffee25 100644
--- a/runner.ubuntu/Dockerfile
+++ b/runner/Dockerfile.ubuntu
@@ -22,3 +22,6 @@ RUN mkdir /storage /cache && \
 
 WORKDIR /home/pkgeval
 USER pkgeval
+
+COPY ./entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/runner/entrypoint.sh b/runner/entrypoint.sh
new file mode 100755
index 000000000..c9acea935
--- /dev/null
+++ b/runner/entrypoint.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -ue
+
+DEPOT=$1
+shift
+
+mkdir $DEPOT
+
+mkdir -p /storage/artifacts
+ln -s /storage/artifacts $DEPOT/artifacts
+
+# allow identification of PkgEal
+export CI=true
+export PKGEVAL=true
+export JULIA_PKGEVAL=true
+
+# disable system discovery of Python and R
+export PYTHON=""
+export R_HOME="*"
+
+# no automatic precompilation
+export JULIA_PKG_PRECOMPILE_AUTO=0
+
+exec "$@"
diff --git a/src/run.jl b/src/run.jl
index 3fb833124..f5b79e084 100644
--- a/src/run.jl
+++ b/src/run.jl
@@ -1,9 +1,9 @@
 export Configuration
 
 function prepare_runner()
-    for runner in ("ubuntu", "arch")
-        cd(joinpath(dirname(@__DIR__), "runner.$runner")) do
-            cmd = `docker build --tag newpkgeval:$runner .`
+    cd(joinpath(dirname(@__DIR__), "runner")) do
+        for runner in ("ubuntu", "arch")
+            cmd = `docker build --tag newpkgeval:$runner --file Dockerfile.$runner .`
             if !isdebug(:docker)
                 cmd = pipeline(cmd, stdout=devnull, stderr=devnull)
             end
@@ -44,7 +44,9 @@ function runner_sandboxed_julia(install::String, args=``; interactive=true, tty=
                                 storage=nothing, cache=nothing, sysimage=nothing,
                                 runner="ubuntu", depot="/home/pkgeval/.julia",
                                 xvfb::Bool=true, init::Bool=true)
-    cmd = `docker run`
+    ## Docker args
+
+    cmd = `docker run --rm`
 
     # expose any available GPUs if they are available
     if find_library("libcuda") != ""
@@ -59,16 +61,9 @@ function runner_sandboxed_julia(install::String, args=``; interactive=true, tty=
     cmd = ```$cmd --mount type=bind,source=$julia_path,target=/opt/julia,readonly
                   --mount type=bind,source=$registry_path,target=/usr/local/share/julia/registries,readonly
                   --env JULIA_DEPOT_PATH="$depot:/usr/local/share/julia"
-                  --env JULIA_PKG_PRECOMPILE_AUTO=0
                   --env JULIA_PKG_SERVER
           ```
 
-    # allow identification of PkgEval
-    cmd = `$cmd --env CI=true --env PKGEVAL=true --env JULIA_PKGEVAL=true`
-
-    # disable system discovery of Python and R
-    cmd = `$cmd --env PYTHON="" --env R_HOME="*"`
-
     if storage !== nothing
         cmd = `$cmd --mount type=bind,source=$storage,target=/storage`
     end
@@ -77,10 +72,6 @@ function runner_sandboxed_julia(install::String, args=``; interactive=true, tty=
         cmd = `$cmd --mount type=bind,source=$cache,target=/cache`
     end
 
-    if sysimage !== nothing
-        args = `--sysimage=$sysimage $args`
-    end
-
     # mount working directory in tmpfs
     if tmpfs
         cmd = `$cmd --tmpfs /home/pkgeval:exec,uid=1000,gid=1000`
@@ -110,10 +101,19 @@ function runner_sandboxed_julia(install::String, args=``; interactive=true, tty=
         cmd = `$cmd --init`
     end
 
+    cmd = `$cmd newpkgeval:$runner`
+
+
+    ## Julia args
+
+    if sysimage !== nothing
+        args = `--sysimage=$sysimage $args`
+    end
+
     if xvfb
-        `$cmd --rm newpkgeval:$runner xvfb-run /opt/julia/bin/julia $args`
+        `$cmd $depot xvfb-run /opt/julia/bin/julia $args`
     else
-        `$cmd --rm newpkgeval:$runner /opt/julia/bin/julia $args`
+        `$cmd $depot /opt/julia/bin/julia $args`
     end
 end
 
@@ -146,12 +146,6 @@ function run_sandboxed_test(install::String, pkg; log_limit = 2^20 #= 1 MB =#,
             versioninfo()
             println()
 
-            mkpath(first(DEPOT_PATH))
-
-            # global storage of downloaded artifacts
-            mkpath("/storage/artifacts")
-            symlink("/storage/artifacts", joinpath(first(DEPOT_PATH), "artifacts"))
-
             using Pkg
             Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true
 
@@ -385,12 +379,6 @@ function run_compiled_test(install::String, pkg; compile_time_limit=30*60, cache
         versioninfo()
         println()
 
-        mkpath(first(DEPOT_PATH))
-
-        # global storage of downloaded artifacts
-        mkpath("/storage/artifacts")
-        symlink("/storage/artifacts", joinpath(first(DEPOT_PATH), "artifacts"))
-
         using Pkg
         Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true
 
@@ -504,6 +492,7 @@ function run(configs::Vector{Configuration}, pkgs::Vector;
     for (config, (install,cache)) in instantiated_configs
         Base.run(```docker run --mount type=bind,source=$storage,target=/storage
                                --mount type=bind,source=$cache,target=/cache
+                               --entrypoint=''
                                newpkgeval:ubuntu
                                sudo chown -R pkgeval:pkgeval /storage /cache```)
     end
@@ -744,6 +733,7 @@ function run(configs::Vector{Configuration}, pkgs::Vector;
             uid = ccall(:getuid, Cint, ())
             gid = ccall(:getgid, Cint, ())
             Base.run(```docker run --mount type=bind,source=$cache,target=/cache
+                                   --entrypoint=''
                                    newpkgeval:ubuntu
                                    sudo chown -R $uid:$gid /cache```)
             rm(cache; recursive=true)