Test rendering of environments (#44)

* Add dummy X server * Check rendering * Amend Dockerfile to play nice with CircleCI * Use larger resource class as X server is dieing on small machines, remove sharding which is not really justified given small test suite * Fix Xorg config path * Workaround MuJoCo bug * Explicitly wrap in Xdummy-entrypoint.py (CircleCI ignores Dockerifle entrypoint) * Fix typo * Fix typo in Dockerfile * Copy needed dependencies in Dockerfile for python-req stage * Throttle number of CPUs to see if this resolves spurious failure * Decrease Xorg size, bump CPUs back up to 4, warn of memory leak * Fix type issue * Improve docstring * Fix typo * Bump version
HumanCompatibleAI · Nov 10, 2020 · 728d449 · 728d449
1 parent 8f468f3
commit 728d449
Show file tree

Hide file tree

Showing 8 changed files with 276 additions and 25 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,24 +6,34 @@ version: 2.1  # Version of CircleCI config format
 orbs:
   codecov: codecov/[email protected]  # support for uploading code coverage to codecov
 
+defaults: &defaults
+  docker:
+    - image: humancompatibleai/seals:base
+      auth:
+        username: $DOCKERHUB_USERNAME
+        password: $DOCKERHUB_PASSWORD
+  working_directory: /seals
+
 executors:
-  my-executor:
-    docker:
-      - image: humancompatibleai/seals:base
-        # Credentials defined in `docker-hub-creds` context (database of environment variables).
-        # We authenticate to avoid Dockerhub per-IP rate limiting of anonymous users.
-        auth:
-          username: $DOCKERHUB_USERNAME
-          password: $DOCKERHUB_PASSWORD
-    working_directory: /seals
+  unit-test:
+    <<: *defaults
+    resource_class: large
+    environment:
+      # Don't use auto-detect since it sees all CPUs available, but container is throttled.
+      NUM_CPUS: 4
+  lintandtype:
+    <<: *defaults
+    resource_class: medium
     environment:
       # If you change these, also change ci/code_checks.sh
       LINT_FILES: src/ tests/ docs/conf.py setup.py  # files we lint
       # Files we statically type check. Source files like src/ should almost always be present.
       # In this repo we also typecheck tests/ -- but sometimes you may want to exclude these
       # if they do strange things with types (e.g. mocking).
       TYPECHECK_FILES: src/ tests/ setup.py
-      NUM_CPUS: 2  # more CPUs visible but we're throttled to 2, which breaks auto-detect
+      # Don't use auto-detect since it sees all CPUs available, but container is throttled.
+      NUM_CPUS: 2
+
 
 commands:
   # Define common function to install dependencies and seals, used in the jobs defined in the next section
@@ -76,7 +86,7 @@ commands:
 jobs:
   # `lintandtype` installs dependencies + `seals`, lints the code, builds the docs, and runs type checks.
   lintandtype:
-    executor: my-executor
+    executor: lintandtype
 
     steps:
       - dependencies
@@ -106,8 +116,7 @@ jobs:
 
   # `unit-test` runs the unit tests in `tests/`.
   unit-test:
-    executor: my-executor
-    parallelism: 3
+    executor: unit-test
     steps:
       - dependencies
 
@@ -132,16 +141,16 @@ jobs:
       - run:
           name: run tests
           command: |
-            export DISPLAY=:0  # we need an X11 display for tests of environment rendering
+            # Xdummy-entrypoint.py: starts an X server and sets DISPLAY, then runs wrapped command.
+            # pytest arguments:
             # --cov specifies which directories to report code coverage for
             # Since we test the installed `seals`, our source files live in `venv`, not in `src/seals`.
             # --junitxml records test results in JUnit format. We upload this file using `store_test_results`
             # later, and CircleCI then parses this to pretty-print results.
             # --shard-id and --num-shards are used to split tests across parallel executors using `pytest-shard`.
             # -n uses `pytest-xdist` to parallelize tests within a single instance.
-            pytest --cov=/venv/lib/python3.7/site-packages/seals --cov=tests \
+            Xdummy-entrypoint.py pytest --cov=/venv/lib/python3.7/site-packages/seals --cov=tests \
                    --junitxml=/tmp/test-reports/junit.xml \
-                   --shard-id=${CIRCLE_NODE_INDEX} --num-shards=${CIRCLE_NODE_TOTAL} \
                     -n ${NUM_CPUS} -vv tests/
             # Following two lines rewrite paths from venv/ to src/, based on `coverage:paths` in `setup.cfg`
             # This is needed to avoid confusing Codecov

diff --git a/Dockerfile b/Dockerfile
@@ -44,30 +44,36 @@ RUN    mkdir -p /root/.mujoco \
 ENV PATH="/venv/bin:$PATH"
 ENV LD_LIBRARY_PATH /root/.mujoco/mjpro150/bin:${LD_LIBRARY_PATH}
 
+# Run Xdummy mock X server by default so that rendering will work.
+COPY ci/xorg.conf /etc/dummy_xorg.conf
+COPY ci/Xdummy-entrypoint.py /usr/bin/Xdummy-entrypoint.py
+ENTRYPOINT ["/usr/bin/Xdummy-entrypoint.py"]
+
 # python-req stage contains Python venv, but not code.
 # It is useful for development purposes: you can mount
 # code from outside the Docker container.
 FROM base as python-req
 
-WORKDIR /benchmark-environments
+WORKDIR /seals
 # Copy only necessary dependencies to build virtual environment.
 # This minimizes how often this layer needs to be rebuilt.
 COPY ./setup.py ./setup.py
-COPY ./src/imitation/__init__.py ./src/imitation/__init__.py
+COPY ./README.md ./README.md
+COPY ./src/seals/version.py ./src/seals/version.py
 COPY ./ci/build_venv.sh ./ci/build_venv.sh
 
 # mjkey.txt needs to exist for build, but doesn't need to be a real key
-RUN touch /root/.mujoco/mjkey.txt && /benchmark-environments/scripts/build_venv.sh /venv
+RUN touch /root/.mujoco/mjkey.txt && /seals/ci/build_venv.sh /venv
 
 # full stage contains everything.
 # Can be used for deployment and local testing.
 FROM python-req as full
 
 # Delay copying (and installing) the code until the very end
-COPY . /benchmark-environments
+COPY . /seals
 # Build a wheel then install to avoid copying whole directory (pip issue #2195)
 RUN python setup.py sdist bdist_wheel
-RUN pip install dist/evaluating_rewards-*.whl
+RUN pip install dist/seals-*.whl
 
 # Default entrypoints
 CMD ["pytest", "-n", "auto", "-vv", "tests/"]
diff --git a/ci/Xdummy-entrypoint.py b/ci/Xdummy-entrypoint.py
@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+# Adapted from https://github.com/openai/mujoco-py/blob/master/vendor/Xdummy-entrypoint
+# Copyright OpenAI; MIT License
+
+import argparse
+import os
+import sys
+import subprocess
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    args, extra_args = parser.parse_known_args()
+
+    subprocess.Popen(
+        [
+            "nohup",
+            "Xorg",
+            "-noreset",
+            "+extension",
+            "GLX",
+            "+extension",
+            "RANDR",
+            "+extension",
+            "RENDER",
+            "-logfile",
+            "/tmp/xdummy.log",
+            "-config",
+            "/etc/dummy_xorg.conf",
+            ":0",
+        ]
+    )
+    subprocess.Popen(
+        ["nohup", "Xdummy"],
+        stdout=open("/dev/null", "w"),
+        stderr=open("/dev/null", "w"),
+    )
+    os.environ["DISPLAY"] = ":0"
+
+    if not extra_args:
+        argv = ["/bin/bash"]
+    else:
+        argv = extra_args
+
+    # Explicitly flush right before the exec since otherwise things might get
+    # lost in Python's buffers around stdout/stderr (!).
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    os.execvpe(argv[0], argv, os.environ)
diff --git a/ci/xorg.conf b/ci/xorg.conf
@@ -0,0 +1,138 @@
+# This xorg configuration file is meant to be used by xpra
+# to start a dummy X11 server.
+# For details, please see:
+# https://xpra.org/Xdummy.html
+
+Section "ServerFlags"
+  Option "DontVTSwitch" "true"
+  Option "AllowMouseOpenFail" "true"
+  Option "PciForceNone" "true"
+  Option "AutoEnableDevices" "false"
+  Option "AutoAddDevices" "false"
+EndSection
+
+Section "InputDevice"
+  Identifier "dummy_mouse"
+  Option "CorePointer" "true"
+  Driver "void"
+EndSection
+
+Section "InputDevice"
+  Identifier "dummy_keyboard"
+  Option "CoreKeyboard" "true"
+  Driver "void"
+EndSection
+
+Section "Device"
+  Identifier "dummy_videocard"
+  Driver "dummy"
+  Option "ConstantDPI" "true"
+  #VideoRam 4096000
+  #VideoRam 256000
+  VideoRam 192000
+EndSection
+
+Section "Monitor"
+  Identifier "dummy_monitor"
+  HorizSync   5.0 - 1000.0
+  VertRefresh 5.0 - 200.0
+  #This can be used to get a specific DPI, but only for the default resolution:
+  #DisplaySize 508 317
+  #NOTE: the highest modes will not work without increasing the VideoRam
+  # for the dummy video card.
+  Modeline "32768x32768" 15226.50 32768 35800 39488 46208 32768 32771 32781 32953
+  Modeline "32768x16384" 7516.25 32768 35544 39192 45616 16384 16387 16397 16478
+  Modeline "16384x8192" 2101.93 16384 16416 24400 24432 8192 8390 8403 8602
+  Modeline "8192x4096" 424.46 8192 8224 9832 9864 4096 4195 4202 4301
+  Modeline "5496x1200" 199.13 5496 5528 6280 6312 1200 1228 1233 1261
+  Modeline "5280x1080" 169.96 5280 5312 5952 5984 1080 1105 1110 1135
+  Modeline "5280x1200" 191.40 5280 5312 6032 6064 1200 1228 1233 1261
+  Modeline "5120x3200" 199.75 5120 5152 5904 5936 3200 3277 3283 3361
+  Modeline "4800x1200" 64.42 4800 4832 5072 5104 1200 1229 1231 1261
+  Modeline "3840x2880" 133.43 3840 3872 4376 4408 2880 2950 2955 3025
+  Modeline "3840x2560" 116.93 3840 3872 4312 4344 2560 2622 2627 2689
+  Modeline "3840x2048" 91.45 3840 3872 4216 4248 2048 2097 2101 2151
+  Modeline "3840x1080" 100.38 3840 3848 4216 4592 1080 1081 1084 1093
+  Modeline "3600x1200" 106.06 3600 3632 3984 4368 1200 1201 1204 1214
+  Modeline "3288x1080" 39.76 3288 3320 3464 3496 1080 1106 1108 1135
+  Modeline "2048x2048" 49.47 2048 2080 2264 2296 2048 2097 2101 2151
+  Modeline "2048x1536" 80.06 2048 2104 2312 2576 1536 1537 1540 1554
+  Modeline "2560x1600" 47.12 2560 2592 2768 2800 1600 1639 1642 1681
+  Modeline "2560x1440" 42.12 2560 2592 2752 2784 1440 1475 1478 1513
+  Modeline "1920x1440" 69.47 1920 1960 2152 2384 1440 1441 1444 1457
+  Modeline "1920x1200" 26.28 1920 1952 2048 2080 1200 1229 1231 1261
+  Modeline "1920x1080" 23.53 1920 1952 2040 2072 1080 1106 1108 1135
+  Modeline "1680x1050" 20.08 1680 1712 1784 1816 1050 1075 1077 1103
+  Modeline "1600x1200" 22.04 1600 1632 1712 1744 1200 1229 1231 1261
+  Modeline "1600x900" 33.92 1600 1632 1760 1792 900 921 924 946
+  Modeline "1440x900" 30.66 1440 1472 1584 1616 900 921 924 946
+  ModeLine "1366x768" 72.00 1366 1414 1446 1494  768 771 777 803
+  Modeline "1280x1024" 31.50 1280 1312 1424 1456 1024 1048 1052 1076
+  Modeline "1280x800" 24.15 1280 1312 1400 1432 800 819 822 841
+  Modeline "1280x768" 23.11 1280 1312 1392 1424 768 786 789 807
+  Modeline "1360x768" 24.49 1360 1392 1480 1512 768 786 789 807
+  Modeline "1024x768" 18.71 1024 1056 1120 1152 768 786 789 807
+  Modeline "768x1024" 19.50 768 800 872 904 1024 1048 1052 1076
+
+
+  #common resolutions for android devices (both orientations):
+  Modeline "800x1280" 25.89 800 832 928 960 1280 1310 1315 1345
+  Modeline "1280x800" 24.15 1280 1312 1400 1432 800 819 822 841
+  Modeline "720x1280" 30.22 720 752 864 896 1280 1309 1315 1345
+  Modeline "1280x720" 27.41 1280 1312 1416 1448 720 737 740 757
+  Modeline "768x1024" 24.93 768 800 888 920 1024 1047 1052 1076
+  Modeline "1024x768" 23.77 1024 1056 1144 1176 768 785 789 807
+  Modeline "600x1024" 19.90 600 632 704 736 1024 1047 1052 1076
+  Modeline "1024x600" 18.26 1024 1056 1120 1152 600 614 617 631
+  Modeline "536x960" 16.74 536 568 624 656 960 982 986 1009
+  Modeline "960x536" 15.23 960 992 1048 1080 536 548 551 563
+  Modeline "600x800" 15.17 600 632 688 720 800 818 822 841
+  Modeline "800x600" 14.50 800 832 880 912 600 614 617 631
+  Modeline "480x854" 13.34 480 512 560 592 854 873 877 897
+  Modeline "848x480" 12.09 848 880 920 952 480 491 493 505
+  Modeline "480x800" 12.43 480 512 552 584 800 818 822 841
+  Modeline "800x480" 11.46 800 832 872 904 480 491 493 505
+  #resolutions for android devices (both orientations)
+  #minus the status bar
+  #38px status bar (and width rounded up)
+  Modeline "800x1242" 25.03 800 832 920 952 1242 1271 1275 1305
+  Modeline "1280x762" 22.93 1280 1312 1392 1424 762 780 783 801
+  Modeline "720x1242" 29.20 720 752 856 888 1242 1271 1276 1305
+  Modeline "1280x682" 25.85 1280 1312 1408 1440 682 698 701 717
+  Modeline "768x986" 23.90 768 800 888 920 986 1009 1013 1036
+  Modeline "1024x730" 22.50 1024 1056 1136 1168 730 747 750 767
+  Modeline "600x986" 19.07 600 632 704 736 986 1009 1013 1036
+  Modeline "1024x562" 17.03 1024 1056 1120 1152 562 575 578 591
+  Modeline "536x922" 16.01 536 568 624 656 922 943 947 969
+  Modeline "960x498" 14.09 960 992 1040 1072 498 509 511 523
+  Modeline "600x762" 14.39 600 632 680 712 762 779 783 801
+  Modeline "800x562" 13.52 800 832 880 912 562 575 578 591
+  Modeline "480x810" 12.59 480 512 552 584 810 828 832 851
+  Modeline "848x442" 11.09 848 880 920 952 442 452 454 465
+  Modeline "480x762" 11.79 480 512 552 584 762 779 783 801
+EndSection
+
+Section "Screen"
+  Identifier "dummy_screen"
+  Device "dummy_videocard"
+  Monitor "dummy_monitor"
+  DefaultDepth 24
+  SubSection "Display"
+    Viewport 0 0
+    Depth 24
+    #Modes "32768x32768" "32768x16384" "16384x8192" "8192x4096" "5120x3200" "3840x2880" "3840x2560" "3840x2048" "2048x2048" "2560x1600" "1920x1440" "1920x1200" "1920x1080" "1600x1200" "1680x1050" "1600x900" "1400x1050" "1440x900" "1280x1024" "1366x768" "1280x800" "1024x768" "1024x600" "800x600" "320x200"
+    Modes "5120x3200" "3840x2880" "3840x2560" "3840x2048" "2048x2048" "2560x1600" "1920x1440" "1920x1200" "1920x1080" "1600x1200" "1680x1050" "1600x900" "1400x1050" "1440x900" "1280x1024" "1366x768" "1280x800" "1024x768" "1024x600" "800x600" "320x200"
+    #Virtual 32000 32000
+    #Virtual 16384 8192
+    # 1024x768 is big enough for testing, but small enough it won't eat up lots of RAM
+    Virtual 1024 768
+    #Virtual 5120 3200
+  EndSubSection
+EndSection
+
+Section "ServerLayout"
+  Identifier   "dummy_layout"
+  Screen       "dummy_screen"
+  InputDevice  "dummy_mouse"
+  InputDevice  "dummy_keyboard"
+EndSection
diff --git a/setup.py b/setup.py
@@ -46,7 +46,6 @@ def get_readme() -> str:
     "pydocstyle",
     "pytest",
     "pytest-cov",
-    "pytest-shard",
     "pytest-xdist",
     "pytype",
     "stable-baselines3>=0.9.0",

diff --git a/src/seals/testing/envs.py b/src/seals/testing/envs.py
@@ -18,6 +18,7 @@
 )
 
 import gym
+from gym.envs.mujoco import mujoco_env
 import numpy as np
 
 Step = Tuple[Any, Optional[float], bool, Mapping[str, Any]]
@@ -223,7 +224,7 @@ def test_premature_step(env: gym.Env, skip_fn, raises_fn) -> None:
     """Test that you must call reset() before calling step().
 
     Example usage in pytest:
-        test_premature_step(env, skip_fn=pytest.skip, exception_fn=pytest.raises)
+        test_premature_step(env, skip_fn=pytest.skip, raises_fn=pytest.raises)
 
     Args:
         env: The environment to test.
@@ -243,6 +244,49 @@ def test_premature_step(env: gym.Env, skip_fn, raises_fn) -> None:
         env.step(act)
 
 
+def test_render(env: gym.Env, raises_fn) -> None:
+    """Test that render() supports the modes declared.
+
+    Example usage in pytest:
+        test_render(env, raises_fn=pytest.raises)
+
+    Args:
+        env: The environment to test.
+        raises_fn: Context manager to check NotImplementedError is thrown when
+            environment metadata indicates modes are supported.
+
+    Raises:
+        AssertionError: if test fails. This occurs if:
+            (a) `env.render(mode=mode)` fails for any mode declared supported
+            in `env.metadata["render.modes"]`; (b) env.render() *succeeds* when
+            `env.metadata["render.modes"]` is empty; (c) `env.render(mode="rgb_array")`
+            returns different values at the same time step.
+    """
+    env.reset()  # make sure environment is in consistent state
+
+    render_modes = env.metadata["render.modes"]
+    if not render_modes:
+        # No modes supported -- render() should fail.
+        with raises_fn(NotImplementedError):
+            env.render()
+    else:
+        for mode in render_modes:
+            env.render(mode=mode)
+
+        # WARNING(adam): there seems to be a memory leak with Gym 0.17.3
+        # & MuJoCoPy 1.50.1.68. `MujocoEnv.close()` does not call `finish()`
+        # on the viewer (commented out) so the resources are not released.
+        # For now this is OK, but may bite if we end up testing a lot of
+        # MuJoCo environments.
+        is_mujoco = isinstance(env.unwrapped, mujoco_env.MujocoEnv)
+        if "rgb_array" in render_modes and not is_mujoco:
+            # Render should not change without calling `step()`.
+            # MuJoCo rendering fails this check, ignore -- not much we can do.
+            resa = env.render(mode="rgb_array")
+            resb = env.render(mode="rgb_array")
+            assert np.allclose(resa, resb)
+
+
 class CountingEnv(gym.Env):
     """At timestep `t` of each episode, has `t == obs == reward / 10`.
 

diff --git a/src/seals/version.py b/src/seals/version.py
@@ -1,3 +1,3 @@
 """Project version. Keep it separate from __init__.py so setup.py can import it."""
 
-VERSION = "0.1.0"
+VERSION = "0.1.1"