diff --git a/.circleci/config.yml b/.circleci/config.yml index 1446d75..7f65c46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,16 +6,24 @@ version: 2.1 # Version of CircleCI config format orbs: codecov: codecov/codecov@1.1.0 # support for uploading code coverage to codecov +defaults: &defaults + docker: + - image: humancompatibleai/seals:base + auth: + username: $DOCKERHUB_USERNAME + password: $DOCKERHUB_PASSWORD + working_directory: /seals + executors: - my-executor: - docker: - - image: humancompatibleai/seals:base - # Credentials defined in `docker-hub-creds` context (database of environment variables). - # We authenticate to avoid Dockerhub per-IP rate limiting of anonymous users. - auth: - username: $DOCKERHUB_USERNAME - password: $DOCKERHUB_PASSWORD - working_directory: /seals + unit-test: + <<: *defaults + resource_class: large + environment: + # Don't use auto-detect since it sees all CPUs available, but container is throttled. + NUM_CPUS: 4 + lintandtype: + <<: *defaults + resource_class: medium environment: # If you change these, also change ci/code_checks.sh LINT_FILES: src/ tests/ docs/conf.py setup.py # files we lint @@ -23,7 +31,9 @@ executors: # In this repo we also typecheck tests/ -- but sometimes you may want to exclude these # if they do strange things with types (e.g. mocking). TYPECHECK_FILES: src/ tests/ setup.py - NUM_CPUS: 2 # more CPUs visible but we're throttled to 2, which breaks auto-detect + # Don't use auto-detect since it sees all CPUs available, but container is throttled. + NUM_CPUS: 2 + commands: # Define common function to install dependencies and seals, used in the jobs defined in the next section @@ -76,7 +86,7 @@ commands: jobs: # `lintandtype` installs dependencies + `seals`, lints the code, builds the docs, and runs type checks. lintandtype: - executor: my-executor + executor: lintandtype steps: - dependencies @@ -106,8 +116,7 @@ jobs: # `unit-test` runs the unit tests in `tests/`. unit-test: - executor: my-executor - parallelism: 3 + executor: unit-test steps: - dependencies @@ -132,16 +141,16 @@ jobs: - run: name: run tests command: | - export DISPLAY=:0 # we need an X11 display for tests of environment rendering + # Xdummy-entrypoint.py: starts an X server and sets DISPLAY, then runs wrapped command. + # pytest arguments: # --cov specifies which directories to report code coverage for # Since we test the installed `seals`, our source files live in `venv`, not in `src/seals`. # --junitxml records test results in JUnit format. We upload this file using `store_test_results` # later, and CircleCI then parses this to pretty-print results. # --shard-id and --num-shards are used to split tests across parallel executors using `pytest-shard`. # -n uses `pytest-xdist` to parallelize tests within a single instance. - pytest --cov=/venv/lib/python3.7/site-packages/seals --cov=tests \ + Xdummy-entrypoint.py pytest --cov=/venv/lib/python3.7/site-packages/seals --cov=tests \ --junitxml=/tmp/test-reports/junit.xml \ - --shard-id=${CIRCLE_NODE_INDEX} --num-shards=${CIRCLE_NODE_TOTAL} \ -n ${NUM_CPUS} -vv tests/ # Following two lines rewrite paths from venv/ to src/, based on `coverage:paths` in `setup.cfg` # This is needed to avoid confusing Codecov diff --git a/Dockerfile b/Dockerfile index 409361c..ba0c227 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,30 +44,36 @@ RUN mkdir -p /root/.mujoco \ ENV PATH="/venv/bin:$PATH" ENV LD_LIBRARY_PATH /root/.mujoco/mjpro150/bin:${LD_LIBRARY_PATH} +# Run Xdummy mock X server by default so that rendering will work. +COPY ci/xorg.conf /etc/dummy_xorg.conf +COPY ci/Xdummy-entrypoint.py /usr/bin/Xdummy-entrypoint.py +ENTRYPOINT ["/usr/bin/Xdummy-entrypoint.py"] + # python-req stage contains Python venv, but not code. # It is useful for development purposes: you can mount # code from outside the Docker container. FROM base as python-req -WORKDIR /benchmark-environments +WORKDIR /seals # Copy only necessary dependencies to build virtual environment. # This minimizes how often this layer needs to be rebuilt. COPY ./setup.py ./setup.py -COPY ./src/imitation/__init__.py ./src/imitation/__init__.py +COPY ./README.md ./README.md +COPY ./src/seals/version.py ./src/seals/version.py COPY ./ci/build_venv.sh ./ci/build_venv.sh # mjkey.txt needs to exist for build, but doesn't need to be a real key -RUN touch /root/.mujoco/mjkey.txt && /benchmark-environments/scripts/build_venv.sh /venv +RUN touch /root/.mujoco/mjkey.txt && /seals/ci/build_venv.sh /venv # full stage contains everything. # Can be used for deployment and local testing. FROM python-req as full # Delay copying (and installing) the code until the very end -COPY . /benchmark-environments +COPY . /seals # Build a wheel then install to avoid copying whole directory (pip issue #2195) RUN python setup.py sdist bdist_wheel -RUN pip install dist/evaluating_rewards-*.whl +RUN pip install dist/seals-*.whl # Default entrypoints CMD ["pytest", "-n", "auto", "-vv", "tests/"] diff --git a/ci/Xdummy-entrypoint.py b/ci/Xdummy-entrypoint.py new file mode 100755 index 0000000..f1abd5c --- /dev/null +++ b/ci/Xdummy-entrypoint.py @@ -0,0 +1,50 @@ +#!/usr/bin/python + +# Adapted from https://github.com/openai/mujoco-py/blob/master/vendor/Xdummy-entrypoint +# Copyright OpenAI; MIT License + +import argparse +import os +import sys +import subprocess + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + args, extra_args = parser.parse_known_args() + + subprocess.Popen( + [ + "nohup", + "Xorg", + "-noreset", + "+extension", + "GLX", + "+extension", + "RANDR", + "+extension", + "RENDER", + "-logfile", + "/tmp/xdummy.log", + "-config", + "/etc/dummy_xorg.conf", + ":0", + ] + ) + subprocess.Popen( + ["nohup", "Xdummy"], + stdout=open("/dev/null", "w"), + stderr=open("/dev/null", "w"), + ) + os.environ["DISPLAY"] = ":0" + + if not extra_args: + argv = ["/bin/bash"] + else: + argv = extra_args + + # Explicitly flush right before the exec since otherwise things might get + # lost in Python's buffers around stdout/stderr (!). + sys.stdout.flush() + sys.stderr.flush() + + os.execvpe(argv[0], argv, os.environ) diff --git a/ci/xorg.conf b/ci/xorg.conf new file mode 100644 index 0000000..a1bce88 --- /dev/null +++ b/ci/xorg.conf @@ -0,0 +1,138 @@ +# This xorg configuration file is meant to be used by xpra +# to start a dummy X11 server. +# For details, please see: +# https://xpra.org/Xdummy.html + +Section "ServerFlags" + Option "DontVTSwitch" "true" + Option "AllowMouseOpenFail" "true" + Option "PciForceNone" "true" + Option "AutoEnableDevices" "false" + Option "AutoAddDevices" "false" +EndSection + +Section "InputDevice" + Identifier "dummy_mouse" + Option "CorePointer" "true" + Driver "void" +EndSection + +Section "InputDevice" + Identifier "dummy_keyboard" + Option "CoreKeyboard" "true" + Driver "void" +EndSection + +Section "Device" + Identifier "dummy_videocard" + Driver "dummy" + Option "ConstantDPI" "true" + #VideoRam 4096000 + #VideoRam 256000 + VideoRam 192000 +EndSection + +Section "Monitor" + Identifier "dummy_monitor" + HorizSync 5.0 - 1000.0 + VertRefresh 5.0 - 200.0 + #This can be used to get a specific DPI, but only for the default resolution: + #DisplaySize 508 317 + #NOTE: the highest modes will not work without increasing the VideoRam + # for the dummy video card. + Modeline "32768x32768" 15226.50 32768 35800 39488 46208 32768 32771 32781 32953 + Modeline "32768x16384" 7516.25 32768 35544 39192 45616 16384 16387 16397 16478 + Modeline "16384x8192" 2101.93 16384 16416 24400 24432 8192 8390 8403 8602 + Modeline "8192x4096" 424.46 8192 8224 9832 9864 4096 4195 4202 4301 + Modeline "5496x1200" 199.13 5496 5528 6280 6312 1200 1228 1233 1261 + Modeline "5280x1080" 169.96 5280 5312 5952 5984 1080 1105 1110 1135 + Modeline "5280x1200" 191.40 5280 5312 6032 6064 1200 1228 1233 1261 + Modeline "5120x3200" 199.75 5120 5152 5904 5936 3200 3277 3283 3361 + Modeline "4800x1200" 64.42 4800 4832 5072 5104 1200 1229 1231 1261 + Modeline "3840x2880" 133.43 3840 3872 4376 4408 2880 2950 2955 3025 + Modeline "3840x2560" 116.93 3840 3872 4312 4344 2560 2622 2627 2689 + Modeline "3840x2048" 91.45 3840 3872 4216 4248 2048 2097 2101 2151 + Modeline "3840x1080" 100.38 3840 3848 4216 4592 1080 1081 1084 1093 + Modeline "3600x1200" 106.06 3600 3632 3984 4368 1200 1201 1204 1214 + Modeline "3288x1080" 39.76 3288 3320 3464 3496 1080 1106 1108 1135 + Modeline "2048x2048" 49.47 2048 2080 2264 2296 2048 2097 2101 2151 + Modeline "2048x1536" 80.06 2048 2104 2312 2576 1536 1537 1540 1554 + Modeline "2560x1600" 47.12 2560 2592 2768 2800 1600 1639 1642 1681 + Modeline "2560x1440" 42.12 2560 2592 2752 2784 1440 1475 1478 1513 + Modeline "1920x1440" 69.47 1920 1960 2152 2384 1440 1441 1444 1457 + Modeline "1920x1200" 26.28 1920 1952 2048 2080 1200 1229 1231 1261 + Modeline "1920x1080" 23.53 1920 1952 2040 2072 1080 1106 1108 1135 + Modeline "1680x1050" 20.08 1680 1712 1784 1816 1050 1075 1077 1103 + Modeline "1600x1200" 22.04 1600 1632 1712 1744 1200 1229 1231 1261 + Modeline "1600x900" 33.92 1600 1632 1760 1792 900 921 924 946 + Modeline "1440x900" 30.66 1440 1472 1584 1616 900 921 924 946 + ModeLine "1366x768" 72.00 1366 1414 1446 1494 768 771 777 803 + Modeline "1280x1024" 31.50 1280 1312 1424 1456 1024 1048 1052 1076 + Modeline "1280x800" 24.15 1280 1312 1400 1432 800 819 822 841 + Modeline "1280x768" 23.11 1280 1312 1392 1424 768 786 789 807 + Modeline "1360x768" 24.49 1360 1392 1480 1512 768 786 789 807 + Modeline "1024x768" 18.71 1024 1056 1120 1152 768 786 789 807 + Modeline "768x1024" 19.50 768 800 872 904 1024 1048 1052 1076 + + + #common resolutions for android devices (both orientations): + Modeline "800x1280" 25.89 800 832 928 960 1280 1310 1315 1345 + Modeline "1280x800" 24.15 1280 1312 1400 1432 800 819 822 841 + Modeline "720x1280" 30.22 720 752 864 896 1280 1309 1315 1345 + Modeline "1280x720" 27.41 1280 1312 1416 1448 720 737 740 757 + Modeline "768x1024" 24.93 768 800 888 920 1024 1047 1052 1076 + Modeline "1024x768" 23.77 1024 1056 1144 1176 768 785 789 807 + Modeline "600x1024" 19.90 600 632 704 736 1024 1047 1052 1076 + Modeline "1024x600" 18.26 1024 1056 1120 1152 600 614 617 631 + Modeline "536x960" 16.74 536 568 624 656 960 982 986 1009 + Modeline "960x536" 15.23 960 992 1048 1080 536 548 551 563 + Modeline "600x800" 15.17 600 632 688 720 800 818 822 841 + Modeline "800x600" 14.50 800 832 880 912 600 614 617 631 + Modeline "480x854" 13.34 480 512 560 592 854 873 877 897 + Modeline "848x480" 12.09 848 880 920 952 480 491 493 505 + Modeline "480x800" 12.43 480 512 552 584 800 818 822 841 + Modeline "800x480" 11.46 800 832 872 904 480 491 493 505 + #resolutions for android devices (both orientations) + #minus the status bar + #38px status bar (and width rounded up) + Modeline "800x1242" 25.03 800 832 920 952 1242 1271 1275 1305 + Modeline "1280x762" 22.93 1280 1312 1392 1424 762 780 783 801 + Modeline "720x1242" 29.20 720 752 856 888 1242 1271 1276 1305 + Modeline "1280x682" 25.85 1280 1312 1408 1440 682 698 701 717 + Modeline "768x986" 23.90 768 800 888 920 986 1009 1013 1036 + Modeline "1024x730" 22.50 1024 1056 1136 1168 730 747 750 767 + Modeline "600x986" 19.07 600 632 704 736 986 1009 1013 1036 + Modeline "1024x562" 17.03 1024 1056 1120 1152 562 575 578 591 + Modeline "536x922" 16.01 536 568 624 656 922 943 947 969 + Modeline "960x498" 14.09 960 992 1040 1072 498 509 511 523 + Modeline "600x762" 14.39 600 632 680 712 762 779 783 801 + Modeline "800x562" 13.52 800 832 880 912 562 575 578 591 + Modeline "480x810" 12.59 480 512 552 584 810 828 832 851 + Modeline "848x442" 11.09 848 880 920 952 442 452 454 465 + Modeline "480x762" 11.79 480 512 552 584 762 779 783 801 +EndSection + +Section "Screen" + Identifier "dummy_screen" + Device "dummy_videocard" + Monitor "dummy_monitor" + DefaultDepth 24 + SubSection "Display" + Viewport 0 0 + Depth 24 + #Modes "32768x32768" "32768x16384" "16384x8192" "8192x4096" "5120x3200" "3840x2880" "3840x2560" "3840x2048" "2048x2048" "2560x1600" "1920x1440" "1920x1200" "1920x1080" "1600x1200" "1680x1050" "1600x900" "1400x1050" "1440x900" "1280x1024" "1366x768" "1280x800" "1024x768" "1024x600" "800x600" "320x200" + Modes "5120x3200" "3840x2880" "3840x2560" "3840x2048" "2048x2048" "2560x1600" "1920x1440" "1920x1200" "1920x1080" "1600x1200" "1680x1050" "1600x900" "1400x1050" "1440x900" "1280x1024" "1366x768" "1280x800" "1024x768" "1024x600" "800x600" "320x200" + #Virtual 32000 32000 + #Virtual 16384 8192 + # 1024x768 is big enough for testing, but small enough it won't eat up lots of RAM + Virtual 1024 768 + #Virtual 5120 3200 + EndSubSection +EndSection + +Section "ServerLayout" + Identifier "dummy_layout" + Screen "dummy_screen" + InputDevice "dummy_mouse" + InputDevice "dummy_keyboard" +EndSection diff --git a/setup.py b/setup.py index ac7b88e..e65a5aa 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,6 @@ def get_readme() -> str: "pydocstyle", "pytest", "pytest-cov", - "pytest-shard", "pytest-xdist", "pytype", "stable-baselines3>=0.9.0", diff --git a/src/seals/testing/envs.py b/src/seals/testing/envs.py index c7c7b48..21dde25 100644 --- a/src/seals/testing/envs.py +++ b/src/seals/testing/envs.py @@ -18,6 +18,7 @@ ) import gym +from gym.envs.mujoco import mujoco_env import numpy as np Step = Tuple[Any, Optional[float], bool, Mapping[str, Any]] @@ -223,7 +224,7 @@ def test_premature_step(env: gym.Env, skip_fn, raises_fn) -> None: """Test that you must call reset() before calling step(). Example usage in pytest: - test_premature_step(env, skip_fn=pytest.skip, exception_fn=pytest.raises) + test_premature_step(env, skip_fn=pytest.skip, raises_fn=pytest.raises) Args: env: The environment to test. @@ -243,6 +244,49 @@ def test_premature_step(env: gym.Env, skip_fn, raises_fn) -> None: env.step(act) +def test_render(env: gym.Env, raises_fn) -> None: + """Test that render() supports the modes declared. + + Example usage in pytest: + test_render(env, raises_fn=pytest.raises) + + Args: + env: The environment to test. + raises_fn: Context manager to check NotImplementedError is thrown when + environment metadata indicates modes are supported. + + Raises: + AssertionError: if test fails. This occurs if: + (a) `env.render(mode=mode)` fails for any mode declared supported + in `env.metadata["render.modes"]`; (b) env.render() *succeeds* when + `env.metadata["render.modes"]` is empty; (c) `env.render(mode="rgb_array")` + returns different values at the same time step. + """ + env.reset() # make sure environment is in consistent state + + render_modes = env.metadata["render.modes"] + if not render_modes: + # No modes supported -- render() should fail. + with raises_fn(NotImplementedError): + env.render() + else: + for mode in render_modes: + env.render(mode=mode) + + # WARNING(adam): there seems to be a memory leak with Gym 0.17.3 + # & MuJoCoPy 1.50.1.68. `MujocoEnv.close()` does not call `finish()` + # on the viewer (commented out) so the resources are not released. + # For now this is OK, but may bite if we end up testing a lot of + # MuJoCo environments. + is_mujoco = isinstance(env.unwrapped, mujoco_env.MujocoEnv) + if "rgb_array" in render_modes and not is_mujoco: + # Render should not change without calling `step()`. + # MuJoCo rendering fails this check, ignore -- not much we can do. + resa = env.render(mode="rgb_array") + resb = env.render(mode="rgb_array") + assert np.allclose(resa, resb) + + class CountingEnv(gym.Env): """At timestep `t` of each episode, has `t == obs == reward / 10`. diff --git a/src/seals/version.py b/src/seals/version.py index 698ec57..0fdb4d8 100644 --- a/src/seals/version.py +++ b/src/seals/version.py @@ -1,3 +1,3 @@ """Project version. Keep it separate from __init__.py so setup.py can import it.""" -VERSION = "0.1.0" +VERSION = "0.1.1" diff --git a/tests/test_envs.py b/tests/test_envs.py index 0a40d23..4e8c69b 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -3,6 +3,7 @@ from typing import List import gym +from gym.envs import registration import pytest import seals # noqa: F401 required for env registration @@ -10,7 +11,7 @@ ENV_NAMES: List[str] = [ env_spec.id - for env_spec in gym.envs.registration.registry.all() + for env_spec in registration.registry.all() if env_spec.id.startswith("seals/") ] @@ -41,3 +42,7 @@ def test_premature_step(self, env: gym.Env): def test_rollout_schema(self, env: gym.Env): """Tests if environments have correct types on `step()` and `reset()`.""" envs.test_rollout_schema(env) + + def test_render(self, env: gym.Env): + """Tests `render()` supports modes specified in environment metadata.""" + envs.test_render(env, raises_fn=pytest.raises)