diff --git a/.travis.yml b/.travis.yml
index b0e33ae37..6aa45e954 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,6 +33,8 @@ env:
         - CONDA_DEPENDENCIES='openjpeg Cython jinja2 scipy matplotlib mock requests beautifulsoup4 sqlalchemy scikit-image pytest-mock pandas nomkl pytest-cov coverage glymur suds-jurko sphinx'
         - PIP_DEPENDENCIES='sunpy-sphinx-theme pytest-astropy'
         - CONDA_CHANNELS='sunpy'
+        # Add a matplotlib backend for generating plots in tests.
+        - MPLBACKEND='Agg'
     matrix:
         # Make sure that egg_info works without dependencies
         - PYTHON_VERSION=3.5 SETUP_CMD='egg_info'
@@ -54,9 +56,6 @@ matrix:
 
     include:
 
-        # Add a matplotlib backend for generating plots in tests.
-        - env: MPLBACKEND='Agg'
-
         # Do a basic test.
         - os: linux
           stage: Initial tests
@@ -102,11 +101,6 @@ install:
     - git clone git://github.com/astropy/ci-helpers.git
     - source ci-helpers/travis/setup_conda.sh
 
-before_script:
-    - "export DISPLAY=:99.0"
-    - "sh -e /etc/init.d/xvfb start"
-    - sleep 3 # give xvfb some time to start
-
 script:
     - $MAIN_CMD $SETUP_CMD