Cache properly for conda env, name jobs pretty and separate dependency installation and run tests in Github Actions

.github/workflows/master.yml

@@ -9,52 +9,58 @@ on:
       - master
 
 jobs:
-  python35:
+  pip_python35:
+    name: PIP, Python (3.5)
     runs-on: ubuntu-latest
     env:
       SPARK_VERSION: 2.3.4
       PANDAS_VERSION: 0.23.4
       PYARROW_VERSION: 0.10.0
+      # DISPLAY=0.0 does not work in Github Actions with Python 3.5. Here we work around wtih  xvfb-run
       PYTHON_EXECUTABLE: xvfb-run python
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-java@v1
       with:
         java-version: 1.8
-    - name: Using cache
-      uses: actions/cache@v1
+    - uses: actions/cache@v1
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-
-    # Setup Python3.5 via `apt-get install` since the dafault Python3.5 from `actions/setup-python`
-    # seems to have some problem with Tkinter, so we should manually install the python3.5-tk.
-    # for this, we should use the Python manually installed, not the default one from `actions/setup-python`
+    # Setup Python 3.5 via `apt-get install` since the dafault Python 3.5 from `actions/setup-python`
+    # seems to have some problems with Tkinter, so we should manually install the python3.5-tk.
+    # For this, we should use the Python manually installed, not the default one from `actions/setup-python`
     - name: Setup Python 3.5
       run: |
         sudo add-apt-repository ppa:deadsnakes/ppa
         sudo apt-get install tk-dev python3.5-tk python3.5
         sudo rm /usr/bin/python
         sudo ln -s /usr/bin/python3.5 /usr/bin/python
-        # Below command is required for ensuring pip installed executables to be in the path.
+        # Below command is required for ensuring PIP-installed executables to be in the path.
         echo "::add-path::/home/runner/.local/bin"
-    - name: Install dependencies & run tests
+    - name: Install dependencies
       run: |
         ./dev/download_travis_dependencies.sh
         sudo apt-get install xclip
         pip install setuptools
         pip install -r requirements-dev.txt
         pip install pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION
         pip list
+    - name: Run tests
+      run: |
+        # .cache directory is for Travis CI. Once we remove Travis CI, we should download
+        # Spark to a directory with a different name to prevent confusion.
         export SPARK_HOME="$HOME/.cache/spark-versions/spark-$SPARK_VERSION-bin-hadoop2.7"
         ./dev/lint-python
         ./dev/pytest
     # Push the results back to codecov
     - name: Update Codecov
       run: bash <(curl -s https://codecov.io/bash)
 
-  python:
+  conda_python:
+    name: Conda, Python
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -74,27 +80,27 @@ jobs:
       SPARK_VERSION: ${{ matrix.spark-version }}
       PANDAS_VERSION: ${{ matrix.pandas-version }}
       PYARROW_VERSION: ${{ matrix.pyarrow-version }}
-      DISPLAY: 0.0
       # `QT_QPA_PLATFORM` for resolving 'QXcbConnection: Could not connect to display :0.0'
+      DISPLAY: 0.0
       QT_QPA_PLATFORM: offscreen
       KOALAS_USAGE_LOGGER: ${{ matrix.logger }}
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-java@v1
       with:
         java-version: 1.8
-    - name: Using cache
-      uses: actions/cache@v1
+    - uses: actions/cache@v1
       with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }}
+        path: /home/runner/miniconda/envs
+        key: ${{ runner.os }}-conda-${{ hashFiles('**/requirements-dev.txt') }}
         restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Install dependencies & run tests
+          ${{ runner.os }}-conda-
+    - name: Install dependencies
       run: |
         ./dev/download_travis_dependencies.sh
         curl -s https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh
         bash miniconda.sh -b -p $HOME/miniconda
+        # See also https://github.com/conda/conda/issues/7980
         . $HOME/miniconda/etc/profile.d/conda.sh
         hash -r
         conda config --set always_yes yes --set changeps1 no
@@ -111,9 +117,20 @@ jobs:
         conda install -c conda-forge --yes pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION
         conda install -c conda-forge --yes --freeze-installed --file requirements-dev.txt
         conda list
+    - name: Run tests
+      run: |
+        # The environment should be initialized newly between Github Actions steps. Also,
+        # for some reasons we should add enviornment directory manually to allow for
+        # 'test-enviornment' to be found. See also https://github.com/conda/conda/issues/7980
+        . $HOME/miniconda/etc/profile.d/conda.sh
+        conda config --prepend envs_dirs /home/runner/miniconda/envs
+        conda activate test-environment
+        # .cache directory is for Travis CI. Once we remove Travis CI, we should download
+        # Spark to a directory with a different name to prevent confusion.
         export SPARK_HOME="$HOME/.cache/spark-versions/spark-$SPARK_VERSION-bin-hadoop2.7"
         ./dev/lint-python
         ./dev/pytest
     # Push the results back to codecov
     - name: Update Codecov
       run: bash <(curl -s https://codecov.io/bash)
+