Merge pull request #2 from grondo/more-travis-fixes

travis-ci: fix docker tag deploy for v0.11 branch, backport other docker fixes

.travis.yml

@@ -75,15 +75,15 @@ jobs:
       compiler: gcc
       env:
        - ARGS="--with-flux-security --enable-caliper --prefix=/usr"
-       - IMG=centos7-base
+       - IMG=centos7
        - DOCKER_TAG=t
        - PYTHON_VERSION=2.7
     - name: "Centos 7: py3.4 --with-flux-security"
       stage: test
       compiler: gcc
       env:
        - ARGS="--with-flux-security --prefix=/usr"
-       - IMG=centos7-base
+       - IMG=centos7
        - PYTHON_VERSION=3.4
 
 stages:
@@ -112,15 +112,15 @@ before_install:
  - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); exit(flags&os.O_NONBLOCK);'
  # coveralls-lcov required only for coveralls upload:
  - if test "$COVERAGE" = "t" ; then gem install coveralls-lcov; fi
- - if test -z "${IMG}"; then IMG="bionic-base"; fi
+ - if test -z "${IMG}"; then IMG="bionic"; fi
  #
  #  Tag image if this build is on master or result of a tag:
  - |
   if test "$DOCKER_TAG" = "t" \
-    -a "$TRAVIS_REPO_SLUG" = "flux-framework/flux-core" \
+    -a "$TRAVIS_REPO_SLUG" = "flux-framework/flux-core-v0.11" \
     -a "$TRAVIS_PULL_REQUEST" = "false" \
     -a \( "$TRAVIS_BRANCH" = "master" -o -n "$TRAVIS_TAG" \); then
-     export TAGNAME="${DOCKERREPO}:${IMG}-${TRAVIS_TAG:-latest}"
+     export TAGNAME="${DOCKERREPO}:${IMG}${TRAVIS_TAG:+-${TRAVIS_TAG}}"
      echo "Tagging new image $TAGNAME"
   fi
 
@@ -149,7 +149,7 @@ after_success:
        docker login -u "$DOCKER_USERNAME" --password-stdin && \
      docker push ${TAGNAME}
      # If this is the bionic-base build, then also tag without image name:
-     if echo "$TAGNAME" | grep -q "bionic-base"; then
+     if echo "$TAGNAME" | grep -q "bionic"; then
        t="${DOCKERREPO}:${TRAVIS_TAG:${DOCKER_STABLE_PREFIX}-latest}"
        docker tag "$TAGNAME" ${t} && \
        docker push ${t}

src/test/docker/README.md

@@ -0,0 +1,57 @@
+### Docker images for flux-core
+
+The Dockerfiles, resulting docker images, and `docker-run-checks.sh`
+script contained herein are used as part of the strategy for CI testing
+of Flux Framework projects under [Travis CI](https://travis-ci.org).
+
+Docker is used under Travis to speed up deployment of an
+environment with correct build dependencies and to keep a docker
+image deployed at `fluxrm/flux-core` DockerHub with latest master build
+(`fluxrm/flux-core:latest`) and tagged builds (`fluxrm/flux-core:v<tag>`),
+which can be used by other framework projects to build against the latest
+or a tagged version of flux-core.
+
+#### fluxrm/testenv Docker images
+
+The Dockerfiles under `bionic/Dockerfile` and
+`centos7/Dockerfile` describe the images built under the
+`fluxrm/testenv:bionic` and `fluxrm/testenv:centos7`
+respectively, and include the base dependencies required to build
+flux-core. These images are updated manually by flux-core maintainers, but
+the Dockerfiles should be kept up to date for a single point of management.
+
+#### The travis build Dockerfile
+
+A secondary Dockerfile exists under `./travis/Dockerfile` which is used
+to customize the `fluxrm/testenv` before building. Without this secondary
+`docker build` stage, there would be no way for PRs on GitHub to add
+new dependencies for users that are not core maintainers (or the "base"
+images would need to be completely rebuilt on each CI run). For now,
+`flux-security` is also built manually within the `travis/Dockerfile`
+because it is assumed that package will be rapidly changing, and it
+would not make sense to be constantly updating the base `fluxrm/testenv`
+Docker images.
+
+#### Adding a new dependency
+
+When constructing a PR that adds new dependency, the dependency should
+be added (for both CentOS and Ubuntu) in `travis/Dockerfile`. This will
+result in a temporary docker image being created during testing of the
+PR with the dependency installed.
+
+Later, a flux-core maintainer can move the dependency into the `testenv`
+Docker images `bionic/Dockerfile` and `centos7/Dockerfile`.
+These docker images should then be built by hand and manually
+pushed to DockerHub at `fluxrm/testenv:bionic` and
+`fluxrm/testenv:centos7`. Be sure to test that the `docker-run-test.sh`
+script still runs against the new `testenv` images, e.g.:
+
+```
+$ for i in bionic centos7; do
+    make clean &&
+    docker build --no-cache -t fluxrm/testenv:$i src/test/docker/$i &&
+    src/test/docker/docker-run-checks.sh -j 4 --image=$i &&
+    docker push fluxrm/testenv:$i
+   done
+```
+

src/test/docker/docker-run-checks.sh

@@ -7,7 +7,7 @@
 #
 #
 # option Defaults:
-IMAGE=bionic-base
+IMAGE=bionic
 FLUX_SECURITY_VERSION=0.2.0
 JOBS=2
 MOUNT_HOME_ARGS="--volume=$HOME:/home/$USER -e HOME"

src/test/docker/travis/Dockerfile

@@ -46,10 +46,15 @@ RUN case $BASE_IMAGE in \
      *) (>&2 echo "Unknown BASE_IMAGE") ;; \
     esac
 
-# Make sure user in appropriate group for sudo on different platorms
+# Install extra dependencies if necessary here.
+#
+# Do not forget to run `apt update` on Ubuntu/bionic.
+# Do NOT run `yum upgrade` on CentOS (this will unnecessarily upgrade
+#  existing packages)
+#
 RUN case $BASE_IMAGE in \
-     bionic*) apt-get -qq install -y --no-install-recommends python-six python3-six ;; \
-     centos*) yum -y install python-six python34-devel python34-cffi python34-six ;; \
+     bionic*) apt update && apt install -yy --no-install-recommends libyaml-cpp-dev;; \
+     centos*) yum -y update && yum -y install yaml-cpp-devel ;; \
      *) (>&2 echo "Unknown BASE_IMAGE") ;; \
     esac
 

src/test/travis_run.sh

@@ -93,6 +93,10 @@ fi
 export FLUX_TEST_SIZE_MAX=5
 
 # Invoke MPI tests
+# CentOS 7: mpich only available via environment-module:
+if test -f /usr/share/Modules/init/bash; then
+    . /usr/share/Modules/init/bash && module load mpi
+fi
 export TEST_MPI=t
 
 # Generate logfiles from sharness tests for extra information: