diff --git a/.travis.yml b/.travis.yml index db66fa44ecad..f8f634337f10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,7 +75,7 @@ jobs: compiler: gcc env: - ARGS="--with-flux-security --enable-caliper --prefix=/usr" - - IMG=centos7-base + - IMG=centos7 - DOCKER_TAG=t - PYTHON_VERSION=2.7 - name: "Centos 7: py3.4 --with-flux-security" @@ -83,7 +83,7 @@ jobs: compiler: gcc env: - ARGS="--with-flux-security --prefix=/usr" - - IMG=centos7-base + - IMG=centos7 - PYTHON_VERSION=3.4 stages: @@ -112,15 +112,15 @@ before_install: - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); exit(flags&os.O_NONBLOCK);' # coveralls-lcov required only for coveralls upload: - if test "$COVERAGE" = "t" ; then gem install coveralls-lcov; fi - - if test -z "${IMG}"; then IMG="bionic-base"; fi + - if test -z "${IMG}"; then IMG="bionic"; fi # # Tag image if this build is on master or result of a tag: - | if test "$DOCKER_TAG" = "t" \ - -a "$TRAVIS_REPO_SLUG" = "flux-framework/flux-core" \ + -a "$TRAVIS_REPO_SLUG" = "flux-framework/flux-core-v0.11" \ -a "$TRAVIS_PULL_REQUEST" = "false" \ -a \( "$TRAVIS_BRANCH" = "master" -o -n "$TRAVIS_TAG" \); then - export TAGNAME="${DOCKERREPO}:${IMG}-${TRAVIS_TAG:-latest}" + export TAGNAME="${DOCKERREPO}:${IMG}${TRAVIS_TAG:+-${TRAVIS_TAG}}" echo "Tagging new image $TAGNAME" fi @@ -149,7 +149,7 @@ after_success: docker login -u "$DOCKER_USERNAME" --password-stdin && \ docker push ${TAGNAME} # If this is the bionic-base build, then also tag without image name: - if echo "$TAGNAME" | grep -q "bionic-base"; then + if echo "$TAGNAME" | grep -q "bionic"; then t="${DOCKERREPO}:${TRAVIS_TAG:${DOCKER_STABLE_PREFIX}-latest}" docker tag "$TAGNAME" ${t} && \ docker push ${t} diff --git a/src/test/docker/README.md b/src/test/docker/README.md new file mode 100644 index 000000000000..dae98d12b11c --- /dev/null +++ b/src/test/docker/README.md @@ -0,0 +1,57 @@ +### Docker images for flux-core + +The Dockerfiles, resulting docker images, and `docker-run-checks.sh` +script contained herein are used as part of the strategy for CI testing +of Flux Framework projects under [Travis CI](https://travis-ci.org). + +Docker is used under Travis to speed up deployment of an +environment with correct build dependencies and to keep a docker +image deployed at `fluxrm/flux-core` DockerHub with latest master build +(`fluxrm/flux-core:latest`) and tagged builds (`fluxrm/flux-core:v`), +which can be used by other framework projects to build against the latest +or a tagged version of flux-core. + +#### fluxrm/testenv Docker images + +The Dockerfiles under `bionic/Dockerfile` and +`centos7/Dockerfile` describe the images built under the +`fluxrm/testenv:bionic` and `fluxrm/testenv:centos7` +respectively, and include the base dependencies required to build +flux-core. These images are updated manually by flux-core maintainers, but +the Dockerfiles should be kept up to date for a single point of management. + +#### The travis build Dockerfile + +A secondary Dockerfile exists under `./travis/Dockerfile` which is used +to customize the `fluxrm/testenv` before building. Without this secondary +`docker build` stage, there would be no way for PRs on GitHub to add +new dependencies for users that are not core maintainers (or the "base" +images would need to be completely rebuilt on each CI run). For now, +`flux-security` is also built manually within the `travis/Dockerfile` +because it is assumed that package will be rapidly changing, and it +would not make sense to be constantly updating the base `fluxrm/testenv` +Docker images. + +#### Adding a new dependency + +When constructing a PR that adds new dependency, the dependency should +be added (for both CentOS and Ubuntu) in `travis/Dockerfile`. This will +result in a temporary docker image being created during testing of the +PR with the dependency installed. + +Later, a flux-core maintainer can move the dependency into the `testenv` +Docker images `bionic/Dockerfile` and `centos7/Dockerfile`. +These docker images should then be built by hand and manually +pushed to DockerHub at `fluxrm/testenv:bionic` and +`fluxrm/testenv:centos7`. Be sure to test that the `docker-run-test.sh` +script still runs against the new `testenv` images, e.g.: + +``` +$ for i in bionic centos7; do + make clean && + docker build --no-cache -t fluxrm/testenv:$i src/test/docker/$i && + src/test/docker/docker-run-checks.sh -j 4 --image=$i && + docker push fluxrm/testenv:$i + done +``` + diff --git a/src/test/docker/bionic-base/Dockerfile b/src/test/docker/bionic/Dockerfile similarity index 100% rename from src/test/docker/bionic-base/Dockerfile rename to src/test/docker/bionic/Dockerfile diff --git a/src/test/docker/centos7-base/Dockerfile b/src/test/docker/centos7/Dockerfile similarity index 100% rename from src/test/docker/centos7-base/Dockerfile rename to src/test/docker/centos7/Dockerfile diff --git a/src/test/docker/centos7-base/config.site b/src/test/docker/centos7/config.site similarity index 100% rename from src/test/docker/centos7-base/config.site rename to src/test/docker/centos7/config.site diff --git a/src/test/docker/docker-run-checks.sh b/src/test/docker/docker-run-checks.sh index 0923912bcd7e..6e134c4a3ceb 100755 --- a/src/test/docker/docker-run-checks.sh +++ b/src/test/docker/docker-run-checks.sh @@ -7,7 +7,7 @@ # # # option Defaults: -IMAGE=bionic-base +IMAGE=bionic FLUX_SECURITY_VERSION=0.2.0 JOBS=2 MOUNT_HOME_ARGS="--volume=$HOME:/home/$USER -e HOME" diff --git a/src/test/docker/travis/Dockerfile b/src/test/docker/travis/Dockerfile index b0eb81907cd3..bb348a4bab24 100644 --- a/src/test/docker/travis/Dockerfile +++ b/src/test/docker/travis/Dockerfile @@ -46,10 +46,15 @@ RUN case $BASE_IMAGE in \ *) (>&2 echo "Unknown BASE_IMAGE") ;; \ esac -# Make sure user in appropriate group for sudo on different platorms +# Install extra dependencies if necessary here. +# +# Do not forget to run `apt update` on Ubuntu/bionic. +# Do NOT run `yum upgrade` on CentOS (this will unnecessarily upgrade +# existing packages) +# RUN case $BASE_IMAGE in \ - bionic*) apt-get -qq install -y --no-install-recommends python-six python3-six ;; \ - centos*) yum -y install python-six python34-devel python34-cffi python34-six ;; \ + bionic*) apt update && apt install -yy --no-install-recommends libyaml-cpp-dev;; \ + centos*) yum -y update && yum -y install yaml-cpp-devel ;; \ *) (>&2 echo "Unknown BASE_IMAGE") ;; \ esac diff --git a/src/test/travis_run.sh b/src/test/travis_run.sh index c3a12b0243d3..4a61c1e7fdfc 100755 --- a/src/test/travis_run.sh +++ b/src/test/travis_run.sh @@ -93,6 +93,10 @@ fi export FLUX_TEST_SIZE_MAX=5 # Invoke MPI tests +# CentOS 7: mpich only available via environment-module: +if test -f /usr/share/Modules/init/bash; then + . /usr/share/Modules/init/bash && module load mpi +fi export TEST_MPI=t # Generate logfiles from sharness tests for extra information: