Merge pull request #147 from /issues/125

Issues/125

CHANGES.rst

@@ -10,6 +10,7 @@ Unreleased Changes
 * `PR #141 <https://github.com/jantman/biweeklybudget/pull/141>`_ - Switch acceptance tests from PhantomJS to headless Chrome.
 * Switch docs build screenshot script to use headless Chrome instead of PhantomJS.
 * `Issue #142 <https://github.com/jantman/biweeklybudget/issues/142>`_ - Speed up acceptance tests. The acceptance tests recently crossed the 20-minute barrier, which is unacceptable. This makes some improvements to the tests, mainly around combining classes that can be combined and also using mysql/mysqldump to refresh the DB, instead of refreshing and recreating via the ORM. That offers a approximately 50-90% speed improvement for each of the 43 refreshes. Unfortunately, it seems that the majority of time is taken up by pytest-selenium; see Issue 142 for further information.
+* `Issue #125 <https://github.com/jantman/biweeklybudget/issues/125>`_ - Switch Docker image base from ``python:3.6.1`` (Debian) to ``python:3.6.3-alpine3.4`` (Alpine Linux); drops final image size from 876MB to 274MB. (*Note:* Alpine linux does not have ``/bin/bash``.)
 
 0.5.0 (2017-10-28)
 ------------------

biweeklybudget/tests/conftest.py

@@ -162,24 +162,27 @@ def testflask():
     This is a version of pytest-flask's live_server fixture, modified for
     session use.
     """
-    # Bind to an open port
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.bind(('', 0))
-    port = s.getsockname()[1]
-    s.close()
-
-    from biweeklybudget.flaskapp.app import app  # noqa
-    server = LiveServer(app, port)
-    server.start()
-    yield(server)
-    server.stop()
+    if 'BIWEEKLYBUDGET_TEST_BASE_URL' not in os.environ:
+        # Bind to an open port
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.bind(('', 0))
+        port = s.getsockname()[1]
+        s.close()
+        from biweeklybudget.flaskapp.app import app  # noqa
+        server = LiveServer(app, port)
+        server.start()
+        yield(server)
+        server.stop()
 
 
 @pytest.fixture(scope="session")
 def base_url(testflask):
     """
     Simple fixture to return ``testflask`` base URL
     """
+    url = os.environ.get('BIWEEKLYBUDGET_TEST_BASE_URL', None)
+    if url is not None:
+        return url
     return testflask.url()
 
 

biweeklybudget/tests/docker_build.py

@@ -82,37 +82,49 @@
 
 DOCKERFILE_TEMPLATE = """
 # biweeklybudget Dockerfile - http://github.com/jantman/biweeklybudget
-FROM python:3.6.1
+FROM python:3.6.3-alpine3.4
 
 ARG version
 USER root
 
-COPY tini_0.14.0.deb /tmp/tini_0.14.0.deb
 COPY requirements.txt /tmp/requirements.txt
 COPY entrypoint.sh /tmp/entrypoint.sh
 {copy}
 
-RUN /usr/bin/dpkg -i /tmp/tini_0.14.0.deb
 RUN /usr/local/bin/pip install virtualenv
-RUN /usr/local/bin/virtualenv /app && \
-    /app/bin/pip install {install} && \
-    /app/bin/pip install gunicorn==19.7.1 wishlist
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-# install phantomjs and locales; setup locales
-# phantomjs installation from:
-# https://github.com/josefcs/debian-phantomjs/blob/master/Dockerfile
-# @ 1e5bd20c8c51ec9a2e54a765594eea79aa0b9aed
-RUN apt-get update && \
-    apt-get --assume-yes install locales wget ca-certificates \
-    fontconfig bzip2 && \
-    apt-get clean && cd / && \
-    wget -qO- {phantomjs_url} | tar xvj && \
-    cp /phantomjs-*/bin/phantomjs /usr/bin/phantomjs && \
-    rm -rf /phantomjs* /var/lib/apt/lists/* && \
-    echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen && /usr/sbin/locale-gen && \
-    install -g root -o root -m 755 /tmp/entrypoint.sh /app/bin/entrypoint.sh
+RUN /usr/local/bin/virtualenv /app
+RUN set -ex \
+    && apk add --no-cache \
+        fontconfig \
+        libxml2 \
+        libxml2-dev \
+        libxslt \
+        libxslt-dev \
+        tini \
+    && apk add --no-cache --virtual .build-deps \
+        gcc \
+        libffi-dev \
+        linux-headers \
+        make \
+        musl-dev \
+        openssl-dev \
+    && /app/bin/pip install {install} \
+    && /app/bin/pip install gunicorn==19.7.1 \
+    && apk del .build-deps
+
+# install phantomjs per https://github.com/fgrehm/docker-phantomjs2
+RUN set -ex \
+    && apk add --no-cache --virtual .fetch-deps \
+        curl \
+    && mkdir -p /usr/share \
+    && cd /usr/share \
+    && curl -Ls https://github.com/fgrehm/docker-phantomjs2/releases/download/\
+v2.0.0-20150722/dockerized-phantomjs.tar.gz | tar xz -C / \
+    && ln -s /usr/local/bin/phantomjs /usr/bin/phantomjs \
+    && apk del .fetch-deps \
+    && phantomjs --version
+
+RUN install -g root -o root -m 755 /tmp/entrypoint.sh /app/bin/entrypoint.sh
 
 # default to using settings_example.py, and user can override as needed
 ENV SETTINGS_MODULE=biweeklybudget.settings_example
@@ -123,18 +135,15 @@
 LABEL homepage "http://github.com/jantman/biweeklybudget"
 
 EXPOSE 80
-ENTRYPOINT ["/usr/bin/tini", "--"]
+ENTRYPOINT ["/sbin/tini", "--"]
 CMD ["/app/bin/entrypoint.sh"]
 """
 
-PHANTOMJS_URL = 'https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-' \
-    '2.1.1-linux-x86_64.tar.bz2'
-
 
 class DockerImageBuilder(object):
 
     image_name = 'jantman/biweeklybudget'
-    _phantomjs_version = '2.1.1'
+    _phantomjs_version = '2.0.0'
 
     def __init__(self, toxinidir, distdir):
         """
@@ -392,7 +401,7 @@ def _test_phantomjs(self, container):
         :type container: ``docker.models.containers.Container``
         """
         cmd = [
-            '/bin/bash',
+            '/bin/sh',
             '-c',
             '/usr/bin/phantomjs --version'
         ]
@@ -417,7 +426,7 @@ def _test_phantomjs(self, container):
         phantom_script = self._string_to_tarfile('phantomtest.js', phantomtest)
         container.put_archive('/', phantom_script)
         cmd = [
-            '/bin/bash',
+            '/bin/sh',
             '-c',
             '/usr/bin/phantomjs /phantomtest.js; '
             'echo "exitcode=$?"'
@@ -544,11 +553,16 @@ def _build_image(self, tag):
         logger.info('Running docker build with args: %s', kwargs)
         res = self._docker.api.build(**kwargs)
         logger.info('Build running; output:')
+        error = None
         for line in res:
+            if 'errorDetail' in line:
+                error = line['errorDetail']
             try:
                 print(line['stream'])
             except Exception:
                 print("\t%s" % line)
+        if error is not None:
+            raise RuntimeError(str(error))
         logger.info('Build complete for image: %s', tag)
         return tag
 
@@ -689,8 +703,7 @@ def _dockerfile(self):
             s_install = 'biweeklybudget==%s' % self.build_ver
         s = DOCKERFILE_TEMPLATE.format(
             copy=s_copy,
-            install=s_install,
-            phantomjs_url=PHANTOMJS_URL
+            install=s_install
         )
         logger.debug("Dockerfile:\n%s", s)
         return s
@@ -702,7 +715,7 @@ def _entrypoint(self):
         :return: entrypoint script contents
         :rtype: str
         """
-        s = "#!/bin/bash -ex\n"
+        s = "#!/bin/sh -ex\n"
         s += "export PYTHONUNBUFFERED=true\n"
         s += "/app/bin/python /app/bin/initdb -vv \n"
         s += "/app/bin/gunicorn -w 4 -b :80 --log-file=- --access-logfile=- " \

docs/source/development.rst

@@ -85,6 +85,44 @@ If you want to run the acceptance tests without dumping and refreshing the test
 environment variable will prevent refreshing the DB after classes that manipulate data;
 this will cause subsequent tests to fail but can be useful for debugging.
 
+Running Acceptance Tests Against Docker
++++++++++++++++++++++++++++++++++++++++
+
+The acceptance tests have a "hidden" hook to run against an already-running Flask application,
+such as testing the Docker containers. **Be warned** that the acceptance tests modify data,
+so they should never be run against a real database. This hook is controlled via the
+``BIWEEKLYBUDGET_TEST_BASE_URL`` environment variable. If this variable is set, the acceptance
+tests will not start a Flask server, but will instead use the specified URL. The URL must not
+end with a trailing slash.
+
+An example of testing a recently-built Docker container is below:
+
+```bash
+tox -e docker
+# lots of output; ending with something like: Image "<DOCKER_TAG>" built and tested.
+# get a timestamp to use in the container names
+TS=$(date +%s)
+# run mysql container
+docker run -d --name biweeklybudget-mariadb-${TS} -e MYSQL_ROOT_PASSWORD=root -p 3306 mariadb:5.5.56
+# wait a few seconds for the DB to start up. If this next command fails, wait a bit and try again
+docker exec -it biweeklybudget-mariadb-${TS} /usr/bin/mysql -uroot -proot -e "CREATE DATABASE budgetfoo;"
+# Find the port number (<DockerPort>) that the MySQL container is bound to on the host
+export DB_CONNSTRING='mysql+pymysql://root:[email protected]:<DockerPort>/budgetfoo?charset=utf8mb4'
+# run docker container
+docker run -d --name biweeklybudget-test-${TS} -e DB_CONNSTRING='mysql+pymysql://root:root@mysql:3306/budgetfoo?charset=utf8mb4' --link biweeklybudget-mariadb-${TS}:mysql -p 80 jantman/biweeklybudget:<DOCKER_TAG>
+docker ps
+# in the docker ps output, find the host port (<DockerPort>) that is bound to port 80 on the biweeklybudget container
+export BIWEEKLYBUDGET_TEST_BASE_URL=http://127.0.0.1:<DockerPort>
+# if you want, run `docker logs -f biweeklybudget-test-${TS}` in another shell
+tox -e acceptance36
+# when the tests finish:
+docker stop biweeklybudget-test-${TS} biweeklybudget-mariadb-${TS}
+docker rm biweeklybudget-test-${TS} biweeklybudget-mariadb-${TS}
+```
+
+To ensure that the tests are running against the appropriate container, you may want
+to watch the Docker container's logs during the test run.
+
 .. _development.alembic:
 
 Alembic DB Migrations
@@ -150,7 +188,8 @@ Release Checklist
 2. Verify whether or not DB migrations are needed. If they are, ensure they've been created, tested and verified.
 3. Confirm that there are CHANGES.rst entries for all major changes.
 4. Rebuild documentation and javascript documentation locally: ``tox -e jsdoc,docs``. Commit any changes.
-5. Run the Docker image build and tests locally: ``tox -e docker``.
+5. Run the Docker image build and tests locally: ``tox -e docker``. If the pull request includes changes to the Dockerfile
+   or the container build process, run acceptance tests against the newly-built container as described above.
 6. Ensure that Travis tests passing in all environments.
 7. Ensure that test coverage is no less than the last release, and that there are acceptance tests for any non-trivial changes.
 8. If there have been any major visual or functional changes to the UI, regenerate screenshots via ``tox -e screenshots``.