Merge pull request freedomofpress#4239 from freedomofpress/pyyy3

Make application code Python 3 compliant
rmol · Apr 16, 2019 · 4410a6c · 4410a6c
2 parents 7729d35 + 44e8d06
commit 4410a6c
Show file tree

Hide file tree

Showing 57 changed files with 493 additions and 363 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -13,7 +13,7 @@ common-steps:
 
   - &restorecache
     restore_cache:
-      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/Dockerfile" }}
+      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
       paths:
         - /caches/layers.tar.gz
 
@@ -30,19 +30,19 @@ common-steps:
       command: |
         set +o pipefail
         docker images
-        fromtag=$(docker images |grep securedrop-test-xenial |head -n1 |awk '{print $2}')
-        cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial:${fromtag:-latest}" ./bin/dev-shell true
+        fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
+        cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" ./bin/dev-shell true
 
   - &saveimagelayers
     run:
       name: Save Docker image layer cache
       command: |
         docker images
-        docker save -o /caches/layers.tar securedrop-test-xenial:latest
+        docker save -o /caches/layers.tar securedrop-test-xenial-py2:latest
 
   - &savecache
     save_cache:
-      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/Dockerfile" }}
+      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python2/Dockerfile" }}
       paths:
         - /caches/layers.tar
 
@@ -66,22 +66,14 @@ jobs:
     steps:
       - checkout
       - *rebaseontarget
-
-      - run:
-          name: Ensure cache dir exists and permissions are good
-          command: |
-            sudo mkdir -p /caches && sudo chown circleci: -R /caches
+      - *createcachedir
 
       - restore_cache:
-          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/trusty/Dockerfile" }}
+          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/trusty/python2/Dockerfile" }}
           paths:
             - /caches/layers.tar.gz
 
-      - run:
-          name: Load image layer cache
-          command: |
-            set +o pipefail
-            docker load -i /caches/layers.tar |true
+      - *loadimagelayers
 
       - run:
           name: Build Docker image
@@ -98,7 +90,7 @@ jobs:
             docker save -o /caches/layers.tar securedrop-test-trusty:latest
 
       - save_cache:
-          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/trusty/Dockerfile" }}
+          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/trusty/python2/Dockerfile" }}
           paths:
             - /caches/layers.tar
 
@@ -145,9 +137,66 @@ jobs:
           name: Run tests
           command: |
             export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
-            docker rm -f securedrop-test-xenial || true
-            fromtag=$(docker images |grep securedrop-test-xenial |head -n1 |awk '{print $2}')
-            cd securedrop && DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial:${fromtag:-latest}" make test
+            docker rm -f securedrop-test-xenial-py2 || true
+            fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
+            cd securedrop && DOCKER_RUN_ARGUMENTS=$(bash <(curl -s https://codecov.io/env)) DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make test
+
+      - store_test_results:
+          path: ~/test-results
+
+      - store_artifacts:
+          path: ~/test-results
+
+  python3-app-tests:
+    machine:
+      enabled: true
+    environment:
+      DOCKER_API_VERSION: 1.23
+      BASE_OS: xenial
+      PYTHON_VERSION: 3
+    parallelism: 3
+    steps:
+      - checkout
+      - *rebaseontarget
+      - *createcachedir
+
+      - restore_cache:
+          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
+          paths:
+            - /caches/layers.tar.gz
+
+      - *loadimagelayers
+
+      - run:
+          name: Build Docker images
+          command: |
+            set +o pipefail
+            docker images
+            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
+            cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true
+
+      - run:
+          name: Save Docker image layer cache
+          command: |
+            docker images
+            docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest
+
+      - save_cache:
+          key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
+          paths:
+            - /caches/layers.tar
+
+      - run:
+          name: Make test results directory
+          command: mkdir -p ~/test-results
+
+      - run:
+          name: Run tests
+          command: |
+            export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
+            docker rm -f securedrop-test-xenial-py3 || true
+            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
+            cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test
 
       - store_test_results:
           path: ~/test-results
@@ -180,9 +229,9 @@ jobs:
           name: Run tests
           command: |
             export TESTFILES=$(cd securedrop; circleci tests glob 'tests/pageslayout/test*py' |circleci tests split --split-by=timings |xargs echo)
-            docker rm -f securedrop-test-xenial || true
-            fromtag=$(docker images |grep securedrop-test-xenial |head -n1 |awk '{print $2}')
-            cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial:${fromtag:-latest}" make translation-test
+            docker rm -f securedrop-test-xenial-py2 || true
+            fromtag=$(docker images |grep securedrop-test-xenial-py2 |head -n1 |awk '{print $2}')
+            cd securedrop && DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py2:${fromtag:-latest}" make translation-test
 
       - store_test_results:
           path: ~/test-results
@@ -284,6 +333,12 @@ workflows:
               ignore:
                 - /docs-.*/
                 - /i18n-.*/
+      - python3-app-tests:
+          filters:
+            branches:
+              ignore:
+                - /docs-.*/
+                - /i18n-.*/
       - admin-tests:
           filters:
             branches:

diff --git a/docs/development/setup_development.rst b/docs/development/setup_development.rst
@@ -106,6 +106,16 @@ To get started, you can try the following:
    bin/dev-shell bin/run-test tests/functional # functional tests only
    bin/dev-shell bash                          # shell inside the container
 
+To specify the version of Python you want to use, set the ``PYTHON_VERSION``
+environmental variable, like so:
+
+.. code:: sh
+
+    PYTHON_VERSION=3 make test    # Run tests on Python 3
+    PYTHON_VERSION=2 make test    # Run tests on Python 2
+    PYTHON_VERSION=3 make dev     # Run dev container on Python 3
+    PYTHON_VERSION=2 make dev     # Run dev container on Python 2
+
 .. tip:: The interactive shell in the container does not run
          ``redis``, ``Xvfb`` etc.  However you can import shell helper
          functions with ``source bin/dev-deps`` and call ``run_xvfb``,

diff --git a/molecule/testinfra/staging/app/test_apparmor.py b/molecule/testinfra/staging/app/test_apparmor.py
@@ -105,8 +105,8 @@ def test_apparmor_total_profiles(host):
     """ Ensure number of total profiles is sum of enforced and
         complaining profiles """
     with host.sudo():
-        total_expected = str((len(sdvars.apparmor_enforce)
-                             + len(sdvars.apparmor_complain)))
+        total_expected = str(len(sdvars.apparmor_enforce)
+                             + len(sdvars.apparmor_complain))
         # Trusty has ~10, Xenial about ~20 profiles, so let's expect
         # *at least* the sum.
         assert host.check_output("aa-status --profiled") >= total_expected

diff --git a/securedrop/Makefile b/securedrop/Makefile
@@ -24,6 +24,10 @@ test: ## Run the test suite in a Ubuntu 16.04 (Xenial) dockerized environment
 test-trusty: ## Run the test suite in a Ubuntu 14.04 (Trusty) dockerized environment (to be removed April 30, 2019)
 	BASE_OS=trusty ./bin/dev-shell ./bin/run-test -v $${TESTFILES:-tests}
 
+.PHONY: test-python3
+test-python3: ## Run the Python 3 test suite in a Ubuntu 16.04 (Xenial) dockerized environment
+	PYTHON_VERSION=3 BASE_OS=xenial ./bin/dev-shell ./bin/run-test -v $${TESTFILES:-tests}
+
 .PHONY: translation-test
 translation-test: ## Run all pages-layout tests in all supported languages
 	./bin/dev-shell ./bin/translation-test $${TESTFILES:-tests/pageslayout}
@@ -43,7 +47,7 @@ test-config: ## Generate the test config
 	python -c 'import os; from jinja2 import Environment, FileSystemLoader; \
          env = Environment(loader=FileSystemLoader(".")); \
          ctx = {"securedrop_app_gpg_fingerprint": "65A1B5FF195B56353CC63DFFCC40EF1228271441"}; \
-         ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.iteritems())); \
+         ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.items())); \
          ctx = open("config.py", "w").write(env.get_template("config.py.example").render(ctx))'
 	@echo >> config.py
 	@echo "SUPPORTED_LOCALES = ['ar', 'de_DE', 'es_ES', 'en_US', 'el', 'fr_FR', 'it_IT', 'nb_NO', 'nl', 'pt_BR', 'tr', 'zh_Hant']"  >> config.py

diff --git a/securedrop/bin/dev-shell b/securedrop/bin/dev-shell
@@ -7,13 +7,16 @@
 set -eu
 
 TOPLEVEL=$(git rev-parse --show-toplevel)
-source "${BASH_SOURCE%/*}/../../devops/scripts/ticker"
 
 if ! test -n "${BASE_OS:-}" ; then
     # If no base OS was specified, then we use Xenial
     BASE_OS=xenial
 fi
 
+if ! test -n "${PYTHON_VERSION:-}" ; then
+    PYTHON_VERSION=2
+fi
+
 function exit_if_not_supported_base_image() {
     # Currently we only support Xenial or Trusty.
     if [[ "$1" != "xenial" && "$1" != "trusty" ]]
@@ -23,37 +26,51 @@ function exit_if_not_supported_base_image() {
     fi
 }
 
+function validate_python_version() {
+    # Trusty will be EOL April 2019. Orgs must be on Xenial to upgrade to Python 3.
+    if [[ "$1" != "xenial" && "$2" != "2" ]]
+    then
+        echo "For Ubuntu Trusty, PYTHON_VERSION must be 2"
+        exit 1
+    fi
+
+    if [[ "$2" != "2" && "$2" != "3" ]]
+    then
+        echo "PYTHON_VERSION must be 2 or 3"
+        exit 1
+    fi
+}
+
 function docker_image() {
     exit_if_not_supported_base_image $1
+    validate_python_version $1 $2
 
     docker build \
            ${DOCKER_BUILD_ARGUMENTS:-} \
-       --build-arg=USER_ID="$(id -u)" \
-       --build-arg=USER_NAME="${USER:-root}" \
-           -t "securedrop-test-${1}" \
-       --file "${TOPLEVEL}/securedrop/dockerfiles/${1}/Dockerfile" \
+	   --build-arg=USER_ID="$(id -u)" \
+	   --build-arg=USER_NAME="${USER:-root}" \
+           -t "securedrop-test-${1}-py${2}" \
+       --file "${TOPLEVEL}/securedrop/dockerfiles/${1}/python${2}/Dockerfile" \
        "${TOPLEVEL}/securedrop"
 }
 
 function docker_run() {
     exit_if_not_supported_base_image $1
+    validate_python_version $1 $2
 
     find . \( -name '*.pyc' -o -name __pycache__ \) -delete
     docker run \
        -p 127.0.0.1:5901:5901 \
-       --rm \
+	   --rm \
+	   --user "${USER:-root}" \
+	   --volume "${TOPLEVEL}:${TOPLEVEL}" \
+	   --workdir "${TOPLEVEL}/securedrop" \
        -e NUM_SOURCES \
-       --user "${USER:-root}" \
-       --volume "${TOPLEVEL}:${TOPLEVEL}" \
-       --workdir "${TOPLEVEL}/securedrop" \
+       -e LC_ALL=C.UTF-8 \
+       -e LANG=C.UTF-8 \
        --name securedrop-dev \
-       -ti ${DOCKER_RUN_ARGUMENTS:-} "securedrop-test-${1}" "${@:2}"
+	   -ti ${DOCKER_RUN_ARGUMENTS:-} "securedrop-test-${1}-py${2}" "${@:3}"
 }
 
-if test -n "${CIRCLE_SHA1:-}" ; then
-   docker_image $BASE_OS
-else
-   ticker docker_image $BASE_OS
-fi
-
-docker_run $BASE_OS "$@"
+docker_image $BASE_OS $PYTHON_VERSION
+docker_run $BASE_OS $PYTHON_VERSION "$@"
diff --git a/securedrop/bin/run-test b/securedrop/bin/run-test
@@ -27,7 +27,7 @@ mkdir -p "/tmp/test-results/logs"
 : "${PAGE_LAYOUT_LOCALES:=en_US,ar,fr_FR}"
 export PAGE_LAYOUT_LOCALES
 
-pytest \
+py.test \
   --page-layout \
   --durations 10 \
   --junitxml=/tmp/test-results/junit.xml \

diff --git a/securedrop/create-dev-data.py b/securedrop/create-dev-data.py
@@ -102,10 +102,9 @@ def create_source_and_submissions(num_submissions=2, num_replies=2):
 
     db.session.commit()
 
-    print(("Test source (codename: '{}', journalist designation '{}') "
-           "added with {} submissions and {} replies")
-          .format(codename, journalist_designation, num_submissions,
-                  num_replies))
+    print("Test source (codename: '{}', journalist designation '{}') "
+          "added with {} submissions and {} replies").format(
+              codename, journalist_designation, num_submissions, num_replies)
 
 
 if __name__ == "__main__":  # pragma: no cover

diff --git a/securedrop/crypto_util.py b/securedrop/crypto_util.py
@@ -5,6 +5,7 @@
 import pretty_bad_protocol as gnupg
 import os
 import io
+import six
 import scrypt
 import subprocess
 from random import SystemRandom
@@ -162,7 +163,7 @@ def hash_codename(self, codename, salt=None):
             salt = self.scrypt_id_pepper
         return b32encode(scrypt.hash(clean(codename),
                          salt,
-                         **self.scrypt_params))
+                         **self.scrypt_params)).decode('utf-8')
 
     def genkeypair(self, name, secret):
         """Generate a GPG key through batch file key generation. A source's
@@ -259,7 +260,11 @@ def decrypt(self, secret, ciphertext):
         """
         hashed_codename = self.hash_codename(secret,
                                              salt=self.scrypt_gpg_pepper)
-        return self.gpg.decrypt(ciphertext, passphrase=hashed_codename).data
+        data = self.gpg.decrypt(ciphertext, passphrase=hashed_codename).data
+
+        if not six.PY2:  # Python3
+            return data.decode('utf-8')
+        return data
 
 
 def clean(s, also=''):

diff --git a/securedrop/dockerfiles/trusty/Dockerfile → ...rop/dockerfiles/trusty/python2/Dockerfile b/securedrop/dockerfiles/trusty/Dockerfile → ...rop/dockerfiles/trusty/python2/Dockerfile
diff --git a/securedrop/dockerfiles/xenial/Dockerfile → ...rop/dockerfiles/xenial/python2/Dockerfile b/securedrop/dockerfiles/xenial/Dockerfile → ...rop/dockerfiles/xenial/python2/Dockerfile
@@ -14,8 +14,7 @@ RUN apt-get update && \
 RUN apt-get install -y devscripts vim \
                        python-pip libpython2.7-dev libssl-dev secure-delete \
                        gnupg2 ruby redis-server firefox git xvfb haveged curl \
-                       gettext paxctl x11vnc enchant libffi-dev sqlite3 gettext sudo \
-                       libgtk2.0
+                       gettext paxctl x11vnc enchant libffi-dev sqlite3 gettext sudo
 
 ENV FIREFOX_CHECKSUM=88d25053306d33658580973b063cd459a56e3596a3a298c1fb8ab1d52171d860
 RUN curl -LO https://launchpad.net/~ubuntu-mozilla-security/+archive/ubuntu/ppa/+build/9727836/+files/firefox_46.0.1+build1-0ubuntu0.14.04.3_amd64.deb && \
@@ -27,7 +26,6 @@ RUN curl -LO https://launchpad.net/~ubuntu-mozilla-security/+archive/ubuntu/ppa/
 RUN gem install sass -v 3.4.23
 
 COPY requirements requirements
-
 RUN pip install -r requirements/securedrop-app-code-requirements.txt && \
     pip install -r requirements/test-requirements.txt