Py3 integration tests (#917)

* feat: Port integration tests to py3, and run them in the same docker container as the rust server.

Fix #911

.circleci/config.yml

@@ -29,7 +29,7 @@ commands:
           name: Setup python
           command: |
             sudo apt-get update && sudo apt-get install -y python3-dev python3-pip
-            pip3 install tokenlib
+            pip3 install hawkauthlib konfig pyramid pyramid_hawkauth requests simplejson tokenlib unittest2 WebTest WSGIProxy2
   rust-check:
     steps:
       - run:
@@ -88,7 +88,7 @@ commands:
                -f docker-compose.yaml
                -f docker-compose.e2e.yaml
                up
-               --exit-code-from server-syncstorage
+               --exit-code-from e2e-tests
                --abort-on-container-exit
           environment:
               SYNCSTORAGE_RS_IMAGE: app:build

Dockerfile

@@ -6,9 +6,7 @@ ENV PATH=$PATH:/root/.cargo/bin
 RUN apt-get -q update && \
     apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go python3-dev python3-pip && \
     pip3 install tokenlib && \
-    rm -rf /var/lib/apt/lists/* && \
-    cd /app && \
-    mkdir -m 755 bin
+    rm -rf /var/lib/apt/lists/*
 
 RUN \
     cargo --version && \
@@ -30,6 +28,7 @@ COPY --from=builder /app/bin /app/bin
 COPY --from=builder /app/version.json /app
 COPY --from=builder /app/spanner_config.ini /app
 COPY --from=builder /app/tools/spanner /app/tools/spanner
+COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
 
 USER app:app
 

docker-compose.e2e.yaml

@@ -2,6 +2,8 @@ version: '3'
 services:
     db:
     syncstorage-rs:
+        depends_on:
+          - db
         # TODO: either syncstorage-rs should retry the db connection
         # itself a few times or should include a wait-for-it.sh script
         # inside its docker that would do this for us. Same (probably
@@ -11,12 +13,21 @@ services:
             sleep 15;
             /app/bin/syncstorage;
           "
-    server-syncstorage:
-        image: mozilla/server-syncstorage:latest
+    e2e-tests:
         depends_on:
           - syncstorage-rs
+        image: app:build
+        privileged: true
+        user: root
+        environment:
+          SYNC_HOST: 0.0.0.0
+          SYNC_MASTER_SECRET: secret0
+          SYNC_DATABASE_URL: mysql://test:test@db:3306/syncstorage
+          SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver
+          SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ
+          SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB
+          SYNC_FXA_METRICS_HASH_SECRET: insecure
         entrypoint: >
-          /bin/ash -c "
-            sleep 18;
-            /app/docker-entrypoint.sh test_endpoint http://syncstorage-rs:8000#secret0;
+          /bin/sh -c "
+            sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
           "

tests.ini

@@ -0,0 +1,23 @@
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+[app:main]
+use = egg:SyncStorage
+
+[storage]
+backend = syncstorage.storage.sql.SQLStorage
+sqluri = ${MOZSVC_SQLURI}
+standard_collections = true
+quota_size = 5242880
+pool_size = 100
+pool_recycle = 3600
+reset_on_return = true
+create_tables = true
+max_post_records = 4000
+batch_upload_enabled = true
+force_consistent_sort_order = true
+
+[hawkauth]
+secret = "TED KOPPEL IS A ROBOT"

tools/integration_tests/requirements.txt

@@ -0,0 +1,10 @@
+hawkauthlib
+konfig
+pyramid
+pyramid_hawkauth
+requests
+simplejson
+tokenlib
+unittest2
+webtest
+wsgiproxy2

tools/integration_tests/run.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import atexit
+import os.path
+import subprocess
+import sys
+from test_storage import TestStorage
+from test_support import run_live_functional_tests
+import time
+
+
+DEBUG_BUILD = 'target/debug/syncstorage'
+RELEASE_BUILD = '/app/bin/syncstorage'
+
+if __name__ == "__main__":
+    # When run as a script, this file will execute the
+    # functional tests against a live webserver.
+    target_binary = None
+    if os.path.exists(DEBUG_BUILD):
+        target_binary = DEBUG_BUILD
+    elif os.path.exists(RELEASE_BUILD):
+        target_binary = RELEASE_BUILD
+    else:
+        raise RuntimeError("Neither target/debug/syncstorage nor /app/bin/syncstorage were found.")
+    the_server_subprocess = subprocess.Popen('SYNC_MASTER_SECRET=secret0 ' + target_binary, shell=True)
+    ## TODO we should change this to watch for a log message on startup to know when to continue instead of sleeping for a fixed amount
+    time.sleep(20)
+
+    def stop_subprocess():
+        the_server_subprocess.terminate()
+        the_server_subprocess.wait()
+
+    atexit.register(stop_subprocess)
+
+    res = run_live_functional_tests(TestStorage, sys.argv)
+    sys.exit(res)