test: add Tokenserver integration tests to CI (#1180)

Closes #1174

.circleci/config.yml

@@ -57,7 +57,7 @@ commands:
     steps:
       - run:
           name: cargo build
-          command: cargo build
+          command: cargo build --features tokenserver_test_mode
   setup-gcp-grpc:
     steps:
       - run:

Cargo.toml

@@ -89,6 +89,7 @@ jsonwebtoken = "7.2.0"
 
 [features]
 no_auth = []
+tokenserver_test_mode = []
 
 [[bin]]
 name = "purge_ttl"

Dockerfile

@@ -4,24 +4,33 @@ ADD . /app
 ENV PATH=$PATH:/root/.cargo/bin
 # temp removed --no-install-recommends due to CI docker build issue
 RUN apt-get -q update && \
-    apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools && \
+    apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \
     pip3 install -r requirements.txt && \
     rm -rf /var/lib/apt/lists/*
 
 RUN \
     cargo --version && \
     rustc --version && \
-    cargo install --path . --locked --root /app && \
-    cargo install --path . --bin purge_ttl --locked --root /app
+    cargo install --features tokenserver_test_mode --path . --locked --root /app && \
+    cargo install --features tokenserver_test_mode --path . --bin purge_ttl --locked --root /app
 
 FROM debian:buster-slim
 WORKDIR /app
 COPY --from=builder /app/requirements.txt /app
+# Due to a build error that occurs with the Python cryptography package, we
+# have to set this env var to prevent the cryptography package from building
+# with Rust. See this link for more information:
+# https://pythonshowcase.com/question/problem-installing-cryptography-on-raspberry-pi
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
 RUN \
     groupadd --gid 10001 app && \
     useradd --uid 10001 --gid 10001 --home /app --create-home app && \
     apt-get -q update && \
-    apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-dev python3-pip python3-setuptools curl jq && \
+    apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev libffi-dev ca-certificates libcurl4 python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \
+    # The python3-cryptography debian package installs version 2.6.1, but we
+    # we want to use the version specified in requirements.txt. To do this,
+    # we have to remove the python3-cryptography package here.
+    apt-get -q remove -y python3-cryptography && \
     pip3 install -r /app/requirements.txt && \
     rm -rf /var/lib/apt/lists/*
 

Makefile

@@ -15,7 +15,7 @@ PATH_TO_GRPC_CERT = ../server-syncstorage/local/lib/python2.7/site-packages/grpc
 
 clippy:
 	# Matches what's run in circleci
-	cargo clippy --all --all-targets -- -D warnings
+	cargo clippy --all --all-targets --all-features -- -D warnings
 
 docker_start_mysql:
 	docker-compose -f docker-compose.mysql.yaml up -d
@@ -36,10 +36,10 @@ docker_stop_spanner:
 	docker-compose -f docker-compose.spanner.yaml down
 
 run:
-	RUST_LOG=debug RUST_BACKTRACE=full cargo run -- --config config/local.toml
+	RUST_LOG=debug RUST_BACKTRACE=full cargo run -- --config config/local.toml --features tokenserver_test_mode
 
 run_spanner:
 	GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) make run
 
 test:
-	SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) SYNC_TOKENSERVER__DATABASE_URL=$(SYNC_TOKENSERVER__DATABASE_URL) RUST_TEST_THREADS=1 cargo test
+	SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) SYNC_TOKENSERVER__DATABASE_URL=$(SYNC_TOKENSERVER__DATABASE_URL) RUST_TEST_THREADS=1 cargo test --features tokenserver_test_mode

docker-compose.e2e.mysql.yaml

@@ -25,7 +25,12 @@ services:
           SYNC_HOST: 0.0.0.0
           SYNC_MASTER_SECRET: secret0
           SYNC_DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage
-          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3307/tokenserver
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
+          SYNC_TOKENSERVER__ENABLED: "true"
+          SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: api-accounts.stage.mozaws.net
+          SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET: secret0
+          SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: https://oauth.stage.mozaws.net
+          TOKENSERVER_HOST: http://localhost:8000
         entrypoint: >
           /bin/sh -c "
             sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'

docker-compose.e2e.spanner.yaml

@@ -25,8 +25,13 @@ services:
           SYNC_HOST: 0.0.0.0
           SYNC_MASTER_SECRET: secret0
           SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
-          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
           SYNC_SPANNER_EMULATOR_HOST: sync-db:9010
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
+          SYNC_TOKENSERVER__ENABLED: "true"
+          SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: api-accounts.stage.mozaws.net
+          SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET: secret0
+          SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: https://oauth.stage.mozaws.net
+          TOKENSERVER_HOST: http://localhost:8000
         entrypoint: >
           /bin/sh -c "
             sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'

docker-compose.mysql.yaml

@@ -20,7 +20,7 @@ services:
             - tokenserver_db_data:/var/lib/mysql
         restart: always
         ports:
-            - "3307"
+            - "3306"
         environment:
             #MYSQL_RANDOM_ROOT_PASSWORD: yes
             MYSQL_ROOT_PASSWORD: random
@@ -40,7 +40,7 @@ services:
           SYNC_HOST: 0.0.0.0
           SYNC_MASTER_SECRET: secret0
           SYNC_DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage
-          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3307/tokenserver
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
 
 volumes:
     sync_db_data:

requirements.txt

@@ -1,2 +1,8 @@
+# We pin cryptography to 3.4.8 due to a build error that occurs with the
+# current version (36.0.0 at the time of this commit). In short, building
+# the cryptography package with Rust results in a segfault, so we need to
+# set the CRYPTOGRAPHY_DONT_BUILD_RUST env var to force the package to build
+# with C instead. This env var is only present in cryptography<3.5.
+cryptography==3.4.8
 pyfxa==0.7.7
 tokenlib==2.0.0

src/server/mod.rs

@@ -128,12 +128,11 @@ macro_rules! build_app {
                     .route(web::get().to(handlers::get_bso))
                     .route(web::put().to(handlers::put_bso)),
             )
-            // XXX: This route will be enabled when we are ready to roll out Tokenserver
             // Tokenserver
-            // .service(
-            //     web::resource("/1.0/{application}/{version}")
-            //         .route(web::get().to(tokenserver::handlers::get_tokenserver_result)),
-            // )
+            .service(
+                web::resource("/1.0/{application}/{version}")
+                    .route(web::get().to(tokenserver::handlers::get_tokenserver_result)),
+            )
             // Dockerflow
             // Remember to update .::web::middleware::DOCKER_FLOW_ENDPOINTS
             // when applying changes to endpoint names.

src/tokenserver/mod.rs

@@ -26,10 +26,10 @@ pub struct ServerState {
 impl ServerState {
     pub fn from_settings(settings: &Settings) -> Result<Self, ApiError> {
         let oauth_verifier: Box<dyn VerifyToken> = if settings.test_mode_enabled {
-            #[cfg(debug_assertions)]
+            #[cfg(feature = "tokenserver_test_mode")]
             let oauth_verifier = Box::new(TestModeOAuthVerifier);
 
-            #[cfg(not(debug_assertions))]
+            #[cfg(not(feature = "tokenserver_test_mode"))]
             let oauth_verifier = Box::new(OAuthVerifier {
                 fxa_oauth_server_url: settings.fxa_oauth_server_url.clone(),
             });

tools/integration_tests/requirements.txt

@@ -1,3 +1,4 @@
+cryptography==3.4.8
 hawkauthlib
 konfig
 mysqlclient

tools/integration_tests/run.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import atexit
 import os.path
 import psutil
 import signal
@@ -9,6 +8,7 @@
 from test_storage import TestStorage
 from test_support import run_live_functional_tests
 import time
+from tokenserver.run import run_end_to_end_tests, run_local_tests
 
 DEBUG_BUILD = "target/debug/syncstorage"
 RELEASE_BUILD = "/app/bin/syncstorage"
@@ -50,9 +50,20 @@ def start_server():
     os.environ.setdefault("SYNC_MASTER_SECRET", "secret0")
     os.environ.setdefault("SYNC_CORS_MAX_AGE", "555")
     os.environ.setdefault("SYNC_CORS_ALLOWED_ORIGIN", "localhost")
+    os.environ['SYNC_TOKENSERVER__TEST_MODE_ENABLED'] = 'true'
+    the_server_subprocess = start_server()
+    try:
+        res = 0
+        res |= run_live_functional_tests(TestStorage, sys.argv)
+        res |= run_local_tests()
+    finally:
+        terminate_process(the_server_subprocess)
 
+    os.environ['SYNC_TOKENSERVER__TEST_MODE_ENABLED'] = 'false'
     the_server_subprocess = start_server()
-    atexit.register(lambda: terminate_process(the_server_subprocess))
-    res = run_live_functional_tests(TestStorage, sys.argv)
+    try:
+        res |= run_end_to_end_tests()
+    finally:
+        terminate_process(the_server_subprocess)
 
     sys.exit(res)

tools/integration_tests/tokenserver/test_e2e.py

@@ -124,7 +124,7 @@ def _fxa_metrics_hash(self, value):
         hasher.update(value.encode('utf-8'))
         return hasher.hexdigest()
 
-    def _derive_secret(self, master_secret, node):
+    def _derive_secret(self, master_secret):
         info = "services.mozilla.com/mozsvc/v1/node_secret/%s" % self.NODE_URL
         hkdf_params = {
             "salt": None,
@@ -205,8 +205,7 @@ def test_valid_request(self):
 
         signing_secret = binascii.b2a_hex(
             self.TOKEN_SIGNING_SECRET.encode("utf-8")).decode()
-        node_specific_secret = self._derive_secret(signing_secret,
-                                                   self.NODE_URL)
+        node_specific_secret = self._derive_secret(signing_secret)
         expected_token = tokenlib.make_token(payload_dict,
                                              secret=node_specific_secret)
         expected_signature = urlsafe_b64decode(expected_token)[-32:]

tools/integration_tests/tokenserver/test_support.py

@@ -15,13 +15,13 @@
 
 class TestCase:
     FXA_EMAIL_DOMAIN = 'api-accounts.stage.mozaws.net'
-    FXA_METRICS_HASH_SECRET = 'secret'
+    FXA_METRICS_HASH_SECRET = 'secret0'
     NODE_ID = 800
     NODE_URL = 'https://example.com'
     SYNC_1_1_SERVICE_ID = 1
     SYNC_1_5_SERVICE_ID = 2
     SYNC_1_5_SERVICE_NAME = 'sync-1.5'
-    TOKEN_SIGNING_SECRET = 'secret'
+    TOKEN_SIGNING_SECRET = 'secret0'
     TOKENSERVER_HOST = os.environ['TOKENSERVER_HOST']
 
     def setUp(self):