test: Add Tokenserver integration tests

src/error.rs

@@ -187,13 +187,18 @@ impl ApiError {
     }
 
     pub fn render_404<B>(res: ServiceResponse<B>) -> Result<ErrorHandlerResponse<B>> {
-        // Replace the outbound error message with our own.
-        let resp =
-            HttpResponseBuilder::new(StatusCode::NOT_FOUND).json(WeaveError::UnknownError as u32);
-        Ok(ErrorHandlerResponse::Response(ServiceResponse::new(
-            res.request().clone(),
-            resp.into_body(),
-        )))
+        if res.request().path().starts_with("/1.0/") {
+            // Do not use a custom response for Tokenserver requests.
+            Ok(ErrorHandlerResponse::Response(res))
+        } else {
+            // Replace the outbound error message with our own for Sync requests.
+            let resp = HttpResponseBuilder::new(StatusCode::NOT_FOUND)
+                .json(WeaveError::UnknownError as u32);
+            Ok(ErrorHandlerResponse::Response(ServiceResponse::new(
+                res.request().clone(),
+                resp.into_body(),
+            )))
+        }
     }
 }
 

src/server/mod.rs

@@ -141,7 +141,7 @@ macro_rules! build_app {
             // XXX: This route will be enabled when we are ready to roll out Tokenserver
             // Tokenserver
             // .service(
-            //     web::resource("/1.0/{application}/{version}".to_string())
+            //     web::resource("/1.0/{application}/{version}")
             //         .route(web::get().to(tokenserver::handlers::get_tokenserver_result)),
             // )
             // Dockerflow

src/tokenserver/error.rs

@@ -29,23 +29,23 @@ impl Default for TokenserverError {
 
 impl TokenserverError {
     pub fn invalid_generation() -> Self {
-        TokenserverError {
+        Self {
             status: "invalid-generation",
             location: ErrorLocation::Body,
             ..Self::default()
         }
     }
 
     pub fn invalid_keys_changed_at() -> Self {
-        TokenserverError {
+        Self {
             status: "invalid-keysChangedAt",
             location: ErrorLocation::Body,
             ..Self::default()
         }
     }
 
     pub fn invalid_key_id(description: &'static str) -> Self {
-        TokenserverError {
+        Self {
             status: "invalid-key-id",
             description,
             ..Self::default()
@@ -87,6 +87,14 @@ impl TokenserverError {
             ..Self::default()
         }
     }
+
+    pub fn unauthorized(description: &'static str) -> Self {
+        Self {
+            status: "error",
+            description,
+            ..Self::default()
+        }
+    }
 }
 
 #[derive(Clone, Copy, Debug)]

src/tokenserver/extractors.rs

@@ -179,7 +179,7 @@ impl FromRequest for TokenData {
             // The request must use Bearer auth
             if let Some((auth_type, _)) = authorization_header.split_once(" ") {
                 if auth_type.to_ascii_lowercase() != "bearer" {
-                    return Err(TokenserverError::invalid_credentials("Unsupported").into());
+                    return Err(TokenserverError::unauthorized("Unsupported").into());
                 }
             }
 

src/tokenserver/handlers.rs

@@ -21,9 +21,10 @@ use crate::{
 pub struct TokenserverResult {
     id: String,
     key: String,
-    uid: String,
+    uid: i64,
     api_endpoint: String,
     duration: u64,
+    hashed_fxa_uid: String,
 }
 
 pub async fn get_tokenserver_result(
@@ -110,9 +111,10 @@ pub async fn get_tokenserver_result(
     let result = TokenserverResult {
         id: token,
         key: derived_secret,
-        uid: tokenserver_request.fxa_uid,
+        uid: tokenserver_user.uid,
         api_endpoint,
         duration: tokenserver_request.duration,
+        hashed_fxa_uid: hashed_fxa_uid.to_owned(),
     };
 
     Ok(HttpResponse::build(StatusCode::OK).json(result))

src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `nodes` ADD CONSTRAINT `nodes_ibfk_1` FOREIGN KEY (`service`) REFERENCES `services` (`id`);
+ALTER TABLE `users` ADD CONSTRAINT `users_ibfk_1` FOREIGN KEY (`nodeid`) REFERENCES `nodes` (`id`);

src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `nodes` DROP FOREIGN KEY `nodes_ibfk_1`;
+ALTER TABLE `users` DROP FOREIGN KEY `users_ibfk_1`;

src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/down.sql

@@ -0,0 +1,5 @@
+ALTER TABLE `nodes` ALTER `available` SET DEFAULT 0;
+ALTER TABLE `nodes` ALTER `current_load` SET DEFAULT 0;
+ALTER TABLE `nodes` ALTER `capacity` SET DEFAULT 0;
+ALTER TABLE `nodes` ALTER `downed` SET DEFAULT 0;
+ALTER TABLE `nodes` ALTER `backoff` SET DEFAULT 0;

src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/up.sql

@@ -0,0 +1,5 @@
+ALTER TABLE `nodes` ALTER `available` DROP DEFAULT;
+ALTER TABLE `nodes` ALTER `current_load` DROP DEFAULT;
+ALTER TABLE `nodes` ALTER `capacity` DROP DEFAULT;
+ALTER TABLE `nodes` ALTER `downed` DROP DEFAULT;
+ALTER TABLE `nodes` ALTER `backoff` DROP DEFAULT;

src/tokenserver/migrations/2021-09-30-142746_add_indexes/down.sql

@@ -0,0 +1,4 @@
+ALTER TABLE `nodes` DROP INDEX `unique_idx`;
+ALTER TABLE `users` DROP INDEX `lookup_idx`;
+ALTER TABLE `users` DROP INDEX `replaced_at_idx`;
+ALTER TABLE `users` DROP INDEX `node_idx`;

src/tokenserver/migrations/2021-09-30-142746_add_indexes/up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE `nodes` ADD UNIQUE KEY `unique_idx` (`service`, `node`);
+ALTER TABLE `users` ADD INDEX `lookup_idx` (`email`, `service`, `created_at`);
+ALTER TABLE `users` ADD INDEX `replaced_at_idx` (`service`, `replaced_at`);
+ALTER TABLE `users` ADD INDEX `node_idx` (`nodeid`);

src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql

@@ -0,0 +1 @@
+ALTER TABLE `nodes` ADD KEY `service` (`service`);

src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql

@@ -0,0 +1 @@
+ALTER TABLE `nodes` DROP KEY `service`;

src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql

@@ -0,0 +1 @@
+ALTER TABLE `users` ADD KEY `nodeid` (`nodeid`);

src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql

@@ -0,0 +1 @@
+ALTER TABLE `users` DROP KEY `nodeid`;

src/tokenserver/verify.py

@@ -2,12 +2,14 @@
 from fxa.errors import ClientError, TrustError
 import json
 
+DEFAULT_OAUTH_SCOPE = 'https://identity.mozilla.com/apps/oldsync'
+
 
 def verify_token(token, server_url=None):
     client = Client(server_url=server_url)
 
     try:
-        token_data = client.verify_token(token)
+        token_data = client.verify_token(token, DEFAULT_OAUTH_SCOPE)
 
         # Serialize the data to make it easier to parse in Rust
         return json.dumps(token_data)

tokenserver_rs.sql

@@ -0,0 +1,102 @@
+-- MySQL dump 10.13  Distrib 8.0.26, for Linux (x86_64)
+--
+-- Host: localhost    Database: tokenserver_rs
+-- ------------------------------------------------------
+-- Server version	8.0.26-0ubuntu0.20.04.2
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `__diesel_schema_migrations`
+--
+
+DROP TABLE IF EXISTS `__diesel_schema_migrations`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `__diesel_schema_migrations` (
+  `version` varchar(50) NOT NULL,
+  `run_on` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`version`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `nodes`
+--
+
+DROP TABLE IF EXISTS `nodes`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `nodes` (
+  `id` bigint NOT NULL AUTO_INCREMENT,
+  `service` int NOT NULL,
+  `node` varchar(64) NOT NULL,
+  `available` int NOT NULL,
+  `current_load` int NOT NULL,
+  `capacity` int NOT NULL,
+  `downed` int NOT NULL,
+  `backoff` int NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `unique_idx` (`service`,`node`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `services`
+--
+
+DROP TABLE IF EXISTS `services`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `services` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `service` varchar(30) DEFAULT NULL,
+  `pattern` varchar(128) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `service` (`service`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `users`
+--
+
+DROP TABLE IF EXISTS `users`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `users` (
+  `uid` bigint NOT NULL AUTO_INCREMENT,
+  `service` int NOT NULL,
+  `email` varchar(255) NOT NULL,
+  `generation` bigint NOT NULL,
+  `client_state` varchar(32) NOT NULL,
+  `created_at` bigint NOT NULL,
+  `replaced_at` bigint DEFAULT NULL,
+  `nodeid` bigint NOT NULL,
+  `keys_changed_at` bigint DEFAULT NULL,
+  PRIMARY KEY (`uid`),
+  KEY `lookup_idx` (`email`,`service`,`created_at`),
+  KEY `replaced_at_idx` (`service`,`replaced_at`),
+  KEY `node_idx` (`nodeid`)
+) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb3;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2021-09-30 15:18:17

tools/integration_tests/requirements.txt

@@ -1,7 +1,11 @@
 hawkauthlib
 konfig
+mysql-connector-python
+psutil
+pyjwt
 pyramid
 pyramid_hawkauth
+pyfxa
 requests
 simplejson
 tokenlib

tools/integration_tests/run.py

@@ -2,16 +2,26 @@
 
 import atexit
 import os.path
+import psutil
+import signal
 import subprocess
 import sys
 from test_storage import TestStorage
 from test_support import run_live_functional_tests
 import time
 
-
 DEBUG_BUILD = "target/debug/syncstorage"
 RELEASE_BUILD = "/app/bin/syncstorage"
 
+
+def terminate_process(process):
+    proc = psutil.Process(pid=process.pid)
+    child_proc = proc.children(recursive=True)
+    for p in [proc] + child_proc:
+        os.kill(p.pid, signal.SIGTERM)
+    process.wait()
+
+
 if __name__ == "__main__":
     # When run as a script, this file will execute the
     # functional tests against a live webserver.
@@ -25,18 +35,21 @@
             "Neither target/debug/syncstorage \
                 nor /app/bin/syncstorage were found."
         )
-    the_server_subprocess = subprocess.Popen(
-        "SYNC_MASTER_SECRET=secret0 " + target_binary, shell=True
-    )
-    # TODO we should change this to watch for a log message on startup
-    # to know when to continue instead of sleeping for a fixed amount
-    time.sleep(20)
 
-    def stop_subprocess():
-        the_server_subprocess.terminate()
-        the_server_subprocess.wait()
+    def start_server():
+        the_server_subprocess = subprocess.Popen(target_binary,
+                                                 shell=True,
+                                                 env=os.environ)
+
+        # TODO we should change this to watch for a log message on startup
+        # to know when to continue instead of sleeping for a fixed amount
+        time.sleep(20)
 
-    atexit.register(stop_subprocess)
+        return the_server_subprocess
 
+    os.environ.setdefault('SYNC_MASTER_SECRET', 'secret0')
+    the_server_subprocess = start_server()
+    atexit.register(lambda: terminate_process(the_server_subprocess))
     res = run_live_functional_tests(TestStorage, sys.argv)
+
     sys.exit(res)

tools/integration_tests/tokenserver/run.py

@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+import unittest
+
+from tokenserver.test_authorization import TestAuthorization
+from tokenserver.test_e2e import TestE2e
+from tokenserver.test_misc import TestMisc
+from tokenserver.test_node_assignment import TestNodeAssignment
+
+
+def run_local_tests():
+    return run_tests([TestAuthorization, TestMisc, TestNodeAssignment])
+
+
+def run_end_to_end_tests():
+    return run_tests([TestE2e])
+
+
+def run_tests(test_cases):
+    loader = unittest.TestLoader()
+    success = True
+
+    for test_case in test_cases:
+        suite = loader.loadTestsFromTestCase(test_case)
+        runner = unittest.TextTestRunner()
+        res = runner.run(suite)
+        success = success and res.wasSuccessful()
+
+    if success:
+        return 0
+    else:
+        return 1