refactor: Tokenserver: Add mature MySQL adapter (#1119)

Closes #1054
mozilla-services · Jul 29, 2021 · 503d1aa · 503d1aa
1 parent 7ab3729
commit 503d1aa
Show file tree

Hide file tree

Showing 26 changed files with 742 additions and 126 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -62,6 +62,18 @@ commands:
       - run:
           name: Install grpcio dependencies
           command: sudo apt-get update && sudo apt-get install -y cmake golang-go
+  setup-mysql:
+    steps:
+      - run:
+          name: Install MySQL client
+          command: sudo apt-get update && sudo apt-get install -y default-mysql-client
+  create-tokenserver-database:
+    steps:
+      - run:
+          name: Create Tokenserver database
+          command: |
+            mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;'   
+            mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';"
 
   write-version:
     steps:
@@ -163,6 +175,7 @@ jobs:
           password: $DOCKER_PASS
         environment:
             SYNC_DATABASE_URL: mysql://test:[email protected]/syncstorage
+            SYNC_TOKENSERVER__DATABASE_URL: mysql://test:[email protected]/tokenserver
             RUST_BACKTRACE: 1
             # XXX: begin_test_transaction doesn't play nice over threaded tests
             RUST_TEST_THREADS: 1
@@ -171,7 +184,7 @@ jobs:
           username: $DOCKER_USER
           password: $DOCKER_PASS
         environment:
-            MYSQL_ROOT_PASSWORD: random
+            MYSQL_ROOT_PASSWORD: password
             MYSQL_USER: test
             MYSQL_PASSWORD: test
             MYSQL_DATABASE: syncstorage
@@ -190,6 +203,8 @@ jobs:
       - setup-rust
       - setup-python
       - setup-gcp-grpc
+      - setup-mysql
+      - create-tokenserver-database
       # XXX: currently the time needed to setup-sccache negates its savings
       #- setup-sccache
       #- restore-sccache-cache

diff --git a/Makefile b/Makefile
@@ -3,6 +3,7 @@
 ##
 
 SYNC_DATABASE_URL = 'mysql://sample_user:sample_password@localhost/syncstorage_rs'
+SYNC_TOKENSERVER__DATABASE_URL = 'mysql://sample_user:sample_password@localhost/tokenserver_rs'
 
 # This key can live anywhere on your machine. Adjust path as needed.
 PATH_TO_SYNC_SPANNER_KEYS = `pwd`/service-account.json
@@ -41,4 +42,4 @@ run_spanner:
 	GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) make run
 
 test:
-	SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) RUST_TEST_THREADS=1 cargo test
+	SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) SYNC_TOKENSERVER__DATABASE_URL=$(SYNC_TOKENSERVER__DATABASE_URL) RUST_TEST_THREADS=1 cargo test
diff --git a/docker-compose.e2e.mysql.yaml b/docker-compose.e2e.mysql.yaml
@@ -1,9 +1,11 @@
 version: '3'
 services:
-    db:
+    sync-db:
+    tokenserver-db:
     syncstorage-rs:
         depends_on:
-          - db
+          - sync-db
+          - tokenserver-db
         # TODO: either syncstorage-rs should retry the db connection
         # itself a few times or should include a wait-for-it.sh script
         # inside its docker that would do this for us. Same (probably
@@ -22,11 +24,8 @@ services:
         environment:
           SYNC_HOST: 0.0.0.0
           SYNC_MASTER_SECRET: secret0
-          SYNC_DATABASE_URL: mysql://test:test@db:3306/syncstorage
-          SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver
-          SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ
-          SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB
-          SYNC_FXA_METRICS_HASH_SECRET: insecure
+          SYNC_DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3307/tokenserver
         entrypoint: >
           /bin/sh -c "
             sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'

diff --git a/docker-compose.e2e.spanner.yaml b/docker-compose.e2e.spanner.yaml
@@ -1,10 +1,11 @@
 version: '3'
 services:
-    db:
-    db-setup:
+    sync-db:
+    sync-db-setup:
+    tokenserver-db:
     syncstorage-rs:
         depends_on:
-          - db-setup
+          - sync-db-setup
         # TODO: either syncstorage-rs should retry the db connection
         # itself a few times or should include a wait-for-it.sh script
         # inside its docker that would do this for us. Same (probably
@@ -24,11 +25,8 @@ services:
           SYNC_HOST: 0.0.0.0
           SYNC_MASTER_SECRET: secret0
           SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
-          SYNC_SPANNER_EMULATOR_HOST: db:9010
-          SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver
-          SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ
-          SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB
-          SYNC_FXA_METRICS_HASH_SECRET: insecure
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
+          SYNC_SPANNER_EMULATOR_HOST: sync-db:9010
         entrypoint: >
           /bin/sh -c "
             sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'

diff --git a/docker-compose.mysql.yaml b/docker-compose.mysql.yaml
@@ -1,9 +1,9 @@
 version: '3'
 services:
-    db:
+    sync-db:
         image: mysql:5.7
         volumes:
-            - db_data:/var/lib/mysql
+            - sync_db_data:/var/lib/mysql
         restart: always
         ports:
             - "3306"
@@ -14,24 +14,37 @@ services:
             MYSQL_USER: test
             MYSQL_PASSWORD: test
 
+    tokenserver-db:
+        image: mysql:5.7
+        volumes:
+            - tokenserver_db_data:/var/lib/mysql
+        restart: always
+        ports:
+            - "3307"
+        environment:
+            #MYSQL_RANDOM_ROOT_PASSWORD: yes
+            MYSQL_ROOT_PASSWORD: random
+            MYSQL_DATABASE: tokenserver
+            MYSQL_USER: test
+            MYSQL_PASSWORD: test
+
     syncstorage-rs:
         image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest}
         restart: always
         ports:
           - "8000:8000"
         depends_on:
-          - db
+          - sync-db
+          - tokenserver-db
         environment:
           SYNC_HOST: 0.0.0.0
           SYNC_MASTER_SECRET: secret0
-          SYNC_DATABASE_URL: mysql://test:test@db:3306/syncstorage
-          SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver
-          SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ
-          SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB
-          SYNC_FXA_METRICS_HASH_SECRET: insecure
+          SYNC_DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3307/tokenserver
 
 volumes:
-    db_data:
+    sync_db_data:
+    tokenserver_db_data:
 
 # Application runs off of port 8000.
 # you can test if it's available with

diff --git a/docker-compose.spanner.yaml b/docker-compose.spanner.yaml
@@ -1,39 +1,49 @@
 version: '3'
 services:
-    db:
+    sync-db:
         image: gcr.io/cloud-spanner-emulator/emulator
         ports:
             - "9010:9010"
             - "9020:9020"
         environment:
             PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-    db-setup:
+    sync-db-setup:
         image: app:build
         depends_on:
-          - db
+          - sync-db
         restart: "no"
         entrypoint: "/app/scripts/prepare-spanner.sh"
         environment:
-            SYNC_SPANNER_EMULATOR_HOST: db:9020
+            SYNC_SPANNER_EMULATOR_HOST: sync-db:9020
+    tokenserver-db:
+        image: mysql:5.7
+        volumes:
+            - tokenserver_db_data:/var/lib/mysql
+        restart: always
+        ports:
+            - "3306"
+        environment:
+            #MYSQL_RANDOM_ROOT_PASSWORD: yes
+            MYSQL_ROOT_PASSWORD: random
+            MYSQL_DATABASE: tokenserver
+            MYSQL_USER: test
+            MYSQL_PASSWORD: test
     syncstorage-rs:
         image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest}
         restart: always
         ports:
           - "8000:8000"
         depends_on:
-          - db-setup
+          - sync-db-setup
         environment:
           SYNC_HOST: 0.0.0.0
           SYNC_MASTER_SECRET: secret0
           SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
-          SYNC_SPANNER_EMULATOR_HOST: db:9010
-          SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver
-          SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ
-          SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB
-          SYNC_FXA_METRICS_HASH_SECRET: insecure
+          SYNC_SPANNER_EMULATOR_HOST: sync-db:9010
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
 
 volumes:
-    db_data:
+    tokenserver_db_data:
 
 # Application runs off of port 8000.
 # you can test if it's available with

diff --git a/src/db/mysql/mod.rs b/src/db/mysql/mod.rs
@@ -8,3 +8,5 @@ mod schema;
 mod test;
 
 pub use self::pool::MysqlDbPool;
+#[cfg(test)]
+pub use self::test::TestTransactionCustomizer;
diff --git a/src/db/mysql/models.rs b/src/db/mysql/models.rs
@@ -994,6 +994,7 @@ impl MysqlDb {
     }
 }
 
+#[macro_export]
 macro_rules! sync_db_method {
     ($name:ident, $sync_name:ident, $type:ident) => {
         sync_db_method!($name, $sync_name, $type, results::$type);

diff --git a/src/db/mysql/pool.rs b/src/db/mysql/pool.rs
@@ -35,8 +35,8 @@ embed_migrations!();
 ///
 /// Mysql DDL statements implicitly commit which could disrupt MysqlPool's
 /// begin_test_transaction during tests. So this runs on its own separate conn.
-pub fn run_embedded_migrations(settings: &Settings) -> Result<()> {
-    let conn = MysqlConnection::establish(&settings.database_url)?;
+pub fn run_embedded_migrations(database_url: &str) -> Result<()> {
+    let conn = MysqlConnection::establish(database_url)?;
     #[cfg(test)]
     // XXX: this doesn't show the DDL statements
     // https://github.com/shssoichiro/diesel-logger/issues/1
@@ -63,7 +63,7 @@ impl MysqlDbPool {
     ///
     /// Also initializes the Mysql db, ensuring all migrations are ran.
     pub fn new(settings: &Settings, metrics: &Metrics) -> Result<Self> {
-        run_embedded_migrations(settings)?;
+        run_embedded_migrations(&settings.database_url)?;
         Self::new_without_migrations(settings, metrics)
     }
 

diff --git a/src/server/mod.rs b/src/server/mod.rs
@@ -14,7 +14,7 @@ use crate::db::{pool_from_settings, spawn_pool_periodic_reporter, DbPool};
 use crate::error::ApiError;
 use crate::server::metrics::Metrics;
 use crate::settings::{Deadman, Secrets, ServerLimits, Settings};
-use crate::tokenserver::{self, OAuthVerifier, VerifyToken};
+use crate::tokenserver;
 use crate::web::{handlers, middleware};
 
 pub const BSO_ID_REGEX: &str = r"[ -~]{1,64}";
@@ -41,12 +41,10 @@ pub struct ServerState {
     /// Secrets used during Hawk authentication.
     pub secrets: Arc<Secrets>,
 
-    // TODO: These will eventually be added as settings passed to a more mature
-    // database adapter (which will be added in #1054)
-    pub tokenserver_database_url: Option<String>,
-    pub fxa_metrics_hash_secret: Option<String>, // SYNC_FXA_METRICS_HASH_SECRET
-
-    pub tokenserver_oauth_verifier: Box<dyn VerifyToken>,
+    // XXX: This is only any Option temporarily. Once Tokenserver is rolled out to production,
+    // it will always be enabled, and syncstorage will always have state associated with
+    // Tokenserver.
+    pub tokenserver_state: Option<tokenserver::ServerState>,
 
     /// Metric reporting
     pub metrics: Box<StatsdClient>,
@@ -140,11 +138,12 @@ macro_rules! build_app {
                     .route(web::get().to(handlers::get_bso))
                     .route(web::put().to(handlers::put_bso)),
             )
+            // XXX: This route will be enabled when we are ready to roll out Tokenserver
             // Tokenserver
-            .service(
-                web::resource("/1.0/sync/1.5".to_string())
-                    .route(web::get().to(tokenserver::handlers::get_tokenserver_result)),
-            )
+            // .service(
+            //     web::resource("/1.0/sync/1.5".to_string())
+            //         .route(web::get().to(tokenserver::handlers::get_tokenserver_result)),
+            // )
             // Dockerflow
             // Remember to update .::web::middleware::DOCKER_FLOW_ENDPOINTS
             // when applying changes to endpoint names.
@@ -182,15 +181,19 @@ impl Server {
         let secrets = Arc::new(settings.master_secret);
         let host = settings.host.clone();
         let port = settings.port;
-        let tokenserver_database_url = Arc::new(settings.tokenserver_database_url.clone());
-        let fxa_oauth_server_url = settings.fxa_oauth_server_url;
-        let fxa_metrics_hash_secret = Arc::new(settings.fxa_metrics_hash_secret.clone());
         let quota_enabled = settings.enable_quota;
         let actix_keep_alive = settings.actix_keep_alive;
         let deadman = Arc::new(RwLock::new(Deadman {
             max_size: settings.database_pool_max_size,
             ..Default::default()
         }));
+        let tokenserver_state = if settings.tokenserver.enabled {
+            Some(tokenserver::ServerState::from_settings(
+                &settings.tokenserver,
+            )?)
+        } else {
+            None
+        };
 
         spawn_pool_periodic_reporter(Duration::from_secs(10), metrics.clone(), db_pool.clone())?;
 
@@ -201,11 +204,7 @@ impl Server {
                 limits: Arc::clone(&limits),
                 limits_json: limits_json.clone(),
                 secrets: Arc::clone(&secrets),
-                tokenserver_database_url: (*tokenserver_database_url).clone(),
-                fxa_metrics_hash_secret: (*fxa_metrics_hash_secret).clone(),
-                tokenserver_oauth_verifier: Box::new(OAuthVerifier {
-                    fxa_oauth_server_url: fxa_oauth_server_url.clone(),
-                }),
+                tokenserver_state: tokenserver_state.clone(),
                 metrics: Box::new(metrics.clone()),
                 port,
                 quota_enabled,

diff --git a/src/server/test.rs b/src/server/test.rs
@@ -24,7 +24,6 @@ use crate::db::pool_from_settings;
 use crate::db::results::{DeleteBso, GetBso, PostBsos, PutBso};
 use crate::db::util::SyncTimestamp;
 use crate::settings::{test_settings, Secrets, ServerLimits};
-use crate::tokenserver::MockOAuthVerifier;
 use crate::web::{auth::HawkPayload, extractors::BsoBody, X_LAST_MODIFIED};
 
 lazy_static! {
@@ -70,9 +69,7 @@ async fn get_test_state(settings: &Settings) -> ServerState {
         limits: Arc::clone(&SERVER_LIMITS),
         limits_json: serde_json::to_string(&**SERVER_LIMITS).unwrap(),
         secrets: Arc::clone(&SECRETS),
-        tokenserver_database_url: None,
-        fxa_metrics_hash_secret: None,
-        tokenserver_oauth_verifier: Box::new(MockOAuthVerifier::default()),
+        tokenserver_state: None,
         metrics: Box::new(metrics),
         port: settings.port,
         quota_enabled: settings.enable_quota,