diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e803aac58..9810fa8cb0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,6 +23,13 @@ commands: rustup component add rustfmt cargo install cargo-audit rustup component add clippy + setup-python: + steps: + - run: + name: Setup python + command: | + sudo apt-get update && sudo apt-get install -y python3-dev python3-pip + pip3 install tokenlib rust-check: steps: - run: @@ -157,6 +164,7 @@ jobs: fi - checkout - setup-rust + - setup-python - setup-gcp-grpc # XXX: currently the time needed to setup-sccache negates its savings #- setup-sccache diff --git a/Cargo.lock b/Cargo.lock index 838378ba70..75b3c0f254 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,6 +275,19 @@ dependencies = [ "syn", ] +[[package]] +name = "actix-web-httpauth" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536a75d767c5c2b3e64d3f569621f38ed7609359a0c82d149c88290a6ba41b22" +dependencies = [ + "actix-service", + "actix-web", + "base64 0.12.3", + "bytes 0.5.6", + "futures-util", +] + [[package]] name = "addr2line" version = "0.14.0" @@ -742,6 +755,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "ctor" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbaabec2c953050352311293be5c6aba8e141ba19d6811862b232d6fd020484" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "curl" version = "0.4.34" @@ -1157,6 +1180,17 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "ghost" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5bcf1bbeab73aa4cf2fde60a846858dc036163c7c33bec309f8d17de785479" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "gimli" version = "0.23.0" @@ -1425,6 +1459,29 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47741a8bc60fb26eb8d6e0238bbb26d8575ff623fdc97b1a2c00c050b9684ed8" +dependencies = [ + "indoc-impl", + "proc-macro-hack", +] + +[[package]] +name = "indoc-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce046d161f000fffde5f432a0d034d0341dc152643b2598ed5bfce44c4f3a8f0" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", + "unindent", +] + [[package]] name = "instant" version = "0.1.8" @@ -1434,6 +1491,28 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "inventory" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fedd49de24d8c263613701406611410687148ae8c37cd6452650b250f753a0dd" +dependencies = [ + "ctor", + "ghost", + "inventory-impl", +] + +[[package]] +name = "inventory-impl" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddead8880bc50f57fcd3b5869a7f6ff92570bb4e8f6870c22e2483272f2256da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "iovec" version = "0.1.4" @@ -1476,6 +1555,20 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afabcc15e437a6484fc4f12d0fd63068fe457bf93f1c148d3d9649c60b103f32" +dependencies = [ + "base64 0.12.3", + "pem", + "ring", + "serde 1.0.117", + "serde_json", + "simple_asn1", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -1764,6 +1857,17 @@ dependencies = [ "version_check 0.9.2", ] +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits 0.2.14", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1903,12 +2007,42 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + [[package]] name = "peeking_take_while" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pem" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59698ea79df9bf77104aefd39cc3ec990cb9693fb59c3b0a70ddf2646fdffb4b" +dependencies = [ + "base64 0.12.3", + "once_cell", + "regex", +] + [[package]] name = "percent-encoding" version = "1.0.1" @@ -2036,6 +2170,44 @@ version = "2.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d147edb77bcccbfc81fabffdc7bd50c13e103b15ca1e27515fe40de69a5776b" +[[package]] +name = "pyo3" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b90d637542bbf29b140fdd38fa308424073fd2cdf641a5680aed8020145e3c" +dependencies = [ + "ctor", + "indoc", + "inventory", + "libc", + "parking_lot 0.11.0", + "paste", + "pyo3cls", + "unindent", +] + +[[package]] +name = "pyo3-derive-backend" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee2c9fb095acb885ab7e85acc7c8e95da8c4bc7cc4b4ea64b566dfc8c91046a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pyo3cls" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12fdd8a2f217d003c93f9819e3db1717b2e89530171edea4c0deadd90206f50" +dependencies = [ + "pyo3-derive-backend", + "quote", + "syn", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2546,6 +2718,17 @@ dependencies = [ "libc", ] +[[package]] +name = "simple_asn1" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b" +dependencies = [ + "chrono", + "num-bigint", + "num-traits 0.2.14", +] + [[package]] name = "sized-chunks" version = "0.6.2" @@ -2764,6 +2947,7 @@ dependencies = [ "actix-http", "actix-rt", "actix-web", + "actix-web-httpauth", "async-trait", "base64 0.13.0", "bb8", @@ -2785,11 +2969,13 @@ dependencies = [ "hkdf", "hmac", "hostname", + "jsonwebtoken", "lazy_static", "log", "mime", "num_cpus", "protobuf", + "pyo3", "rand", "regex", "scheduled-thread-pool", @@ -3189,6 +3375,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "unindent" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" + [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 4169ea3926..32da602d10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ debug = 1 [dependencies] actix-http = "2" actix-web = "3" +actix-web-httpauth = "0.5" actix-rt = "1" actix-cors = "0.5" async-trait = "0.1.40" @@ -40,10 +41,12 @@ googleapis-raw = { version = "0", path = "vendor/mozilla-rust-sdk/googleapis-raw # `cargo build --features grpcio/openssl ...` grpcio = { version = "0.6.0" } lazy_static = "1.4.0" +pyo3 = "0.12.1" hawk = "3.2" hostname = "0.3.1" hkdf = "0.10" hmac = "0.10" +jsonwebtoken = "7.2.0" log = { version = "0.4", features = ["max_level_info", "release_max_level_info"] } mime = "0.3" num_cpus = "1" diff --git a/Dockerfile b/Dockerfile index e058426dfb..4562cfa26c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ ADD . /app ENV PATH=$PATH:/root/.cargo/bin # temp removed --no-install-recommends due to CI docker build issue RUN apt-get -q update && \ - apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go && \ + apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go python3-dev python3-pip && \ + pip3 install tokenlib && \ rm -rf /var/lib/apt/lists/* && \ cd /app && \ mkdir -m 755 bin @@ -21,7 +22,8 @@ RUN \ groupadd --gid 10001 app && \ useradd --uid 10001 --gid 10001 --home /app --create-home app && \ apt-get -q update && \ - apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 && \ + apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-dev python3-pip && \ + pip3 install tokenlib && \ rm -rf /var/lib/apt/lists/* COPY --from=builder /app/bin /app/bin diff --git a/src/main.rs b/src/main.rs index 97e824542c..ad4347eea6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,7 @@ extern crate slog_scope; use std::{error::Error, sync::Arc}; use docopt::Docopt; -use serde_derive::Deserialize; +use serde::Deserialize; use logging::init_logging; use syncstorage::{logging, server, settings}; diff --git a/src/server/mod.rs b/src/server/mod.rs index 87714e9fed..3f3c660a72 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -128,7 +128,7 @@ macro_rules! build_app { ) // Tokenserver .service( - web::resource(&cfg_path("/1.0/sync/1.5")).route(web::get().to(tokenserver::get)), + web::resource("/1.0/sync/1.5".to_string()).route(web::get().to(tokenserver::get)), ) // Dockerflow // Remember to update .::web::middleware::DOCKER_FLOW_ENDPOINTS diff --git a/src/web/extractors.rs b/src/web/extractors.rs index ad827da164..94d37bb393 100644 --- a/src/web/extractors.rs +++ b/src/web/extractors.rs @@ -36,7 +36,6 @@ use crate::web::{ error::{HawkErrorKind, ValidationErrorKind}, X_WEAVE_RECORDS, }; - const BATCH_MAX_IDS: usize = 100; // BSO const restrictions @@ -1635,23 +1634,6 @@ where Ok(None) } -// Tokenserver extractor -#[derive(Debug, Default, Clone, Deserialize)] -pub struct TokenServerRequest { - // TODO extract required headers from the request into this struct. -} - -impl FromRequest for TokenServerRequest { - type Config = (); - type Error = Error; - type Future = LocalBoxFuture<'static, Result>; - - /// Extract and validate the precondition headers - fn from_request(_req: &HttpRequest, _payload: &mut Payload) -> Self::Future { - Box::pin(async move { Ok(Self {}) }) - } -} - #[cfg(test)] mod tests { use actix_http::h1; diff --git a/src/web/tokenserver.rs b/src/web/tokenserver.rs index 8b4b4e4033..561675dc6e 100644 --- a/src/web/tokenserver.rs +++ b/src/web/tokenserver.rs @@ -1,26 +1,173 @@ use actix_web::error::BlockingError; use actix_web::web::block; use actix_web::HttpResponse; +use actix_web_httpauth::extractors::bearer::BearerAuth; use futures::future::{Future, TryFutureExt}; -use crate::error::ApiError; -use crate::web::extractors::TokenServerRequest; +use crate::error::{ApiError, ApiErrorKind}; -pub struct TokenServerResult {} +use diesel::mysql::MysqlConnection; +use diesel::prelude::*; +use diesel::sql_types::*; +use diesel::RunQueryDsl; +use std::env; + +use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation}; +use pyo3::prelude::*; +use pyo3::types::IntoPyDict; + +#[derive(Debug)] +enum MyError { + EnvError(env::VarError), +} + +impl From for MyError { + fn from(error: env::VarError) -> Self { + MyError::EnvError(error) + } +} + +#[derive(Debug, QueryableByName)] +struct TokenserverUser { + #[sql_type = "Bigint"] + uid: i64, + // This is no longer used. Was for making more than just sync tokens. + #[sql_type = "Text"] + pattern: String, + #[sql_type = "Text"] + email: String, + #[sql_type = "Bigint"] + generation: i64, + #[sql_type = "Text"] + client_state: String, + #[sql_type = "Bigint"] + created_at: i64, + #[sql_type = "Nullable"] + replaced_at: Option, + #[sql_type = "Text"] + node: String, + #[sql_type = "Nullable"] + keys_changed_at: Option, +} + +#[derive(serde::Serialize)] +pub struct TokenServerResult { + id: String, + key: String, + uid: String, + api_endpoint: String, + duration: String, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] +pub struct Claims { + pub sub: String, + pub iat: i64, + pub exp: i64, +} pub fn get( - request: TokenServerRequest, + auth: BearerAuth, ) -> impl Future>> { - block(move || get_sync(request).map_err(Into::into)).map_ok(move |_result| { - // TODO turn _result into a json response. + block(move || get_sync(&auth).map_err(Into::into)).map_ok(move |result| { HttpResponse::Ok() .content_type("application/json") - .body("{}") + .body(serde_json::to_string(&result).unwrap()) }) } -pub fn get_sync(_request: TokenServerRequest) -> Result { - // TODO Perform any blocking calls needed to respond to the tokenserver request. - Ok(TokenServerResult {}) +pub fn get_sync(auth: &BearerAuth) -> Result { + // the public rsa components come from + // https://oauth.accounts.firefox.com/v1/jwks + // TODO we should fetch it from an environment var instead of hardcoding it. + let token_data = decode::( + &auth.token(), + &DecodingKey::from_rsa_components("2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ", "AQAB"), + &Validation::new(Algorithm::RS256), + ).map_err(|ee| { + ApiError::from(ApiErrorKind::Internal(format!("Unable to decode token_data: {:}", ee))) + })?; + let email = format!("{:}@api.accounts.firefox.com", token_data.claims.sub); + + // TODO pull out of settings instead + let shared_secret = env::var("SYNC_MASTER_SECRET").expect("SYNC_MASTER_SECRET must be set"); + let database_url = + env::var("TOKENSERVER_DATABASE_URL").expect("TOKENSERVER_DATABASE_URL must be set"); + + let connection = MysqlConnection::establish(&database_url) + .unwrap_or_else(|_| panic!("Error connecting to {}", database_url)); + let user_record = diesel::sql_query( + r#"SELECT users.uid, services.pattern, users.email, users.generation, + users.client_state, users.created_at, users.replaced_at, + nodes.node, users.keys_changed_at from users, services, + nodes + WHERE users.email = ? + AND services.id = users.service + AND nodes.id = users.nodeid + AND nodes.service = services.id"#, + ) + .bind::(email) + .load::(&connection) + .unwrap(); + let (python_result, python_derived_result) = Python::with_gil(|py| { + let tokenlib = PyModule::from_code( + py, + r#" +import tokenlib + + +def make_token(plaintext, shared_secret): + return tokenlib.make_token(plaintext, secret=shared_secret) + + +def get_derived_secret(plaintext, shared_secret): + return tokenlib.get_derived_secret(plaintext, secret=shared_secret) +"#, + "main.py", + "main", + ) + .map_err(|e| { + e.print_and_set_sys_last_vars(py); + e + })?; + let thedict = [ + ("node", user_record[0].node.as_ref()), + ("uid", token_data.claims.sub.as_ref()), + ("fxa_kid", "asdf"), // userid component of authorization email + ("fxa_uid", "qwer"), + ("hashed_device_id", "..."), + ("hashed_fxa_uid", "..."), + ] + .into_py_dict(py); + // todo don't hardcode + // we're supposed to check the "duration" query + // param and use that if present (for testing) + thedict.set_item("expires", 300).unwrap(); + let result = match tokenlib.call1("make_token", (thedict, &shared_secret)) { + Err(e) => { + e.print_and_set_sys_last_vars(py); + return Err(e); + } + Ok(x) => x.extract::().unwrap(), + }; + let derived_result = match tokenlib.call1("get_derived_secret", (&result, &shared_secret)) { + Err(e) => { + e.print_and_set_sys_last_vars(py); + return Err(e); + } + Ok(x) => x.extract::().unwrap(), + }; + //assert_eq!(result, false); + Ok((result, derived_result)) + }) + .unwrap(); + let api_endpoint = format!("{:}/1.5/{:}", user_record[0].node, user_record[0].uid); + Ok(TokenServerResult { + id: python_result, + key: python_derived_result, + uid: token_data.claims.sub, + api_endpoint, + duration: "300".to_string(), + }) }