diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a8215b898..e36dce7102 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,7 @@ commands: - run: name: Core Python Checks command: | - flake8 syncstorage/src/tokenserver + flake8 syncserver/src/tokenserver flake8 tools/integration_tests flake8 tools/tokenserver rust-clippy: @@ -48,7 +48,7 @@ commands: - run: name: Rust Clippy command: | - cargo clippy --all --all-targets --all-features -- -D warnings + cargo clippy --workspace --all-targets --all-features -- -D warnings cargo-build: steps: - run: @@ -83,18 +83,18 @@ commands: "$CIRCLE_TAG" \ "$CIRCLE_PROJECT_USERNAME" \ "$CIRCLE_PROJECT_REPONAME" \ - "$CIRCLE_BUILD_URL" > syncstorage/version.json + "$CIRCLE_BUILD_URL" > syncserver/version.json run-tests: steps: - run: name: cargo test - command: cargo test --all --verbose + command: cargo test --workspace --verbose - run: name: quota test - command: cargo test --all --verbose + command: cargo test --workspace --verbose environment: - SYNC_ENFORCE_QUOTA: 1 + SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 run-e2e-mysql-tests: steps: @@ -181,7 +181,7 @@ jobs: username: $DOCKER_USER password: $DOCKER_PASS environment: - SYNC_DATABASE_URL: mysql://test:test@127.0.0.1/syncstorage + SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@127.0.0.1/syncstorage SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver RUST_BACKTRACE: 1 # XXX: begin_test_transaction doesn't play nice over threaded tests diff --git a/Cargo.lock b/Cargo.lock index d34ad1d831..12aeb26c5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3000,12 +3000,13 @@ dependencies = [ ] [[package]] -name = "syncstorage" +name = "syncserver" version = "0.12.4" dependencies = [ "actix-cors", "actix-http", "actix-rt", + "actix-service", "actix-web", "async-trait", "backtrace", @@ -3014,7 +3015,6 @@ dependencies = [ "bytes 1.1.0", "cadence", "chrono", - "config 0.11.0", "deadpool", "diesel", "diesel_logger", @@ -3027,7 +3027,6 @@ dependencies = [ "grpcio", "hawk", "hex", - "hkdf", "hmac", "hostname", "http", @@ -3055,11 +3054,14 @@ dependencies = [ "slog-scope", "slog-stdlog", "slog-term", - "syncstorage-common", - "syncstorage-db-common", + "syncserver-common", + "syncserver-db-common", + "syncserver-settings", + "syncstorage-settings", "thiserror", "time 0.3.9", "tokenserver-common", + "tokenserver-settings", "tokio", "url 2.2.2", "urlencoding", @@ -3070,11 +3072,15 @@ dependencies = [ ] [[package]] -name = "syncstorage-common" +name = "syncserver-common" version = "0.12.4" +dependencies = [ + "hkdf", + "sha2", +] [[package]] -name = "syncstorage-db-common" +name = "syncserver-db-common" version = "0.12.4" dependencies = [ "async-trait", @@ -3090,8 +3096,33 @@ dependencies = [ "lazy_static", "serde 1.0.135", "serde_json", - "syncstorage-common", + "syncserver-common", "thiserror", + "url 2.2.2", +] + +[[package]] +name = "syncserver-settings" +version = "0.12.3" +dependencies = [ + "config 0.11.0", + "num_cpus", + "serde 1.0.135", + "slog-scope", + "syncserver-common", + "syncstorage-settings", + "tokenserver-settings", + "url 2.2.2", +] + +[[package]] +name = "syncstorage-settings" +version = "0.12.3" +dependencies = [ + "rand 0.8.5", + "serde 1.0.135", + "syncserver-common", + "time 0.3.9", ] [[package]] @@ -3273,11 +3304,19 @@ dependencies = [ "backtrace", "serde 1.0.135", "serde_json", - "syncstorage-common", - "syncstorage-db-common", + "syncserver-common", + "syncserver-db-common", "thiserror", ] +[[package]] +name = "tokenserver-settings" +version = "0.12.3" +dependencies = [ + "serde 1.0.135", + "tokenserver-common", +] + [[package]] name = "tokio" version = "0.2.25" diff --git a/Cargo.toml b/Cargo.toml index 92d47af867..56eced59d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,15 @@ [workspace] resolver = "2" members = [ - "syncstorage-db-common", - "syncstorage-common", + "syncserver-settings", + "syncserver-common", + "syncserver-db-common", + "syncstorage-settings", "tokenserver-common", - "syncstorage", + "tokenserver-settings", + "syncserver", ] -default-members = ["syncstorage"] +default-members = ["syncserver"] [profile.release] # Enables line numbers in Sentry reporting diff --git a/Dockerfile b/Dockerfile index f33ae83605..f358d37995 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,8 @@ RUN apt-get -q update && \ RUN \ cargo --version && \ rustc --version && \ - cargo install --path ./syncstorage --locked --root /app && \ - cargo install --path ./syncstorage --locked --root /app --bin purge_ttl + cargo install --path ./syncserver --locked --root /app && \ + cargo install --path ./syncserver --locked --root /app --bin purge_ttl FROM debian:buster-slim WORKDIR /app @@ -35,14 +35,14 @@ RUN \ rm -rf /var/lib/apt/lists/* COPY --from=builder /app/bin /app/bin -COPY --from=builder /app/syncstorage/version.json /app +COPY --from=builder /app/syncserver/version.json /app COPY --from=builder /app/spanner_config.ini /app COPY --from=builder /app/tools/spanner /app/tools/spanner COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests COPY --from=builder /app/tools/tokenserver/process_account_events.py /app/tools/tokenserver/process_account_events.py COPY --from=builder /app/tools/tokenserver/requirements.txt /app/tools/tokenserver/requirements.txt COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh -COPY --from=builder /app/syncstorage/src/db/spanner/schema.ddl /app/schema.ddl +COPY --from=builder /app/syncserver/src/db/spanner/schema.ddl /app/schema.ddl RUN chmod +x /app/scripts/prepare-spanner.sh RUN pip3 install -r /app/tools/integration_tests/requirements.txt @@ -50,4 +50,4 @@ RUN pip3 install -r /app/tools/tokenserver/requirements.txt USER app:app -ENTRYPOINT ["/app/bin/syncstorage", "--config=spanner_config.ini"] +ENTRYPOINT ["/app/bin/syncserver", "--config=spanner_config.ini"] diff --git a/Makefile b/Makefile index baf0e07cce..39629617c5 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,6 @@ # Collection of helper scripts used for local dev. ## -SYNC_DATABASE_URL = 'mysql://sample_user:sample_password@localhost/syncstorage_rs' -SYNC_TOKENSERVER__DATABASE_URL = 'mysql://sample_user:sample_password@localhost/tokenserver_rs' - # This key can live anywhere on your machine. Adjust path as needed. PATH_TO_SYNC_SPANNER_KEYS = `pwd`/service-account.json @@ -15,7 +12,7 @@ PATH_TO_GRPC_CERT = ../server-syncstorage/local/lib/python2.7/site-packages/grpc clippy: # Matches what's run in circleci - cargo clippy --all --all-targets --all-features -- -D warnings + cargo clippy --workspace --all-targets --all-features -- -D warnings clean: cargo clean @@ -50,4 +47,7 @@ run_spanner: GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) make run test: - SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) SYNC_TOKENSERVER__DATABASE_URL=$(SYNC_TOKENSERVER__DATABASE_URL) RUST_TEST_THREADS=1 cargo test + SYNC_SYNCSTORAGE__DATABASE_URL=mysql://sample_user:sample_password@localhost/syncstorage_rs \ + SYNC_TOKENSERVER__DATABASE_URL=mysql://sample_user:sample_password@localhost/tokenserver_rs \ + RUST_TEST_THREADS=1 \ + cargo test diff --git a/README.md b/README.md index 420b019d85..d9f63b0d4b 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ Note that, again, you may set `DATABASE_ID` to your liking. Make sure that the ` To run an application server that points to the local Spanner emulator: ```sh -SYNC_SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner +SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner ``` ### Running via Docker @@ -166,7 +166,7 @@ This requires access to the mozilla-rust-sdk which is now available at `/vendor/ 1. Make sure you have [Docker installed](https://docs.docker.com/install/) locally. 2. Copy the contents of mozilla-rust-sdk into top level root dir here. 3. Change cargo.toml mozilla-rust-sdk entry to point to `"path = "mozilla-rust-sdk/googleapis-raw"` instead of the parent dir. -4. Comment out the `image` value under `syncstorage-rs` in either docker-compose.mysql.yml or docker-compose.spanner.yml (depending on which database backend you want to run), and add this instead: +4. Comment out the `image` value under `syncserver` in either docker-compose.mysql.yml or docker-compose.spanner.yml (depending on which database backend you want to run), and add this instead: ```yml build: context: . @@ -220,7 +220,7 @@ We use [env_logger](https://crates.io/crates/env_logger): set the `RUST_LOG` env ### Unit tests -`make test` - open the Makefile to adjust your `SYNC_DATABASE_URL` as needed. +`make test` - open the Makefile to adjust your `SYNC_SYNCSTORAGE__DATABASE_URL` as needed. #### Debugging unit test state @@ -229,7 +229,7 @@ default, we use the diesel test_transaction functionality to ensure test data is not committed to the database. Therefore, there is an environment variable which can be used to turn off test_transaction. - SYNC_DATABASE_USE_TEST_TRANSACTIONS=false cargo test [testname] + SYNC_SYNCSTORAGE__DATABASE_USE_TEST_TRANSACTIONS=false cargo test [testname] Note that you will almost certainly want to pass a single test name. When running the entire test suite, data from previous tests will cause future tests to fail. diff --git a/docker-compose.e2e.mysql.yaml b/docker-compose.e2e.mysql.yaml index c1b9be51fa..bb5d6e99a2 100644 --- a/docker-compose.e2e.mysql.yaml +++ b/docker-compose.e2e.mysql.yaml @@ -2,23 +2,22 @@ version: '3' services: sync-db: tokenserver-db: - syncstorage-rs: + syncserver: depends_on: - sync-db - tokenserver-db - # TODO: either syncstorage-rs should retry the db connection + # TODO: either syncserver should retry the db connection # itself a few times or should include a wait-for-it.sh script - # inside its docker that would do this for us. Same (probably - # the latter solution) for server-syncstorage below + # inside its docker that would do this for us. entrypoint: > /bin/sh -c " sleep 15; - /app/bin/syncstorage; + /app/bin/syncserver; " e2e-tests: depends_on: - mock-fxa-server - - syncstorage-rs + - syncserver image: app:build privileged: true user: root @@ -26,7 +25,7 @@ services: MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000 SYNC_HOST: 0.0.0.0 SYNC_MASTER_SECRET: secret0 - SYNC_DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage + SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver SYNC_TOKENSERVER__ENABLED: "true" SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: "https://token.stage.mozaws.net/" diff --git a/docker-compose.e2e.spanner.yaml b/docker-compose.e2e.spanner.yaml index 71b5d88a2b..b431276b09 100644 --- a/docker-compose.e2e.spanner.yaml +++ b/docker-compose.e2e.spanner.yaml @@ -3,22 +3,21 @@ services: sync-db: sync-db-setup: tokenserver-db: - syncstorage-rs: + syncserver: depends_on: - sync-db-setup - # TODO: either syncstorage-rs should retry the db connection + # TODO: either syncserver should retry the db connection # itself a few times or should include a wait-for-it.sh script - # inside its docker that would do this for us. Same (probably - # the latter solution) for server-syncstorage below + # inside its docker that would do this for us. entrypoint: > /bin/sh -c " sleep 15; - /app/bin/syncstorage; + /app/bin/syncserver; " e2e-tests: depends_on: - mock-fxa-server - - syncstorage-rs + - syncserver image: app:build privileged: true user: root @@ -26,8 +25,8 @@ services: MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000 SYNC_HOST: 0.0.0.0 SYNC_MASTER_SECRET: secret0 - SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database - SYNC_SPANNER_EMULATOR_HOST: sync-db:9010 + SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database + SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010 SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver SYNC_TOKENSERVER__ENABLED: "true" SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: "https://token.stage.mozaws.net/" diff --git a/docker-compose.mysql.yaml b/docker-compose.mysql.yaml index 3ea3d72e55..f2e5239c3b 100644 --- a/docker-compose.mysql.yaml +++ b/docker-compose.mysql.yaml @@ -1,3 +1,7 @@ +# Application runs off of port 8000. +# you can test if it's available with +# curl "http://localhost:8000/__heartbeat__" + version: '3' services: sync-db: @@ -36,25 +40,25 @@ services: MOCK_FXA_SERVER_HOST: 0.0.0.0 MOCK_FXA_SERVER_PORT: 6000 - syncstorage-rs: + syncserver: + # NOTE: The naming in the rest of this repository has been updated to reflect the fact + # that Syncstorage and Tokenserver are now part of one repository/server called + # "Syncserver" (updated from "syncstorage-rs"). We keep the legacy naming below for + # backwards compatibility with previous Docker images. image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest} restart: always ports: - - "8000:8000" + - "8000:8000" depends_on: - - sync-db - - tokenserver-db + - sync-db + - tokenserver-db environment: - SYNC_HOST: 0.0.0.0 - SYNC_MASTER_SECRET: secret0 - SYNC_DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage - SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver - SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" + SYNC_HOST: 0.0.0.0 + SYNC_MASTER_SECRET: secret0 + SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage + SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver + SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" volumes: sync_db_data: tokenserver_db_data: - -# Application runs off of port 8000. -# you can test if it's available with -# curl "http://localhost:8000/__heartbeat__" diff --git a/docker-compose.spanner.yaml b/docker-compose.spanner.yaml index 3f510e697e..a313e37813 100644 --- a/docker-compose.spanner.yaml +++ b/docker-compose.spanner.yaml @@ -14,7 +14,7 @@ services: restart: "no" entrypoint: "/app/scripts/prepare-spanner.sh" environment: - SYNC_SPANNER_EMULATOR_HOST: sync-db:9020 + SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9020 tokenserver-db: image: mysql:5.7 volumes: @@ -35,7 +35,11 @@ services: environment: MOCK_FXA_SERVER_HOST: 0.0.0.0 MOCK_FXA_SERVER_PORT: 6000 - syncstorage-rs: + syncserver: + # NOTE: The naming in the rest of this repository has been updated to reflect the fact + # that Syncstorage and Tokenserver are now part of one repository/server called + # "Syncserver" (updated from "syncstorage-rs"). We keep the legacy naming below for + # backwards compatibility with previous Docker images. image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest} restart: always ports: @@ -45,8 +49,8 @@ services: environment: SYNC_HOST: 0.0.0.0 SYNC_MASTER_SECRET: secret0 - SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database - SYNC_SPANNER_EMULATOR_HOST: sync-db:9010 + SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database + SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010 SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" diff --git a/docs/config.md b/docs/config.md index f1466b659f..ab606762df 100644 --- a/docs/config.md +++ b/docs/config.md @@ -10,13 +10,15 @@ In addition, durable sync configuration options can either be specified as envir For example the following are equivalent: ```bash -$ SYNC_HOST=0.0.0.0 SYNC_MASTER_SECRET="SuperSikkr3t" SYNC_DATABASE_URL=mysql://scott:tiger@localhost/syncstorage cargo run +$ SYNC_HOST=0.0.0.0 SYNC_MASTER_SECRET="SuperSikkr3t" SYNC_SYNCSTORAGE__DATABASE_URL=mysql://scott:tiger@localhost/syncstorage cargo run ``` ```bash $ cat sync.ini HOST=0.0.0.0 MASTER_SECRET=SuperSikkr3t + +[syncstorage] DATABASE_URL=mysql://scott:tiger@localhost/syncstorage $ cargo run -- --config sync.ini ``` diff --git a/scripts/prepare-spanner.sh b/scripts/prepare-spanner.sh index 986444ad7d..8195d7cf92 100755 --- a/scripts/prepare-spanner.sh +++ b/scripts/prepare-spanner.sh @@ -35,13 +35,13 @@ DDL_STATEMENTS=$( ) curl -sS --request POST \ - "$SYNC_SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances" \ + "$SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances" \ --header 'Accept: application/json' \ --header 'Content-Type: application/json' \ --data "{\"instance\":{\"config\":\"emulator-test-config\",\"nodeCount\":1,\"displayName\":\"Test Instance\"},\"instanceId\":\"$INSTANCE_ID\"}" curl -sS --request POST \ - "$SYNC_SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances/$INSTANCE_ID/databases" \ + "$SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances/$INSTANCE_ID/databases" \ --header 'Accept: application/json' \ --header 'Content-Type: application/json' \ --data "{\"createStatement\":\"CREATE DATABASE \`$DATABASE_ID\`\",\"extraStatements\":$DDL_STATEMENTS}" diff --git a/syncstorage-common/Cargo.toml b/syncserver-common/Cargo.toml similarity index 53% rename from syncstorage-common/Cargo.toml rename to syncserver-common/Cargo.toml index fd4f049b23..438d939686 100644 --- a/syncstorage-common/Cargo.toml +++ b/syncserver-common/Cargo.toml @@ -1,6 +1,8 @@ [package] -name = "syncstorage-common" +name = "syncserver-common" version = "0.12.4" edition = "2021" [dependencies] +hkdf = "0.11" +sha2 = "0.9" diff --git a/syncserver-common/src/lib.rs b/syncserver-common/src/lib.rs new file mode 100644 index 0000000000..c3f0e2dfba --- /dev/null +++ b/syncserver-common/src/lib.rs @@ -0,0 +1,52 @@ +use hkdf::Hkdf; +use sha2::Sha256; + +// header statics must be lower case, numbers and symbols per the RFC spec. This reduces chance of error. +pub static X_LAST_MODIFIED: &str = "x-last-modified"; +pub static X_WEAVE_TIMESTAMP: &str = "x-weave-timestamp"; +pub static X_WEAVE_NEXT_OFFSET: &str = "x-weave-next-offset"; +pub static X_WEAVE_RECORDS: &str = "x-weave-records"; +pub static X_WEAVE_BYTES: &str = "x-weave-bytes"; +pub static X_WEAVE_TOTAL_RECORDS: &str = "x-weave-total-records"; +pub static X_WEAVE_TOTAL_BYTES: &str = "x-weave-total-bytes"; +pub static X_VERIFY_CODE: &str = "x-verify-code"; + +// max load size in bytes +pub const MAX_SPANNER_LOAD_SIZE: usize = 100_000_000; + +/// Helper function for [HKDF](https://tools.ietf.org/html/rfc5869) expansion to 32 bytes. +pub fn hkdf_expand_32(info: &[u8], salt: Option<&[u8]>, key: &[u8]) -> Result<[u8; 32], String> { + let mut result = [0u8; 32]; + let hkdf = Hkdf::::new(salt, key); + hkdf.expand(info, &mut result) + .map_err(|e| format!("HKDF Error: {:?}", e))?; + Ok(result) +} + +#[macro_export] +macro_rules! from_error { + ($from:ty, $to:ty, $to_kind:expr) => { + impl From<$from> for $to { + fn from(inner: $from) -> $to { + $to_kind(inner).into() + } + } + }; +} + +#[macro_export] +macro_rules! impl_fmt_display { + ($error:ty, $kind:ty) => { + impl fmt::Display for $error { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.kind, formatter) + } + } + }; +} + +pub trait ReportableError { + fn error_backtrace(&self) -> String; + fn is_sentry_event(&self) -> bool; + fn metric_label(&self) -> Option; +} diff --git a/syncstorage-db-common/Cargo.toml b/syncserver-db-common/Cargo.toml similarity index 83% rename from syncstorage-db-common/Cargo.toml rename to syncserver-db-common/Cargo.toml index 667d951123..fb32ad1812 100644 --- a/syncstorage-db-common/Cargo.toml +++ b/syncserver-db-common/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "syncstorage-db-common" +name = "syncserver-db-common" version = "0.12.4" edition = "2021" @@ -14,7 +14,7 @@ deadpool = { git = "https://github.com/mozilla-services/deadpool", branch = "dea diesel = { version = "1.4", features = ["mysql", "r2d2"] } diesel_migrations = { version = "1.4.0", features = ["mysql"] } # Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause -# syncserver to either fail to either compile, or start. In those cases, try +# syncstorage to either fail to either compile, or start. In those cases, try # `cargo build --features grpcio/openssl ...` grpcio = { version = "0.9" } hostname = "0.3.1" @@ -23,5 +23,6 @@ futures = { version = "0.3", features = ["compat"] } lazy_static = "1.4.0" serde = "1.0" serde_json = { version = "1.0", features = ["arbitrary_precision"] } -syncstorage-common = { path = "../syncstorage-common" } +syncserver-common = { path = "../syncserver-common" } thiserror = "1.0.26" +url = "2.1" diff --git a/syncstorage-db-common/src/error.rs b/syncserver-db-common/src/error.rs similarity index 97% rename from syncstorage-db-common/src/error.rs rename to syncserver-db-common/src/error.rs index d7526c40f5..a9aac86c07 100644 --- a/syncstorage-db-common/src/error.rs +++ b/syncserver-db-common/src/error.rs @@ -2,7 +2,7 @@ use std::fmt; use backtrace::Backtrace; use http::StatusCode; -use syncstorage_common::{from_error, impl_fmt_display}; +use syncserver_common::{from_error, impl_fmt_display}; use thiserror::Error; #[derive(Debug)] @@ -47,7 +47,7 @@ pub enum DbErrorKind { #[error("Database integrity error: {}", _0)] Integrity(String), - #[error("Invalid SYNC_DATABASE_URL: {}", _0)] + #[error("Invalid database URL: {}", _0)] InvalidUrl(String), #[error("Unexpected error: {}", _0)] diff --git a/syncstorage-db-common/src/lib.rs b/syncserver-db-common/src/lib.rs similarity index 100% rename from syncstorage-db-common/src/lib.rs rename to syncserver-db-common/src/lib.rs diff --git a/syncstorage-db-common/src/params.rs b/syncserver-db-common/src/params.rs similarity index 100% rename from syncstorage-db-common/src/params.rs rename to syncserver-db-common/src/params.rs diff --git a/syncstorage-db-common/src/results.rs b/syncserver-db-common/src/results.rs similarity index 100% rename from syncstorage-db-common/src/results.rs rename to syncserver-db-common/src/results.rs diff --git a/syncstorage-db-common/src/util.rs b/syncserver-db-common/src/util.rs similarity index 100% rename from syncstorage-db-common/src/util.rs rename to syncserver-db-common/src/util.rs diff --git a/syncserver-settings/Cargo.toml b/syncserver-settings/Cargo.toml new file mode 100644 index 0000000000..a1e1f22815 --- /dev/null +++ b/syncserver-settings/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "syncserver-settings" +version = "0.12.3" +edition = "2021" + +[dependencies] +config = "0.11" +num_cpus = "1" +serde = "1.0" +slog-scope = "4.3" +syncserver-common = { path = "../syncserver-common" } +syncstorage-settings = { path = "../syncstorage-settings" } +tokenserver-settings = { path = "../tokenserver-settings" } +url = "2.1" diff --git a/syncserver-settings/src/lib.rs b/syncserver-settings/src/lib.rs new file mode 100644 index 0000000000..3a9a8343d8 --- /dev/null +++ b/syncserver-settings/src/lib.rs @@ -0,0 +1,284 @@ +#[macro_use] +extern crate slog_scope; + +use std::env::{self, VarError}; + +use config::{Config, ConfigError, Environment, File}; +use serde::{Deserialize, Deserializer}; +use syncserver_common::{ + X_LAST_MODIFIED, X_VERIFY_CODE, X_WEAVE_BYTES, X_WEAVE_NEXT_OFFSET, X_WEAVE_RECORDS, + X_WEAVE_TIMESTAMP, X_WEAVE_TOTAL_BYTES, X_WEAVE_TOTAL_RECORDS, +}; +use syncstorage_settings::Settings as SyncstorageSettings; +use tokenserver_settings::Settings as TokenserverSettings; +use url::Url; + +pub static PREFIX: &str = "sync"; + +#[derive(Clone, Deserialize)] +#[serde(default)] +pub struct Settings { + pub port: u16, + pub host: String, + pub actix_keep_alive: Option, + /// The master secret, from which are derived + /// the signing secret and token secret + /// that are used during Hawk authentication. + pub master_secret: Secrets, + + pub human_logs: bool, + + pub statsd_host: Option, + pub statsd_port: u16, + + /// Cors Settings + pub cors_allowed_origin: Option, + pub cors_max_age: Option, + pub cors_allowed_methods: Option>, + pub cors_allowed_headers: Option>, + + // TOOD: Eventually, the below settings will be enabled or disabled via Cargo features + pub syncstorage: SyncstorageSettings, + pub tokenserver: TokenserverSettings, +} + +impl Settings { + /// Load the settings from the config file if supplied, then the environment. + pub fn with_env_and_config_file(filename: Option<&str>) -> Result { + let mut s = Config::default(); + + // Merge the config file if supplied + if let Some(config_filename) = filename { + s.merge(File::with_name(config_filename))?; + } + + // Merge the environment overrides + // While the prefix is currently case insensitive, it's traditional that + // environment vars be UPPERCASE, this ensures that will continue should + // Environment ever change their policy about case insensitivity. + // This will accept environment variables specified as + // `SYNC_FOO__BAR_VALUE="gorp"` as `foo.bar_value = "gorp"` + s.merge(Environment::with_prefix(&PREFIX.to_uppercase()).separator("__"))?; + + match s.try_into::() { + Ok(mut s) => { + s.syncstorage.normalize(); + + if matches!(env::var("ACTIX_THREADPOOL"), Err(VarError::NotPresent)) { + // Db backends w/ blocking calls block via + // actix-threadpool: grow its size to accommodate the + // full number of connections + let total_db_pool_size = { + let syncstorage_pool_max_size = + if s.syncstorage.uses_spanner() || !s.syncstorage.enabled { + 0 + } else { + s.syncstorage.database_pool_max_size + }; + + let tokenserver_pool_max_size = if s.tokenserver.enabled { + s.tokenserver.database_pool_max_size + } else { + 0 + }; + + syncstorage_pool_max_size + tokenserver_pool_max_size + }; + + let fxa_threads = if s.tokenserver.enabled + && s.tokenserver.fxa_oauth_primary_jwk.is_none() + && s.tokenserver.fxa_oauth_secondary_jwk.is_none() + { + s.tokenserver + .additional_blocking_threads_for_fxa_requests + .ok_or_else(|| { + println!( + "If the Tokenserver OAuth JWK is not cached, additional blocking \ + threads must be used to handle the requests to FxA." + ); + + let setting_name = + "tokenserver.additional_blocking_threads_for_fxa_requests"; + ConfigError::NotFound(String::from(setting_name)) + })? + } else { + 0 + }; + + env::set_var( + "ACTIX_THREADPOOL", + ((total_db_pool_size + fxa_threads) as usize) + .max(num_cpus::get() * 5) + .to_string(), + ); + } + + Ok(s) + } + // Configuration errors are not very sysop friendly, Try to make them + // a bit more 3AM useful. + Err(ConfigError::Message(v)) => { + println!("Bad configuration: {:?}", &v); + println!("Please set in config file or use environment variable."); + println!( + "For example to set `database_url` use env var `{}_DATABASE_URL`\n", + PREFIX.to_uppercase() + ); + error!("Configuration error: Value undefined {:?}", &v); + Err(ConfigError::NotFound(v)) + } + Err(e) => { + error!("Configuration error: Other: {:?}", &e); + Err(e) + } + } + } + + pub fn test_settings() -> Self { + let mut settings = + Self::with_env_and_config_file(None).expect("Could not get Settings in test_settings"); + settings.port = 8000; + settings.syncstorage.database_pool_max_size = 1; + settings.syncstorage.database_use_test_transactions = true; + settings.syncstorage.database_pool_connection_max_idle = Some(300); + settings.syncstorage.database_pool_connection_lifespan = Some(300); + settings + } + + pub fn banner(&self) -> String { + let quota = if self.syncstorage.enable_quota { + format!( + "Quota: {} bytes ({}enforced)", + self.syncstorage.limits.max_quota_limit, + if !self.syncstorage.enforce_quota { + "un" + } else { + "" + } + ) + } else { + "No quota".to_owned() + }; + let db = Url::parse(&self.syncstorage.database_url) + .map(|url| url.scheme().to_owned()) + .unwrap_or_else(|_| "".to_owned()); + format!("http://{}:{} ({}) {}", self.host, self.port, db, quota) + } +} + +impl Default for Settings { + fn default() -> Settings { + Settings { + port: 8000, + host: "127.0.0.1".to_string(), + actix_keep_alive: None, + master_secret: Secrets::default(), + statsd_host: Some("localhost".to_owned()), + statsd_port: 8125, + human_logs: false, + cors_allowed_origin: None, + cors_allowed_methods: Some( + ["DELETE", "GET", "POST", "PUT"] + .into_iter() + .map(String::from) + .collect(), + ), + cors_allowed_headers: Some( + [ + "Authorization", + "Content-Type", + "UserAgent", + X_LAST_MODIFIED, + X_WEAVE_TIMESTAMP, + X_WEAVE_NEXT_OFFSET, + X_WEAVE_RECORDS, + X_WEAVE_BYTES, + X_WEAVE_TOTAL_RECORDS, + X_WEAVE_TOTAL_BYTES, + X_VERIFY_CODE, + "TEST_IDLES", + ] + .into_iter() + .map(String::from) + .collect(), + ), + cors_max_age: None, + syncstorage: SyncstorageSettings::default(), + tokenserver: TokenserverSettings::default(), + } + } +} + +/// Secrets used during Hawk authentication. +#[derive(Clone, Debug)] +pub struct Secrets { + /// The master secret in byte array form. + /// + /// The signing secret and token secret are derived from this. + pub master_secret: Vec, + + /// The signing secret used during Hawk authentication. + pub signing_secret: [u8; 32], +} + +impl Secrets { + /// Decode the master secret to a byte array + /// and derive the signing secret from it. + pub fn new(master_secret: &str) -> Result { + let master_secret = master_secret.as_bytes().to_vec(); + let signing_secret = syncserver_common::hkdf_expand_32( + b"services.mozilla.com/tokenlib/v1/signing", + None, + &master_secret, + )?; + Ok(Self { + master_secret, + signing_secret, + }) + } +} + +impl Default for Secrets { + /// Create a (useless) default `Secrets` instance. + fn default() -> Self { + Self { + master_secret: vec![], + signing_secret: [0u8; 32], + } + } +} + +impl<'d> Deserialize<'d> for Secrets { + /// Deserialize the master secret and signing secret byte arrays + /// from a single master secret string. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'d>, + { + let master_secret: String = Deserialize::deserialize(deserializer)?; + Secrets::new(&master_secret) + .map_err(|e| serde::de::Error::custom(format!("error: {:?}", e))) + } +} + +#[cfg(test)] +mod test { + use std::env; + + use super::*; + + #[test] + fn test_environment_variable_prefix() { + // Setting an environment variable with the correct prefix correctly sets the setting + // (note that the default value for the settings.tokenserver.enabled setting is false) + env::set_var("SYNC_TOKENSERVER__ENABLED", "true"); + let settings = Settings::with_env_and_config_file(None).unwrap(); + assert!(settings.tokenserver.enabled); + + // Setting an environment variable with the incorrect prefix does not set the setting + env::remove_var("SYNC_TOKENSERVER__ENABLED"); + env::set_var("SYNC__TOKENSERVER__ENABLED", "true"); + let settings = Settings::with_env_and_config_file(None).unwrap(); + assert!(!settings.tokenserver.enabled); + } +} diff --git a/syncstorage/Cargo.toml b/syncserver/Cargo.toml similarity index 84% rename from syncstorage/Cargo.toml rename to syncserver/Cargo.toml index 98535ee101..a288821a05 100644 --- a/syncstorage/Cargo.toml +++ b/syncserver/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "syncstorage" +name = "syncserver" version = "0.12.4" license = "MPL-2.0" authors = [ @@ -8,21 +8,21 @@ authors = [ "Mozilla Services Engineering ", ] edition = "2018" -default-run = "syncstorage" +default-run = "syncserver" [dependencies] actix-http = "2" actix-web = "3" -actix-rt = "1" # Pin to 1.0, due to dependencies on Tokio +actix-rt = "1" # Pin to 1.0, due to dependencies on Tokio actix-cors = "0.5" +actix-service = "1.0.6" async-trait = "0.1.40" backtrace = "0.3.61" base64 = "0.13" -bb8 = "0.4.1" # pin to 0.4 due to dependencies on Tokio +bb8 = "0.4.1" # pin to 0.4 due to dependencies on Tokio bytes = "1.0" cadence = "0.26" chrono = "0.4" -config = "0.11" # Pin to 0.5 for now, to keep it under tokio 0.2 (issue977). # Fix for #803 (deadpool#92) points to our fork for now #deadpool = "0.5" # pin to 0.5 @@ -43,7 +43,6 @@ lazy_static = "1.4.0" hawk = "3.2" hex = "0.4.3" hostname = "0.3.1" -hkdf = "0.11" hmac = "0.11" http = "0.2.5" log = { version = "0.4", features = [ @@ -79,11 +78,14 @@ slog-mozlog-json = "0.1" slog-scope = "4.3" slog-stdlog = "4.1" slog-term = "2.6" -syncstorage-db-common = { path = "../syncstorage-db-common" } -syncstorage-common = { path = "../syncstorage-common" } +syncserver-settings = { path = "../syncserver-settings" } +syncserver-db-common = { path = "../syncserver-db-common" } +syncserver-common = { path = "../syncserver-common" } +syncstorage-settings = { path = "../syncstorage-settings" } time = "^0.3" thiserror = "1.0.26" tokenserver-common = { path = "../tokenserver-common" } +tokenserver-settings = { path = "../tokenserver-settings" } # pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.) tokio = { version = "0.2.4", features = ["macros", "sync"] } url = "2.1" diff --git a/syncstorage/src/bin/purge_ttl.rs b/syncserver/src/bin/purge_ttl.rs similarity index 99% rename from syncstorage/src/bin/purge_ttl.rs rename to syncserver/src/bin/purge_ttl.rs index 975008f293..fa672deefb 100644 --- a/syncstorage/src/bin/purge_ttl.rs +++ b/syncserver/src/bin/purge_ttl.rs @@ -278,7 +278,7 @@ fn main() -> Result<(), Box> { .unwrap_or(false); info!("INCREMENTAL: {:?}", incremental); - const DB_ENV: &str = "SYNC_DATABASE_URL"; + const DB_ENV: &str = "SYNC_SYNCSTORAGE__DATABASE_URL"; let db_url = env::var(DB_ENV).map_err(|_| format!("Invalid or undefined {}", DB_ENV))?; let url = Url::parse(&db_url).map_err(|e| format!("Invalid {}: {}", DB_ENV, e))?; if url.scheme() != "spanner" || url.host() != Some(Host::Domain("projects")) { diff --git a/syncstorage/src/db/mock.rs b/syncserver/src/db/mock.rs similarity index 99% rename from syncstorage/src/db/mock.rs rename to syncserver/src/db/mock.rs index c786165aca..af13bfb7b9 100644 --- a/syncstorage/src/db/mock.rs +++ b/syncserver/src/db/mock.rs @@ -2,7 +2,7 @@ #![allow(clippy::new_without_default)] use async_trait::async_trait; use futures::future; -use syncstorage_db_common::{ +use syncserver_db_common::{ error::DbError, params, results, util::SyncTimestamp, Db, DbFuture, DbPool, GetPoolState, PoolState, }; diff --git a/syncstorage/src/db/mod.rs b/syncserver/src/db/mod.rs similarity index 97% rename from syncstorage/src/db/mod.rs rename to syncserver/src/db/mod.rs index 598abc3af5..3277af34e2 100644 --- a/syncstorage/src/db/mod.rs +++ b/syncserver/src/db/mod.rs @@ -11,15 +11,15 @@ use std::time::Duration; use actix_web::{error::BlockingError, web}; use cadence::{Gauged, StatsdClient}; -use syncstorage_db_common::{ +use syncserver_db_common::{ error::{DbError, DbErrorKind}, results, DbPool, GetPoolState, PoolState, }; +use syncstorage_settings::Settings; use tokio::{self, time}; use url::Url; use crate::server::metrics::Metrics; -use crate::settings::Settings; /// Create/initialize a pool of managed Db connections pub async fn pool_from_settings( diff --git a/syncstorage/src/db/mysql/batch.rs b/syncserver/src/db/mysql/batch.rs similarity index 99% rename from syncstorage/src/db/mysql/batch.rs rename to syncserver/src/db/mysql/batch.rs index ef7ab50991..4616c9b76b 100644 --- a/syncstorage/src/db/mysql/batch.rs +++ b/syncserver/src/db/mysql/batch.rs @@ -9,7 +9,7 @@ use diesel::{ sql_types::{BigInt, Integer}, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, }; -use syncstorage_db_common::{ +use syncserver_db_common::{ error::{DbError, DbErrorKind}, params, results, UserIdentifier, BATCH_LIFETIME, }; diff --git a/syncstorage/src/db/mysql/batch_commit.sql b/syncserver/src/db/mysql/batch_commit.sql similarity index 100% rename from syncstorage/src/db/mysql/batch_commit.sql rename to syncserver/src/db/mysql/batch_commit.sql diff --git a/syncstorage/src/db/mysql/diesel_ext.rs b/syncserver/src/db/mysql/diesel_ext.rs similarity index 100% rename from syncstorage/src/db/mysql/diesel_ext.rs rename to syncserver/src/db/mysql/diesel_ext.rs diff --git a/syncstorage/src/db/mysql/mod.rs b/syncserver/src/db/mysql/mod.rs similarity index 100% rename from syncstorage/src/db/mysql/mod.rs rename to syncserver/src/db/mysql/mod.rs diff --git a/syncstorage/src/db/mysql/models.rs b/syncserver/src/db/mysql/models.rs similarity index 99% rename from syncstorage/src/db/mysql/models.rs rename to syncserver/src/db/mysql/models.rs index e842006e86..fe03ca4a69 100644 --- a/syncstorage/src/db/mysql/models.rs +++ b/syncserver/src/db/mysql/models.rs @@ -15,12 +15,13 @@ use diesel::{ }; #[cfg(test)] use diesel_logger::LoggingConnection; -use syncstorage_db_common::{ +use syncserver_db_common::{ error::{DbError, DbErrorKind}, params, results, util::SyncTimestamp, Db, DbFuture, Sorting, UserIdentifier, DEFAULT_BSO_TTL, }; +use syncstorage_settings::{Quota, DEFAULT_MAX_TOTAL_RECORDS}; use super::{ batch, @@ -30,7 +31,6 @@ use super::{ }; use crate::db; use crate::server::metrics::Metrics; -use crate::settings::{Quota, DEFAULT_MAX_TOTAL_RECORDS}; use crate::web::tags::Tags; pub type Result = std::result::Result; diff --git a/syncstorage/src/db/mysql/pool.rs b/syncserver/src/db/mysql/pool.rs similarity index 97% rename from syncstorage/src/db/mysql/pool.rs rename to syncserver/src/db/mysql/pool.rs index 45130b12ce..1dde134a9d 100644 --- a/syncstorage/src/db/mysql/pool.rs +++ b/syncserver/src/db/mysql/pool.rs @@ -14,14 +14,14 @@ use diesel::{ }; #[cfg(test)] use diesel_logger::LoggingConnection; -use syncstorage_db_common::{error::DbError, Db, DbPool, GetPoolState, PoolState, STD_COLLS}; +use syncserver_db_common::{error::DbError, Db, DbPool, GetPoolState, PoolState, STD_COLLS}; +use syncstorage_settings::{Quota, Settings}; use super::models::{MysqlDb, Result}; #[cfg(test)] use super::test::TestTransactionCustomizer; use crate::db; use crate::server::metrics::Metrics; -use crate::settings::{Quota, Settings}; embed_migrations!(); diff --git a/syncstorage/src/db/mysql/schema.rs b/syncserver/src/db/mysql/schema.rs similarity index 100% rename from syncstorage/src/db/mysql/schema.rs rename to syncserver/src/db/mysql/schema.rs diff --git a/syncstorage/src/db/mysql/test.rs b/syncserver/src/db/mysql/test.rs similarity index 89% rename from syncstorage/src/db/mysql/test.rs rename to syncserver/src/db/mysql/test.rs index a26ba6f2c2..0b425e3e51 100644 --- a/syncstorage/src/db/mysql/test.rs +++ b/syncserver/src/db/mysql/test.rs @@ -9,6 +9,8 @@ use diesel::{ QueryDsl, RunQueryDsl, }; +use syncserver_settings::Settings as SyncserverSettings; +use syncstorage_settings::Settings as SyncstorageSettings; use url::Url; use crate::db::mysql::{ @@ -17,7 +19,6 @@ use crate::db::mysql::{ schema::collections, }; use crate::server::metrics; -use crate::settings::{test_settings, Settings}; #[derive(Debug)] pub struct TestTransactionCustomizer; @@ -28,9 +29,9 @@ impl CustomizeConnection for TestTransactionCustomiz } } -pub fn db(settings: &Settings) -> Result { +pub fn db(settings: &SyncstorageSettings) -> Result { let _ = env_logger::try_init(); - // inherit SYNC_DATABASE_URL from the env + // inherit SYNC_SYNCSTORAGE__DATABASE_URL from the env let pool = MysqlDbPool::new(settings, &metrics::Metrics::noop())?; pool.get_sync() @@ -38,7 +39,7 @@ pub fn db(settings: &Settings) -> Result { #[test] fn static_collection_id() -> Result<()> { - let settings = test_settings(); + let settings = SyncserverSettings::test_settings().syncstorage; if Url::parse(&settings.database_url).unwrap().scheme() != "mysql" { // Skip this test if we're not using mysql return Ok(()); diff --git a/syncstorage/src/db/spanner/BATCH_COMMIT.txt b/syncserver/src/db/spanner/BATCH_COMMIT.txt similarity index 100% rename from syncstorage/src/db/spanner/BATCH_COMMIT.txt rename to syncserver/src/db/spanner/BATCH_COMMIT.txt diff --git a/syncstorage/src/db/spanner/batch.rs b/syncserver/src/db/spanner/batch.rs similarity index 99% rename from syncstorage/src/db/spanner/batch.rs rename to syncserver/src/db/spanner/batch.rs index c13af9da51..c649cfef9d 100644 --- a/syncstorage/src/db/spanner/batch.rs +++ b/syncserver/src/db/spanner/batch.rs @@ -8,7 +8,7 @@ use protobuf::{ well_known_types::{ListValue, Value}, RepeatedField, }; -use syncstorage_db_common::{ +use syncserver_db_common::{ error::{DbError, DbErrorKind}, params, results, util::to_rfc3339, diff --git a/syncstorage/src/db/spanner/batch_commit_insert.sql b/syncserver/src/db/spanner/batch_commit_insert.sql similarity index 100% rename from syncstorage/src/db/spanner/batch_commit_insert.sql rename to syncserver/src/db/spanner/batch_commit_insert.sql diff --git a/syncstorage/src/db/spanner/batch_commit_update.sql b/syncserver/src/db/spanner/batch_commit_update.sql similarity index 100% rename from syncstorage/src/db/spanner/batch_commit_update.sql rename to syncserver/src/db/spanner/batch_commit_update.sql diff --git a/syncstorage/src/db/spanner/batch_index.sql b/syncserver/src/db/spanner/batch_index.sql similarity index 100% rename from syncstorage/src/db/spanner/batch_index.sql rename to syncserver/src/db/spanner/batch_index.sql diff --git a/syncstorage/src/db/spanner/insert_standard_collections.sql b/syncserver/src/db/spanner/insert_standard_collections.sql similarity index 100% rename from syncstorage/src/db/spanner/insert_standard_collections.sql rename to syncserver/src/db/spanner/insert_standard_collections.sql diff --git a/syncstorage/src/db/spanner/macros.rs b/syncserver/src/db/spanner/macros.rs similarity index 100% rename from syncstorage/src/db/spanner/macros.rs rename to syncserver/src/db/spanner/macros.rs diff --git a/syncstorage/src/db/spanner/manager/bb8.rs b/syncserver/src/db/spanner/manager/bb8.rs similarity index 100% rename from syncstorage/src/db/spanner/manager/bb8.rs rename to syncserver/src/db/spanner/manager/bb8.rs diff --git a/syncstorage/src/db/spanner/manager/deadpool.rs b/syncserver/src/db/spanner/manager/deadpool.rs similarity index 95% rename from syncstorage/src/db/spanner/manager/deadpool.rs rename to syncserver/src/db/spanner/manager/deadpool.rs index b2455ec95f..c62a99b7df 100644 --- a/syncstorage/src/db/spanner/manager/deadpool.rs +++ b/syncserver/src/db/spanner/manager/deadpool.rs @@ -3,9 +3,10 @@ use std::{fmt, sync::Arc}; use async_trait::async_trait; use deadpool::managed::{Manager, RecycleError, RecycleResult}; use grpcio::{EnvBuilder, Environment}; -use syncstorage_db_common::error::{DbError, DbErrorKind}; +use syncserver_db_common::error::{DbError, DbErrorKind}; +use syncstorage_settings::Settings; -use crate::{server::metrics::Metrics, settings::Settings}; +use crate::server::metrics::Metrics; use super::session::{create_spanner_session, recycle_spanner_session, SpannerSession}; diff --git a/syncstorage/src/db/spanner/manager/mod.rs b/syncserver/src/db/spanner/manager/mod.rs similarity index 100% rename from syncstorage/src/db/spanner/manager/mod.rs rename to syncserver/src/db/spanner/manager/mod.rs diff --git a/syncstorage/src/db/spanner/manager/session.rs b/syncserver/src/db/spanner/manager/session.rs similarity index 99% rename from syncstorage/src/db/spanner/manager/session.rs rename to syncserver/src/db/spanner/manager/session.rs index b7767d55e1..65a07a9c9f 100644 --- a/syncstorage/src/db/spanner/manager/session.rs +++ b/syncserver/src/db/spanner/manager/session.rs @@ -4,7 +4,7 @@ use google_cloud_rust_raw::spanner::v1::{ }; use grpcio::{CallOption, ChannelBuilder, ChannelCredentials, Environment, MetadataBuilder}; use std::sync::Arc; -use syncstorage_db_common::error::{DbError, DbErrorKind}; +use syncserver_db_common::error::{DbError, DbErrorKind}; use crate::db::{self, spanner::now}; use crate::server::metrics::Metrics; diff --git a/syncstorage/src/db/spanner/mod.rs b/syncserver/src/db/spanner/mod.rs similarity index 100% rename from syncstorage/src/db/spanner/mod.rs rename to syncserver/src/db/spanner/mod.rs diff --git a/syncstorage/src/db/spanner/models.rs b/syncserver/src/db/spanner/models.rs similarity index 99% rename from syncstorage/src/db/spanner/models.rs rename to syncserver/src/db/spanner/models.rs index 91dc87dd23..7ef38a7e88 100644 --- a/syncstorage/src/db/spanner/models.rs +++ b/syncserver/src/db/spanner/models.rs @@ -22,14 +22,16 @@ use protobuf::{ well_known_types::{ListValue, Value}, Message, RepeatedField, }; -use syncstorage_db_common::{ +use syncserver_common::MAX_SPANNER_LOAD_SIZE; +use syncserver_db_common::{ error::{DbError, DbErrorKind}, params, results, util::SyncTimestamp, Db, DbFuture, Sorting, UserIdentifier, DEFAULT_BSO_TTL, FIRST_CUSTOM_COLLECTION_ID, }; +use syncstorage_settings::Quota; -use crate::{db::spanner::now, server::metrics::Metrics, settings::Quota, web::tags::Tags}; +use crate::{db::spanner::now, server::metrics::Metrics, web::tags::Tags}; use super::{ batch, @@ -52,9 +54,6 @@ pub const TOMBSTONE: i32 = 0; pub const PRETOUCH_TS: &str = "0001-01-01T00:00:00.00Z"; -// max load size in bytes -pub const MAX_SPANNER_LOAD_SIZE: usize = 100_000_000; - /// Per session Db metadata #[derive(Debug, Default)] struct SpannerDbSession { @@ -1674,7 +1673,7 @@ impl SpannerDb { // NOTE: Currently this put_bso_async_test impl. is only used during db tests, // see above for the non-tests version pub async fn put_bso_async_test(&self, bso: params::PutBso) -> Result { - use syncstorage_db_common::util::to_rfc3339; + use syncserver_db_common::util::to_rfc3339; let collection_id = self .get_or_create_collection_id_async(&bso.collection) .await?; diff --git a/syncstorage/src/db/spanner/pool.rs b/syncserver/src/db/spanner/pool.rs similarity index 97% rename from syncstorage/src/db/spanner/pool.rs rename to syncserver/src/db/spanner/pool.rs index 688b4429dc..5261192aca 100644 --- a/syncstorage/src/db/spanner/pool.rs +++ b/syncserver/src/db/spanner/pool.rs @@ -2,13 +2,11 @@ use std::{collections::HashMap, fmt, sync::Arc, time::Duration}; use async_trait::async_trait; use bb8::ErrorSink; -use syncstorage_db_common::{error::DbError, Db, DbPool, GetPoolState, PoolState, STD_COLLS}; +use syncserver_db_common::{error::DbError, Db, DbPool, GetPoolState, PoolState, STD_COLLS}; +use syncstorage_settings::{Quota, Settings}; use tokio::sync::RwLock; -use crate::{ - server::metrics::Metrics, - settings::{Quota, Settings}, -}; +use crate::server::metrics::Metrics; pub use super::manager::Conn; use super::{ diff --git a/syncstorage/src/db/spanner/schema.ddl b/syncserver/src/db/spanner/schema.ddl similarity index 100% rename from syncstorage/src/db/spanner/schema.ddl rename to syncserver/src/db/spanner/schema.ddl diff --git a/syncstorage/src/db/spanner/support.rs b/syncserver/src/db/spanner/support.rs similarity index 99% rename from syncstorage/src/db/spanner/support.rs rename to syncserver/src/db/spanner/support.rs index 01d7d37adf..5abeddcc25 100644 --- a/syncstorage/src/db/spanner/support.rs +++ b/syncserver/src/db/spanner/support.rs @@ -15,7 +15,7 @@ use protobuf::{ well_known_types::{ListValue, NullValue, Struct, Value}, RepeatedField, }; -use syncstorage_db_common::{ +use syncserver_db_common::{ error::{DbError, DbErrorKind}, params, results, util::to_rfc3339, diff --git a/syncstorage/src/db/tests/batch.rs b/syncserver/src/db/tests/batch.rs similarity index 96% rename from syncstorage/src/db/tests/batch.rs rename to syncserver/src/db/tests/batch.rs index 795f0f5825..8efa1613a7 100644 --- a/syncstorage/src/db/tests/batch.rs +++ b/syncserver/src/db/tests/batch.rs @@ -1,5 +1,6 @@ use log::debug; -use syncstorage_db_common::{params, results, util::SyncTimestamp, BATCH_LIFETIME}; +use syncserver_db_common::{params, results, util::SyncTimestamp, BATCH_LIFETIME}; +use syncserver_settings::Settings; use super::support::{db_pool, gbso, hid, pbso, postbso, test_db, Result}; @@ -157,7 +158,7 @@ async fn append_commit() -> Result<()> { #[tokio::test] async fn quota_test_create_batch() -> Result<()> { - let mut settings = crate::settings::test_settings(); + let mut settings = Settings::test_settings().syncstorage; if !settings.enable_quota { debug!("[test] Skipping test"); @@ -199,7 +200,7 @@ async fn quota_test_create_batch() -> Result<()> { #[tokio::test] async fn quota_test_append_batch() -> Result<()> { - let mut settings = crate::settings::test_settings(); + let mut settings = Settings::test_settings().syncstorage; if !settings.enable_quota { debug!("[test] Skipping test"); @@ -244,7 +245,7 @@ async fn quota_test_append_batch() -> Result<()> { #[tokio::test] async fn test_append_async_w_null() -> Result<()> { - let settings = crate::settings::test_settings(); + let settings = Settings::test_settings().syncstorage; let pool = db_pool(Some(settings)).await?; let db = test_db(pool.as_ref()).await?; // Remember: TTL is seconds to live, not an expiry date diff --git a/syncstorage/src/db/tests/db.rs b/syncserver/src/db/tests/db.rs similarity index 98% rename from syncstorage/src/db/tests/db.rs rename to syncserver/src/db/tests/db.rs index 0f811d7fd8..346501e149 100644 --- a/syncstorage/src/db/tests/db.rs +++ b/syncserver/src/db/tests/db.rs @@ -3,12 +3,10 @@ use std::collections::HashMap; use lazy_static::lazy_static; use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use syncstorage_db_common::{ - params, util::SyncTimestamp, Sorting, UserIdentifier, DEFAULT_BSO_TTL, -}; +use syncserver_db_common::{params, util::SyncTimestamp, Sorting, UserIdentifier, DEFAULT_BSO_TTL}; +use syncserver_settings::Settings; use super::support::{db_pool, dbso, dbsos, gbso, gbsos, hid, pbso, postbso, test_db, Result}; -use crate::settings::test_settings; // distant future (year 2099) timestamp for tests const MAX_TIMESTAMP: u64 = 4_070_937_600_000; @@ -642,8 +640,8 @@ async fn get_collection_usage() -> Result<()> { let sum = expected.values().sum::(); let total = db.get_storage_usage(hid(uid)).await?; assert_eq!(total, sum as u64); - let settings = test_settings(); - if settings.enable_quota { + let settings = Settings::test_settings(); + if settings.syncstorage.enable_quota { let collection_id = db.get_collection_id("bookmarks".to_owned()).await?; let quota = db .get_quota_usage(params::GetQuotaUsage { @@ -663,9 +661,9 @@ async fn get_collection_usage() -> Result<()> { #[tokio::test] async fn test_quota() -> Result<()> { - let settings = crate::settings::test_settings(); + let settings = Settings::test_settings(); - if !settings.enable_quota { + if !settings.syncstorage.enable_quota { debug!("[test] Skipping test"); return Ok(()); } diff --git a/syncstorage/src/db/tests/mod.rs b/syncserver/src/db/tests/mod.rs similarity index 100% rename from syncstorage/src/db/tests/mod.rs rename to syncserver/src/db/tests/mod.rs diff --git a/syncstorage/src/db/tests/support.rs b/syncserver/src/db/tests/support.rs similarity index 77% rename from syncstorage/src/db/tests/support.rs rename to syncserver/src/db/tests/support.rs index 71d1bd4702..8822c5f985 100644 --- a/syncstorage/src/db/tests/support.rs +++ b/syncserver/src/db/tests/support.rs @@ -1,31 +1,28 @@ use std::str::FromStr; -use syncstorage_db_common::{params, util::SyncTimestamp, Db, Sorting, UserIdentifier}; +use syncserver_db_common::{params, util::SyncTimestamp, Db, Sorting, UserIdentifier}; +use syncserver_settings::Settings as SyncserverSettings; +use syncstorage_settings::Settings as SyncstorageSettings; use crate::db::DbPool; use crate::error::ApiResult; -use crate::{ - db::pool_from_settings, - error::ApiError, - server::metrics, - settings::{test_settings, Settings}, -}; +use crate::{db::pool_from_settings, error::ApiError, server::metrics}; pub type Result = std::result::Result; #[cfg(test)] -pub async fn db_pool(settings: Option) -> Result> { +pub async fn db_pool(settings: Option) -> Result> { let _ = env_logger::try_init(); - // The default for SYNC_DATABASE_USE_TEST_TRANSACTIONS is false, - // but we want the mysql default to be true, so let's check explicitly - // the env var because we can't rely on the default value or the env - // var passed through to settings. - let use_test_transactions = std::env::var("SYNC_DATABASE_USE_TEST_TRANSACTIONS") + // The default for SYNC_SYNCSTORAGE__DATABASE_USE_TEST_TRANSACTIONS is + // false, but we want the mysql default to be true, so let's check + // explicitly the env var because we can't rely on the default value or + // the env var passed through to settings. + let use_test_transactions = std::env::var("SYNC_SYNCSTORAGE__DATABASE_USE_TEST_TRANSACTIONS") .unwrap_or_else(|_| "true".to_string()) .eq("true"); - // inherit SYNC_DATABASE_URL from the env - let mut settings = settings.unwrap_or_else(test_settings); + // inherit SYNC_SYNCSTORAGE__DATABASE_URL from the env + let mut settings = settings.unwrap_or_else(|| SyncserverSettings::test_settings().syncstorage); settings.database_use_test_transactions = use_test_transactions; let metrics = metrics::Metrics::noop(); diff --git a/syncstorage/src/db/transaction.rs b/syncserver/src/db/transaction.rs similarity index 98% rename from syncstorage/src/db/transaction.rs rename to syncserver/src/db/transaction.rs index de43c0f688..bd5450686c 100644 --- a/syncstorage/src/db/transaction.rs +++ b/syncserver/src/db/transaction.rs @@ -9,7 +9,8 @@ use actix_web::web::Data; use actix_web::{FromRequest, HttpRequest, HttpResponse}; use futures::future::LocalBoxFuture; use futures::FutureExt; -use syncstorage_db_common::{params, Db, DbPool, UserIdentifier}; +use syncserver_common::X_LAST_MODIFIED; +use syncserver_db_common::{params, Db, DbPool, UserIdentifier}; use crate::db::results::ConnectionInfo; use crate::error::{ApiError, ApiErrorKind}; @@ -19,7 +20,6 @@ use crate::web::extractors::{ BsoParam, CollectionParam, HawkIdentifier, PreConditionHeader, PreConditionHeaderOpt, }; use crate::web::tags::Tags; -use crate::web::X_LAST_MODIFIED; #[derive(Clone)] pub struct DbTransactionPool { @@ -134,7 +134,8 @@ impl DbTransactionPool { self.collection.clone(), self.bso_opt.clone(), ) - .await?; + .await + .map_err(ApiError::from)?; if let Some(precondition) = &self.precondition.opt { let status = match precondition { diff --git a/syncstorage/src/error.rs b/syncserver/src/error.rs similarity index 98% rename from syncstorage/src/error.rs rename to syncserver/src/error.rs index 37d4754c96..c81ea604d4 100644 --- a/syncstorage/src/error.rs +++ b/syncserver/src/error.rs @@ -21,8 +21,8 @@ use serde::{ Serialize, }; -use syncstorage_common::{from_error, impl_fmt_display, ReportableError}; -use syncstorage_db_common::error::DbError; +use syncserver_common::{from_error, impl_fmt_display, ReportableError}; +use syncserver_db_common::error::DbError; use thiserror::Error; use crate::web::error::{HawkError, ValidationError}; diff --git a/syncstorage/src/lib.rs b/syncserver/src/lib.rs similarity index 94% rename from syncstorage/src/lib.rs rename to syncserver/src/lib.rs index 6e9aaa9c42..90284b151d 100644 --- a/syncstorage/src/lib.rs +++ b/syncserver/src/lib.rs @@ -15,6 +15,5 @@ pub mod error; pub mod db; pub mod logging; pub mod server; -pub mod settings; pub mod tokenserver; pub mod web; diff --git a/syncstorage/src/logging.rs b/syncserver/src/logging.rs similarity index 100% rename from syncstorage/src/logging.rs rename to syncserver/src/logging.rs diff --git a/syncstorage/src/main.rs b/syncserver/src/main.rs similarity index 89% rename from syncstorage/src/main.rs rename to syncserver/src/main.rs index 37c02cf5d4..d247e4efd8 100644 --- a/syncstorage/src/main.rs +++ b/syncserver/src/main.rs @@ -8,7 +8,8 @@ use docopt::Docopt; use serde::Deserialize; use logging::init_logging; -use syncstorage::{logging, server, settings}; +use syncserver::{logging, server}; +use syncserver_settings::Settings; const USAGE: &str = " Usage: syncstorage [options] @@ -28,7 +29,7 @@ async fn main() -> Result<(), Box> { let args: Args = Docopt::new(USAGE) .and_then(|d| d.deserialize()) .unwrap_or_else(|e| e.exit()); - let settings = settings::Settings::with_env_and_config_file(&args.flag_config)?; + let settings = Settings::with_env_and_config_file(args.flag_config.as_deref())?; init_logging(!settings.human_logs).expect("Logging failed to initialize"); debug!("Starting up..."); // Set SENTRY_DSN environment variable to enable Sentry. @@ -47,7 +48,7 @@ async fn main() -> Result<(), Box> { // Setup and run the server let banner = settings.banner(); - let server = if settings.disable_syncstorage { + let server = if !settings.syncstorage.enabled { server::Server::tokenserver_only_with_settings(settings) .await .unwrap() diff --git a/syncstorage/src/server/metrics.rs b/syncserver/src/server/metrics.rs similarity index 100% rename from syncstorage/src/server/metrics.rs rename to syncserver/src/server/metrics.rs diff --git a/syncstorage/src/server/mod.rs b/syncserver/src/server/mod.rs similarity index 88% rename from syncstorage/src/server/mod.rs rename to syncserver/src/server/mod.rs index 4e80d3a5b5..83c276c2f3 100644 --- a/syncstorage/src/server/mod.rs +++ b/syncserver/src/server/mod.rs @@ -2,18 +2,23 @@ use std::{sync::Arc, time::Duration}; +use actix_cors::Cors; use actix_web::{ - dev, http::header::LOCATION, http::StatusCode, middleware::errhandlers::ErrorHandlers, web, - App, HttpRequest, HttpResponse, HttpServer, + dev, + http::StatusCode, + http::{header::LOCATION, Method}, + middleware::errhandlers::ErrorHandlers, + web, App, HttpRequest, HttpResponse, HttpServer, }; use cadence::StatsdClient; -use syncstorage_db_common::DbPool; +use syncserver_db_common::DbPool; +use syncserver_settings::Settings; +use syncstorage_settings::{Deadman, ServerLimits}; use tokio::sync::RwLock; use crate::db::{pool_from_settings, spawn_pool_periodic_reporter}; use crate::error::ApiError; use crate::server::metrics::Metrics; -use crate::settings::{Deadman, ServerLimits, Settings}; use crate::tokenserver; use crate::web::{handlers, middleware}; @@ -230,19 +235,19 @@ impl Server { pub async fn with_settings(settings: Settings) -> Result { let settings_copy = settings.clone(); let metrics = metrics::metrics_from_opts( - &settings.statsd_label, + &settings.syncstorage.statsd_label, settings.statsd_host.as_deref(), settings.statsd_port, )?; let host = settings.host.clone(); let port = settings.port; - let db_pool = pool_from_settings(&settings, &Metrics::from(&metrics)).await?; - let deadman = Arc::new(RwLock::new(Deadman::from(&settings))); - let limits = Arc::new(settings.limits); + let deadman = Arc::new(RwLock::new(Deadman::from(&settings.syncstorage))); + let db_pool = pool_from_settings(&settings.syncstorage, &Metrics::from(&metrics)).await?; + let limits = Arc::new(settings.syncstorage.limits); let limits_json = serde_json::to_string(&*limits).expect("ServerLimits failed to serialize"); let secrets = Arc::new(settings.master_secret); - let quota_enabled = settings.enable_quota; + let quota_enabled = settings.syncstorage.enable_quota; let actix_keep_alive = settings.actix_keep_alive; let tokenserver_state = if settings.tokenserver.enabled { let state = tokenserver::ServerState::from_settings( @@ -283,7 +288,7 @@ impl Server { tokenserver_state.clone(), Arc::clone(&secrets), limits, - settings_copy.build_cors() + build_cors(&settings_copy) ) }); @@ -324,7 +329,7 @@ impl Server { build_app_without_syncstorage!( Some(tokenserver_state.clone()), Arc::clone(&secrets), - settings_copy.build_cors() + build_cors(&settings_copy) ) }); @@ -335,3 +340,34 @@ impl Server { Ok(server) } } + +pub fn build_cors(settings: &Settings) -> Cors { + // Followed by the "official middleware" so they run first. + // actix is getting increasingly tighter about CORS headers. Our server is + // not a huge risk but does deliver XHR JSON content. + // For now, let's be permissive and use NGINX (the wrapping server) + // for finer grained specification. + let mut cors = Cors::default(); + + if let Some(allowed_origin) = &settings.cors_allowed_origin { + cors = cors.allowed_origin(allowed_origin); + } + + if let Some(allowed_methods) = &settings.cors_allowed_methods { + let mut methods = vec![]; + for method_string in allowed_methods { + let method = Method::from_bytes(method_string.as_bytes()).unwrap(); + methods.push(method); + } + cors = cors.allowed_methods(methods); + } + if let Some(allowed_headers) = &settings.cors_allowed_headers { + cors = cors.allowed_headers(allowed_headers); + } + + if let Some(max_age) = &settings.cors_max_age { + cors = cors.max_age(*max_age); + } + + cors +} diff --git a/syncstorage/src/server/test.rs b/syncserver/src/server/test.rs similarity index 94% rename from syncstorage/src/server/test.rs rename to syncserver/src/server/test.rs index 00d12b6041..36a0056684 100644 --- a/syncstorage/src/server/test.rs +++ b/syncserver/src/server/test.rs @@ -9,25 +9,26 @@ use actix_web::{ }; use chrono::offset::Utc; use hawk::{self, Credentials, Key, RequestBuilder}; -use hkdf::Hkdf; use hmac::{Hmac, Mac, NewMac}; use lazy_static::lazy_static; use rand::{thread_rng, Rng}; use serde::de::DeserializeOwned; use serde_json::json; use sha2::Sha256; -use syncstorage_db_common::{ +use syncserver_common::{self, X_LAST_MODIFIED}; +use syncserver_db_common::{ params, results::{DeleteBso, GetBso, PostBsos, PutBso}, util::SyncTimestamp, }; +use syncserver_settings::{Secrets, Settings}; +use syncstorage_settings::ServerLimits; use super::*; use crate::build_app; use crate::db::pool_from_settings; -use crate::settings::{test_settings, Secrets, ServerLimits}; use crate::tokenserver; -use crate::web::{auth::HawkPayload, extractors::BsoBody, X_LAST_MODIFIED}; +use crate::web::{auth::HawkPayload, extractors::BsoBody}; lazy_static! { static ref SERVER_LIMITS: Arc = Arc::new(ServerLimits::default()); @@ -44,7 +45,7 @@ const TEST_PORT: u16 = 8080; /// persist to the db between requests. This can be overridden per test via /// customizing the settings fn get_test_settings() -> Settings { - let mut settings = test_settings(); + let mut settings = Settings::test_settings(); let treq = test::TestRequest::with_uri("/").to_http_request(); let port = treq.uri().port_u16().unwrap_or(TEST_PORT); // Make sure that our poolsize is >= the @@ -59,22 +60,22 @@ fn get_test_settings() -> Settings { .expect("Could not get pool_size in get_test_settings"); settings.port = port; settings.host = host; - settings.database_pool_max_size = pool_size + 1; + settings.syncstorage.database_pool_max_size = pool_size + 1; settings } async fn get_test_state(settings: &Settings) -> ServerState { let metrics = Metrics::sink(); ServerState { - db_pool: pool_from_settings(settings, &Metrics::from(&metrics)) + db_pool: pool_from_settings(&settings.syncstorage, &Metrics::from(&metrics)) .await .expect("Could not get db_pool in get_test_state"), limits: Arc::clone(&SERVER_LIMITS), limits_json: serde_json::to_string(&**SERVER_LIMITS).unwrap(), metrics: Box::new(metrics), port: settings.port, - quota_enabled: settings.enable_quota, - deadman: Arc::new(RwLock::new(Deadman::from(settings))), + quota_enabled: settings.syncstorage.enable_quota, + deadman: Arc::new(RwLock::new(Deadman::from(&settings.syncstorage))), } } @@ -88,13 +89,13 @@ macro_rules! init_app { ($settings:expr) => { async { crate::logging::init_logging(false).unwrap(); - let limits = Arc::new($settings.limits.clone()); + let limits = Arc::new($settings.syncstorage.limits.clone()); test::init_service(build_app!( get_test_state(&$settings).await, None::, Arc::clone(&SECRETS), limits, - $settings.build_cors() + build_cors(&$settings) )) .await } @@ -158,11 +159,12 @@ fn create_hawk_header(method: &str, port: u16, path: &str) -> String { id.extend(payload.as_bytes()); id.extend_from_slice(&signature); let id = base64::encode_config(&id, base64::URL_SAFE); - let token_secret = hkdf_expand_32( + let token_secret = syncserver_common::hkdf_expand_32( format!("services.mozilla.com/tokenlib/v1/derive/{}", id).as_bytes(), Some(b"wibble"), &SECRETS.master_secret, - ); + ) + .expect("hkdf_expand_32 failed in create_hawk_header"); let token_secret = base64::encode_config(&token_secret, base64::URL_SAFE); let request = RequestBuilder::new(method, host, port, path).request(); let credentials = Credentials { @@ -176,14 +178,6 @@ fn create_hawk_header(method: &str, port: u16, path: &str) -> String { format!("Hawk {}", header) } -fn hkdf_expand_32(info: &[u8], salt: Option<&[u8]>, key: &[u8]) -> [u8; 32] { - let mut result = [0u8; 32]; - let hkdf = Hkdf::::new(salt, key); - hkdf.expand(info, &mut result) - .expect("Could not hkdf.expand in hkdf_expand_32"); - result -} - async fn test_endpoint( method: http::Method, path: &str, @@ -212,13 +206,13 @@ where T: DeserializeOwned, { let settings = get_test_settings(); - let limits = Arc::new(settings.limits.clone()); + let limits = Arc::new(settings.syncstorage.limits.clone()); let mut app = test::init_service(build_app!( get_test_state(&settings).await, None::, Arc::clone(&SECRETS), limits, - settings.build_cors() + build_cors(&settings) )) .await; @@ -253,13 +247,13 @@ async fn test_endpoint_with_body( body: serde_json::Value, ) -> Bytes { let settings = get_test_settings(); - let limits = Arc::new(settings.limits.clone()); + let limits = Arc::new(settings.syncstorage.limits.clone()); let mut app = test::init_service(build_app!( get_test_state(&settings).await, None::, Arc::clone(&SECRETS), limits, - settings.build_cors() + build_cors(&settings) )) .await; let req = create_request(method, path, None, Some(body)).to_request(); @@ -659,11 +653,11 @@ async fn info_configuration_xlm() { #[actix_rt::test] async fn overquota() { let mut settings = get_test_settings(); - settings.enable_quota = true; - settings.enforce_quota = true; - settings.limits.max_quota_limit = 5; + settings.syncstorage.enable_quota = true; + settings.syncstorage.enforce_quota = true; + settings.syncstorage.limits.max_quota_limit = 5; // persist the db across requests - settings.database_use_test_transactions = false; + settings.syncstorage.database_use_test_transactions = false; let mut app = init_app!(settings).await; // Clear out any data that's already in the store. @@ -726,7 +720,7 @@ async fn lbheartbeat_max_pool_size_check() { use actix_web::web::Buf; let mut settings = get_test_settings(); - settings.database_pool_max_size = 10; + settings.syncstorage.database_pool_max_size = 10; let mut app = init_app!(settings).await; @@ -781,8 +775,8 @@ async fn lbheartbeat_max_pool_size_check() { #[actix_rt::test] async fn lbheartbeat_ttl_check() { let mut settings = get_test_settings(); - settings.lbheartbeat_ttl = Some(2); - settings.lbheartbeat_ttl_jitter = 60; + settings.syncstorage.lbheartbeat_ttl = Some(2); + settings.syncstorage.lbheartbeat_ttl_jitter = 60; let mut app = init_app!(settings).await; diff --git a/syncstorage/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs similarity index 100% rename from syncstorage/src/server/user_agent.rs rename to syncserver/src/server/user_agent.rs diff --git a/syncstorage/src/tokenserver/README.md b/syncserver/src/tokenserver/README.md similarity index 100% rename from syncstorage/src/tokenserver/README.md rename to syncserver/src/tokenserver/README.md diff --git a/syncstorage/src/tokenserver/auth/browserid.rs b/syncserver/src/tokenserver/auth/browserid.rs similarity index 99% rename from syncstorage/src/tokenserver/auth/browserid.rs rename to syncserver/src/tokenserver/auth/browserid.rs index acf81dd287..8be9bb0f0a 100644 --- a/syncstorage/src/tokenserver/auth/browserid.rs +++ b/syncserver/src/tokenserver/auth/browserid.rs @@ -2,9 +2,9 @@ use async_trait::async_trait; use reqwest::{Client as ReqwestClient, StatusCode}; use serde::{de::Deserializer, Deserialize, Serialize}; use tokenserver_common::error::{ErrorLocation, TokenType, TokenserverError}; +use tokenserver_settings::Settings; use super::VerifyToken; -use crate::tokenserver::settings::Settings; use std::{convert::TryFrom, time::Duration}; diff --git a/syncstorage/src/tokenserver/auth/mod.rs b/syncserver/src/tokenserver/auth/mod.rs similarity index 100% rename from syncstorage/src/tokenserver/auth/mod.rs rename to syncserver/src/tokenserver/auth/mod.rs diff --git a/syncstorage/src/tokenserver/auth/oauth.rs b/syncserver/src/tokenserver/auth/oauth.rs similarity index 99% rename from syncstorage/src/tokenserver/auth/oauth.rs rename to syncserver/src/tokenserver/auth/oauth.rs index edf3e90ab4..bfd6dbf066 100644 --- a/syncstorage/src/tokenserver/auth/oauth.rs +++ b/syncserver/src/tokenserver/auth/oauth.rs @@ -7,10 +7,10 @@ use pyo3::{ use serde::{Deserialize, Serialize}; use serde_json; use tokenserver_common::error::TokenserverError; +use tokenserver_settings::{Jwk, Settings}; use tokio::time; use super::VerifyToken; -use crate::tokenserver::settings::{Jwk, Settings}; use core::time::Duration; use std::convert::TryFrom; diff --git a/syncstorage/src/tokenserver/auth/verify.py b/syncserver/src/tokenserver/auth/verify.py similarity index 100% rename from syncstorage/src/tokenserver/auth/verify.py rename to syncserver/src/tokenserver/auth/verify.py diff --git a/syncstorage/src/tokenserver/db/mock.rs b/syncserver/src/tokenserver/db/mock.rs similarity index 98% rename from syncstorage/src/tokenserver/db/mock.rs rename to syncserver/src/tokenserver/db/mock.rs index ae6441074f..871e6a4e0f 100644 --- a/syncstorage/src/tokenserver/db/mock.rs +++ b/syncserver/src/tokenserver/db/mock.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use futures::future; -use syncstorage_db_common::{error::DbError, GetPoolState, PoolState}; +use syncserver_db_common::{error::DbError, GetPoolState, PoolState}; use super::models::{Db, DbFuture}; use super::params; diff --git a/syncstorage/src/tokenserver/db/mod.rs b/syncserver/src/tokenserver/db/mod.rs similarity index 100% rename from syncstorage/src/tokenserver/db/mod.rs rename to syncserver/src/tokenserver/db/mod.rs diff --git a/syncstorage/src/tokenserver/db/models.rs b/syncserver/src/tokenserver/db/models.rs similarity index 99% rename from syncstorage/src/tokenserver/db/models.rs rename to syncserver/src/tokenserver/db/models.rs index dbd6866a4c..6b1f5ae3f4 100644 --- a/syncstorage/src/tokenserver/db/models.rs +++ b/syncserver/src/tokenserver/db/models.rs @@ -8,7 +8,7 @@ use diesel::{ #[cfg(test)] use diesel_logger::LoggingConnection; use futures::future::LocalBoxFuture; -use syncstorage_db_common::error::DbError; +use syncserver_db_common::error::DbError; use std::{ result, @@ -782,7 +782,8 @@ mod tests { use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH}; - use crate::settings::test_settings; + use syncserver_settings::Settings; + use crate::tokenserver::db::pool::{DbPool, TokenserverPool}; #[tokio::test] @@ -2055,13 +2056,10 @@ mod tests { async fn db_pool() -> DbResult { let _ = env_logger::try_init(); - let tokenserver_settings = test_settings().tokenserver; + let mut settings = Settings::test_settings().tokenserver; + settings.run_migrations = true; let use_test_transactions = true; - TokenserverPool::new( - &tokenserver_settings, - &Metrics::noop(), - use_test_transactions, - ) + TokenserverPool::new(&settings, &Metrics::noop(), use_test_transactions) } } diff --git a/syncstorage/src/tokenserver/db/params.rs b/syncserver/src/tokenserver/db/params.rs similarity index 100% rename from syncstorage/src/tokenserver/db/params.rs rename to syncserver/src/tokenserver/db/params.rs diff --git a/syncstorage/src/tokenserver/db/pool.rs b/syncserver/src/tokenserver/db/pool.rs similarity index 97% rename from syncstorage/src/tokenserver/db/pool.rs rename to syncserver/src/tokenserver/db/pool.rs index aa8de639be..d3c8480dcd 100644 --- a/syncstorage/src/tokenserver/db/pool.rs +++ b/syncserver/src/tokenserver/db/pool.rs @@ -1,17 +1,18 @@ +use std::time::Duration; + use async_trait::async_trait; use diesel::{ mysql::MysqlConnection, r2d2::{ConnectionManager, Pool}, }; use diesel_logger::LoggingConnection; -use std::time::Duration; -use syncstorage_db_common::{error::DbError, GetPoolState, PoolState}; +use syncserver_db_common::{error::DbError, GetPoolState, PoolState}; +use tokenserver_settings::Settings; use super::models::{Db, DbResult, TokenserverDb}; use crate::db; use crate::diesel::Connection; use crate::server::metrics::Metrics; -use crate::tokenserver::settings::Settings; #[cfg(test)] use crate::db::mysql::TestTransactionCustomizer; diff --git a/syncstorage/src/tokenserver/db/results.rs b/syncserver/src/tokenserver/db/results.rs similarity index 100% rename from syncstorage/src/tokenserver/db/results.rs rename to syncserver/src/tokenserver/db/results.rs diff --git a/syncstorage/src/tokenserver/extractors.rs b/syncserver/src/tokenserver/extractors.rs similarity index 98% rename from syncstorage/src/tokenserver/extractors.rs rename to syncserver/src/tokenserver/extractors.rs index 0c21044e83..c0e72d58b2 100644 --- a/syncstorage/src/tokenserver/extractors.rs +++ b/syncserver/src/tokenserver/extractors.rs @@ -19,13 +19,17 @@ use lazy_static::lazy_static; use regex::Regex; use serde::Deserialize; use sha2::Sha256; -use tokenserver_common::error::{ErrorLocation, TokenserverError}; +use syncserver_settings::Secrets; +use tokenserver_common::{ + error::{ErrorLocation, TokenserverError}, + NodeType, +}; use super::{ db::{models::Db, params, pool::DbPool, results}, - LogItemsMutator, NodeType, ServerState, TokenserverMetrics, + LogItemsMutator, ServerState, TokenserverMetrics, }; -use crate::{server::metrics, settings::Secrets, web::tags::Tags}; +use crate::{server::metrics, web::tags::Tags}; lazy_static! { static ref CLIENT_STATE_REGEX: Regex = Regex::new("^[a-zA-Z0-9._-]{1,32}$").unwrap(); @@ -702,9 +706,11 @@ mod tests { use futures::executor::block_on; use lazy_static::lazy_static; use serde_json; + use syncserver_settings::Settings as GlobalSettings; + use syncstorage_settings::ServerLimits; + use tokenserver_settings::Settings as TokenserverSettings; use crate::server::metrics; - use crate::settings::{Secrets, ServerLimits, Settings}; use crate::tokenserver::{ auth::{browserid, oauth, MockVerifier}, db::mock::MockDbPool as MockTokenserverPool, @@ -1323,7 +1329,8 @@ mod tests { oauth_verifier: MockVerifier, browserid_verifier: MockVerifier, ) -> ServerState { - let settings = Settings::default(); + let syncserver_settings = GlobalSettings::default(); + let tokenserver_settings = TokenserverSettings::default(); ServerState { fxa_email_domain: "test.com".to_owned(), @@ -1335,9 +1342,9 @@ mod tests { node_type: NodeType::default(), metrics: Box::new( metrics::metrics_from_opts( - &settings.tokenserver.statsd_label, - settings.statsd_host.as_deref(), - settings.statsd_port, + &tokenserver_settings.statsd_label, + syncserver_settings.statsd_host.as_deref(), + syncserver_settings.statsd_port, ) .unwrap(), ), diff --git a/syncstorage/src/tokenserver/handlers.rs b/syncserver/src/tokenserver/handlers.rs similarity index 99% rename from syncstorage/src/tokenserver/handlers.rs rename to syncserver/src/tokenserver/handlers.rs index 81af88f887..855efa5114 100644 --- a/syncstorage/src/tokenserver/handlers.rs +++ b/syncserver/src/tokenserver/handlers.rs @@ -6,7 +6,7 @@ use std::{ use actix_web::{http::StatusCode, Error, HttpResponse}; use serde::Serialize; use serde_json::Value; -use tokenserver_common::error::TokenserverError; +use tokenserver_common::{error::TokenserverError, NodeType}; use super::{ auth::{MakeTokenPlaintext, Tokenlib, TokenserverOrigin}, @@ -15,7 +15,7 @@ use super::{ params::{GetNodeId, PostUser, PutUser, ReplaceUsers}, }, extractors::TokenserverRequest, - NodeType, TokenserverMetrics, + TokenserverMetrics, }; #[derive(Debug, Serialize)] diff --git a/syncstorage/src/tokenserver/logging.rs b/syncserver/src/tokenserver/logging.rs similarity index 100% rename from syncstorage/src/tokenserver/logging.rs rename to syncserver/src/tokenserver/logging.rs diff --git a/syncstorage/src/tokenserver/migrations/2021-07-16-001122_init/down.sql b/syncserver/src/tokenserver/migrations/2021-07-16-001122_init/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-07-16-001122_init/down.sql rename to syncserver/src/tokenserver/migrations/2021-07-16-001122_init/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-07-16-001122_init/up.sql b/syncserver/src/tokenserver/migrations/2021-07-16-001122_init/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-07-16-001122_init/up.sql rename to syncserver/src/tokenserver/migrations/2021-07-16-001122_init/up.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-08-03-234845_populate_services/down.sql b/syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-08-03-234845_populate_services/down.sql rename to syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-08-03-234845_populate_services/up.sql b/syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-08-03-234845_populate_services/up.sql rename to syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/up.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql b/syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql b/syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/down.sql b/syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/down.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/up.sql b/syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/up.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/up.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-142746_add_indexes/down.sql b/syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-142746_add_indexes/down.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-142746_add_indexes/up.sql b/syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-142746_add_indexes/up.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/up.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql b/syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql b/syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql b/syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql b/syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql rename to syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-12-22-160451_remove_services/down.sql b/syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/down.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-12-22-160451_remove_services/down.sql rename to syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/down.sql diff --git a/syncstorage/src/tokenserver/migrations/2021-12-22-160451_remove_services/up.sql b/syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/up.sql similarity index 100% rename from syncstorage/src/tokenserver/migrations/2021-12-22-160451_remove_services/up.sql rename to syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/up.sql diff --git a/syncstorage/src/tokenserver/mod.rs b/syncserver/src/tokenserver/mod.rs similarity index 94% rename from syncstorage/src/tokenserver/mod.rs rename to syncserver/src/tokenserver/mod.rs index 9892390910..e624160649 100644 --- a/syncstorage/src/tokenserver/mod.rs +++ b/syncserver/src/tokenserver/mod.rs @@ -3,14 +3,15 @@ pub mod db; pub mod extractors; pub mod handlers; pub mod logging; -pub mod settings; use actix_web::{dev::RequestHead, http::header::USER_AGENT, HttpRequest}; use cadence::StatsdClient; use serde::{ ser::{SerializeMap, Serializer}, - Deserialize, Serialize, + Serialize, }; +use tokenserver_common::NodeType; +use tokenserver_settings::Settings; use crate::{ error::ApiError, @@ -21,7 +22,6 @@ use db::{ params, pool::{DbPool, TokenserverPool}, }; -use settings::Settings; use std::{collections::HashMap, convert::TryFrom, fmt}; @@ -83,20 +83,6 @@ impl ServerState { pub struct TokenserverMetrics(Metrics); -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub enum NodeType { - #[serde(rename = "mysql")] - MySql, - #[serde(rename = "spanner")] - Spanner, -} - -impl Default for NodeType { - fn default() -> Self { - Self::Spanner - } -} - #[derive(Clone, Debug)] struct LogItems(HashMap); diff --git a/syncstorage/src/web/auth.rs b/syncserver/src/web/auth.rs similarity index 91% rename from syncstorage/src/web/auth.rs rename to syncserver/src/web/auth.rs index 59c5519485..12523ea8e7 100644 --- a/syncstorage/src/web/auth.rs +++ b/syncserver/src/web/auth.rs @@ -10,10 +10,11 @@ use std::convert::TryInto; use chrono::offset::Utc; use hawk::{self, Header as HawkHeader, Key, RequestBuilder}; -use hkdf::Hkdf; use hmac::{Hmac, Mac, NewMac}; use serde::{Deserialize, Serialize}; use sha2::Sha256; +use syncserver_common; +use syncserver_settings::Secrets; use time::Duration; use actix_web::dev::ConnectionInfo; @@ -25,7 +26,6 @@ use super::{ }; use crate::error::{ApiErrorKind, ApiResult}; use crate::label; -use crate::settings::Secrets; use crate::tokenserver::auth::TokenserverOrigin; /// A parsed and authenticated JSON payload @@ -84,11 +84,12 @@ impl HawkPayload { let payload = HawkPayload::extract_and_validate(id, secrets, expiry)?; - let token_secret = hkdf_expand_32( + let token_secret = syncserver_common::hkdf_expand_32( format!("services.mozilla.com/tokenlib/v1/derive/{}", id).as_bytes(), Some(payload.salt.as_bytes()), &secrets.master_secret, - )?; + ) + .map_err(|e| ApiErrorKind::Internal(format!("HKDF Error: {:?}", e)))?; let token_secret = base64::encode_config(&token_secret, base64::URL_SAFE); let request = RequestBuilder::new(method, host, port, path).request(); @@ -195,16 +196,6 @@ impl HawkPayload { } } -/// Helper function for [HKDF](https://tools.ietf.org/html/rfc5869) expansion to 32 bytes. -pub fn hkdf_expand_32(info: &[u8], salt: Option<&[u8]>, key: &[u8]) -> ApiResult<[u8; 32]> { - let mut result = [0u8; 32]; - let hkdf = Hkdf::::new(salt, key); - // This unwrap will never panic because 32 bytes is a valid size for Hkdf - hkdf.expand(info, &mut result) - .map_err(|e| ApiErrorKind::Internal(format!("HKDF Error: {:?}", e)))?; - Ok(result) -} - /// Helper function for [HMAC](https://tools.ietf.org/html/rfc2104) verification. fn verify_hmac(info: &[u8], key: &[u8], expected: &[u8]) -> ApiResult<()> { let mut hmac = Hmac::::new_from_slice(key)?; @@ -215,7 +206,6 @@ fn verify_hmac(info: &[u8], key: &[u8], expected: &[u8]) -> ApiResult<()> { #[cfg(test)] mod tests { use super::{HawkPayload, Secrets}; - use crate::settings::Settings; #[test] fn valid_header() { @@ -227,7 +217,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -255,7 +245,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -275,7 +265,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -292,7 +282,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -331,7 +321,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -348,7 +338,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64, ); @@ -366,7 +356,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -384,7 +374,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -402,7 +392,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -420,7 +410,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -441,7 +431,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -459,7 +449,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -477,7 +467,7 @@ mod tests { &fixture.request.path, &fixture.request.host, fixture.request.port, - &fixture.settings.master_secret, + &fixture.master_secret, fixture.expected.expires.round() as u64 - 1, ); @@ -488,7 +478,7 @@ mod tests { struct TestFixture { pub header: HawkHeader, pub request: Request, - pub settings: Settings, + pub master_secret: Secrets, pub expected: HawkPayload, } @@ -507,16 +497,7 @@ mod tests { "localhost", 5000, ), - settings: Settings { - debug: false, - port: 0, - host: "127.0.0.1".to_string(), - database_url: "".to_string(), - database_use_test_transactions: false, - limits: Default::default(), - master_secret: Secrets::new("Ted Koppel is a robot").unwrap(), - ..Default::default() - }, + master_secret: Secrets::new("Ted Koppel is a robot").unwrap(), expected: HawkPayload { expires: 1_884_968_439.0, node: "http://localhost:5000".to_string(), diff --git a/syncstorage/src/web/error.rs b/syncserver/src/web/error.rs similarity index 99% rename from syncstorage/src/web/error.rs rename to syncserver/src/web/error.rs index 5f64321e78..27aa6861a0 100644 --- a/syncstorage/src/web/error.rs +++ b/syncserver/src/web/error.rs @@ -13,7 +13,7 @@ use serde::{ Serialize, }; use serde_json::{Error as JsonError, Value}; -use syncstorage_common::{from_error, impl_fmt_display}; +use syncserver_common::{from_error, impl_fmt_display}; use super::extractors::RequestErrorLocation; use crate::error::{ApiError, WeaveError}; diff --git a/syncstorage/src/web/extractors.rs b/syncserver/src/web/extractors.rs similarity index 98% rename from syncstorage/src/web/extractors.rs rename to syncserver/src/web/extractors.rs index 9c8cfc6ee7..4fd3636ab3 100644 --- a/syncstorage/src/web/extractors.rs +++ b/syncserver/src/web/extractors.rs @@ -17,6 +17,7 @@ use actix_web::{ }; use futures::future::{self, FutureExt, LocalBoxFuture, Ready, TryFutureExt}; +use syncserver_settings::Secrets; use lazy_static::lazy_static; use mime::STAR_STAR; @@ -26,7 +27,8 @@ use serde::{ Deserialize, Serialize, }; use serde_json::Value; -use syncstorage_db_common::{ +use syncserver_common::X_WEAVE_RECORDS; +use syncserver_db_common::{ params::{self, PostCollectionBso}, util::SyncTimestamp, DbPool, Sorting, UserIdentifier, @@ -37,13 +39,12 @@ use crate::db::transaction::DbTransactionPool; use crate::error::{ApiError, ApiErrorKind}; use crate::label; use crate::server::{metrics, ServerState, BSO_ID_REGEX, COLLECTION_ID_REGEX}; -use crate::settings::Secrets; use crate::tokenserver::auth::TokenserverOrigin; use crate::web::{ auth::HawkPayload, error::{HawkErrorKind, ValidationErrorKind}, tags::Tags, - DOCKER_FLOW_ENDPOINTS, X_WEAVE_RECORDS, + DOCKER_FLOW_ENDPOINTS, }; const BATCH_MAX_IDS: usize = 100; @@ -1481,14 +1482,14 @@ impl FromRequest for BatchRequestOpt { /// both. /// /// Used with Option to extract a possible PreConditionHeader. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum PreConditionHeader { IfModifiedSince(SyncTimestamp), IfUnmodifiedSince(SyncTimestamp), NoHeader, } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct PreConditionHeaderOpt { pub opt: Option, } @@ -1576,7 +1577,7 @@ impl FromRequest for PreConditionHeaderOpt { } /// Validation Error Location in the request -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq)] +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] pub enum RequestErrorLocation { Body, @@ -1756,14 +1757,16 @@ mod tests { use rand::{thread_rng, Rng}; use serde_json::{self, json}; use sha2::Sha256; - use syncstorage_db_common::Db; + use syncserver_common; + use syncserver_db_common::Db; + use syncserver_settings::Settings as GlobalSettings; + use syncstorage_settings::{Deadman, ServerLimits, Settings as SyncstorageSettings}; use tokio::sync::RwLock; use crate::db::mock::{MockDb, MockDbPool}; use crate::server::{metrics, ServerState}; - use crate::settings::{Deadman, Secrets, ServerLimits, Settings}; - use crate::web::auth::{hkdf_expand_32, HawkPayload}; + use crate::web::auth::HawkPayload; lazy_static! { static ref SERVER_LIMITS: Arc = Arc::new(ServerLimits::default()); @@ -1784,7 +1787,8 @@ mod tests { } fn make_state() -> ServerState { - let settings = Settings::default(); + let syncserver_settings = GlobalSettings::default(); + let syncstorage_settings = SyncstorageSettings::default(); ServerState { db_pool: Box::new(MockDbPool::new()), limits: Arc::clone(&SERVER_LIMITS), @@ -1792,13 +1796,13 @@ mod tests { port: 8000, metrics: Box::new( metrics::metrics_from_opts( - &settings.statsd_label, - settings.statsd_host.as_deref(), - settings.statsd_port, + &syncstorage_settings.statsd_label, + syncserver_settings.statsd_host.as_deref(), + syncserver_settings.statsd_port, ) .unwrap(), ), - quota_enabled: settings.enable_quota, + quota_enabled: syncstorage_settings.enable_quota, deadman: Arc::new(RwLock::new(Deadman::default())), } } @@ -1823,7 +1827,7 @@ mod tests { let mut id = payload.as_bytes().to_vec(); id.extend(payload_hash.to_vec()); let id = base64::encode_config(&id, base64::URL_SAFE); - let token_secret = hkdf_expand_32( + let token_secret = syncserver_common::hkdf_expand_32( format!("services.mozilla.com/tokenlib/v1/derive/{}", id).as_bytes(), Some(salt.as_bytes()), &SECRETS.master_secret, diff --git a/syncstorage/src/web/handlers.rs b/syncserver/src/web/handlers.rs similarity index 98% rename from syncstorage/src/web/handlers.rs rename to syncserver/src/web/handlers.rs index 0e2f88268f..217cb44765 100644 --- a/syncstorage/src/web/handlers.rs +++ b/syncserver/src/web/handlers.rs @@ -5,24 +5,23 @@ use std::convert::Into; use actix_web::{dev::HttpResponseBuilder, http::StatusCode, web::Data, HttpRequest, HttpResponse}; use serde::Serialize; use serde_json::{json, Map, Value}; -use syncstorage_db_common::{ +use syncserver_common::{X_LAST_MODIFIED, X_WEAVE_NEXT_OFFSET, X_WEAVE_RECORDS}; +use syncserver_db_common::{ error::{DbError, DbErrorKind}, params, results::{CreateBatch, Paginated}, Db, }; +use time; use crate::{ db::transaction::DbTransactionPool, error::{ApiError, ApiErrorKind}, server::ServerState, tokenserver, - web::{ - extractors::{ - BsoPutRequest, BsoRequest, CollectionPostRequest, CollectionRequest, EmitApiMetric, - HeartbeatRequest, MetaRequest, ReplyFormat, TestErrorRequest, - }, - X_LAST_MODIFIED, X_WEAVE_NEXT_OFFSET, X_WEAVE_RECORDS, + web::extractors::{ + BsoPutRequest, BsoRequest, CollectionPostRequest, CollectionRequest, EmitApiMetric, + HeartbeatRequest, MetaRequest, ReplyFormat, TestErrorRequest, }, }; @@ -642,7 +641,7 @@ pub async fn lbheartbeat(req: HttpRequest) -> Result { let db_state = if cfg!(test) { use actix_web::http::header::HeaderValue; use std::str::FromStr; - use syncstorage_db_common::PoolState; + use syncserver_db_common::PoolState; let test_pool = PoolState { connections: u32::from_str( diff --git a/syncstorage/src/web/middleware/mod.rs b/syncserver/src/web/middleware/mod.rs similarity index 100% rename from syncstorage/src/web/middleware/mod.rs rename to syncserver/src/web/middleware/mod.rs diff --git a/syncstorage/src/web/middleware/rejectua.rs b/syncserver/src/web/middleware/rejectua.rs similarity index 100% rename from syncstorage/src/web/middleware/rejectua.rs rename to syncserver/src/web/middleware/rejectua.rs diff --git a/syncstorage/src/web/middleware/sentry.rs b/syncserver/src/web/middleware/sentry.rs similarity index 99% rename from syncstorage/src/web/middleware/sentry.rs rename to syncserver/src/web/middleware/sentry.rs index d5cacf7d6b..b2f6b5d4d4 100644 --- a/syncstorage/src/web/middleware/sentry.rs +++ b/syncserver/src/web/middleware/sentry.rs @@ -10,7 +10,7 @@ use actix_web::{ use futures::future::{self, LocalBoxFuture}; use sentry::protocol::Event; use sentry_backtrace::parse_stacktrace; -use syncstorage_common::ReportableError; +use syncserver_common::ReportableError; use tokenserver_common::error::TokenserverError; use crate::error::ApiError; diff --git a/syncstorage/src/web/middleware/weave.rs b/syncserver/src/web/middleware/weave.rs similarity index 97% rename from syncstorage/src/web/middleware/weave.rs rename to syncserver/src/web/middleware/weave.rs index 931d179b07..6f72e8157b 100644 --- a/syncstorage/src/web/middleware/weave.rs +++ b/syncserver/src/web/middleware/weave.rs @@ -9,10 +9,11 @@ use actix_web::{ }; use futures::future::{self, LocalBoxFuture}; -use syncstorage_db_common::util::SyncTimestamp; +use syncserver_common::{X_LAST_MODIFIED, X_WEAVE_TIMESTAMP}; +use syncserver_db_common::util::SyncTimestamp; use crate::error::{ApiError, ApiErrorKind}; -use crate::web::{DOCKER_FLOW_ENDPOINTS, X_LAST_MODIFIED, X_WEAVE_TIMESTAMP}; +use crate::web::DOCKER_FLOW_ENDPOINTS; pub struct WeaveTimestampMiddleware { service: S, diff --git a/syncserver/src/web/mod.rs b/syncserver/src/web/mod.rs new file mode 100644 index 0000000000..f759eee55e --- /dev/null +++ b/syncserver/src/web/mod.rs @@ -0,0 +1,22 @@ +//! Web authentication, handlers, and middleware +pub mod auth; +pub mod error; +pub mod extractors; +pub mod handlers; +pub mod middleware; +pub mod tags; + +// Known DockerFlow commands for Ops callbacks +pub const DOCKER_FLOW_ENDPOINTS: [&str; 4] = [ + "/__heartbeat__", + "/__lbheartbeat__", + "/__version__", + "/__error__", +]; + +#[macro_export] +macro_rules! label { + ($string:expr) => { + Some($string.to_string()) + }; +} diff --git a/syncstorage/src/web/tags.rs b/syncserver/src/web/tags.rs similarity index 100% rename from syncstorage/src/web/tags.rs rename to syncserver/src/web/tags.rs diff --git a/syncstorage/version.json b/syncserver/version.json similarity index 100% rename from syncstorage/version.json rename to syncserver/version.json diff --git a/syncstorage-common/src/lib.rs b/syncstorage-common/src/lib.rs deleted file mode 100644 index 2b508038fd..0000000000 --- a/syncstorage-common/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -#[macro_export] -macro_rules! from_error { - ($from:ty, $to:ty, $to_kind:expr) => { - impl From<$from> for $to { - fn from(inner: $from) -> $to { - $to_kind(inner).into() - } - } - }; -} - -#[macro_export] -macro_rules! impl_fmt_display { - ($error:ty, $kind:ty) => { - impl fmt::Display for $error { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.kind, formatter) - } - } - }; -} - -pub trait ReportableError { - fn error_backtrace(&self) -> String; - fn is_sentry_event(&self) -> bool; - fn metric_label(&self) -> Option; -} diff --git a/syncstorage-settings/Cargo.toml b/syncstorage-settings/Cargo.toml new file mode 100644 index 0000000000..eeacdca5da --- /dev/null +++ b/syncstorage-settings/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "syncstorage-settings" +version = "0.12.3" +edition = "2021" + +[dependencies] +rand = "0.8" +serde = "1.0" +syncserver-common = { path = "../syncserver-common" } +time = "^0.3" diff --git a/syncstorage-settings/src/lib.rs b/syncstorage-settings/src/lib.rs new file mode 100644 index 0000000000..6f03ec5363 --- /dev/null +++ b/syncstorage-settings/src/lib.rs @@ -0,0 +1,191 @@ +//! Application settings objects and initialization + +use std::cmp::min; + +use rand::{thread_rng, Rng}; +use serde::{Deserialize, Serialize}; +use syncserver_common::{self, MAX_SPANNER_LOAD_SIZE}; + +static KILOBYTE: u32 = 1024; +static MEGABYTE: u32 = KILOBYTE * KILOBYTE; +static GIGABYTE: u32 = MEGABYTE * 1_000; +static DEFAULT_MAX_POST_BYTES: u32 = 2 * MEGABYTE; +static DEFAULT_MAX_POST_RECORDS: u32 = 100; +static DEFAULT_MAX_RECORD_PAYLOAD_BYTES: u32 = 2 * MEGABYTE; +static DEFAULT_MAX_REQUEST_BYTES: u32 = DEFAULT_MAX_POST_BYTES + 4 * KILOBYTE; +static DEFAULT_MAX_TOTAL_BYTES: u32 = 100 * DEFAULT_MAX_POST_BYTES; +// also used to determine the max number of records to return for MySQL. +pub static DEFAULT_MAX_TOTAL_RECORDS: u32 = 100 * DEFAULT_MAX_POST_RECORDS; +// Hard spanner limit is 4GB per split (items under a unique index). +// This gives us more than a bit of wiggle room. +static DEFAULT_MAX_QUOTA_LIMIT: u32 = 2 * GIGABYTE; + +#[derive(Clone, Debug, Default, Copy)] +pub struct Quota { + pub size: usize, + pub enabled: bool, + pub enforced: bool, +} + +#[derive(Copy, Clone, Default, Debug)] +/// Deadman configures how the `/__lbheartbeat__` health check endpoint fails +/// for special conditions. +/// +/// We'll fail the check (usually temporarily) due to the db pool maxing out +/// connections, which notifies the orchestration system that it should back +/// off traffic to this instance until the check succeeds. +/// +/// Optionally we can permanently fail the check after a set time period, +/// indicating that this instance should be evicted and replaced. +pub struct Deadman { + pub max_size: u32, + pub previous_count: usize, + pub clock_start: Option, + pub expiry: Option, +} + +impl From<&Settings> for Deadman { + fn from(settings: &Settings) -> Self { + let expiry = settings.lbheartbeat_ttl.map(|lbheartbeat_ttl| { + // jitter's a range of percentage of ttl added to ttl. E.g. a 60s + // ttl w/ a 10% jitter results in a random final ttl between 60-66s + let ttl = lbheartbeat_ttl as f32; + let max_jitter = ttl * (settings.lbheartbeat_ttl_jitter as f32 * 0.01); + let ttl = thread_rng().gen_range(ttl..ttl + max_jitter); + time::Instant::now() + time::Duration::seconds(ttl as i64) + }); + Deadman { + max_size: settings.database_pool_max_size, + expiry, + ..Default::default() + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(default)] +pub struct Settings { + pub database_url: String, + pub database_pool_max_size: u32, + // NOTE: Not supported by deadpool! + pub database_pool_min_idle: Option, + /// Pool timeout when waiting for a slot to become available, in seconds + pub database_pool_connection_timeout: Option, + /// Max age a given connection should live, in seconds + pub database_pool_connection_lifespan: Option, + /// Max time a connection should sit idle before being dropped. + pub database_pool_connection_max_idle: Option, + pub database_use_test_transactions: bool, + + /// Server-enforced limits for request payloads. + pub limits: ServerLimits, + + pub statsd_label: String, + + pub enable_quota: bool, + pub enforce_quota: bool, + + pub spanner_emulator_host: Option, + pub enabled: bool, + + /// Fail the `/__lbheartbeat__` healthcheck after running for this duration + /// of time (in seconds) + jitter + pub lbheartbeat_ttl: Option, + /// Percentage of `lbheartbeat_ttl` time to "jitter" (adds additional, + /// randomized time) + pub lbheartbeat_ttl_jitter: u32, +} + +impl Default for Settings { + fn default() -> Settings { + Settings { + database_url: "mysql://root@127.0.0.1/syncstorage".to_string(), + database_pool_max_size: 10, + database_pool_min_idle: None, + database_pool_connection_lifespan: None, + database_pool_connection_max_idle: None, + database_pool_connection_timeout: Some(30), + database_use_test_transactions: false, + limits: ServerLimits::default(), + statsd_label: "syncstorage".to_string(), + enable_quota: false, + enforce_quota: false, + spanner_emulator_host: None, + enabled: true, + lbheartbeat_ttl: None, + lbheartbeat_ttl_jitter: 25, + } + } +} + +impl Settings { + pub fn normalize(&mut self) { + // Adjust the max values if required. + if self.uses_spanner() { + self.limits.max_total_bytes = + min(self.limits.max_total_bytes, MAX_SPANNER_LOAD_SIZE as u32); + } else { + // No quotas for stand alone servers + self.limits.max_quota_limit = 0; + self.enable_quota = false; + self.enforce_quota = false; + } + } + + pub fn spanner_database_name(&self) -> Option<&str> { + if !self.uses_spanner() { + None + } else { + Some(&self.database_url["spanner://".len()..]) + } + } + + pub fn uses_spanner(&self) -> bool { + self.database_url.as_str().starts_with("spanner://") + } +} + +/// Server-enforced limits for request payloads. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(default)] +pub struct ServerLimits { + /// Maximum combined size of BSO payloads for a single request, in bytes. + pub max_post_bytes: u32, + + /// Maximum BSO count for a single request. + pub max_post_records: u32, + + /// Maximum size of an individual BSO payload, in bytes. + pub max_record_payload_bytes: u32, + + /// Maximum `Content-Length` for all incoming requests, in bytes. + /// + /// Enforced externally to this repo, at the web server level. + /// It's important that nginx (or whatever) + /// really is configured to enforce exactly this limit, + /// otherwise client requests may fail with a 413 + /// before even reaching the API. + pub max_request_bytes: u32, + + /// Maximum combined size of BSO payloads across a batch upload, in bytes. + pub max_total_bytes: u32, + + /// Maximum BSO count across a batch upload. + pub max_total_records: u32, + pub max_quota_limit: u32, +} + +impl Default for ServerLimits { + /// Create a default `ServerLimits` instance. + fn default() -> Self { + Self { + max_post_bytes: DEFAULT_MAX_POST_BYTES, + max_post_records: DEFAULT_MAX_POST_RECORDS, + max_record_payload_bytes: DEFAULT_MAX_RECORD_PAYLOAD_BYTES, + max_request_bytes: DEFAULT_MAX_REQUEST_BYTES, + max_total_bytes: DEFAULT_MAX_TOTAL_BYTES, + max_total_records: DEFAULT_MAX_TOTAL_RECORDS, + max_quota_limit: DEFAULT_MAX_QUOTA_LIMIT, + } + } +} diff --git a/syncstorage/src/settings.rs b/syncstorage/src/settings.rs deleted file mode 100644 index df1dbb5ca5..0000000000 --- a/syncstorage/src/settings.rs +++ /dev/null @@ -1,598 +0,0 @@ -//! Application settings objects and initialization -use std::{ - cmp::min, - env::{self, VarError}, -}; - -use actix_cors::Cors; -use actix_web::http::header::{AUTHORIZATION, CONTENT_TYPE, USER_AGENT}; -use config::{Config, ConfigError, Environment, File}; -use http::method::Method; -use rand::{thread_rng, Rng}; -use serde::{de::Deserializer, Deserialize, Serialize}; -use url::Url; - -use crate::db::spanner::models::MAX_SPANNER_LOAD_SIZE; -use crate::error::ApiError; -use crate::tokenserver::settings::Settings as TokenserverSettings; -use crate::web::auth::hkdf_expand_32; -use crate::web::{ - X_LAST_MODIFIED, X_VERIFY_CODE, X_WEAVE_BYTES, X_WEAVE_NEXT_OFFSET, X_WEAVE_RECORDS, - X_WEAVE_TIMESTAMP, X_WEAVE_TOTAL_BYTES, X_WEAVE_TOTAL_RECORDS, -}; - -static DEFAULT_PORT: u16 = 8000; - -static KILOBYTE: u32 = 1024; -static MEGABYTE: u32 = KILOBYTE * KILOBYTE; -static GIGABYTE: u32 = MEGABYTE * 1_000; -static DEFAULT_MAX_POST_BYTES: u32 = 2 * MEGABYTE; -static DEFAULT_MAX_POST_RECORDS: u32 = 100; -static DEFAULT_MAX_RECORD_PAYLOAD_BYTES: u32 = 2 * MEGABYTE; -static DEFAULT_MAX_REQUEST_BYTES: u32 = DEFAULT_MAX_POST_BYTES + 4 * KILOBYTE; -static DEFAULT_MAX_TOTAL_BYTES: u32 = 100 * DEFAULT_MAX_POST_BYTES; -// also used to determine the max number of records to return for MySQL. -pub static DEFAULT_MAX_TOTAL_RECORDS: u32 = 100 * DEFAULT_MAX_POST_RECORDS; -// Hard spanner limit is 4GB per split (items under a unique index). -// This gives us more than a bit of wiggle room. -static DEFAULT_MAX_QUOTA_LIMIT: u32 = 2 * GIGABYTE; -static PREFIX: &str = "sync"; - -#[derive(Clone, Debug, Default, Copy)] -pub struct Quota { - pub size: usize, - pub enabled: bool, - pub enforced: bool, -} - -#[derive(Copy, Clone, Default, Debug)] -/// Deadman configures how the `/__lbheartbeat__` health check endpoint fails -/// for special conditions. -/// -/// We'll fail the check (usually temporarily) due to the db pool maxing out -/// connections, which notifies the orchestration system that it should back -/// off traffic to this instance until the check succeeds. -/// -/// Optionally we can permanently fail the check after a set time period, -/// indicating that this instance should be evicted and replaced. -pub struct Deadman { - pub max_size: u32, - pub previous_count: usize, - pub clock_start: Option, - pub expiry: Option, -} - -impl From<&Settings> for Deadman { - fn from(settings: &Settings) -> Self { - let expiry = settings.lbheartbeat_ttl.map(|lbheartbeat_ttl| { - // jitter's a range of percentage of ttl added to ttl. E.g. a 60s - // ttl w/ a 10% jitter results in a random final ttl between 60-66s - let ttl = lbheartbeat_ttl as f32; - let max_jitter = ttl * (settings.lbheartbeat_ttl_jitter as f32 * 0.01); - let ttl = thread_rng().gen_range(ttl..ttl + max_jitter); - time::Instant::now() + time::Duration::seconds(ttl as i64) - }); - Deadman { - max_size: settings.database_pool_max_size, - expiry, - ..Default::default() - } - } -} - -#[derive(Clone, Debug, Deserialize)] -pub struct Settings { - pub debug: bool, - pub port: u16, - pub host: String, - pub database_url: String, - pub database_pool_max_size: u32, - // NOTE: Not supported by deadpool! - pub database_pool_min_idle: Option, - /// Pool timeout when waiting for a slot to become available, in seconds - pub database_pool_connection_timeout: Option, - /// Max age a given connection should live, in seconds - pub database_pool_connection_lifespan: Option, - /// Max time a connection should sit idle before being dropped. - pub database_pool_connection_max_idle: Option, - #[cfg(test)] - pub database_use_test_transactions: bool, - - pub actix_keep_alive: Option, - - /// Server-enforced limits for request payloads. - pub limits: ServerLimits, - - /// The master secret, from which are derived - /// the signing secret and token secret - /// that are used during Hawk authentication. - pub master_secret: Secrets, - - pub human_logs: bool, - - pub statsd_host: Option, - pub statsd_port: u16, - pub statsd_label: String, - - pub enable_quota: bool, - pub enforce_quota: bool, - - pub spanner_emulator_host: Option, - - /// Disable all of the endpoints related to syncstorage. To be used when running Tokenserver - /// in isolation. - pub disable_syncstorage: bool, - - /// Settings specific to Tokenserver - pub tokenserver: TokenserverSettings, - - /// Cors Settings - pub cors_allowed_origin: Option, - pub cors_max_age: Option, - pub cors_allowed_methods: Option>, - pub cors_allowed_headers: Option>, - - /// Fail the `/__lbheartbeat__` healthcheck after running for this duration - /// of time (in seconds) + jitter - pub lbheartbeat_ttl: Option, - /// Percentage of `lbheartbeat_ttl` time to "jitter" (adds additional, - /// randomized time) - pub lbheartbeat_ttl_jitter: u32, -} - -impl Default for Settings { - fn default() -> Settings { - Settings { - debug: false, - port: DEFAULT_PORT, - host: "127.0.0.1".to_string(), - database_url: "mysql://root@127.0.0.1/syncstorage".to_string(), - database_pool_max_size: 10, - database_pool_min_idle: None, - database_pool_connection_lifespan: None, - database_pool_connection_max_idle: None, - database_pool_connection_timeout: Some(30), - #[cfg(test)] - database_use_test_transactions: false, - actix_keep_alive: None, - limits: ServerLimits::default(), - master_secret: Secrets::default(), - statsd_host: None, - statsd_port: 8125, - statsd_label: "syncstorage".to_string(), - human_logs: false, - enable_quota: false, - enforce_quota: false, - spanner_emulator_host: None, - disable_syncstorage: false, - tokenserver: TokenserverSettings::default(), - cors_allowed_origin: Some("*".to_owned()), - cors_allowed_methods: Some(vec![ - "DELETE".to_owned(), - "GET".to_owned(), - "POST".to_owned(), - "PUT".to_owned(), - ]), - cors_allowed_headers: Some(vec![ - AUTHORIZATION.to_string(), - CONTENT_TYPE.to_string(), - USER_AGENT.to_string(), - X_LAST_MODIFIED.to_owned(), - X_WEAVE_TIMESTAMP.to_owned(), - X_WEAVE_NEXT_OFFSET.to_owned(), - X_WEAVE_RECORDS.to_owned(), - X_WEAVE_BYTES.to_owned(), - X_WEAVE_TOTAL_RECORDS.to_owned(), - X_WEAVE_TOTAL_BYTES.to_owned(), - X_VERIFY_CODE.to_owned(), - "TEST_IDLES".to_owned(), - ]), - cors_max_age: Some(1728000), - lbheartbeat_ttl: None, - lbheartbeat_ttl_jitter: 25, - } - } -} - -impl Settings { - /// Load the settings from the config file if supplied, then the environment. - pub fn with_env_and_config_file(filename: &Option) -> Result { - let mut s = Config::default(); - // Set our defaults, this can be fixed up drastically later after: - // https://github.com/mehcode/config-rs/issues/60 - s.set_default("debug", false)?; - s.set_default("port", i64::from(DEFAULT_PORT))?; - s.set_default("host", "127.0.0.1")?; - s.set_default("human_logs", false)?; - #[cfg(test)] - s.set_default("database_pool_connection_timeout", Some(30))?; - s.set_default("database_pool_max_size", 10)?; - // Max lifespan a connection should have. - s.set_default::>("database_connection_lifespan", None)?; - // Max time a connection should be idle before dropping. - s.set_default::>("database_connection_max_idle", None)?; - s.set_default("database_use_test_transactions", false)?; - s.set_default("master_secret", "")?; - // Each backend does their own default process, so specifying a "universal" value - // for database_pool_max_size doesn't quite work. Generally the max pool size is - // 10. - s.set_default("master_secret", "")?; - s.set_default("limits.max_post_bytes", i64::from(DEFAULT_MAX_POST_BYTES))?; - s.set_default( - "limits.max_post_records", - i64::from(DEFAULT_MAX_POST_RECORDS), - )?; - s.set_default( - "limits.max_record_payload_bytes", - i64::from(DEFAULT_MAX_RECORD_PAYLOAD_BYTES), - )?; - s.set_default( - "limits.max_request_bytes", - i64::from(DEFAULT_MAX_REQUEST_BYTES), - )?; - s.set_default("limits.max_total_bytes", i64::from(DEFAULT_MAX_TOTAL_BYTES))?; - s.set_default( - "limits.max_total_records", - i64::from(DEFAULT_MAX_TOTAL_RECORDS), - )?; - s.set_default("limits.max_quota_limit", i64::from(DEFAULT_MAX_QUOTA_LIMIT))?; - - s.set_default("statsd_host", "localhost")?; - s.set_default("statsd_port", 8125)?; - s.set_default("statsd_label", "syncstorage")?; - s.set_default("enable_quota", false)?; - s.set_default("enforce_quota", false)?; - s.set_default("disable_syncstorage", false)?; - - // Set Tokenserver defaults - s.set_default( - "tokenserver.database_url", - "mysql://root@127.0.0.1/tokenserver", - )?; - s.set_default("tokenserver.database_pool_max_size", 10)?; - s.set_default("tokenserver.enabled", false)?; - s.set_default( - "tokenserver.fxa_browserid_audience", - "https://token.stage.mozaws.net", - )?; - s.set_default( - "tokenserver.fxa_browserid_issuer", - "api-accounts.stage.mozaws.net", - )?; - s.set_default( - "tokenserver.fxa_browserid_server_url", - "https://verifier.stage.mozaws.net/v2", - )?; - s.set_default("tokenserver.fxa_browserid_request_timeout", 10)?; - s.set_default("tokenserver.fxa_browserid_connect_timeout", 5)?; - s.set_default( - "tokenserver.fxa_email_domain", - "api-accounts.stage.mozaws.net", - )?; - s.set_default("tokenserver.fxa_metrics_hash_secret", "secret")?; - s.set_default( - "tokenserver.fxa_oauth_server_url", - "https://oauth.stage.mozaws.net", - )?; - s.set_default("tokenserver.fxa_oauth_request_timeout", 10)?; - - // The type parameter for None:: below would more appropriately be `Jwk`, but due - // to constraints imposed by version 0.11 of the config crate, it is not possible to - // implement `ValueKind: From`. The next best thing would be to use `ValueKind`, - // but `ValueKind` is private in this version of config. We use `bool` as a placeholder, - // since `ValueKind: From` is implemented, and None:: for all T is simply - // converted to ValueKind::Nil (see below link). - // https://github.com/mehcode/config-rs/blob/0.11.0/src/value.rs#L35 - s.set_default("tokenserver.fxa_oauth_primary_jwk", None::)?; - s.set_default("tokenserver.fxa_oauth_secondary_jwk", None::)?; - - s.set_default("tokenserver.node_type", "spanner")?; - s.set_default("tokenserver.statsd_label", "syncstorage.tokenserver")?; - s.set_default("tokenserver.run_migrations", cfg!(test))?; - s.set_default("tokenserver.token_duration", 3600)?; - - // Set Cors defaults - s.set_default( - "cors_allowed_headers", - Some(vec![ - AUTHORIZATION.to_string().as_str(), - CONTENT_TYPE.to_string().as_str(), - USER_AGENT.to_string().as_str(), - X_LAST_MODIFIED, - X_WEAVE_TIMESTAMP, - X_WEAVE_NEXT_OFFSET, - X_WEAVE_RECORDS, - X_WEAVE_BYTES, - X_WEAVE_TOTAL_RECORDS, - X_WEAVE_TOTAL_BYTES, - X_VERIFY_CODE, - "TEST_IDLES", - ]), - )?; - s.set_default( - "cors_allowed_methods", - Some(vec!["DELETE", "GET", "POST", "PUT"]), - )?; - s.set_default("cors_allowed_origin", Some("*"))?; - s.set_default("lbheartbeat_ttl_jitter", 25)?; - - // Merge the config file if supplied - if let Some(config_filename) = filename { - s.merge(File::with_name(config_filename))?; - } - - // Merge the environment overrides - // While the prefix is currently case insensitive, it's traditional that - // environment vars be UPPERCASE, this ensures that will continue should - // Environment ever change their policy about case insensitivity. - // This will accept environment variables specified as - // `SYNC_FOO__BAR_VALUE="gorp"` as `foo.bar_value = "gorp"` - s.merge(Environment::with_prefix(&PREFIX.to_uppercase()).separator("__"))?; - - Ok(match s.try_into::() { - Ok(mut s) => { - // Adjust the max values if required. - if s.uses_spanner() { - let mut ms = s; - ms.limits.max_total_bytes = - min(ms.limits.max_total_bytes, MAX_SPANNER_LOAD_SIZE as u32); - return Ok(ms); - } else { - // No quotas for stand alone servers - s.limits.max_quota_limit = 0; - s.enable_quota = false; - s.enforce_quota = false; - } - if s.limits.max_quota_limit == 0 { - s.enable_quota = false - } - if s.enforce_quota { - s.enable_quota = true - } - - if matches!(env::var("ACTIX_THREADPOOL"), Err(VarError::NotPresent)) { - // Db backends w/ blocking calls block via - // actix-threadpool: grow its size to accommodate the - // full number of connections - let total_db_pool_size = { - let syncstorage_pool_max_size = if s.uses_spanner() || s.disable_syncstorage - { - 0 - } else { - s.database_pool_max_size - }; - - let tokenserver_pool_max_size = if s.tokenserver.enabled { - s.tokenserver.database_pool_max_size - } else { - 0 - }; - - syncstorage_pool_max_size + tokenserver_pool_max_size - }; - - let fxa_threads = if s.tokenserver.enabled - && s.tokenserver.fxa_oauth_primary_jwk.is_none() - && s.tokenserver.fxa_oauth_secondary_jwk.is_none() - { - s.tokenserver - .additional_blocking_threads_for_fxa_requests - .ok_or_else(|| { - println!( - "If the Tokenserver OAuth JWK is not cached, additional blocking \ - threads must be used to handle the requests to FxA." - ); - - let setting_name = - "tokenserver.additional_blocking_threads_for_fxa_requests"; - ConfigError::NotFound(String::from(setting_name)) - })? - } else { - 0 - }; - - env::set_var( - "ACTIX_THREADPOOL", - ((total_db_pool_size + fxa_threads) as usize) - .max(num_cpus::get() * 5) - .to_string(), - ); - } - - s - } - Err(e) => match e { - // Configuration errors are not very sysop friendly, Try to make them - // a bit more 3AM useful. - ConfigError::Message(v) => { - println!("Bad configuration: {:?}", &v); - println!("Please set in config file or use environment variable."); - println!( - "For example to set `database_url` use env var `{}_DATABASE_URL`\n", - PREFIX.to_uppercase() - ); - error!("Configuration error: Value undefined {:?}", &v); - return Err(ConfigError::NotFound(v)); - } - _ => { - error!("Configuration error: Other: {:?}", &e); - return Err(e); - } - }, - }) - } - - pub fn uses_spanner(&self) -> bool { - self.database_url.as_str().starts_with("spanner://") - } - - pub fn spanner_database_name(&self) -> Option<&str> { - if !self.uses_spanner() { - None - } else { - Some(&self.database_url["spanner://".len()..]) - } - } - - /// A simple banner for display of certain settings at startup - pub fn banner(&self) -> String { - let quota = if self.enable_quota { - format!( - "Quota: {} bytes ({}enforced)", - self.limits.max_quota_limit, - if !self.enforce_quota { "un" } else { "" } - ) - } else { - "No quota".to_owned() - }; - let db = Url::parse(&self.database_url) - .map(|url| url.scheme().to_owned()) - .unwrap_or_else(|_| "".to_owned()); - format!("http://{}:{} ({}) {}", self.host, self.port, db, quota) - } - - pub fn build_cors(&self) -> Cors { - // Followed by the "official middleware" so they run first. - // actix is getting increasingly tighter about CORS headers. Our server is - // not a huge risk but does deliver XHR JSON content. - // For now, let's be permissive and use NGINX (the wrapping server) - // for finer grained specification. - let mut cors = Cors::default(); - - if let Some(allowed_methods) = &self.cors_allowed_methods { - let mut methods = vec![]; - for method_string in allowed_methods { - let method = Method::from_bytes(method_string.as_bytes()).unwrap(); - methods.push(method); - } - cors = cors.allowed_methods(methods); - } - if let Some(allowed_headers) = &self.cors_allowed_headers { - cors = cors.allowed_headers(allowed_headers); - } - - if let Some(max_age) = &self.cors_max_age { - cors = cors.max_age(*max_age); - } - // explicitly set the CORS allow origin, since Default does not - // appear to set the `allow-origins: *` header. - if let Some(origin) = &self.cors_allowed_origin { - if origin == "*" { - cors = cors.allow_any_origin(); - } else { - cors = cors.allowed_origin(origin); - } - } - - cors - } -} - -/// Server-enforced limits for request payloads. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ServerLimits { - /// Maximum combined size of BSO payloads for a single request, in bytes. - pub max_post_bytes: u32, - - /// Maximum BSO count for a single request. - pub max_post_records: u32, - - /// Maximum size of an individual BSO payload, in bytes. - pub max_record_payload_bytes: u32, - - /// Maximum `Content-Length` for all incoming requests, in bytes. - /// - /// Enforced externally to this repo, at the web server level. - /// It's important that nginx (or whatever) - /// really is configured to enforce exactly this limit, - /// otherwise client requests may fail with a 413 - /// before even reaching the API. - pub max_request_bytes: u32, - - /// Maximum combined size of BSO payloads across a batch upload, in bytes. - pub max_total_bytes: u32, - - /// Maximum BSO count across a batch upload. - pub max_total_records: u32, - pub max_quota_limit: u32, -} - -impl Default for ServerLimits { - /// Create a default `ServerLimits` instance. - fn default() -> Self { - Self { - max_post_bytes: DEFAULT_MAX_POST_BYTES, - max_post_records: DEFAULT_MAX_POST_RECORDS, - max_record_payload_bytes: DEFAULT_MAX_RECORD_PAYLOAD_BYTES, - max_request_bytes: DEFAULT_MAX_REQUEST_BYTES, - max_total_bytes: DEFAULT_MAX_TOTAL_BYTES, - max_total_records: DEFAULT_MAX_TOTAL_RECORDS, - max_quota_limit: DEFAULT_MAX_QUOTA_LIMIT, - } - } -} - -/// Secrets used during Hawk authentication. -#[derive(Clone, Debug)] -pub struct Secrets { - /// The master secret in byte array form. - /// - /// The signing secret and token secret are derived from this. - pub master_secret: Vec, - - /// The signing secret used during Hawk authentication. - pub signing_secret: [u8; 32], -} - -impl Secrets { - /// Decode the master secret to a byte array - /// and derive the signing secret from it. - pub fn new(master_secret: &str) -> Result { - let master_secret = master_secret.as_bytes().to_vec(); - let signing_secret = hkdf_expand_32( - b"services.mozilla.com/tokenlib/v1/signing", - None, - &master_secret, - )?; - Ok(Self { - master_secret, - signing_secret, - }) - } -} - -impl Default for Secrets { - /// Create a (useless) default `Secrets` instance. - fn default() -> Self { - Self { - master_secret: vec![], - signing_secret: [0u8; 32], - } - } -} - -impl<'d> Deserialize<'d> for Secrets { - /// Deserialize the master secret and signing secret byte arrays - /// from a single master secret string. - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'d>, - { - let master_secret: String = Deserialize::deserialize(deserializer)?; - Secrets::new(&master_secret) - .map_err(|e| serde::de::Error::custom(format!("error: {:?}", e))) - } -} - -#[cfg(test)] -pub fn test_settings() -> Settings { - let mut settings = Settings::with_env_and_config_file(&None) - .expect("Could not get Settings in get_test_settings"); - settings.debug = true; - settings.port = 8000; - settings.database_pool_max_size = 1; - settings.database_use_test_transactions = true; - settings.database_pool_connection_max_idle = Some(300); - settings.database_pool_connection_lifespan = Some(300); - settings -} diff --git a/syncstorage/src/web/mod.rs b/syncstorage/src/web/mod.rs deleted file mode 100644 index 9f2ea9bb44..0000000000 --- a/syncstorage/src/web/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Web authentication, handlers, and middleware -pub mod auth; -pub mod error; -pub mod extractors; -pub mod handlers; -pub mod middleware; -pub mod tags; - -// header statics must be lower case, numbers and symbols per the RFC spec. This reduces chance of error. -pub static X_LAST_MODIFIED: &str = "x-last-modified"; -pub static X_WEAVE_TIMESTAMP: &str = "x-weave-timestamp"; -pub static X_WEAVE_NEXT_OFFSET: &str = "x-weave-next-offset"; -pub static X_WEAVE_RECORDS: &str = "x-weave-records"; -pub static X_WEAVE_BYTES: &str = "x-weave-bytes"; -pub static X_WEAVE_TOTAL_RECORDS: &str = "x-weave-total-records"; -pub static X_WEAVE_TOTAL_BYTES: &str = "x-weave-total-bytes"; -pub static X_VERIFY_CODE: &str = "x-verify-code"; - -// Known DockerFlow commands for Ops callbacks -pub const DOCKER_FLOW_ENDPOINTS: [&str; 4] = [ - "/__heartbeat__", - "/__lbheartbeat__", - "/__version__", - "/__error__", -]; - -#[macro_export] -macro_rules! label { - ($string:expr) => { - Some($string.to_string()) - }; -} diff --git a/tokenserver-common/Cargo.toml b/tokenserver-common/Cargo.toml index 3b9b601b8c..da54201fff 100644 --- a/tokenserver-common/Cargo.toml +++ b/tokenserver-common/Cargo.toml @@ -8,6 +8,6 @@ actix-web = "3" backtrace = "0.3.61" serde = "1.0" serde_json = { version = "1.0", features = ["arbitrary_precision"] } -syncstorage-common = { path = "../syncstorage-common" } -syncstorage-db-common = { path = "../syncstorage-db-common" } +syncserver-common = { path = "../syncserver-common" } +syncserver-db-common = { path = "../syncserver-db-common" } thiserror = "1.0.26" diff --git a/tokenserver-common/src/error.rs b/tokenserver-common/src/error.rs index 9784dfb5c0..250eb3613d 100644 --- a/tokenserver-common/src/error.rs +++ b/tokenserver-common/src/error.rs @@ -6,8 +6,8 @@ use serde::{ ser::{SerializeMap, Serializer}, Serialize, }; -use syncstorage_common::ReportableError; -use syncstorage_db_common::error::DbError; +use syncserver_common::ReportableError; +use syncserver_db_common::error::DbError; #[derive(Clone, Debug)] pub struct TokenserverError { diff --git a/tokenserver-common/src/lib.rs b/tokenserver-common/src/lib.rs index a91e735174..ab76d983f0 100644 --- a/tokenserver-common/src/lib.rs +++ b/tokenserver-common/src/lib.rs @@ -1 +1,23 @@ pub mod error; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum NodeType { + #[serde(rename = "mysql")] + MySql, + #[serde(rename = "spanner")] + Spanner, +} + +impl NodeType { + pub fn spanner() -> Self { + Self::Spanner + } +} + +impl Default for NodeType { + fn default() -> Self { + Self::Spanner + } +} diff --git a/tokenserver-settings/Cargo.toml b/tokenserver-settings/Cargo.toml new file mode 100644 index 0000000000..635f91ed26 --- /dev/null +++ b/tokenserver-settings/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "tokenserver-settings" +version = "0.12.3" +edition = "2021" + +[dependencies] +serde = "1.0" +tokenserver-common = { path = "../tokenserver-common" } diff --git a/syncstorage/src/tokenserver/settings.rs b/tokenserver-settings/src/lib.rs similarity index 94% rename from syncstorage/src/tokenserver/settings.rs rename to tokenserver-settings/src/lib.rs index d17d83210c..7d6590f89c 100644 --- a/syncstorage/src/tokenserver/settings.rs +++ b/tokenserver-settings/src/lib.rs @@ -1,8 +1,8 @@ use serde::Deserialize; - -use super::NodeType; +use tokenserver_common::NodeType; #[derive(Clone, Debug, Deserialize)] +#[serde(default)] pub struct Settings { /// The URL of the Tokenserver MySQL database. pub database_url: String, @@ -50,6 +50,7 @@ pub struct Settings { /// The rate at which capacity should be released from nodes that are at capacity. pub node_capacity_release_rate: Option, /// The type of the storage nodes used by this instance of Tokenserver. + #[serde(default = "NodeType::spanner")] pub node_type: NodeType, /// The label to be used when reporting Metrics. pub statsd_label: String, @@ -82,12 +83,12 @@ pub struct Jwk { impl Default for Settings { fn default() -> Settings { Settings { - database_url: "mysql://root@127.0.0.1/tokenserver_rs".to_owned(), + database_url: "mysql://root@127.0.0.1/tokenserver".to_owned(), database_pool_max_size: 10, database_pool_min_idle: None, database_pool_connection_timeout: Some(30), enabled: false, - fxa_email_domain: "api.accounts.firefox.com".to_owned(), + fxa_email_domain: "api-accounts.stage.mozaws.net".to_owned(), fxa_metrics_hash_secret: "secret".to_owned(), fxa_oauth_server_url: "https://oauth.stage.mozaws.net".to_owned(), fxa_oauth_request_timeout: 10, @@ -103,7 +104,7 @@ impl Default for Settings { statsd_label: "syncstorage.tokenserver".to_owned(), run_migrations: cfg!(test), spanner_node_id: None, - additional_blocking_threads_for_fxa_requests: None, + additional_blocking_threads_for_fxa_requests: Some(1), token_duration: 3600, } } diff --git a/tools/integration_tests/run.py b/tools/integration_tests/run.py index c87fba4311..3849cc7069 100644 --- a/tools/integration_tests/run.py +++ b/tools/integration_tests/run.py @@ -10,8 +10,8 @@ import time from tokenserver.run import run_end_to_end_tests, run_local_tests -DEBUG_BUILD = "target/debug/syncstorage" -RELEASE_BUILD = "/app/bin/syncstorage" +DEBUG_BUILD = "target/debug/syncserver" +RELEASE_BUILD = "/app/bin/syncserver" def terminate_process(process): @@ -32,8 +32,8 @@ def terminate_process(process): target_binary = RELEASE_BUILD else: raise RuntimeError( - "Neither target/debug/syncstorage \ - nor /app/bin/syncstorage were found." + "Neither target/debug/syncserver \ + nor /app/bin/syncserver were found." ) def start_server(): diff --git a/tools/spanner/count_expired_rows.py b/tools/spanner/count_expired_rows.py index f4a9975566..9389381f0a 100644 --- a/tools/spanner/count_expired_rows.py +++ b/tools/spanner/count_expired_rows.py @@ -26,7 +26,7 @@ def from_env(): try: - url = os.environ.get("SYNC_DATABASE_URL") + url = os.environ.get("SYNC_SYNCSTORAGE__DATABASE_URL") if not url: raise Exception("no url") purl = parse.urlparse(url) diff --git a/tools/spanner/count_users.py b/tools/spanner/count_users.py index d1dd8ef51c..40fb0b1680 100644 --- a/tools/spanner/count_users.py +++ b/tools/spanner/count_users.py @@ -26,7 +26,7 @@ def from_env(): try: - url = os.environ.get("SYNC_DATABASE_URL") + url = os.environ.get("SYNC_SYNCSTORAGE__DATABASE_URL") if not url: raise Exception("no url") purl = parse.urlparse(url) diff --git a/tools/spanner/purge_ttl.py b/tools/spanner/purge_ttl.py index 6f94c8ebb7..b3d0cf0149 100644 --- a/tools/spanner/purge_ttl.py +++ b/tools/spanner/purge_ttl.py @@ -176,7 +176,7 @@ def get_args(): parser.add_argument( "-u", "--sync_database_url", - default=os.environ.get("SYNC_DATABASE_URL"), + default=os.environ.get("SYNC_SYNCSTORAGE__DATABASE_URL"), help="Spanner Database DSN" ) parser.add_argument( diff --git a/tools/spanner/write_batch.py b/tools/spanner/write_batch.py index f71fb18aa5..0a795a01c9 100644 --- a/tools/spanner/write_batch.py +++ b/tools/spanner/write_batch.py @@ -178,7 +178,7 @@ def create_user(txn): def from_env(): try: - url = os.environ.get("SYNC_DATABASE_URL") + url = os.environ.get("SYNC_SYNCSTORAGE__DATABASE_URL") if not url: raise Exception("no url") purl = parse.urlparse(url)