refactor: add settings crates (#1306)

This is a breaking change. This commit separates syncstorage and tokenserver settings into separate structs that are contained by a parent `Settings` struct. This means that any env vars that hold settings specific to syncstorage (e.g. `SYNC_DATABASE_URL`) have been renamed to `SYNC_SYNCSTORAGE__DATABASE_URL`. Any settings that were moved from the top-level `Settings` struct to the lower level, syncstorage-specific struct will now have a `SYNC_SYNCSTORAGE__` prefix instead of a `SYNC_` prefix.

Closes #1276

.circleci/config.yml

@@ -40,15 +40,15 @@ commands:
       - run:
           name: Core Python Checks
           command: |
-            flake8 syncstorage/src/tokenserver
+            flake8 syncserver/src/tokenserver
             flake8 tools/integration_tests
             flake8 tools/tokenserver
   rust-clippy:
     steps:
       - run:
           name: Rust Clippy
           command: |
-            cargo clippy --all --all-targets --all-features -- -D warnings
+            cargo clippy --workspace --all-targets --all-features -- -D warnings
   cargo-build:
     steps:
       - run:
@@ -83,18 +83,18 @@ commands:
             "$CIRCLE_TAG" \
             "$CIRCLE_PROJECT_USERNAME" \
             "$CIRCLE_PROJECT_REPONAME" \
-            "$CIRCLE_BUILD_URL" > syncstorage/version.json
+            "$CIRCLE_BUILD_URL" > syncserver/version.json
 
   run-tests:
     steps:
       - run:
           name: cargo test
-          command: cargo test --all --verbose
+          command: cargo test --workspace --verbose
       - run:
           name: quota test
-          command: cargo test --all --verbose
+          command: cargo test --workspace --verbose
           environment:
-              SYNC_ENFORCE_QUOTA: 1
+              SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1
 
   run-e2e-mysql-tests:
     steps:
@@ -181,7 +181,7 @@ jobs:
           username: $DOCKER_USER
           password: $DOCKER_PASS
         environment:
-            SYNC_DATABASE_URL: mysql://test:[email protected]/syncstorage
+            SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:[email protected]/syncstorage
             SYNC_TOKENSERVER__DATABASE_URL: mysql://test:[email protected]/tokenserver
             RUST_BACKTRACE: 1
             # XXX: begin_test_transaction doesn't play nice over threaded tests

Cargo.toml

@@ -1,12 +1,15 @@
 [workspace]
 resolver = "2"
 members = [
-    "syncstorage-db-common",
-    "syncstorage-common",
+    "syncserver-settings",
+    "syncserver-common",
+    "syncserver-db-common",
+    "syncstorage-settings",
     "tokenserver-common",
-    "syncstorage",
+    "tokenserver-settings",
+    "syncserver",
 ]
-default-members = ["syncstorage"]
+default-members = ["syncserver"]
 
 [profile.release]
 # Enables line numbers in Sentry reporting

Dockerfile

@@ -11,8 +11,8 @@ RUN apt-get -q update && \
 RUN \
     cargo --version && \
     rustc --version && \
-    cargo install --path ./syncstorage --locked --root /app && \
-    cargo install --path ./syncstorage --locked --root /app --bin purge_ttl
+    cargo install --path ./syncserver --locked --root /app && \
+    cargo install --path ./syncserver --locked --root /app --bin purge_ttl
 
 FROM debian:buster-slim
 WORKDIR /app
@@ -35,19 +35,19 @@ RUN \
     rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /app/bin /app/bin
-COPY --from=builder /app/syncstorage/version.json /app
+COPY --from=builder /app/syncserver/version.json /app
 COPY --from=builder /app/spanner_config.ini /app
 COPY --from=builder /app/tools/spanner /app/tools/spanner
 COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
 COPY --from=builder /app/tools/tokenserver/process_account_events.py /app/tools/tokenserver/process_account_events.py
 COPY --from=builder /app/tools/tokenserver/requirements.txt /app/tools/tokenserver/requirements.txt
 COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh
-COPY --from=builder /app/syncstorage/src/db/spanner/schema.ddl /app/schema.ddl
+COPY --from=builder /app/syncserver/src/db/spanner/schema.ddl /app/schema.ddl
 
 RUN chmod +x /app/scripts/prepare-spanner.sh
 RUN pip3 install -r /app/tools/integration_tests/requirements.txt
 RUN pip3 install -r /app/tools/tokenserver/requirements.txt
 
 USER app:app
 
-ENTRYPOINT ["/app/bin/syncstorage", "--config=spanner_config.ini"]
+ENTRYPOINT ["/app/bin/syncserver", "--config=spanner_config.ini"]

Makefile

@@ -2,9 +2,6 @@
 # Collection of helper scripts used for local dev.
 ##
 
-SYNC_DATABASE_URL = 'mysql://sample_user:sample_password@localhost/syncstorage_rs'
-SYNC_TOKENSERVER__DATABASE_URL = 'mysql://sample_user:sample_password@localhost/tokenserver_rs'
-
 # This key can live anywhere on your machine. Adjust path as needed.
 PATH_TO_SYNC_SPANNER_KEYS = `pwd`/service-account.json
 
@@ -15,7 +12,7 @@ PATH_TO_GRPC_CERT = ../server-syncstorage/local/lib/python2.7/site-packages/grpc
 
 clippy:
 	# Matches what's run in circleci
-	cargo clippy --all --all-targets --all-features -- -D warnings
+	cargo clippy --workspace --all-targets --all-features -- -D warnings
 
 clean:
 	cargo clean
@@ -50,4 +47,7 @@ run_spanner:
 	GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) make run
 
 test:
-	SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) SYNC_TOKENSERVER__DATABASE_URL=$(SYNC_TOKENSERVER__DATABASE_URL) RUST_TEST_THREADS=1 cargo test
+	SYNC_SYNCSTORAGE__DATABASE_URL=mysql://sample_user:sample_password@localhost/syncstorage_rs \
+		SYNC_TOKENSERVER__DATABASE_URL=mysql://sample_user:sample_password@localhost/tokenserver_rs \
+		RUST_TEST_THREADS=1 \
+		cargo test

README.md

@@ -157,7 +157,7 @@ Note that, again, you may set `DATABASE_ID` to your liking. Make sure that the `
 
 To run an application server that points to the local Spanner emulator:
 ```sh
-SYNC_SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner
+SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner
 ```
 
 ### Running via Docker
@@ -166,7 +166,7 @@ This requires access to the mozilla-rust-sdk which is now available at `/vendor/
 1. Make sure you have [Docker installed](https://docs.docker.com/install/) locally.
 2. Copy the contents of mozilla-rust-sdk into top level root dir here.
 3. Change cargo.toml mozilla-rust-sdk entry to point to `"path = "mozilla-rust-sdk/googleapis-raw"` instead of the parent dir.
-4. Comment out the `image` value under `syncstorage-rs` in either docker-compose.mysql.yml or docker-compose.spanner.yml (depending on which database backend you want to run), and add this instead:
+4. Comment out the `image` value under `syncserver` in either docker-compose.mysql.yml or docker-compose.spanner.yml (depending on which database backend you want to run), and add this instead:
     ```yml
       build:
         context: .
@@ -220,7 +220,7 @@ We use [env_logger](https://crates.io/crates/env_logger): set the `RUST_LOG` env
 
 ### Unit tests
 
-`make test` - open the Makefile to adjust your `SYNC_DATABASE_URL` as needed.
+`make test` - open the Makefile to adjust your `SYNC_SYNCSTORAGE__DATABASE_URL` as needed.
 
 #### Debugging unit test state
 
@@ -229,7 +229,7 @@ default, we use the diesel test_transaction functionality to ensure test data
 is not committed to the database. Therefore, there is an environment variable
 which can be used to turn off test_transaction.
 
-        SYNC_DATABASE_USE_TEST_TRANSACTIONS=false cargo test [testname]
+        SYNC_SYNCSTORAGE__DATABASE_USE_TEST_TRANSACTIONS=false cargo test [testname]
 
 Note that you will almost certainly want to pass a single test name. When running
 the entire test suite, data from previous tests will cause future tests to fail.

docker-compose.e2e.mysql.yaml

@@ -2,31 +2,30 @@ version: '3'
 services:
   sync-db:
   tokenserver-db:
-  syncstorage-rs:
+  syncserver:
     depends_on:
       - sync-db
       - tokenserver-db
-    # TODO: either syncstorage-rs should retry the db connection
+    # TODO: either syncserver should retry the db connection
     # itself a few times or should include a wait-for-it.sh script
-    # inside its docker that would do this for us. Same (probably
-    # the latter solution) for server-syncstorage below
+    # inside its docker that would do this for us.
     entrypoint: >
       /bin/sh -c "
         sleep 15;
-        /app/bin/syncstorage;
+        /app/bin/syncserver;
       "
   e2e-tests:
     depends_on:
       - mock-fxa-server
-      - syncstorage-rs
+      - syncserver
     image: app:build
     privileged: true
     user: root
     environment:
       MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000
       SYNC_HOST: 0.0.0.0
       SYNC_MASTER_SECRET: secret0
-      SYNC_DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage
+      SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage
       SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
       SYNC_TOKENSERVER__ENABLED: "true"
       SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: "https://token.stage.mozaws.net/"

docker-compose.e2e.spanner.yaml

@@ -3,31 +3,30 @@ services:
   sync-db:
   sync-db-setup:
   tokenserver-db:
-  syncstorage-rs:
+  syncserver:
     depends_on:
       - sync-db-setup
-    # TODO: either syncstorage-rs should retry the db connection
+    # TODO: either syncserver should retry the db connection
     # itself a few times or should include a wait-for-it.sh script
-    # inside its docker that would do this for us. Same (probably
-    # the latter solution) for server-syncstorage below
+    # inside its docker that would do this for us.
     entrypoint: >
       /bin/sh -c "
         sleep 15;
-        /app/bin/syncstorage;
+        /app/bin/syncserver;
       "
   e2e-tests:
     depends_on:
       - mock-fxa-server
-      - syncstorage-rs
+      - syncserver
     image: app:build
     privileged: true
     user: root
     environment:
       MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000
       SYNC_HOST: 0.0.0.0
       SYNC_MASTER_SECRET: secret0
-      SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
-      SYNC_SPANNER_EMULATOR_HOST: sync-db:9010
+      SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
+      SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010
       SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
       SYNC_TOKENSERVER__ENABLED: "true"
       SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: "https://token.stage.mozaws.net/"