From fdb3811dfdfb598bb536b38739f7b983a7444e5f Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 19 Jun 2018 15:36:04 -0700 Subject: [PATCH] bug: doc fixup * Fix out of date documentation * Fix tests to use new local env var (like we do for endpoint) * Add rust migration document * Fix autokey to return something useful * Point out that docker_compose does bad things to your expectations about CRYPTO_KEY Closes #1260 Issue mozilla-services/autopush-rs#7 --- autokey.py | 2 +- autopush/config.py | 6 +++-- autopush/db.py | 6 ++++- autopush/tests/__init__.py | 7 +---- autopush/tests/test_db.py | 2 ++ configs/autopush_shared.ini.sample | 3 +++ docker-compose.yml | 6 +++-- docs/api/main.rst | 5 ---- docs/index.rst | 6 +++++ docs/running.rst | 33 +++++++++++------------ docs/rust.rst | 42 ++++++++++++++++++++++++++++++ 11 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 docs/rust.rst diff --git a/autokey.py b/autokey.py index 3633df23..c63c70da 100644 --- a/autokey.py +++ b/autokey.py @@ -3,4 +3,4 @@ def main(): - print "Key = %s" % Fernet.generate_key() + print "CRYPTO_KEY=\"%s\"" % Fernet.generate_key() diff --git a/autopush/config.py b/autopush/config.py index a7798394..40a0e0a1 100644 --- a/autopush/config.py +++ b/autopush/config.py @@ -43,7 +43,10 @@ def _init_crypto_key(ck): # type: (Optional[Union[str, List[str]]]) -> List[str] """Provide a default or ensure the provided's a list""" - if ck is None: + # if CRYPTO_KEY is not set by docker, it may pass an empty string, + # which is converted into an Array element and prevents the config + # file value from being read + if ck is None or ck == ['']: return [Fernet.generate_key()] return ck if isinstance(ck, list) else [ck] @@ -300,7 +303,6 @@ def from_argparse(cls, ns, **kwargs): ami_id = get_amid() or "Unknown" allow_table_rotation = not ns.no_table_rotation - return cls( crypto_key=ns.crypto_key, datadog_api_key=ns.datadog_api_key, diff --git a/autopush/db.py b/autopush/db.py index e9918838..ab7ca4d4 100644 --- a/autopush/db.py +++ b/autopush/db.py @@ -467,7 +467,11 @@ class DynamoDBResource(threading.local): def __init__(self, **kwargs): conf = kwargs if not conf.get("endpoint_url"): - conf["endpoint_url"] = os.getenv("AWS_LOCAL_DYNAMODB") + if os.getenv("AWS_LOCAL_DYNAMODB"): + conf.update(dict( + endpoint_url=os.getenv("AWS_LOCAL_DYNAMODB"), + aws_access_key_id="Bogus", + aws_secret_access_key="Bogus")) # If there is no endpoint URL, we must delete the entry if "endpoint_url" in conf and not conf["endpoint_url"]: del(conf["endpoint_url"]) diff --git a/autopush/tests/__init__.py b/autopush/tests/__init__.py index df072c2d..ccb1efea 100644 --- a/autopush/tests/__init__.py +++ b/autopush/tests/__init__.py @@ -28,12 +28,7 @@ def setUp(): ddb_process = subprocess.Popen(cmd, shell=True, env=os.environ) if os.getenv("AWS_LOCAL_DYNAMODB") is None: os.environ["AWS_LOCAL_DYNAMODB"] = "http://127.0.0.1:8000" - ddb_session_args = dict( - endpoint_url=os.getenv("AWS_LOCAL_DYNAMODB"), - aws_access_key_id="BogusKey", - aws_secret_access_key="BogusKey", - ) - boto_resource = DynamoDBResource(**ddb_session_args) + boto_resource = DynamoDBResource() # Setup the necessary message tables message_table = os.environ.get("MESSAGE_TABLE", "message_int_test") create_rotating_message_table(prefix=message_table, delta=-1, diff --git a/autopush/tests/test_db.py b/autopush/tests/test_db.py index 4f8659cf..d7bb773b 100644 --- a/autopush/tests/test_db.py +++ b/autopush/tests/test_db.py @@ -132,6 +132,8 @@ def test_ddb_no_endpoint(self, mresource): del(os.environ["AWS_LOCAL_DYNAMODB"]) DynamoDBResource(region_name="us-east-1") assert mresource.call_args[0] == ('dynamodb',) + resource = DynamoDBResource(endpoint_url="") + assert resource.conf == {} finally: if safe: # pragma: nocover os.environ["AWS_LOCAL_DYNAMODB"] = safe diff --git a/configs/autopush_shared.ini.sample b/configs/autopush_shared.ini.sample index b0111951..fb0b1a1d 100644 --- a/configs/autopush_shared.ini.sample +++ b/configs/autopush_shared.ini.sample @@ -19,6 +19,9 @@ ; endpoint nodes in the same cluster; otherwise, endpoints won't be ; able to decrypt tokens generated by the connection node. ; Keys should be sorted [newest,oldest] +; +; *Note*: `docker_compose` overrides this config option. Be sure to +; use the `CRYPTO_KEY` env value prefix when running via `docker_compose`. crypto_key = [] ; The key_hash is the key used to hash the UAID for storage. diff --git a/docker-compose.yml b/docker-compose.yml index bddab427..3f5198f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,8 @@ version: '2' services: autopush: - image: bbangert/autopush + # image: bbangert/autopush + build: . hostname: autopush environment: - LOCAL_HOSTNAME=localhost @@ -17,7 +18,8 @@ services: volumes: - ./boto-compose.cfg:/etc/boto.cfg:ro autoendpoint: - image: bbangert/autopush + # image: bbangert/autopush + build: . hostname: autoendpoint command: autoendpoint environment: diff --git a/docs/api/main.rst b/docs/api/main.rst index dfc35e02..9887ed18 100644 --- a/docs/api/main.rst +++ b/docs/api/main.rst @@ -18,11 +18,6 @@ Daemon Script Entry Points :private-members: :member-order: bysource -.. autoclass:: RustConnectionApplication - :members: - :private-members: - :member-order: bysource - Common Root +++++++++++ diff --git a/docs/index.rst b/docs/index.rst index 694c3b3f..01e4b671 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,6 +10,11 @@ autopush Mozilla Push server and Push Endpoint utilizing PyPy, twisted, rust, and DynamoDB. +*Please note*: The python components of this server are being deprecated in favor of +`a pure rust `_ implementation. +We are planning on moving the websocket handler first, followed by the endpoint +handlers. See :ref:`Migrating to Rust ` for details. + This is the third generation of Push server built in Mozilla Services, first to handle Push for FirefoxOS clients, then extended for push notifications for Firefox (via the `W3C Push spec `_.) @@ -137,6 +142,7 @@ Reference :hidden: glossary + rust License ======= diff --git a/docs/running.rst b/docs/running.rst index 8aae94be..ac9874fe 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -47,15 +47,6 @@ daemons use other ports. $ curl -O https://raw.githubusercontent.com/mozilla-services/autopush/master/docker-compose.yml -5. Fetch the latest ``boto-compose.cfg`` file: - - .. code-block:: bash - - $ curl -O https://raw.githubusercontent.com/mozilla-services/autopush/master/boto-compose.cfg - -The ``boto-compose.cfg`` file will be mounted inside the Autopush docker -containers when running to point Autopush at the locally running DynamoDB - .. note:: The docker images used take approximately 1.5 GB of disk-space, make sure @@ -69,10 +60,10 @@ run both of the Autopush daemons. To generate one with the docker image: .. code-block:: bash - $ docker run -t -i ~/autopush autokey - Key = hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk= + $ docker run -t -i bbangert/autopush autokey + CRYPTO_KEY="hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk=" -Store the key for later use (including the trailing ``=``). +Store the key for later use (including any trailing ``=``). Start Autopush ============== @@ -82,7 +73,7 @@ Autopush with a single command: .. code-block:: bash - $ CRYPTO_KEY=hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk= docker-compose up + $ CRYPTO_KEY="hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk=" docker-compose up `docker-compose`_ will start up three containers, two for each Autopush daemon, and a third for DynamoDB. @@ -93,8 +84,11 @@ By default, the following services will be exposed: ``http://localhost:8082/`` - HTTP Endpoint Server (See :ref:`the HTTP API `) -You could set the ``CRYPTO_KEY`` as an environment variable, or setup a more -thorough configuration using config files as documented below. +You could set the ``CRYPTO_KEY`` as an environment variable if you are using Docker. +If you are running these programs "stand-alone" or outside of docker-compose, you may +setup a more thorough configuration using config files as documented below. + +*Note*: The load-tester can be run against it or you can run Firefox with the local Autopush per the :ref:`test-with-firefox` docs. @@ -118,7 +112,7 @@ it is either True if enabled, or False if disabled. The configuration files are usually richly commented, and you're encouraged to read them to learn how to set up your installation of autopush. -Please note: any line that does not begin with a `#` or `;` is considered an option +*Note*: any line that does not begin with a `#` or `;` is considered an option line. if an unexpected option is present in a configuration file, the application will fail to start. @@ -172,6 +166,13 @@ as ``/etc/autopush_connection.ini``, update the ``autopush`` section of the Autopush automatically searches for a configuration file at this location so nothing else is needed. +*Note*: The `docker-compose.yml` file provides a number of overrides as environment +variables, such as `CRYPTO_KEY`. If these values are not defined, they are submitted +as `""`, which will prevent values from being read from the config files. In the case +of `CRYPTO_KEY`, a new, random key is automatically generated, which will result in +existing endpoints no longer being valid. It is recommended that for docker based +images, that you ***always*** supply a `CRYPTO_KEY` as part of the run command. + Notes on GCM/FCM support ------------------------ diff --git a/docs/rust.rst b/docs/rust.rst new file mode 100644 index 00000000..bcd8d9ef --- /dev/null +++ b/docs/rust.rst @@ -0,0 +1,42 @@ +.. _rust: + +================= +Migrating to Rust +================= + +Progress never comes from resting. One of the significant considerations of running a service that needs to communicate +with hundreds of millions of clients is cost. We are forced to continually evaluate and optimize. When a lower cost +option is presented, we seriously consider it. + +There is some risk, of course, so rapid change is avoided and testing is strongly encouraged. As of early 2018, the +decision was made to move the costlier elements of the server to Rust. The rust based application is at +`autopush-rs`_. + +Why Rust? +========= + +Rust is a strongly typed, memory efficient language. It has matured rapidly and offers structure that vastly reduces +the memory requirements for running connections. As a bonus, it’s also forced us to handle potential bugs, making the +service more reliable. + +The current python environment we use (pypy) continues to improve as well, but does not offer the sort of improvements +that rust does when it comes to handling socket connections. + +To that end we’re continuing to use pypy for the endpoint connection management for the time being. + +When is the switch going to happen? +=================================== + +As of the end of June 2018, our rust handler is in testing. We expect to deploy it soon, but since this deployment +should not impact external users, we’re not rushing to deploy just to hit an arbitrary milestone. It will be deployed +when all parties have determined it’s ready. + +What will happen to autopush? +============================= + +Currently, the plan is to maintain it so long as it’s in production use. Since we plan on continuing to have autopush +handle endpoints for some period, even after autopush-rs has been deployed to production and is handling connections. +However, we do reserve the right to archive this repo at some future date. + + +.. _`autopush-rs`: https://github.com/mozilla-services/autopush-rs