Merge pull request #1261 from mozilla-services/docs

bug: doc fixup

autokey.py

@@ -3,4 +3,4 @@
 
 
 def main():
-    print "Key = %s" % Fernet.generate_key()
+    print "CRYPTO_KEY=\"%s\"" % Fernet.generate_key()

autopush/config.py

@@ -43,7 +43,10 @@
 def _init_crypto_key(ck):
     # type: (Optional[Union[str, List[str]]]) -> List[str]
     """Provide a default or ensure the provided's a list"""
-    if ck is None:
+    # if CRYPTO_KEY is not set by docker, it may pass an empty string,
+    # which is converted into an Array element and prevents the config
+    # file value from being read
+    if ck is None or ck == ['']:
         return [Fernet.generate_key()]
     return ck if isinstance(ck, list) else [ck]
 
@@ -300,7 +303,6 @@ def from_argparse(cls, ns, **kwargs):
             ami_id = get_amid() or "Unknown"
 
         allow_table_rotation = not ns.no_table_rotation
-
         return cls(
             crypto_key=ns.crypto_key,
             datadog_api_key=ns.datadog_api_key,

autopush/db.py

@@ -467,7 +467,11 @@ class DynamoDBResource(threading.local):
     def __init__(self, **kwargs):
         conf = kwargs
         if not conf.get("endpoint_url"):
-            conf["endpoint_url"] = os.getenv("AWS_LOCAL_DYNAMODB")
+            if os.getenv("AWS_LOCAL_DYNAMODB"):
+                conf.update(dict(
+                    endpoint_url=os.getenv("AWS_LOCAL_DYNAMODB"),
+                    aws_access_key_id="Bogus",
+                    aws_secret_access_key="Bogus"))
         # If there is no endpoint URL, we must delete the entry
         if "endpoint_url" in conf and not conf["endpoint_url"]:
             del(conf["endpoint_url"])

autopush/tests/__init__.py

@@ -28,12 +28,7 @@ def setUp():
     ddb_process = subprocess.Popen(cmd, shell=True, env=os.environ)
     if os.getenv("AWS_LOCAL_DYNAMODB") is None:
         os.environ["AWS_LOCAL_DYNAMODB"] = "http://127.0.0.1:8000"
-    ddb_session_args = dict(
-        endpoint_url=os.getenv("AWS_LOCAL_DYNAMODB"),
-        aws_access_key_id="BogusKey",
-        aws_secret_access_key="BogusKey",
-    )
-    boto_resource = DynamoDBResource(**ddb_session_args)
+    boto_resource = DynamoDBResource()
     # Setup the necessary message tables
     message_table = os.environ.get("MESSAGE_TABLE", "message_int_test")
     create_rotating_message_table(prefix=message_table, delta=-1,

autopush/tests/test_db.py

@@ -132,6 +132,8 @@ def test_ddb_no_endpoint(self, mresource):
             del(os.environ["AWS_LOCAL_DYNAMODB"])
             DynamoDBResource(region_name="us-east-1")
             assert mresource.call_args[0] == ('dynamodb',)
+            resource = DynamoDBResource(endpoint_url="")
+            assert resource.conf == {}
         finally:
             if safe:  # pragma: nocover
                 os.environ["AWS_LOCAL_DYNAMODB"] = safe

configs/autopush_shared.ini.sample

@@ -19,6 +19,9 @@
 ; endpoint nodes in the same cluster; otherwise, endpoints won't be
 ; able to decrypt tokens generated by the connection node.
 ; Keys should be sorted [newest,oldest]
+;
+; *Note*: `docker_compose` overrides this config option. Be sure to
+; use the `CRYPTO_KEY` env value prefix when running via `docker_compose`.
 crypto_key = []
 
 ; The key_hash is the key used to hash the UAID for storage.

docker-compose.yml

@@ -1,7 +1,12 @@
 version: '2'
 services:
   autopush:
-    image: bbangert/autopush
+    build: .
+    # If you do not wish to use a local build, comment out
+    # the previous 'build:' line and uncomment the following
+    # 'image:' line. This will use the latest production
+    # built image.
+    #image: mozilla/autopush
     hostname: autopush
     environment:
       - LOCAL_HOSTNAME=localhost
@@ -17,7 +22,12 @@ services:
     volumes:
       - ./boto-compose.cfg:/etc/boto.cfg:ro
   autoendpoint:
-    image: bbangert/autopush
+    build: .
+    # If you do not wish to use a local build, comment out
+    # the previous 'build:' line and uncomment the following
+    # 'image:' line. This will use the latest production
+    # built image.
+    #image: mozilla/autopush
     hostname: autoendpoint
     command: autoendpoint
     environment:

docs/api/main.rst

@@ -18,11 +18,6 @@ Daemon Script Entry Points
     :private-members:
     :member-order: bysource
 
-.. autoclass:: RustConnectionApplication
-    :members:
-    :private-members:
-    :member-order: bysource
-
 Common Root
 +++++++++++
 

docs/index.rst

@@ -8,7 +8,12 @@ autopush
 .. image:: https://codecov.io/github/mozilla-services/autopush/coverage.svg
   :target: https://codecov.io/github/mozilla-services/autopush
 
-Mozilla Push server and Push Endpoint utilizing PyPy, twisted, rust, and DynamoDB.
+Mozilla Push server and Push Endpoint utilizing PyPy, twisted, and DynamoDB.
+
+*Please note*: The python components of this server are being deprecated in favor of
+`a pure rust <https://github.com/mozilla-services/autopush-rs>`_ implementation.
+We are planning on moving the websocket handler first, followed by the endpoint
+handlers. See :ref:`Migrating to Rust <rust>` for details.
 
 This is the third generation of Push server built in Mozilla Services, first
 to handle Push for FirefoxOS clients, then extended for push notifications for
@@ -137,6 +142,7 @@ Reference
    :hidden:
 
    glossary
+   rust
 
 License
 =======

docs/running.rst

@@ -47,15 +47,6 @@ daemons use other ports.
 
         $ curl -O https://raw.githubusercontent.com/mozilla-services/autopush/master/docker-compose.yml
 
-5. Fetch the latest ``boto-compose.cfg`` file:
-
-    .. code-block:: bash
-
-        $ curl -O https://raw.githubusercontent.com/mozilla-services/autopush/master/boto-compose.cfg
-
-The ``boto-compose.cfg`` file will be mounted inside the Autopush docker
-containers when running to point Autopush at the locally running DynamoDB
-
 .. note::
 
     The docker images used take approximately 1.5 GB of disk-space, make sure
@@ -69,10 +60,10 @@ run both of the Autopush daemons. To generate one with the docker image:
 
 .. code-block:: bash
 
-    $ docker run -t -i ~/autopush autokey
-    Key = hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk=
+    $ docker run -t -i bbangert/autopush autokey
+    CRYPTO_KEY="hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk="
 
-Store the key for later use (including the trailing ``=``).
+Store the key for later use (including any trailing ``=``).
 
 Start Autopush
 ==============
@@ -82,7 +73,7 @@ Autopush with a single command:
 
 .. code-block:: bash
 
-    $ CRYPTO_KEY=hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk= docker-compose up
+    $ CRYPTO_KEY="hkclU1V37Dnp-0DMF9HLe_40Nnr8kDTYVbo2yxuylzk=" docker-compose up
 
 `docker-compose`_ will start up three containers, two for each Autopush daemon,
 and a third for DynamoDB.
@@ -93,8 +84,11 @@ By default, the following services will be exposed:
 
 ``http://localhost:8082/`` - HTTP Endpoint Server (See :ref:`the HTTP API <http>`)
 
-You could set the ``CRYPTO_KEY`` as an environment variable, or setup a more
-thorough configuration using config files as documented below.
+You could set the ``CRYPTO_KEY`` as an environment variable if you are using Docker.
+If you are running these programs "stand-alone" or outside of docker-compose, you may
+setup a more thorough configuration using config files as documented below.
+
+*Note*:
 
 The load-tester can be run against it or you can run Firefox with the
 local Autopush per the :ref:`test-with-firefox` docs.
@@ -118,7 +112,7 @@ it is either True if enabled, or False if disabled. The configuration files are
 usually richly commented, and you're encouraged to read them to learn how to
 set up your installation of autopush.
 
-Please note: any line that does not begin with a `#` or `;` is considered an option
+*Note*: any line that does not begin with a `#` or `;` is considered an option
 line. if an unexpected option is present in a configuration file, the application
 will fail to start.
 
@@ -172,6 +166,13 @@ as ``/etc/autopush_connection.ini``, update the ``autopush`` section of the
 Autopush automatically searches for a configuration file at this location so
 nothing else is needed.
 
+*Note*: The `docker-compose.yml` file provides a number of overrides as environment
+variables, such as `CRYPTO_KEY`. If these values are not defined, they are submitted
+as `""`, which will prevent values from being read from the config files. In the case
+of `CRYPTO_KEY`, a new, random key is automatically generated, which will result in
+existing endpoints no longer being valid. It is recommended that for docker based
+images, that you ***always*** supply a `CRYPTO_KEY` as part of the run command.
+
 Notes on GCM/FCM support
 ------------------------
 

docs/rust.rst

@@ -0,0 +1,42 @@
+.. _rust:
+
+=================
+Migrating to Rust
+=================
+
+Progress never comes from resting. One of the significant considerations of running a service that needs to communicate
+with hundreds of millions of clients is cost. We are forced to continually evaluate and optimize. When a lower cost
+option is presented, we seriously consider it.
+
+There is some risk, of course, so rapid change is avoided and testing is strongly encouraged. As of early 2018, the
+decision was made to move the costlier elements of the server to Rust. The rust based application is at
+`autopush-rs`_.
+
+Why Rust?
+=========
+
+Rust is a strongly typed, memory efficient language. It has matured rapidly and offers structure that vastly reduces
+the memory requirements for running connections. As a bonus, it’s also forced us to handle potential bugs, making the
+service more reliable.
+
+The current python environment we use (pypy) continues to improve as well, but does not offer the sort of improvements
+that rust does when it comes to handling socket connections.
+
+To that end we’re continuing to use pypy for the endpoint connection management for the time being.
+
+When is the switch going to happen?
+===================================
+
+As of the end of June 2018, our rust handler is in testing. We expect to deploy it soon, but since this deployment
+should not impact external users, we’re not rushing to deploy just to hit an arbitrary milestone. It will be deployed
+when all parties have determined it’s ready.
+
+What will happen to autopush?
+=============================
+
+Currently, the plan is to maintain it so long as it’s in production use. Since we plan on continuing to have autopush
+handle endpoints for some period, even after autopush-rs has been deployed to production and is handling connections.
+However, we do reserve the right to archive this repo at some future date.
+
+
+.. _`autopush-rs`: https://github.com/mozilla-services/autopush-rs