fix: bind autopush_rs to all interfaces to match python

cleanup instance id or hostname used for logging/metrics and duplicate
some of that setup in rust (which'll eventually need it anyway)

and some mypy fixes

Closes #1113

autopush/config.py

@@ -291,7 +291,7 @@ def from_argparse(cls, ns, **kwargs):
         # Not a fan of double negatives, but this makes more
         # understandable args
         if not ns.no_aws:
-            ami_id = get_amid()
+            ami_id = get_amid() or "Unknown"
 
         return cls(
             crypto_key=ns.crypto_key,

autopush/db.py

@@ -95,6 +95,7 @@
 
 # Typing
 T = TypeVar('T')  # noqa
+TableFunc = Callable[[str, int, int, ServiceResource], Table]
 
 key_hash = ""
 TRACK_DB_CALLS = False
@@ -143,10 +144,11 @@ def make_rotating_tablename(prefix, delta=0, date=None):
 def create_rotating_message_table(
         prefix="message",     # type: str
         delta=0,              # type: int
-        date=None,            # type: Optional[datetime.date]]
+        date=None,            # type: Optional[datetime.date]
         read_throughput=5,    # type: int
         write_throughput=5,   # type: int
-        boto_resource=None):  # type: DynamoDBResource
+        boto_resource=None    # type: DynamoDBResource
+        ):
     # type: (...) -> Table  # noqa
     """Create a new message table for webpush style message storage"""
     tablename = make_rotating_tablename(prefix, delta, date)
@@ -205,7 +207,8 @@ def get_rotating_message_tablename(
         date=None,                   # type: Optional[datetime.date]
         message_read_throughput=5,   # type: int
         message_write_throughput=5,  # type: int
-        boto_resource=None):         # type: DynamoDBResource
+        boto_resource=None           # type: DynamoDBResource
+        ):
     # type: (...) -> str  # noqa
     """Gets the message table for the current month."""
     tablename = make_rotating_tablename(prefix, delta, date)
@@ -306,7 +309,7 @@ def _drop_table(tablename, boto_resource):
 
 
 def _make_table(
-        table_func,        # type: Callable[[str, int, int, ServiceResource]]
+        table_func,        # type: TableFunc
         tablename,         # type: str
         read_throughput,   # type: int
         write_throughput,  # type: int
@@ -555,7 +558,7 @@ def unregister_channel(self, uaid, channel_id, **kwargs):
 
     @track_provisioned
     def all_channels(self, uaid):
-        # type: (str, str) -> Tuple[bool, Set[str]]
+        # type: (str) -> Tuple[bool, Set[str]]
         """Retrieve a list of all channels for a given uaid"""
 
         # Note: This only returns the chids associated with the UAID.
@@ -752,7 +755,7 @@ def table_status(self):
         return self.table.table_status
 
     def get_uaid(self, uaid):
-        # type: (str) -> Item
+        # type: (str) -> Dict[str, Any]
         """Get the database record for the UAID
 
         :raises:

autopush/logging.py

@@ -25,7 +25,7 @@
 )
 from zope.interface import implementer
 
-HOSTNAME = socket.getfqdn()
+from autopush.utils import get_ec2_instance_id
 
 # A complete set of keys we don't include in Fields from a log event
 IGNORED_KEYS = frozenset([
@@ -59,6 +59,9 @@
 # should only be called once
 began_logging = False
 
+# an ec2 instance id or falling back to the hostname
+instance_id_or_hostname = None
+
 
 def begin_or_register(observer, redirectStandardIO=False, **kwargs):
     # type: (Any, bool, **Any) -> None
@@ -179,7 +182,7 @@ def to_fields(kv):
             return reply
 
         msg = {
-            "Hostname": HOSTNAME,
+            "Hostname": instance_id_or_hostname,
             "Timestamp": ts * 1000 * 1000 * 1000,
             "Type": "twisted:log",
             "Severity": event.get("severity") or severity,
@@ -222,7 +225,13 @@ def stop(self):
     @classmethod
     def setup_logging(cls, logger_name, log_level="info", log_format="json",
                       log_output="stdout", sentry_dsn=None,
-                      firehose_delivery_stream=None):
+                      firehose_delivery_stream=None,
+                      no_aws=False):
+        global instance_id_or_hostname
+        if not instance_id_or_hostname:
+            instance_id = None if no_aws else get_ec2_instance_id()
+            instance_id_or_hostname = instance_id or socket.getfqdn()
+
         pl = cls(logger_name, log_level=log_level, log_format=log_format,
                  log_output=log_output, sentry_dsn=sentry_dsn,
                  firehose_delivery_stream=firehose_delivery_stream)

autopush/main.py

@@ -28,8 +28,6 @@
     MemUsageHTTPFactory,
     agent_from_config
 )
-import autopush.utils as utils
-import autopush.logging as logging
 from autopush.config import AutopushConfig
 from autopush.db import DatabaseManager, DynamoDBResource  # noqa
 from autopush.exceptions import InvalidConfig
@@ -138,15 +136,14 @@ def main(cls, args=None, use_files=True, resource=None):
 
         """
         ns = cls.parse_args(cls.config_files if use_files else [], args)
-        if not ns.no_aws:
-            logging.HOSTNAME = utils.get_ec2_instance_id()
         PushLogger.setup_logging(
             cls.logger_name,
             log_level=ns.log_level or ("debug" if ns.debug else "info"),
             log_format="text" if ns.human_logs else "json",
             log_output=ns.log_output,
             sentry_dsn=bool(os.environ.get("SENTRY_DSN")),
-            firehose_delivery_stream=ns.firehose_stream_name
+            firehose_delivery_stream=ns.firehose_stream_name,
+            no_aws=ns.no_aws
         )
         try:
             app = cls.from_argparse(ns, resource=resource)

autopush/memusage.py

@@ -26,8 +26,8 @@
 
 
 def memusage(do_dump_rpy_heap=True, do_objgraph=True):
-    """Returning a str of memory usage stats"""
     # type: (Optional[bool], Optional[bool]) -> str
+    """Returning a str of memory usage stats"""
     def trap_err(func, *args, **kwargs):
         try:
             return func(*args, **kwargs)

autopush/metrics.py

@@ -1,5 +1,10 @@
 """Metrics interface and implementations"""
-from typing import TYPE_CHECKING, Sequence, Any  # noqa
+from typing import (  # noqa
+    TYPE_CHECKING,
+    Any,
+    Optional,
+    Sequence
+)
 
 from twisted.internet import reactor
 from txstatsd.client import StatsDClientProtocol, TwistedStatsDClient
@@ -8,7 +13,7 @@
 import datadog
 from datadog import ThreadStats
 
-from autopush.utils import get_ec2_instance_id
+from autopush import logging
 
 if TYPE_CHECKING:  # pragma: nocover
     from autopush.config import AutopushConfig  # noqa
@@ -119,7 +124,7 @@ def from_config(conf):
     """Create an IMetrics from the given config"""
     if conf.datadog_api_key:
         return DatadogMetrics(
-            hostname=get_ec2_instance_id() if conf.ami_id else
+            hostname=logging.instance_id_or_hostname if conf.ami_id else
             conf.hostname,
             api_key=conf.datadog_api_key,
             app_key=conf.datadog_app_key,

autopush/tests/test_utils.py

@@ -45,7 +45,7 @@ def test_get_amid_unknown(self, request_mock):
 
         request_mock.side_effect = requests.HTTPError
         result = get_amid()
-        assert result == "Unknown"
+        assert result is None
 
     @patch("requests.get")
     def test_get_ec2_instance_id_unknown(self, request_mock):
@@ -54,7 +54,7 @@ def test_get_ec2_instance_id_unknown(self, request_mock):
 
         request_mock.side_effect = requests.HTTPError
         result = get_ec2_instance_id()
-        assert result == "Unknown"
+        assert result is None
 
     @patch("requests.get")
     def test_get_ec2_instance_id(self, request_mock):

autopush/utils.py

@@ -135,7 +135,7 @@ def base64url_decode(string):
 
 
 def get_amid():
-    # type: () -> str
+    # type: () -> Optional[str]
     """Fetch the AMI instance ID
 
     """
@@ -145,11 +145,11 @@ def get_amid():
             timeout=1)
         return resp.content
     except (requests.HTTPError, requests.ConnectionError):
-        return "Unknown"
+        return None
 
 
 def get_ec2_instance_id():
-    # type: () -> str
+    # type: () -> Optional[str]
     """Fetch the EC2 instance-id
 
     """
@@ -159,7 +159,7 @@ def get_ec2_instance_id():
             timeout=1)
         return resp.content
     except (requests.HTTPError, requests.ConnectionError):
-        return "Unknown"
+        return None
 
 
 def decipher_public_key(key_data):

autopush/webpush_server.py

@@ -232,7 +232,7 @@ class StoreMessagesResponse(OutputCommand):
 ###############################################################################
 class WebPushServer(object):
     def __init__(self, conf, db, num_threads=10):
-        # type: (AutopushConfig, DatabaseManager, int) -> WebPushServer
+        # type: (AutopushConfig, DatabaseManager, int) -> None
         self.conf = conf
         self.db = db
         self.db.setup_tables()
@@ -244,7 +244,7 @@ def __init__(self, conf, db, num_threads=10):
         self.running = False
 
     def start(self):
-        # type: (int) -> None
+        # type: () -> None
         self.running = True
         for _ in range(self.num_threads):
             self.workers.append(
@@ -291,7 +291,7 @@ def spawn(self, func, *args, **kwargs):
 
 class CommandProcessor(object):
     def __init__(self, conf, db):
-        # type: (AutopushConfig, DatabaseManager) -> CommandProcessor
+        # type: (AutopushConfig, DatabaseManager) -> None
         self.conf = conf
         self.db = db
         self.hello_processor = HelloCommand(conf, db)