feat: add metric tags

relies on cadence dev w/ new tag support

also move ua parsing into utils w/ setup of metrics' specific values
and a couple other fixes

Closes #1054

autopush/main.py

@@ -322,9 +322,12 @@ class RustConnectionApplication(AutopushMultiService):
     logger_name = "AutopushRust"
     push_server = None
 
-    def __init__(self, conf):
-        # type: (AutopushConfig) -> None
-        super(RustConnectionApplication, self).__init__(conf)
+    def __init__(self, conf, resource=None):
+        # type: (AutopushConfig, DynamoDBResource) -> None
+        super(RustConnectionApplication, self).__init__(
+            conf,
+            resource=resource
+        )
 
     def setup(self, rotate_tables=True):
         super(RustConnectionApplication, self).setup(rotate_tables)

autopush_rs/Cargo.toml

@@ -8,7 +8,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 bytes = "0.4.6"
-cadence = "0.12.2"
+cadence = { git = "https://github.com/tshlabs/cadence", rev = "1a3c90f73c88cab0"}
 chrono = "0.4.0"
 env_logger = { version = "0.5.3", default-features = false }
 error-chain = "0.11.0"

autopush_rs/__init__.py

@@ -39,19 +39,14 @@ def __init__(self, conf, queue):
         cfg.statsd_host = ffi_from_buffer(conf.statsd_host)
         cfg.statsd_port = conf.statsd_port
 
-        self.cfg = cfg
         ptr = _call(lib.autopush_server_new, cfg)
         self.ffi = ffi.gc(ptr, lib.autopush_server_free)
         self.queue = queue
 
     def startService(self):
-        try:
-            _call(lib.autopush_server_start,
-                  self.ffi,
-                  self.queue.ffi)
-        except Exception as ex:
-            import pdb; pdb.set_trace()
-            print(ex)
+        _call(lib.autopush_server_start,
+              self.ffi,
+              self.queue.ffi)
 
     def stopService(self):
         if self.ffi is None:

autopush_rs/src/client.rs

@@ -16,12 +16,13 @@ use futures::{Stream, Sink, Future, Poll, Async};
 use tokio_core::reactor::Timeout;
 use time;
 use uuid::Uuid;
-use woothee::parser::{Parser, WootheeResult};
+use woothee::parser::Parser;
 
 use call;
 use errors::*;
 use protocol::{ClientAck, ClientMessage, ServerMessage, ServerNotification, Notification};
 use server::Server;
+use util::parse_user_agent;
 
 pub struct RegisteredClient {
     pub uaid: Uuid,
@@ -223,13 +224,12 @@ where
                         if message.topic.is_some() {
                             self.data.srv.metrics.incr("ua.notification.topic")?;
                         }
-                        // XXX: tags
-                        self.data.srv.metrics.count(
-                            "ua.message_data",
-                            message.data.as_ref().map_or(0, |d| {
-                                d.len() as i64
-                            }),
-                        )?;
+                        let mlen = message.data.as_ref().map_or(0, |d| d.len() as i64);
+                        // XXX: not emitted for direct notifications (nor was it in websocket.py)
+                        self.data.srv.metrics
+                            .count_with_tags("ua.message_data", mlen)
+                            .with_tag("source", "Stored")
+                            .send()?;
                         ClientState::FinishSend(
                             Some(ServerMessage::Notification(message)),
                             Some(Box::new(ClientState::SendMessages(if messages.len() > 0 {
@@ -591,7 +591,7 @@ where
                 uaid_reset: reset_uaid,
                 existing_uaid: check_storage,
                 connection_type: String::from("webpush"),
-                host: String::from("unknown"),
+                host: self.host.clone(),
                 direct_acked: 0,
                 direct_storage: 0,
                 stored_retrieved: 0,
@@ -719,8 +719,16 @@ where
 
         let now = time::precise_time_ns() / 1000;
         let elapsed = now - webpush.connected_at;
-        // XXX: tags
-        self.srv.metrics.time("ua.connection.lifespan", elapsed).ok();
+
+        let parser = Parser::new();
+        let (ua_result, metrics_os, metrics_browser) = parse_user_agent(&parser, &self.user_agent);
+        self.srv.metrics
+            .time_with_tags("ua.connection.lifespan", elapsed)
+            .with_tag("ua_os_family", metrics_os)
+            .with_tag("ua_browser_family", metrics_browser)
+            .with_tag("host", &webpush.stats.host)
+            .send()
+            .ok();
 
         // If there's direct unack'd messages, they need to be saved out without blocking
         // here
@@ -743,27 +751,13 @@ where
             )
         }
 
-        // Parse the user-agent string
-        let parser = Parser::new();
-        let ua_result = parser
-            .parse(&self.user_agent)
-            .unwrap_or_else(|| WootheeResult {
-                name: "",
-                category: "",
-                os: "",
-                os_version: "".to_string(),
-                browser_type: "",
-                version: "".to_string(),
-                vendor: "",
-            });
-
         // Log out the final stats message
         info!("Session";
-              "uaid_hash" => stats.uaid.as_str(),
+              "uaid_hash" => &stats.uaid,
               "uaid_reset" => stats.uaid_reset,
               "existing_uaid" => stats.existing_uaid,
-              "connection_type" => stats.connection_type.as_str(),
-              "host" => self.host.clone(),
+              "connection_type" => &stats.connection_type,
+              "host" => &stats.host,
               "ua_name" => ua_result.name,
               "ua_os_family" => ua_result.os,
               "ua_os_ver" => ua_result.os_version,

autopush_rs/src/util/mod.rs

@@ -13,12 +13,14 @@ use tokio_core::reactor::{Handle, Timeout};
 
 use errors::*;
 
-mod send_all;
-mod rc;
 mod autojson;
+mod rc;
+mod send_all;
+mod user_agent;
 
 pub use self::send_all::MySendAll;
 pub use self::rc::RcObject;
+pub use self::user_agent::parse_user_agent;
 
 /// Convenience future to time out the resolution of `f` provided within the
 /// duration provided.

autopush_rs/src/util/user_agent.rs

@@ -0,0 +1,92 @@
+use woothee::parser::{Parser, WootheeResult};
+
+// List of valid user-agent attributes to keep, anything not in this
+// list is considered 'Other'. We log the user-agent on connect always
+// to retain the full string, but for DD more tags are expensive so we
+// limit to these.
+const VALID_UA_BROWSER: &[&str] = &["Chrome", "Firefox", "Safari", "Opera"];
+
+// See dataset.rs in https://github.com/woothee/woothee-rust for the
+// full list (WootheeResult's 'os' field may fall back to its 'name'
+// field). Windows has many values and we only care that its Windows
+const VALID_UA_OS: &[&str] = &["Firefox OS", "Linux", "Mac OSX"];
+
+pub fn parse_user_agent<'a>(
+    parser: &'a Parser,
+    agent: &str,
+) -> (WootheeResult<'a>, &'a str, &'a str) {
+    let wresult = parser.parse(&agent).unwrap_or_else(|| WootheeResult {
+        name: "",
+        category: "",
+        os: "",
+        os_version: "".to_string(),
+        browser_type: "",
+        version: "".to_string(),
+        vendor: "",
+    });
+
+    // Determine a base os/browser for metrics' tags
+    let metrics_os = if wresult.os.starts_with("Windows") {
+        "Windows"
+    } else if VALID_UA_OS.contains(&wresult.os) {
+        wresult.os
+    } else {
+        "Other"
+    };
+    let metrics_browser = if VALID_UA_BROWSER.contains(&wresult.name) {
+        wresult.name
+    } else {
+        "Other"
+    };
+    (wresult, metrics_os, metrics_browser)
+}
+
+#[cfg(test)]
+mod tests {
+    use woothee::parser::Parser;
+
+    use super::parse_user_agent;
+
+    #[test]
+    fn test_linux() {
+        let agent = r#"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.2) Gecko/20090807 Mandriva Linux/1.9.1.2-1.1mud2009.1 (2009.1) Firefox/3.5.2 FirePHP/0.3,gzip(gfe),gzip(gfe)"#;
+        let parser = Parser::new();
+        let (ua_result, metrics_os, metrics_browser) = parse_user_agent(&parser, &agent);
+        assert_eq!(metrics_os, "Linux");
+        assert_eq!(ua_result.os, "Linux");
+        assert_eq!(metrics_browser, "Firefox");
+    }
+
+    #[test]
+    fn test_windows() {
+        let agent = r#"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 3.5.30729)"#;
+        let parser = Parser::new();
+        let (ua_result, metrics_os, metrics_browser) = parse_user_agent(&parser, &agent);
+        assert_eq!(metrics_os, "Windows");
+        assert_eq!(ua_result.os, "Windows 7");
+        assert_eq!(metrics_browser, "Firefox");
+    }
+
+    #[test]
+    fn test_osx() {
+        let agent =
+            r#"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:2.1.1) Gecko/ Firefox/5.0.1"#;
+        let parser = Parser::new();
+        let (ua_result, metrics_os, metrics_browser) = parse_user_agent(&parser, &agent);
+        assert_eq!(metrics_os, "Mac OSX");
+        assert_eq!(ua_result.os, "Mac OSX");
+        assert_eq!(metrics_browser, "Firefox");
+    }
+
+    #[test]
+    fn test_other() {
+        let agent =
+            r#"BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102"#;
+        let parser = Parser::new();
+        let (ua_result, metrics_os, metrics_browser) = parse_user_agent(&parser, &agent);
+        assert_eq!(metrics_os, "Other");
+        assert_eq!(ua_result.os, "BlackBerry");
+        assert_eq!(metrics_browser, "Other");
+        assert_eq!(ua_result.name, "UNKNOWN");
+    }
+}