From 1bd9bef675553bf5b4e4f03b55242dd746a9cac2 Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 7 Oct 2024 09:09:06 -0400 Subject: [PATCH 01/20] enums for device family and os family --- syncserver/src/server/user_agent.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 244e341a37..4be640453e 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -1,3 +1,6 @@ +use std::fmt; +use std::str::FromStr; + use woothee::parser::{Parser, WootheeResult}; // List of valid user-agent attributes to keep, anything not in this @@ -11,6 +14,25 @@ const VALID_UA_BROWSER: &[&str] = &["Chrome", "Firefox", "Safari", "Opera"]; // field). Windows has many values and we only care that its Windows const VALID_UA_OS: &[&str] = &["Firefox OS", "Linux", "Mac OSX"]; +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum DeviceFamily { + Desktop, + Phone, + Tablet, + Other, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum OsFamily { + Windows, + MacOs, + Linux, + IOs, + Android, + ChromeOs, + Other, +} + pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) { let parser = Parser::new(); let wresult = parser.parse(agent).unwrap_or_else(|| WootheeResult { From fe3f24edcddcffea3d37c6598712e45e8cc79efb Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 7 Oct 2024 09:37:31 -0400 Subject: [PATCH 02/20] implement fmt::Display trait for OsDamily and DeviceFamily --- syncserver/src/server/user_agent.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 4be640453e..36248d2928 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -22,6 +22,13 @@ pub enum DeviceFamily { Other, } +impl fmt::Display for DeviceFamily { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = format!("{:?}", self).to_lowercase(); + write!(fmt, "{}", name) + } +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum OsFamily { Windows, @@ -33,6 +40,13 @@ pub enum OsFamily { Other, } +impl fmt::Display for OsFamily { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = format!("{:?}", self).to_lowercase(); + write!(fmt, "{}", name) + } +} + pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) { let parser = Parser::new(); let wresult = parser.parse(agent).unwrap_or_else(|| WootheeResult { From 2b7d08a7c6b6228cef7946c2cc0f4433587ddb1d Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 7 Oct 2024 09:39:40 -0400 Subject: [PATCH 03/20] add device info struct for tracking firefox version --- syncserver/src/server/user_agent.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 36248d2928..e642a7cd5c 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -47,6 +47,13 @@ impl fmt::Display for OsFamily { } } +#[derive(Debug, Eq, PartialEq)] +pub struct DeviceInfo { + pub device_family: DeviceFamily, + pub os_family: OsFamily, + pub firefox_version: u32, +} + pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) { let parser = Parser::new(); let wresult = parser.parse(agent).unwrap_or_else(|| WootheeResult { From 7881d618b3b4272033132dc8dcfd3b87aa6dac4b Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 7 Oct 2024 09:46:40 -0400 Subject: [PATCH 04/20] bool utility functions to determine if mobile or desktop --- syncserver/src/server/user_agent.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index e642a7cd5c..528ed0e9c1 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -34,7 +34,7 @@ pub enum OsFamily { Windows, MacOs, Linux, - IOs, + IOS, Android, ChromeOs, Other, @@ -54,6 +54,23 @@ pub struct DeviceInfo { pub firefox_version: u32, } +impl DeviceInfo { + pub fn is_desktop(&self) -> bool { + matches!(&self.device_family, DeviceFamily::Desktop) + || matches!( + &self.os_family, + OsFamily::MacOs | OsFamily::Windows | OsFamily::Linux + ) + } + + pub fn is_mobile(&self) -> bool { + matches!( + &self.device_family, + DeviceFamily::Phone | DeviceFamily::Tablet + ) || matches!(&self.os_family, OsFamily::Android | OsFamily::IOS) + } +} + pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) { let parser = Parser::new(); let wresult = parser.parse(agent).unwrap_or_else(|| WootheeResult { From 39b080c884491ee15305525b98a96ba9c3d49f23 Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 7 Oct 2024 12:32:32 -0400 Subject: [PATCH 05/20] add get device info method for more advanced parsing --- syncserver/src/server/user_agent.rs | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 528ed0e9c1..1f0fc06568 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -1,6 +1,7 @@ use std::fmt; use std::str::FromStr; +use actix_http::header::ACCESS_CONTROL_REQUEST_HEADERS; use woothee::parser::{Parser, WootheeResult}; // List of valid user-agent attributes to keep, anything not in this @@ -55,6 +56,7 @@ pub struct DeviceInfo { } impl DeviceInfo { + /// Determine if the device is a desktop device based on either the form factor or OS. pub fn is_desktop(&self) -> bool { matches!(&self.device_family, DeviceFamily::Desktop) || matches!( @@ -63,12 +65,66 @@ impl DeviceInfo { ) } + /// Determine if the device is a mobile phone based on either the form factor or OS. pub fn is_mobile(&self) -> bool { matches!( &self.device_family, DeviceFamily::Phone | DeviceFamily::Tablet ) || matches!(&self.os_family, OsFamily::Android | OsFamily::IOS) } + + /// Determine if the device is iOS based on either the form factor or OS. + pub fn is_ios(&self) -> bool { + matches!( + &self.device_family, + DeviceFamily::Phone | DeviceFamily::Tablet + ) || matches!(&self.os_family, OsFamily::Android | OsFamily::IOS) + } + + /// Determine if the device is an android (Fenix) device based on either the form factor or OS. + pub fn is_fenix(&self) -> bool { + matches!( + &self.device_family, + DeviceFamily::Phone | DeviceFamily::Tablet + ) || matches!(&self.os_family, OsFamily::Android) + } +} + +pub fn get_device_info(user_agent: &str) -> Result { + let parser = Parser::new(); + let wresult = parser.parse(user_agent).unwrap_or_else(|| WootheeResult { + name: "", + category: "", + os: "", + os_version: "".into(), + browser_type: "", + version: "", + vendor: "", + }); + + let firefox_version = + u32::from_str(wresult.version.split(".").collect::>()[0]).unwrap_or_default(); + let os = wresult.os.to_lowercase(); + let os_family = match os.as_str() { + _ if os.starts_with("windows") => OsFamily::Windows, + "mac osx" => OsFamily::MacOs, + "linux" => OsFamily::Linux, + "iphone" => OsFamily::IOS, + "android" => OsFamily::Android, + "chromeos" => OsFamily::ChromeOs, + _ => OsFamily::Other, + }; + let device_family = match wresult.category { + "pc" => DeviceFamily::Desktop, + "smartphone" if os.as_str() == "ipad" => DeviceFamily::Tablet, + "smartphone" => DeviceFamily::Phone, + _ => DeviceFamily::Other, + }; + Ok(DeviceInfo { + device_family, + os_family, + firefox_version, + }) } pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) { From 89ca480ad60eb5f7bec222fca59508cf07404e4a Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 8 Oct 2024 22:26:25 -0400 Subject: [PATCH 06/20] update get_device_info --- syncserver/src/server/user_agent.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 1f0fc06568..465deede50 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -90,9 +90,9 @@ impl DeviceInfo { } } -pub fn get_device_info(user_agent: &str) -> Result { +pub fn get_device_info(user_agent: &str) -> DeviceInfo { let parser = Parser::new(); - let wresult = parser.parse(user_agent).unwrap_or_else(|| WootheeResult { + let mut w_result = parser.parse(user_agent).unwrap_or_else(|| WootheeResult { name: "", category: "", os: "", @@ -102,9 +102,20 @@ pub fn get_device_info(user_agent: &str) -> Result { vendor: "", }); + // NOTE: Firefox on iPads report back the Safari "desktop" UA + // (e.g. `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 + // (KHTML, like Gecko) Version/13.1 Safari/605.1.15)` + // therefore we have to accept that one. This does mean that we may presume + // that a mac safari UA is an iPad. + if w_result.name.to_lowercase() == "safari" && !user_agent.to_lowercase().contains("firefox/") { + w_result.name = "firefox"; + w_result.category = "smartphone"; + w_result.os = "ipad"; + } + let firefox_version = - u32::from_str(wresult.version.split(".").collect::>()[0]).unwrap_or_default(); - let os = wresult.os.to_lowercase(); + u32::from_str(w_result.version.split(".").collect::>()[0]).unwrap_or_default(); + let os = w_result.os.to_lowercase(); let os_family = match os.as_str() { _ if os.starts_with("windows") => OsFamily::Windows, "mac osx" => OsFamily::MacOs, @@ -114,17 +125,17 @@ pub fn get_device_info(user_agent: &str) -> Result { "chromeos" => OsFamily::ChromeOs, _ => OsFamily::Other, }; - let device_family = match wresult.category { + let device_family = match w_result.category { "pc" => DeviceFamily::Desktop, "smartphone" if os.as_str() == "ipad" => DeviceFamily::Tablet, "smartphone" => DeviceFamily::Phone, _ => DeviceFamily::Other, }; - Ok(DeviceInfo { + DeviceInfo { device_family, os_family, firefox_version, - }) + } } pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) { From f2d953b3dd6c19dc72c8023c8820063dd29a0330 Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 8 Oct 2024 22:50:25 -0400 Subject: [PATCH 07/20] update form factor to DeviceFamily to only include ios, fenix, desktop and other --- syncserver/src/server/user_agent.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 465deede50..a27ab78492 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -18,8 +18,8 @@ const VALID_UA_OS: &[&str] = &["Firefox OS", "Linux", "Mac OSX"]; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum DeviceFamily { Desktop, - Phone, - Tablet, + IOS, + Android, Other, } @@ -69,24 +69,19 @@ impl DeviceInfo { pub fn is_mobile(&self) -> bool { matches!( &self.device_family, - DeviceFamily::Phone | DeviceFamily::Tablet + DeviceFamily::IOS | DeviceFamily::Android ) || matches!(&self.os_family, OsFamily::Android | OsFamily::IOS) } /// Determine if the device is iOS based on either the form factor or OS. pub fn is_ios(&self) -> bool { - matches!( - &self.device_family, - DeviceFamily::Phone | DeviceFamily::Tablet - ) || matches!(&self.os_family, OsFamily::Android | OsFamily::IOS) + matches!(&self.device_family, DeviceFamily::IOS) && matches!(&self.os_family, OsFamily::IOS) } /// Determine if the device is an android (Fenix) device based on either the form factor or OS. pub fn is_fenix(&self) -> bool { - matches!( - &self.device_family, - DeviceFamily::Phone | DeviceFamily::Tablet - ) || matches!(&self.os_family, OsFamily::Android) + matches!(&self.device_family, DeviceFamily::Android) + && matches!(&self.os_family, OsFamily::Android) } } @@ -127,8 +122,8 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { }; let device_family = match w_result.category { "pc" => DeviceFamily::Desktop, - "smartphone" if os.as_str() == "ipad" => DeviceFamily::Tablet, - "smartphone" => DeviceFamily::Phone, + "smartphone" if os.as_str() == "ipad" => DeviceFamily::IOS, + "smartphone" if os.as_str() == "android" => DeviceFamily::Android, _ => DeviceFamily::Other, }; DeviceInfo { From 30d4686a989205f908852c0ebe9976d08e7eabf7 Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 15 Oct 2024 13:42:37 -0400 Subject: [PATCH 08/20] review comments --- syncserver/src/server/user_agent.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index a27ab78492..554d18cc80 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -1,7 +1,6 @@ use std::fmt; use std::str::FromStr; -use actix_http::header::ACCESS_CONTROL_REQUEST_HEADERS; use woothee::parser::{Parser, WootheeResult}; // List of valid user-agent attributes to keep, anything not in this @@ -86,16 +85,17 @@ impl DeviceInfo { } pub fn get_device_info(user_agent: &str) -> DeviceInfo { - let parser = Parser::new(); - let mut w_result = parser.parse(user_agent).unwrap_or_else(|| WootheeResult { - name: "", - category: "", - os: "", - os_version: "".into(), - browser_type: "", - version: "", - vendor: "", - }); + let mut w_result = Parser::new() + .parse(user_agent) + .unwrap_or_else(|| WootheeResult { + name: "", + category: "", + os: "", + os_version: "".into(), + browser_type: "", + version: "", + vendor: "", + }); // NOTE: Firefox on iPads report back the Safari "desktop" UA // (e.g. `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 From 6c19ee56a5bb96ab41872a8ab7840bae8ffe3934 Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 15 Oct 2024 18:21:58 -0400 Subject: [PATCH 09/20] updated parsing and device info function to support platform --- syncserver/src/server/user_agent.rs | 74 ++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 554d18cc80..de0d943269 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -14,11 +14,26 @@ const VALID_UA_BROWSER: &[&str] = &["Chrome", "Firefox", "Safari", "Opera"]; // field). Windows has many values and we only care that its Windows const VALID_UA_OS: &[&str] = &["Firefox OS", "Linux", "Mac OSX"]; +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum Platform { + FirefoxDesktop, + Fenix, + FirefoxIOS, + Other, +} + +impl fmt::Display for Platform { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = format!("{:?}", self).to_lowercase(); + write!(fmt, "{}", name) + } +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum DeviceFamily { Desktop, - IOS, - Android, + Mobile, + Tablet, Other, } @@ -36,7 +51,6 @@ pub enum OsFamily { Linux, IOS, Android, - ChromeOs, Other, } @@ -49,6 +63,7 @@ impl fmt::Display for OsFamily { #[derive(Debug, Eq, PartialEq)] pub struct DeviceInfo { + pub platform: Platform, pub device_family: DeviceFamily, pub os_family: OsFamily, pub firefox_version: u32, @@ -66,20 +81,19 @@ impl DeviceInfo { /// Determine if the device is a mobile phone based on either the form factor or OS. pub fn is_mobile(&self) -> bool { - matches!( - &self.device_family, - DeviceFamily::IOS | DeviceFamily::Android - ) || matches!(&self.os_family, OsFamily::Android | OsFamily::IOS) + matches!(&self.device_family, DeviceFamily::Mobile) + && matches!(&self.os_family, OsFamily::Android | OsFamily::IOS) } /// Determine if the device is iOS based on either the form factor or OS. pub fn is_ios(&self) -> bool { - matches!(&self.device_family, DeviceFamily::IOS) && matches!(&self.os_family, OsFamily::IOS) + matches!(&self.device_family, DeviceFamily::Mobile) + && matches!(&self.os_family, OsFamily::IOS) } /// Determine if the device is an android (Fenix) device based on either the form factor or OS. pub fn is_fenix(&self) -> bool { - matches!(&self.device_family, DeviceFamily::Android) + matches!(&self.device_family, DeviceFamily::Mobile) && matches!(&self.os_family, OsFamily::Android) } } @@ -96,6 +110,8 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { version: "", vendor: "", }); + let firefox_version = + u32::from_str(w_result.version.split(".").collect::>()[0]).unwrap_or_default(); // NOTE: Firefox on iPads report back the Safari "desktop" UA // (e.g. `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 @@ -108,25 +124,39 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { w_result.os = "ipad"; } - let firefox_version = - u32::from_str(w_result.version.split(".").collect::>()[0]).unwrap_or_default(); let os = w_result.os.to_lowercase(); let os_family = match os.as_str() { _ if os.starts_with("windows") => OsFamily::Windows, "mac osx" => OsFamily::MacOs, "linux" => OsFamily::Linux, - "iphone" => OsFamily::IOS, + "iphone" | "ipad" => OsFamily::IOS, "android" => OsFamily::Android, - "chromeos" => OsFamily::ChromeOs, _ => OsFamily::Other, }; + let device_family = match w_result.category { "pc" => DeviceFamily::Desktop, - "smartphone" if os.as_str() == "ipad" => DeviceFamily::IOS, - "smartphone" if os.as_str() == "android" => DeviceFamily::Android, + "smartphone" if os.as_str() == "ipad" => DeviceFamily::Tablet, + "smartphone" => DeviceFamily::Mobile, _ => DeviceFamily::Other, }; + + let platform = match device_family { + DeviceFamily::Desktop => Platform::FirefoxDesktop, + DeviceFamily::Mobile => match os_family { + OsFamily::IOS => Platform::FirefoxIOS, + OsFamily::Android => Platform::Fenix, + _ => Platform::Other, + }, + DeviceFamily::Tablet => match os_family { + OsFamily::IOS => Platform::FirefoxIOS, + _ => Platform::Other, + }, + DeviceFamily::Other => Platform::Other, + }; + DeviceInfo { + platform, device_family, os_family, firefox_version, @@ -163,7 +193,9 @@ pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) { #[cfg(test)] mod tests { - use super::parse_user_agent; + use crate::server::user_agent::{DeviceFamily, OsFamily, Platform}; + + use super::{get_device_info, parse_user_agent}; #[test] fn test_linux() { @@ -203,4 +235,14 @@ mod tests { assert_eq!(metrics_browser, "Other"); assert_eq!(ua_result.name, "UNKNOWN"); } + + #[test] + fn test_desktop() { + let desktop_user_agent = r#"Firefox/130.0.1 (Windows NT 10.0; Win64; x64) FxSync/1.132.0.20240913135723.desktop"#; + let device_info = get_device_info(desktop_user_agent); + assert_eq!(device_info.platform, Platform::FirefoxDesktop); + assert_eq!(device_info.device_family, DeviceFamily::Desktop); + assert_eq!(device_info.os_family, OsFamily::Windows); + assert_eq!(device_info.firefox_version, 130); + } } From 3bb0a092e6a529f981d5f63e5d163c7928da10c3 Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 15 Oct 2024 18:36:19 -0400 Subject: [PATCH 10/20] add tests for desktop, fenix, ios clients --- syncserver/src/server/user_agent.rs | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index de0d943269..55831c68d7 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -237,7 +237,7 @@ mod tests { } #[test] - fn test_desktop() { + fn test_windows_desktop() { let desktop_user_agent = r#"Firefox/130.0.1 (Windows NT 10.0; Win64; x64) FxSync/1.132.0.20240913135723.desktop"#; let device_info = get_device_info(desktop_user_agent); assert_eq!(device_info.platform, Platform::FirefoxDesktop); @@ -245,4 +245,33 @@ mod tests { assert_eq!(device_info.os_family, OsFamily::Windows); assert_eq!(device_info.firefox_version, 130); } + fn test_macos_desktop() { + let desktop_user_agent = + r#"Firefox/130.0.1 (Intel Mac OS X 10.15) FxSync/1.132.0.20240913135723.desktop"#; + let device_info = get_device_info(desktop_user_agent); + assert_eq!(device_info.platform, Platform::FirefoxDesktop); + assert_eq!(device_info.device_family, DeviceFamily::Desktop); + assert_eq!(device_info.os_family, OsFamily::MacOs); + assert_eq!(device_info.firefox_version, 132); + } + + fn test_fenix() { + let fenix_user_agent = + r#"Mozilla/5.0 (Android 13; Mobile; rv:130.0) Gecko/130.0 Firefox/130.0"#; + let device_info = get_device_info(desktop_user_agent); + assert_eq!(device_info.platform, Platform::Fenix); + assert_eq!(device_info.device_family, DeviceFamily::Mobile); + assert_eq!(device_info.os_family, OsFamily::Android); + assert_eq!(device_info.firefox_version, 130); + } + + fn test_firefox_ios() { + let fenix_user_agent = + r#"Mozilla/5.0 (Android 13; Mobile; rv:130.0) Gecko/130.0 Firefox/130.0"#; + let device_info = get_device_info(desktop_user_agent); + assert_eq!(device_info.platform, Platform::Fenix); + assert_eq!(device_info.device_family, DeviceFamily::Mobile); + assert_eq!(device_info.os_family, OsFamily::IOS); + assert_eq!(device_info.firefox_version, 130); + } } From 6a7a3271053beab687e82b5ed6227cd7e647b859 Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 15 Oct 2024 18:47:48 -0400 Subject: [PATCH 11/20] add logic for unparsable firefox-ios user agent --- syncserver/src/server/user_agent.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 55831c68d7..4919e1ad37 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -1,6 +1,7 @@ use std::fmt; use std::str::FromStr; +use validator::ValidateUrl; use woothee::parser::{Parser, WootheeResult}; // List of valid user-agent attributes to keep, anything not in this @@ -110,8 +111,20 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { version: "", vendor: "", }); - let firefox_version = - u32::from_str(w_result.version.split(".").collect::>()[0]).unwrap_or_default(); + + // Current Firefox-iOS logic outputs the `user_agent` in the following formats: + // Firefox-iOS-Sync/108.1b24234 (iPad; iPhone OS 16.4.1) (Firefox) + // OR + // Firefox-iOS-FxA/24 + // Both contain prefix `Firefox-iOS` and are not successfully parsed by Woothee. + // This custom logic accomodates the current state (Q4 - 2024) + // This may be a discussion point for future client-side adjustment to have a more standardized + // user_agent string. + if user_agent.to_lowercase().starts_with("firefox-ios") { + w_result.name = "firefox"; + w_result.category = "smartphone"; + w_result.os = "iphone"; + } // NOTE: Firefox on iPads report back the Safari "desktop" UA // (e.g. `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 @@ -155,6 +168,9 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { DeviceFamily::Other => Platform::Other, }; + let firefox_version = + u32::from_str(w_result.version.split(".").collect::>()[0]).unwrap_or_default(); + DeviceInfo { platform, device_family, @@ -266,12 +282,10 @@ mod tests { } fn test_firefox_ios() { - let fenix_user_agent = - r#"Mozilla/5.0 (Android 13; Mobile; rv:130.0) Gecko/130.0 Firefox/130.0"#; + let fenix_user_agent = r#"Firefox-iOS-FxA/24"#; let device_info = get_device_info(desktop_user_agent); - assert_eq!(device_info.platform, Platform::Fenix); + assert_eq!(device_info.platform, Platform::FirefoxIOS); assert_eq!(device_info.device_family, DeviceFamily::Mobile); assert_eq!(device_info.os_family, OsFamily::IOS); - assert_eq!(device_info.firefox_version, 130); } } From e160cd4f7e2a04c986829226cb3eeba1310a2adb Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 15 Oct 2024 18:49:22 -0400 Subject: [PATCH 12/20] update test user agent var for all tests --- syncserver/src/server/user_agent.rs | 35 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 4919e1ad37..90602e65fd 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -1,7 +1,6 @@ use std::fmt; use std::str::FromStr; -use validator::ValidateUrl; use woothee::parser::{Parser, WootheeResult}; // List of valid user-agent attributes to keep, anything not in this @@ -169,7 +168,7 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { }; let firefox_version = - u32::from_str(w_result.version.split(".").collect::>()[0]).unwrap_or_default(); + u32::from_str(w_result.version.split('.').collect::>()[0]).unwrap_or_default(); DeviceInfo { platform, @@ -254,36 +253,48 @@ mod tests { #[test] fn test_windows_desktop() { - let desktop_user_agent = r#"Firefox/130.0.1 (Windows NT 10.0; Win64; x64) FxSync/1.132.0.20240913135723.desktop"#; - let device_info = get_device_info(desktop_user_agent); + let user_agent = r#"Firefox/130.0.1 (Windows NT 10.0; Win64; x64) FxSync/1.132.0.20240913135723.desktop"#; + let device_info = get_device_info(user_agent); assert_eq!(device_info.platform, Platform::FirefoxDesktop); assert_eq!(device_info.device_family, DeviceFamily::Desktop); assert_eq!(device_info.os_family, OsFamily::Windows); assert_eq!(device_info.firefox_version, 130); } + + #[test] fn test_macos_desktop() { - let desktop_user_agent = + let user_agent = r#"Firefox/130.0.1 (Intel Mac OS X 10.15) FxSync/1.132.0.20240913135723.desktop"#; - let device_info = get_device_info(desktop_user_agent); + let device_info = get_device_info(user_agent); assert_eq!(device_info.platform, Platform::FirefoxDesktop); assert_eq!(device_info.device_family, DeviceFamily::Desktop); assert_eq!(device_info.os_family, OsFamily::MacOs); - assert_eq!(device_info.firefox_version, 132); + assert_eq!(device_info.firefox_version, 130); } + #[test] fn test_fenix() { - let fenix_user_agent = - r#"Mozilla/5.0 (Android 13; Mobile; rv:130.0) Gecko/130.0 Firefox/130.0"#; - let device_info = get_device_info(desktop_user_agent); + let user_agent = r#"Mozilla/5.0 (Android 13; Mobile; rv:130.0) Gecko/130.0 Firefox/130.0"#; + let device_info = get_device_info(user_agent); assert_eq!(device_info.platform, Platform::Fenix); assert_eq!(device_info.device_family, DeviceFamily::Mobile); assert_eq!(device_info.os_family, OsFamily::Android); assert_eq!(device_info.firefox_version, 130); } + #[test] fn test_firefox_ios() { - let fenix_user_agent = r#"Firefox-iOS-FxA/24"#; - let device_info = get_device_info(desktop_user_agent); + let user_agent = r#"Firefox-iOS-FxA/24"#; + let device_info = get_device_info(user_agent); + assert_eq!(device_info.platform, Platform::FirefoxIOS); + assert_eq!(device_info.device_family, DeviceFamily::Mobile); + assert_eq!(device_info.os_family, OsFamily::IOS); + } + + #[test] + fn test_firefox_ios_alternate_user_agent() { + let user_agent = r#"Firefox-iOS-Sync/115.0b32242 (iPhone; iPhone OS 17.7) (Firefox)"#; + let device_info = get_device_info(user_agent); assert_eq!(device_info.platform, Platform::FirefoxIOS); assert_eq!(device_info.device_family, DeviceFamily::Mobile); assert_eq!(device_info.os_family, OsFamily::IOS); From 9e0f0139af977f83fd549b96f6d67b94aad06f6d Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 18 Oct 2024 15:25:53 -0400 Subject: [PATCH 13/20] update WootheeResult to .unwrap_or_default as WootheeResult implements Default trait --- syncserver/src/server/user_agent.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 90602e65fd..b63181acc6 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -99,17 +99,7 @@ impl DeviceInfo { } pub fn get_device_info(user_agent: &str) -> DeviceInfo { - let mut w_result = Parser::new() - .parse(user_agent) - .unwrap_or_else(|| WootheeResult { - name: "", - category: "", - os: "", - os_version: "".into(), - browser_type: "", - version: "", - vendor: "", - }); + let mut w_result: WootheeResult<'_> = Parser::new().parse(user_agent).unwrap_or_default(); // Current Firefox-iOS logic outputs the `user_agent` in the following formats: // Firefox-iOS-Sync/108.1b24234 (iPad; iPhone OS 16.4.1) (Firefox) From c2ff37cd2d8e8b821d307a553c63aa837d268895 Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 18 Oct 2024 15:35:03 -0400 Subject: [PATCH 14/20] change firefox version logic to be more robust and return a value of 0 if unparsable, updated related tests --- syncserver/src/server/user_agent.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index b63181acc6..cfeb549533 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -157,8 +157,12 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { DeviceFamily::Other => Platform::Other, }; - let firefox_version = - u32::from_str(w_result.version.split('.').collect::>()[0]).unwrap_or_default(); + let firefox_version = w_result + .version + .split('.') + .next() + .and_then(|v| v.parse::().ok()) + .unwrap_or(0); DeviceInfo { platform, @@ -279,6 +283,7 @@ mod tests { assert_eq!(device_info.platform, Platform::FirefoxIOS); assert_eq!(device_info.device_family, DeviceFamily::Mobile); assert_eq!(device_info.os_family, OsFamily::IOS); + assert_eq!(device_info.firefox_version, 0); } #[test] @@ -288,5 +293,6 @@ mod tests { assert_eq!(device_info.platform, Platform::FirefoxIOS); assert_eq!(device_info.device_family, DeviceFamily::Mobile); assert_eq!(device_info.os_family, OsFamily::IOS); + assert_eq!(device_info.firefox_version, 0); } } From 83b72167d09f7f919aa153fc905cafa2c9f2fb3c Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 18 Oct 2024 15:51:30 -0400 Subject: [PATCH 15/20] remove unneeded import --- syncserver/src/server/user_agent.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index cfeb549533..0ead7f37e4 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -1,5 +1,4 @@ use std::fmt; -use std::str::FromStr; use woothee::parser::{Parser, WootheeResult}; From 65a4261539b0f5ddee8aa6221a309e88bf56f3b8 Mon Sep 17 00:00:00 2001 From: Taddes Date: Sat, 19 Oct 2024 10:27:47 -0400 Subject: [PATCH 16/20] add test for platform::other --- syncserver/src/server/user_agent.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 0ead7f37e4..0851ff7e31 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -294,4 +294,14 @@ mod tests { assert_eq!(device_info.os_family, OsFamily::IOS); assert_eq!(device_info.firefox_version, 0); } + + #[test] + fn test_platform_other() { + let user_agent = r#"Mozilla/5.0 (Linux; Android 9; SM-A920F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4216.0 Mobile Safari/537.36"#; + let device_info = get_device_info(user_agent); + assert_eq!(device_info.platform, Platform::Other); + assert_eq!(device_info.device_family, DeviceFamily::Mobile); + assert_eq!(device_info.os_family, OsFamily::Android); + assert_eq!(device_info.firefox_version, 86); + } } From 71a30371b0f5b5084f3245e2f898ea82222026d7 Mon Sep 17 00:00:00 2001 From: Taddes Date: Sat, 19 Oct 2024 10:36:57 -0400 Subject: [PATCH 17/20] documentation of get device info fn --- syncserver/src/server/user_agent.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 0851ff7e31..afc6eddbc2 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -97,6 +97,15 @@ impl DeviceInfo { } } +/// Parses user agents from headers and returns a DeviceInfo struct containing +/// DeviceFamily, OsFamily, Platform, and Firefox Version. +/// +/// Intended to handle standard user agent strings but also accomodates the non-standard, +/// Firefox-specific user agents for iOS and desktop. +/// +/// Parsing logic for non-standard iOS strings are in the form Firefox-iOS-FxA/24 and +/// manually modifies WootheeResult to match with correct enums for iOS platform and OS. +/// FxSync/<...>.desktop result still parses natively with Woothee and doesn't require intervention. pub fn get_device_info(user_agent: &str) -> DeviceInfo { let mut w_result: WootheeResult<'_> = Parser::new().parse(user_agent).unwrap_or_default(); From c6d82823b67d0d5e54aeb5d0970b35111e405fed Mon Sep 17 00:00:00 2001 From: Taddes Date: Sun, 20 Oct 2024 15:37:59 -0400 Subject: [PATCH 18/20] update logic to include default implementations for platform, osfamily, device family and possible non-firefox user agent to return an empty device info --- syncserver/src/server/user_agent.rs | 44 ++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index afc6eddbc2..21503da892 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -28,6 +28,12 @@ impl fmt::Display for Platform { } } +impl Default for Platform { + fn default() -> Platform { + Platform::Other + } +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum DeviceFamily { Desktop, @@ -43,6 +49,12 @@ impl fmt::Display for DeviceFamily { } } +impl Default for DeviceFamily { + fn default() -> DeviceFamily { + DeviceFamily::Other + } +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum OsFamily { Windows, @@ -60,7 +72,13 @@ impl fmt::Display for OsFamily { } } -#[derive(Debug, Eq, PartialEq)] +impl Default for OsFamily { + fn default() -> OsFamily { + OsFamily::Other + } +} + +#[derive(Debug, Default, Eq, PartialEq)] pub struct DeviceInfo { pub platform: Platform, pub device_family: DeviceFamily, @@ -103,12 +121,20 @@ impl DeviceInfo { /// Intended to handle standard user agent strings but also accomodates the non-standard, /// Firefox-specific user agents for iOS and desktop. /// +/// It is theoretically possible to have an invalid user agent that is non-Firefox in the +/// case of an invalid UA, bot, or scraper. +/// There is a check for this to return an empty result as opposed to failing. +/// /// Parsing logic for non-standard iOS strings are in the form Firefox-iOS-FxA/24 and /// manually modifies WootheeResult to match with correct enums for iOS platform and OS. /// FxSync/<...>.desktop result still parses natively with Woothee and doesn't require intervention. pub fn get_device_info(user_agent: &str) -> DeviceInfo { let mut w_result: WootheeResult<'_> = Parser::new().parse(user_agent).unwrap_or_default(); + // Check if the user agent is not Firefox and return empty. + if !["firefox"].contains(&w_result.name.to_lowercase().as_str()) { + return DeviceInfo::default(); + } // Current Firefox-iOS logic outputs the `user_agent` in the following formats: // Firefox-iOS-Sync/108.1b24234 (iPad; iPhone OS 16.4.1) (Firefox) // OR @@ -309,8 +335,18 @@ mod tests { let user_agent = r#"Mozilla/5.0 (Linux; Android 9; SM-A920F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4216.0 Mobile Safari/537.36"#; let device_info = get_device_info(user_agent); assert_eq!(device_info.platform, Platform::Other); - assert_eq!(device_info.device_family, DeviceFamily::Mobile); - assert_eq!(device_info.os_family, OsFamily::Android); - assert_eq!(device_info.firefox_version, 86); + assert_eq!(device_info.device_family, DeviceFamily::Other); + assert_eq!(device_info.os_family, OsFamily::Other); + assert_eq!(device_info.firefox_version, 0); + } + + #[test] + fn test_non_firefox_platform_other() { + let user_agent = r#"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)"#; + let device_info = get_device_info(user_agent); + assert_eq!(device_info.platform, Platform::Other); + assert_eq!(device_info.device_family, DeviceFamily::Other); + assert_eq!(device_info.os_family, OsFamily::Other); + assert_eq!(device_info.firefox_version, 0); } } From 47eae09046f37e72d56e0bb299f6fc6f4c17c7d8 Mon Sep 17 00:00:00 2001 From: Taddes Date: Sun, 20 Oct 2024 15:46:54 -0400 Subject: [PATCH 19/20] clippy :( --- syncserver/src/server/user_agent.rs | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 21503da892..913ffd14e0 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -13,11 +13,12 @@ const VALID_UA_BROWSER: &[&str] = &["Chrome", "Firefox", "Safari", "Opera"]; // field). Windows has many values and we only care that its Windows const VALID_UA_OS: &[&str] = &["Firefox OS", "Linux", "Mac OSX"]; -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub enum Platform { FirefoxDesktop, Fenix, FirefoxIOS, + #[default] Other, } @@ -28,17 +29,12 @@ impl fmt::Display for Platform { } } -impl Default for Platform { - fn default() -> Platform { - Platform::Other - } -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub enum DeviceFamily { Desktop, Mobile, Tablet, + #[default] Other, } @@ -49,19 +45,14 @@ impl fmt::Display for DeviceFamily { } } -impl Default for DeviceFamily { - fn default() -> DeviceFamily { - DeviceFamily::Other - } -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub enum OsFamily { Windows, MacOs, Linux, IOS, Android, + #[default] Other, } @@ -72,12 +63,6 @@ impl fmt::Display for OsFamily { } } -impl Default for OsFamily { - fn default() -> OsFamily { - OsFamily::Other - } -} - #[derive(Debug, Default, Eq, PartialEq)] pub struct DeviceInfo { pub platform: Platform, From 5720d668ce3a2147e3729a7bd0d72b60b2d8020f Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 21 Oct 2024 15:11:33 -0400 Subject: [PATCH 20/20] change check order to not bypass ios user agents --- syncserver/src/server/user_agent.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/syncserver/src/server/user_agent.rs b/syncserver/src/server/user_agent.rs index 913ffd14e0..19a484501c 100644 --- a/syncserver/src/server/user_agent.rs +++ b/syncserver/src/server/user_agent.rs @@ -116,10 +116,6 @@ impl DeviceInfo { pub fn get_device_info(user_agent: &str) -> DeviceInfo { let mut w_result: WootheeResult<'_> = Parser::new().parse(user_agent).unwrap_or_default(); - // Check if the user agent is not Firefox and return empty. - if !["firefox"].contains(&w_result.name.to_lowercase().as_str()) { - return DeviceInfo::default(); - } // Current Firefox-iOS logic outputs the `user_agent` in the following formats: // Firefox-iOS-Sync/108.1b24234 (iPad; iPhone OS 16.4.1) (Firefox) // OR @@ -145,6 +141,11 @@ pub fn get_device_info(user_agent: &str) -> DeviceInfo { w_result.os = "ipad"; } + // Check if the user agent is not Firefox and return empty. + if !["firefox"].contains(&w_result.name.to_lowercase().as_str()) { + return DeviceInfo::default(); + } + let os = w_result.os.to_lowercase(); let os_family = match os.as_str() { _ if os.starts_with("windows") => OsFamily::Windows,