Optimizing Cloudflare undetection (#144)

pyaarlo/__init__.py

@@ -539,6 +539,8 @@ def cameras(self):
         :return: a list of cameras.
         :rtype: list(ArloCamera)
         """
+        if not hasattr(self, '_cameras'):
+            self._cameras = []
         return self._cameras
 
     @property
@@ -566,6 +568,8 @@ def base_stations(self):
         :return: a list of base stations.
         :rtype: list(ArloBase)
         """
+        if not hasattr(self, '_bases'):
+            self._bases = []
         return self._bases
 
     @property

pyaarlo/backend.py

@@ -16,6 +16,7 @@
 
 from .constant import (
     AUTH_FINISH_PATH,
+    AUTH_GET_FACTORID,
     AUTH_GET_FACTORS,
     AUTH_PATH,
     AUTH_START_PATH,
@@ -74,6 +75,7 @@ def __init__(self, arlo):
 
         # login
         self._session = None
+        self._cookies = self._load_cookies()
         self._logged_in = self._login()
         if not self._logged_in:
             self.debug("failed to log in")
@@ -133,6 +135,18 @@ def _save_session(self):
         except Exception as e:
             self._arlo.warning("session file not written" + str(e))
 
+    def _save_cookies(self, requests_cookiejar):
+        with open(self._arlo.cfg.cookies_file, 'wb') as f:
+            pickle.dump(requests_cookiejar, f)
+
+    def _load_cookies(self):
+        try:
+            with open(self._arlo.cfg.cookies_file, 'rb') as f:
+                return pickle.load(f)
+        except Exception as e:
+            self._arlo.info("cookies file not created yet: " + str(e))
+            return None
+
     def _transaction_id(self):
         return 'FE!' + str(uuid.uuid4())
 
@@ -152,6 +166,7 @@ def _request(
         timeout=None,
         host=None,
         authpost=False,
+        cookies=None,
     ):
         if params is None:
             params = {}
@@ -173,30 +188,43 @@ def _request(
                 self.vdebug("request-url={}".format(url))
                 self.vdebug("request-params=\n{}".format(pprint.pformat(params)))
                 self.vdebug("request-headers=\n{}".format(pprint.pformat(headers)))
+
                 if method == "GET":
                     r = self._session.get(
                         url,
                         params=params,
                         headers=headers,
                         stream=stream,
                         timeout=timeout,
+                        cookies=cookies,
                     )
+                    self._save_cookies(r.cookies)
                     if stream is True:
                         return r
                 elif method == "PUT":
                     r = self._session.put(
-                        url, json=params, headers=headers, timeout=timeout
+                        url, json=params, headers=headers, timeout=timeout, cookies=cookies,
                     )
+                    self._save_cookies(r.cookies)
                 elif method == "POST":
                     r = self._session.post(
+                        url, json=params, headers=headers, timeout=timeout, cookies=cookies,
+                    )
+                    self._save_cookies(r.cookies)
+                elif method == "OPTIONS":
+                    r = self._session.options(
                         url, json=params, headers=headers, timeout=timeout
                     )
+                    self._save_cookies(r.cookies)
         except Exception as e:
             self._arlo.warning("request-error={}".format(type(e).__name__))
             return None
 
         try:
-            body = r.json()
+            if "application/json" in r.headers["Content-Type"]:
+                body = r.json()
+            else:
+                body = r.text
             self.vdebug("request-body=\n{}".format(pprint.pformat(body)))
         except Exception as e:
             self._arlo.warning("body-error={}".format(type(e).__name__))
@@ -679,6 +707,8 @@ def _auth_headers(self):
             "Accept-Encoding": "gzip, deflate, br",
             "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8",
             "Cache-Control": "no-cache",
+            "Content-Type": "application/json",
+            "Dnt": "1",
             "Origin": ORIGIN_HOST,
             "Pragma": "no-cache",
             "Referer": REFERER_HOST,
@@ -689,7 +719,7 @@ def _auth_headers(self):
             # "Sec-Fetch-Mode": "cors",
             # "Sec-Fetch-Site": "same-site",
             "User-Agent": self._user_agent,
-            "X-User-Device-Automation-name": "QlJPV1NFUg==",
+            "X-User-Device-Automation-Name": "QlJPV1NFUg==",
             "X-User-Device-Id": self._user_device_id,
             "X-User-Device-Type": "BROWSER",
         }
@@ -711,6 +741,7 @@ def _headers(self):
             "Authorization": self._token,
             "Cache-Control": "no-cache",
             "Content-Type": "application/json; charset=utf-8;",
+            "Dnt": "1",
             "Origin": ORIGIN_HOST,
             "Pragma": "no-cache",
             "Referer": REFERER_HOST,
@@ -733,6 +764,9 @@ def _auth(self):
         while attempt < 3:
             attempt += 1
             self.debug("login attempt #{}".format(attempt))
+            self._options = self.auth_options(AUTH_PATH, headers)
+            self._cookies = self._load_cookies()
+
             body = self.auth_post(
                 AUTH_PATH,
                 {
@@ -742,10 +776,11 @@ def _auth(self):
                     "EnvSource": "prod",
                 },
                 headers,
+                cookies=self._cookies,
             )
             if body is not None:
                 break
-            time.sleep(1)
+            time.sleep(3)
         if body is None:
             self._arlo.error("authentication failed")
             return False
@@ -763,31 +798,48 @@ def _auth(self):
 
             # get available 2fa choices,
             self.debug("getting tfa choices")
-            factors = self.auth_get(
-                AUTH_GET_FACTORS + "?data = {}".format(int(time.time())), {}, headers
-            )
-            if factors is None:
-                self._arlo.error("2fa: no secondary choices available")
-                return False
+
+            self._options = self.auth_options(AUTH_GET_FACTORID, headers)
+            self._cookies = self._load_cookies()
 
             # look for code source choice
             self.debug(f"looking for {self._arlo.cfg.tfa_type}/{self._arlo.cfg.tfa_nickname}")
             factors_of_type = []
             factor_id = None
 
-            for factor in factors["items"]:
-                if factor["factorType"].lower() == self._arlo.cfg.tfa_type:
-                    factors_of_type.append(factor)
+            payload = {
+                "factorType": "BROWSER",
+                "factorData": "",
+                "userId": self._user_id
+            }
+
+            r = self.auth_post(
+                AUTH_GET_FACTORID, payload, headers, cookies=self._cookies
+            )
 
-            if len(factors_of_type) > 0:
-                # Try to match the factorNickname with the tfa_nickname
-                for factor in factors_of_type:
-                    if self._arlo.cfg.tfa_nickname == factor["factorNickname"]:
-                        factor_id = factor["factorId"]
-                        break
-                # Otherwise fallback to using the first option
-                else:
-                    factor_id = factors_of_type[0]["factorId"]
+            if r != None:
+                factor_id = r["factorId"]
+            else:
+                factors = self.auth_get(
+                    AUTH_GET_FACTORS + "?data = {}".format(int(time.time())), {}, headers
+                )
+                if factors is None:
+                    self._arlo.error("2fa: no secondary choices available")
+                    return False
+
+                for factor in factors["items"]:
+                    if factor["factorType"].lower() == self._arlo.cfg.tfa_type:
+                        factors_of_type.append(factor)
+
+                if len(factors_of_type) > 0:
+                    # Try to match the factorNickname with the tfa_nickname
+                    for factor in factors_of_type:
+                        if self._arlo.cfg.tfa_nickname == factor["factorNickname"]:
+                            factor_id = factor["factorId"]
+                            break
+                    # Otherwise fallback to using the first option
+                    else:
+                        factor_id = factors_of_type[0]["factorId"]
 
             if factor_id is None:
                 self._arlo.error("2fa no suitable secondary choice available")
@@ -808,6 +860,7 @@ def _auth(self):
                     "factorType": "BROWSER",
                     "userId": self._user_id
                 }
+                self._options = self.auth_options(AUTH_START_PATH, headers)
                 body = self.auth_post(AUTH_START_PATH, payload, headers)
                 if body is None:
                     self._arlo.error("2fa startAuth failed")
@@ -900,6 +953,12 @@ def _login(self):
         # pickup user configured user agent
         self._user_agent = self.user_agent(self._arlo.cfg.user_agent)
 
+        # If we want to use cloudscraper's random user agent and set the cookies during a first call to the origin host
+        # Used during a debugging session about CloudFlare's 403.
+        # _cookies, self._user_agent = cloudscraper.get_tokens(ORIGIN_HOST)
+        # if self._cookies is None:
+        #     self._cookies = _cookies
+
         # If token looks invalid we'll try the whole process.
         get_new_session = days_until(self._expires_in) < 2
         if get_new_session:
@@ -908,7 +967,15 @@ def _login(self):
             for curve in self._arlo.cfg.ecdh_curves:
                 self.debug(f"CloudFlare curve set to: {curve}")
                 self._session = cloudscraper.create_scraper(
-                    ecdhCurve=curve
+                    # browser={
+                    #     'browser': 'chrome',
+                    #     'platform': 'darwin',
+                    #     'desktop': True,
+                    #     'mobile': False,
+                    # },
+                    disableCloudflareV1=True,
+                    ecdhCurve=curve,
+                    debug=False,
                 )
                 if self._auth() and self._validate():
                     success = True
@@ -917,7 +984,17 @@ def _login(self):
             if not success:
                 return False
         else:
-            self._session = requests.session()
+            # self._session = requests.session()
+            # Use cloudscraper to initiate newer sessions
+            # https://github.com/VeNoMouS/cloudscraper?tab=readme-ov-file#note
+            self._session = cloudscraper.create_scraper(
+                # browser={
+                #     'browser': 'chrome',
+                #     'platform': 'darwin',
+                #     'desktop': True,
+                #     'mobile': False,
+                # }
+            )
             self.debug("newish sessions, re-using")
 
         # save session in case we updated it
@@ -1044,11 +1121,12 @@ def get(
         timeout=None,
         host=None,
         wait_for="response",
+        cookies=None,
     ):
         if wait_for == "response":
             self.vdebug("get+response running")
             return self._request(
-                path, "GET", params, headers, stream, raw, timeout, host
+                path, "GET", params, headers, stream, raw, timeout, host, cookies
             )
         else:
             self.vdebug("get sent")
@@ -1064,10 +1142,11 @@ def put(
         raw=False,
         timeout=None,
         wait_for="response",
+        cookies=None,
     ):
         if wait_for == "response":
             self.vdebug("put+response running")
-            return self._request(path, "PUT", params, headers, False, raw, timeout)
+            return self._request(path, "PUT", params, headers, False, raw, timeout, cookies)
         else:
             self.vdebug("put sent")
             self._arlo.bg.run(
@@ -1083,6 +1162,7 @@ def post(
         timeout=None,
         tid=None,
         wait_for="response",
+        cookies=None,
     ):
         """Post a request to the Arlo servers.
 
@@ -1114,16 +1194,23 @@ def post(
                 self._request, path, "POST", params, headers, False, raw, timeout
             )
 
-    def auth_post(self, path, params=None, headers=None, raw=False, timeout=None):
+    def auth_post(self, path, params=None, headers=None, raw=False, timeout=None, cookies=None):
         return self._request(
-            path, "POST", params, headers, False, raw, timeout, self._arlo.cfg.auth_host, authpost=True
+            path, "POST", params, headers, False, raw, timeout, self._arlo.cfg.auth_host, authpost=True, cookies=cookies
         )
 
     def auth_get(
-        self, path, params=None, headers=None, stream=False, raw=False, timeout=None
+        self, path, params=None, headers=None, stream=False, raw=False, timeout=None, cookies=None
     ):
         return self._request(
-            path, "GET", params, headers, stream, raw, timeout, self._arlo.cfg.auth_host, authpost=True
+            path, "GET", params, headers, stream, raw, timeout, self._arlo.cfg.auth_host, authpost=True, cookies=cookies
+        )
+
+    def auth_options(
+        self, path, headers=None, timeout=None
+     ):
+        return self._request(
+            path, "OPTIONS", headers, timeout, self._arlo.cfg.auth_host, authpost=True
         )
 
     @property

pyaarlo/cfg.py

@@ -240,6 +240,10 @@ def session_file(self):
     def save_session(self):
         return self._kw.get("save_session", True)
 
+    @property
+    def cookies_file(self):
+        return self.storage_dir + "/cookies.txt"
+
     @property
     def dump_file(self):
         if self.dump:

pyaarlo/constant.py

@@ -40,6 +40,7 @@
 AUTH_START_PATH = "/api/startAuth"
 AUTH_FINISH_PATH = "/api/finishAuth"
 AUTH_GET_FACTORS = "/api/getFactors"
+AUTH_GET_FACTORID = "/api/getFactorId"
 AUTH_VALIDATE_PATH = "/api/validateAccessToken"
 
 TFA_CONSOLE_SOURCE = "console"
@@ -300,8 +301,8 @@
         "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) "
         "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1",
     "mac":
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) "
-        "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
     "firefox":
         "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) "
         "Gecko/20100101 Firefox/85.0",
@@ -313,6 +314,6 @@
 CERT_BEGIN = '-----BEGIN CERTIFICATE-----\n'
 CERT_END = '-----END CERTIFICATE-----\n'
 
-ECDH_CURVES = ['prime256v1', 'secp384r1']
+ECDH_CURVES = ['secp384r1', 'prime256v1']
 
 VALID_DEVICE_STATES = ["provisioned", "synced"]

pyaarlo/tfa.py

@@ -61,6 +61,8 @@ def start(self):
                 ctx = None
 
             self._imap = imaplib.IMAP4_SSL(self._arlo.cfg.tfa_host, port=self._arlo.cfg.tfa_port, ssl_context=ctx)
+            if self._arlo._cfg.verbose:
+                self._imap.debug = 4
             res, status = self._imap.login(
                 self._arlo.cfg.tfa_username, self._arlo.cfg.tfa_password
             )