Fix compatibility to support python >=3.9 (#1)

* Update dependencies

* Fix CI

* Fix types to support >=py3.8

* Fix CI to run with ready for review PRs

.github/workflows/main.yml

@@ -5,14 +5,14 @@ on:
     branches:
       - main
   pull_request:
-    types: [opened, synchronize, reopened]
+    types: [opened, synchronize, reopened, ready_for_review]
 
 jobs:
   tox:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.11']
+        python-version: ['3.11', '3.10', '3.9']
       fail-fast: false
     steps:
       - name: Check out

cfcrawler/cipher.py

@@ -1,6 +1,8 @@
+from typing import Dict, List
+
 from cfcrawler.types import Browser
 
-MAP_BROWSER_TO_CIPHER: dict[Browser, list[str]] = {
+MAP_BROWSER_TO_CIPHER: Dict[Browser, List[str]] = {
     Browser.CHROME: [
         "TLS_AES_128_GCM_SHA256",
         "TLS_CHACHA20_POLY1305_SHA256",

cfcrawler/client.py

@@ -22,8 +22,8 @@ class AsyncClient(_AsyncClient):
     def __init__(
         self,
         *,
-        browser: Browser | None = None,
-        default_user_agent: str | None = None,
+        browser: typing.Optional[Browser] = None,
+        default_user_agent: typing.Optional[str] = None,
         auth: typing.Optional[_types.AuthTypes] = None,
         cipher_suite: typing.Optional[str] = None,
         ecdh_curve: typing.Optional[str] = None,
@@ -52,7 +52,7 @@ def __init__(
         self.browser: Browser = browser or random.choice(
             [Browser.CHROME, Browser.FIREFOX]
         )
-        self.ua = FakeUserAgent(self.browser.value.lower())
+        self.ua = FakeUserAgent(self.browser.value)
         transport = CfScrapeTransport(
             browser=self.browser, cipher_suite=cipher_suite, ecdh_curve=ecdh_curve
         )

cfcrawler/transport.py

@@ -1,4 +1,5 @@
 import ssl
+import typing
 
 from httpx import AsyncHTTPTransport
 
@@ -10,8 +11,8 @@ class CfScrapeTransport(AsyncHTTPTransport):
     def __init__(
         self,
         browser: Browser,
-        ecdh_curve: str | None = None,
-        cipher_suite: str | None = None,
+        ecdh_curve: typing.Optional[str] = None,
+        cipher_suite: typing.Optional[str] = None,
         *args,
         **kwargs,
     ) -> None:

cfcrawler/types.py

@@ -1,5 +1,39 @@
-from enum import StrEnum, auto
-from typing import DefaultDict
+from enum import Enum, auto
+
+from typing_extensions import DefaultDict
+
+
+class StrEnum(str, Enum):
+    """
+    Enum where members are also (and must be) strings
+    """
+
+    def __new__(cls, *values):
+        "values must already be of type `str`"
+        if len(values) > 3:
+            raise TypeError("too many arguments for str(): %r" % (values,))
+        if len(values) == 1:
+            # it must be a string
+            if not isinstance(values[0], str):
+                raise TypeError("%r is not a string" % (values[0],))
+        if len(values) >= 2:
+            # check that encoding argument is a string
+            if not isinstance(values[1], str):
+                raise TypeError("encoding must be a string, not %r" % (values[1],))
+        if len(values) == 3:
+            # check that errors argument is a string
+            if not isinstance(values[2], str):
+                raise TypeError("errors must be a string, not %r" % (values[2]))
+        value = str(*values)
+        member = str.__new__(cls, value)
+        member._value_ = value
+        return member
+
+    def _generate_next_value_(name, start, count, last_values):
+        """
+        Return the lower-cased version of the member name.
+        """
+        return name.lower()
 
 
 class ChallengeStatus(StrEnum):

pyproject.toml

@@ -9,12 +9,12 @@ readme = "README.md"
 packages = [{ include = "cfcrawler" }]
 
 [tool.poetry.dependencies]
-python = ">=3.8,<4.0"
+python = ">=3.9,<4.0"
 fake-useragent = "^1.4.0"
 httpx = "^0.25.2"
-pytest-asyncio = "^0.21.1"
 
 [tool.poetry.group.dev.dependencies]
+pytest-asyncio = "^0.21.1"
 pytest = "^7.2.0"
 pytest-cov = "^4.0.0"
 mypy = "^1.5.1"

tox.ini

@@ -1,9 +1,11 @@
 [tox]
 skipsdist = true
-envlist = py311
+envlist = py39,py310,py311
 
 [gh-actions]
 python =
+    3.9: py39
+    3.10: py310
     3.11: py311
 
 [testenv]