From 96f95ae6ffa4e451fff799811704a6be8de08468 Mon Sep 17 00:00:00 2001 From: Mani Mozaffar Date: Sat, 2 Dec 2023 18:54:11 +0300 Subject: [PATCH] Fix compatibility to support python >=3.9 (#1) * Update dependencies * Fix CI * Fix types to support >=py3.8 * Fix CI to run with ready for review PRs --- .github/workflows/main.yml | 4 ++-- cfcrawler/cipher.py | 4 +++- cfcrawler/client.py | 6 +++--- cfcrawler/transport.py | 5 +++-- cfcrawler/types.py | 38 ++++++++++++++++++++++++++++++++++++-- poetry.lock | 30 +++++++++--------------------- pyproject.toml | 4 ++-- tox.ini | 4 +++- 8 files changed, 61 insertions(+), 34 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b4e774..071abae 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,14 +5,14 @@ on: branches: - main pull_request: - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened, ready_for_review] jobs: tox: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.11'] + python-version: ['3.11', '3.10', '3.9'] fail-fast: false steps: - name: Check out diff --git a/cfcrawler/cipher.py b/cfcrawler/cipher.py index c09256a..3965dcf 100644 --- a/cfcrawler/cipher.py +++ b/cfcrawler/cipher.py @@ -1,6 +1,8 @@ +from typing import Dict, List + from cfcrawler.types import Browser -MAP_BROWSER_TO_CIPHER: dict[Browser, list[str]] = { +MAP_BROWSER_TO_CIPHER: Dict[Browser, List[str]] = { Browser.CHROME: [ "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256", diff --git a/cfcrawler/client.py b/cfcrawler/client.py index 83db97c..6cbb6af 100644 --- a/cfcrawler/client.py +++ b/cfcrawler/client.py @@ -22,8 +22,8 @@ class AsyncClient(_AsyncClient): def __init__( self, *, - browser: Browser | None = None, - default_user_agent: str | None = None, + browser: typing.Optional[Browser] = None, + default_user_agent: typing.Optional[str] = None, auth: typing.Optional[_types.AuthTypes] = None, cipher_suite: typing.Optional[str] = None, ecdh_curve: typing.Optional[str] = None, @@ -52,7 +52,7 @@ def __init__( self.browser: Browser = browser or random.choice( [Browser.CHROME, Browser.FIREFOX] ) - self.ua = FakeUserAgent(self.browser.value.lower()) + self.ua = FakeUserAgent(self.browser.value) transport = CfScrapeTransport( browser=self.browser, cipher_suite=cipher_suite, ecdh_curve=ecdh_curve ) diff --git a/cfcrawler/transport.py b/cfcrawler/transport.py index 1af8536..3542964 100644 --- a/cfcrawler/transport.py +++ b/cfcrawler/transport.py @@ -1,4 +1,5 @@ import ssl +import typing from httpx import AsyncHTTPTransport @@ -10,8 +11,8 @@ class CfScrapeTransport(AsyncHTTPTransport): def __init__( self, browser: Browser, - ecdh_curve: str | None = None, - cipher_suite: str | None = None, + ecdh_curve: typing.Optional[str] = None, + cipher_suite: typing.Optional[str] = None, *args, **kwargs, ) -> None: diff --git a/cfcrawler/types.py b/cfcrawler/types.py index 6102ca6..63a64aa 100644 --- a/cfcrawler/types.py +++ b/cfcrawler/types.py @@ -1,5 +1,39 @@ -from enum import StrEnum, auto -from typing import DefaultDict +from enum import Enum, auto + +from typing_extensions import DefaultDict + + +class StrEnum(str, Enum): + """ + Enum where members are also (and must be) strings + """ + + def __new__(cls, *values): + "values must already be of type `str`" + if len(values) > 3: + raise TypeError("too many arguments for str(): %r" % (values,)) + if len(values) == 1: + # it must be a string + if not isinstance(values[0], str): + raise TypeError("%r is not a string" % (values[0],)) + if len(values) >= 2: + # check that encoding argument is a string + if not isinstance(values[1], str): + raise TypeError("encoding must be a string, not %r" % (values[1],)) + if len(values) == 3: + # check that errors argument is a string + if not isinstance(values[2], str): + raise TypeError("errors must be a string, not %r" % (values[2])) + value = str(*values) + member = str.__new__(cls, value) + member._value_ = value + return member + + def _generate_next_value_(name, start, count, last_values): + """ + Return the lower-cased version of the member name. + """ + return name.lower() class ChallengeStatus(StrEnum): diff --git a/poetry.lock b/poetry.lock index f191c99..9174100 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,7 +33,6 @@ files = [ ] [package.dependencies] -pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} setuptools = {version = "*", markers = "python_version >= \"3.12\""} [package.extras] @@ -443,20 +442,20 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.8.0" +version = "6.9.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-6.8.0-py3-none-any.whl", hash = "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb"}, - {file = "importlib_metadata-6.8.0.tar.gz", hash = "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"}, + {file = "importlib_metadata-6.9.0-py3-none-any.whl", hash = "sha256:1c8dc6839ddc9771412596926f24cb5a553bbd40624ee2c7e55e531542bed3b8"}, + {file = "importlib_metadata-6.9.0.tar.gz", hash = "sha256:e8acb523c335a91822674e149b46c0399ec4d328c4d1f6e49c273da5ff0201b9"}, ] [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] @@ -998,17 +997,6 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "pytz" -version = "2023.3.post1" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -1295,13 +1283,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.24.7" +version = "20.25.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.24.7-py3-none-any.whl", hash = "sha256:a18b3fd0314ca59a2e9f4b556819ed07183b3e9a3702ecfe213f593d44f7b3fd"}, - {file = "virtualenv-20.24.7.tar.gz", hash = "sha256:69050ffb42419c91f6c1284a7b24e0475d793447e35929b488bf6a0aade39353"}, + {file = "virtualenv-20.25.0-py3-none-any.whl", hash = "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3"}, + {file = "virtualenv-20.25.0.tar.gz", hash = "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"}, ] [package.dependencies] @@ -1369,5 +1357,5 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" -python-versions = ">=3.8,<4.0" -content-hash = "8e90d075d1668a57902d31db61875abcf6c1f81f49097beab021fe83ae5c0741" +python-versions = ">=3.9,<4.0" +content-hash = "92e63fc167852d3bb7e7d3823d6c0a8282f3e623e38fda412824d545104c9e4a" diff --git a/pyproject.toml b/pyproject.toml index 0aeb2ab..6be6a5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,12 @@ readme = "README.md" packages = [{ include = "cfcrawler" }] [tool.poetry.dependencies] -python = ">=3.8,<4.0" +python = ">=3.9,<4.0" fake-useragent = "^1.4.0" httpx = "^0.25.2" -pytest-asyncio = "^0.21.1" [tool.poetry.group.dev.dependencies] +pytest-asyncio = "^0.21.1" pytest = "^7.2.0" pytest-cov = "^4.0.0" mypy = "^1.5.1" diff --git a/tox.ini b/tox.ini index 2efe7dc..ca8325f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,11 @@ [tox] skipsdist = true -envlist = py311 +envlist = py39,py310,py311 [gh-actions] python = + 3.9: py39 + 3.10: py310 3.11: py311 [testenv]