From 8a3974d8532fa81740ae950afcb32879bf4ca55c Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 14:45:43 +0800 Subject: [PATCH 01/33] Switch to GitHub Actions --- .github/ci.yml | 35 +++++++++++++++++++++++++++++++++++ .travis.yml | 23 ----------------------- 2 files changed, 35 insertions(+), 23 deletions(-) create mode 100644 .github/ci.yml delete mode 100644 .travis.yml diff --git a/.github/ci.yml b/.github/ci.yml new file mode 100644 index 00000000..5160f67d --- /dev/null +++ b/.github/ci.yml @@ -0,0 +1,35 @@ +name: CI +on: [push, pull_request] +jobs: + test-and-coverage: + strategy: + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + python-version: + [ + ["3.8", "py38"], + ["3.9", "py39"], + ["3.10", "py310"], + ["3.11", "py311"], + ["3.12", "py312"], + ["pypy3.8", "pypy3"], + ["3.10", "codestyle"], + ["3.10", "lint"], + ["3.10", "typecheck"], + ] + runs-on: ${{ matrix.os }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version[0] }} + - name: Install Python requirements + run: | + pip install --upgrade pip + pip install --upgrade --editable '.[testing]' + - name: Test + run: tox + env: + TOXENV: ${{ matrix.python-version[1] }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ef2187cc..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -dist: focal -language: python -matrix: - include: - - python: "3.8" - env: TOXENV=py38 - - python: "3.9" - env: TOXENV=py39 - - python: "3.10" - env: TOXENV=py310 - - python: "3.11" - env: TOXENV=py311 - - python: pypy3.8-7.3.9 - dist: xenial - env: TOXENV=pypy3 - - env: TOXENV=codestyle - - env: TOXENV=lint - - env: TOXENV=typecheck -python: "3.10" -install: - - pip install --upgrade pip - - pip install --upgrade --editable '.[testing]' -script: tox From cf29797d84c11ac05f745acfc9c6fda620b76d86 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 14:47:35 +0800 Subject: [PATCH 02/33] workflows folder --- .github/{ => workflows}/ci.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/ci.yml (100%) diff --git a/.github/ci.yml b/.github/workflows/ci.yml similarity index 100% rename from .github/ci.yml rename to .github/workflows/ci.yml From 26bc1b229f6b11a87d7d8e0939d88dbfb5e036af Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 14:53:22 +0800 Subject: [PATCH 03/33] Python 3.8 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5160f67d..0a29d503 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,9 +13,9 @@ jobs: ["3.11", "py311"], ["3.12", "py312"], ["pypy3.8", "pypy3"], - ["3.10", "codestyle"], - ["3.10", "lint"], - ["3.10", "typecheck"], + ["3.8", "codestyle"], + ["3.8", "lint"], + ["3.8", "typecheck"], ] runs-on: ${{ matrix.os }} steps: From 2fca8f79b9ae5f9afd04f5518e243ca71eda3fb8 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 14:54:14 +0800 Subject: [PATCH 04/33] fail-fast --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a29d503..a1781ddc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ on: [push, pull_request] jobs: test-and-coverage: strategy: + fail-fast: false matrix: os: [macos-latest, windows-latest, ubuntu-latest] python-version: From dd2b98d994d34134512d26f41ded343a1cedf7af Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 15:29:38 +0800 Subject: [PATCH 05/33] Support Windows path --- tests/test_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cache.py b/tests/test_cache.py index 981f7604..fda41318 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -56,7 +56,7 @@ def test_get_cache_dir(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("HOME", raising=False) monkeypatch.delenv("XDG_CACHE_HOME", raising=False) monkeypatch.delenv("TLDEXTRACT_CACHE", raising=False) - assert get_cache_dir().endswith("tldextract/.suffix_cache/") + assert get_cache_dir().endswith(os.path.join("tldextract", ".suffix_cache/")) # with home set, but not anything else specified, use XDG_CACHE_HOME default monkeypatch.setenv("HOME", "/home/john") From 841d52c2520ff5abc37694cacbc135abb4308361 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 15:48:29 +0800 Subject: [PATCH 06/33] Support Windows path --- tests/test_cache.py | 10 +++++----- tests/test_parallel.py | 1 - tldextract/cache.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_cache.py b/tests/test_cache.py index fda41318..bce6305b 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -56,14 +56,14 @@ def test_get_cache_dir(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("HOME", raising=False) monkeypatch.delenv("XDG_CACHE_HOME", raising=False) monkeypatch.delenv("TLDEXTRACT_CACHE", raising=False) - assert get_cache_dir().endswith(os.path.join("tldextract", ".suffix_cache/")) + assert get_cache_dir().endswith(str(Path("tldextract", ".suffix_cache"))) # with home set, but not anything else specified, use XDG_CACHE_HOME default monkeypatch.setenv("HOME", "/home/john") monkeypatch.delenv("XDG_CACHE_HOME", raising=False) monkeypatch.delenv("TLDEXTRACT_CACHE", raising=False) - assert get_cache_dir() == os.path.join( - "/home/john", ".cache/python-tldextract", pkg_identifier + assert get_cache_dir() == str( + Path("/home/john", ".cache/python-tldextract", pkg_identifier) ) # if XDG_CACHE_HOME is set, use it @@ -71,8 +71,8 @@ def test_get_cache_dir(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("XDG_CACHE_HOME", "/my/alt/cache") monkeypatch.delenv("TLDEXTRACT_CACHE", raising=False) - assert get_cache_dir() == os.path.join( - "/my/alt/cache/python-tldextract", pkg_identifier + assert get_cache_dir() == str( + Path("/my/alt/cache/python-tldextract", pkg_identifier) ) # if TLDEXTRACT_CACHE is set, use it diff --git a/tests/test_parallel.py b/tests/test_parallel.py index bb332296..265e41b9 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,7 +1,6 @@ """Test ability to run in parallel with shared cache.""" import os -import os.path from multiprocessing import Pool from pathlib import Path diff --git a/tldextract/cache.py b/tldextract/cache.py index d33bc8aa..faeb5140 100644 --- a/tldextract/cache.py +++ b/tldextract/cache.py @@ -87,7 +87,7 @@ def get_cache_dir() -> str: ) # fallback to trying to use package directory itself - return os.path.join(os.path.dirname(__file__), ".suffix_cache/") + return os.path.join(os.path.dirname(__file__), ".suffix_cache") class DiskCache: From f173f583e8e8207939436bcc92d0977291876db7 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 15:51:44 +0800 Subject: [PATCH 07/33] Remove unused import --- tests/test_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cache.py b/tests/test_cache.py index bce6305b..00bd5a39 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,7 +1,6 @@ """Test the caching functionality.""" from __future__ import annotations -import os.path import sys import types from collections.abc import Hashable From d88518fd30d27b7fc1b9daca09c781612726e6d6 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 15:59:48 +0800 Subject: [PATCH 08/33] Support Windows path --- tldextract/cache.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tldextract/cache.py b/tldextract/cache.py index faeb5140..db0be4ed 100644 --- a/tldextract/cache.py +++ b/tldextract/cache.py @@ -7,6 +7,7 @@ import logging import os import os.path +from pathlib import Path import sys from collections.abc import Callable, Hashable, Iterable from typing import ( @@ -79,15 +80,15 @@ def get_cache_dir() -> str: if xdg_cache_home is None: user_home = os.getenv("HOME", None) if user_home: - xdg_cache_home = os.path.join(user_home, ".cache") + xdg_cache_home = str(Path(user_home, ".cache")) if xdg_cache_home is not None: - return os.path.join( + return str(Path( xdg_cache_home, "python-tldextract", get_pkg_unique_identifier() - ) + )) # fallback to trying to use package directory itself - return os.path.join(os.path.dirname(__file__), ".suffix_cache") + return str(Path(os.path.dirname(__file__), ".suffix_cache")) class DiskCache: @@ -153,7 +154,7 @@ def clear(self) -> None: self.file_ext + ".lock" ): try: - os.unlink(os.path.join(root, filename)) + os.unlink(str(Path(root, filename))) except FileNotFoundError: pass except OSError as exc: @@ -165,10 +166,10 @@ def clear(self) -> None: def _key_to_cachefile_path( self, namespace: str, key: str | dict[str, Hashable] ) -> str: - namespace_path = os.path.join(self.cache_dir, namespace) + namespace_path = str(Path(self.cache_dir, namespace)) hashed_key = _make_cache_key(key) - cache_path = os.path.join(namespace_path, hashed_key + self.file_ext) + cache_path = str(Path(namespace_path, hashed_key + self.file_ext)) return cache_path From d0638b4e5fbcc26906acd725714ef8cde5e38513 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 16:02:38 +0800 Subject: [PATCH 09/33] ruff --- tldextract/cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tldextract/cache.py b/tldextract/cache.py index db0be4ed..16f55ee9 100644 --- a/tldextract/cache.py +++ b/tldextract/cache.py @@ -7,9 +7,9 @@ import logging import os import os.path -from pathlib import Path import sys from collections.abc import Callable, Hashable, Iterable +from pathlib import Path from typing import ( TypeVar, cast, @@ -83,9 +83,9 @@ def get_cache_dir() -> str: xdg_cache_home = str(Path(user_home, ".cache")) if xdg_cache_home is not None: - return str(Path( - xdg_cache_home, "python-tldextract", get_pkg_unique_identifier() - )) + return str( + Path(xdg_cache_home, "python-tldextract", get_pkg_unique_identifier()) + ) # fallback to trying to use package directory itself return str(Path(os.path.dirname(__file__), ".suffix_cache")) From 9750c936997cdb5ea31047aedb9e29ad5b152a03 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 16:20:20 +0800 Subject: [PATCH 10/33] Use as_uri() --- tests/custom_suffix_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/custom_suffix_test.py b/tests/custom_suffix_test.py index 1621d0e1..02da2aac 100644 --- a/tests/custom_suffix_test.py +++ b/tests/custom_suffix_test.py @@ -2,12 +2,13 @@ import os import tempfile +from pathlib import Path import tldextract from tldextract.tldextract import ExtractResult -FAKE_SUFFIX_LIST_URL = "file://" + os.path.join( - os.path.dirname(os.path.abspath(__file__)), "fixtures/fake_suffix_list_fixture.dat" +FAKE_SUFFIX_LIST_URL = os.path.join( +Path(os.path.dirname(os.path.abspath(__file__))).as_uri(), "fixtures", "fake_suffix_list_fixture.dat" ) EXTRA_SUFFIXES = ["foo1", "bar1", "baz1"] From e2135ebba5dbe018c99dbdde0c02305e4ab20c62 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 16:40:25 +0800 Subject: [PATCH 11/33] Use as_uri() --- tests/custom_suffix_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/custom_suffix_test.py b/tests/custom_suffix_test.py index 02da2aac..1add1ff6 100644 --- a/tests/custom_suffix_test.py +++ b/tests/custom_suffix_test.py @@ -8,7 +8,11 @@ from tldextract.tldextract import ExtractResult FAKE_SUFFIX_LIST_URL = os.path.join( -Path(os.path.dirname(os.path.abspath(__file__))).as_uri(), "fixtures", "fake_suffix_list_fixture.dat" + Path( + os.path.dirname(os.path.abspath(__file__)), + "fixtures", + "fake_suffix_list_fixture.dat", + ).as_uri(), ) EXTRA_SUFFIXES = ["foo1", "bar1", "baz1"] From f414ffdce877a3c20720549d44e285874ba78bf2 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 17:10:12 +0800 Subject: [PATCH 12/33] Will this windows workaround work? --- tests/test_parallel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 265e41b9..3f272d50 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -42,9 +42,11 @@ def test_cache_cleared_by_other_process( extract("google.com") orig_unlink = os.unlink - def evil_unlink(filename: str) -> None: + def evil_unlink(filename: str | Path) -> None: """Simulate someone deletes the file right before we try to.""" - if filename.startswith(cache_dir): + if (isinstance(filename, str) and filename.startswith(cache_dir)) or ( + isinstance(filename, Path) and filename.is_relative_to(cache_dir) + ): orig_unlink(filename) orig_unlink(filename) From 14d20aaf08f63ea4e48b9d7e016678c948e6246e Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 17:13:52 +0800 Subject: [PATCH 13/33] The future is here --- tests/test_parallel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 3f272d50..8d16d76c 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,5 +1,7 @@ """Test ability to run in parallel with shared cache.""" +from __future__ import annotations + import os from multiprocessing import Pool from pathlib import Path From cd984a42bca68992774d3c56e8536f5d036dd315 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 17:23:55 +0800 Subject: [PATCH 14/33] The future is here --- tests/test_parallel.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 8d16d76c..b3047ab6 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -44,10 +44,22 @@ def test_cache_cleared_by_other_process( extract("google.com") orig_unlink = os.unlink + def is_relative_to(path, other_path): + """Return True if path is relative to other_path or False. + + Taken from standard library (Python 3.9+ required). + Reference: https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.is_relative_to + """ + try: + path.relative_to(other_path) + return True + except ValueError: + return False + def evil_unlink(filename: str | Path) -> None: """Simulate someone deletes the file right before we try to.""" if (isinstance(filename, str) and filename.startswith(cache_dir)) or ( - isinstance(filename, Path) and filename.is_relative_to(cache_dir) + isinstance(filename, Path) and is_relative_to(filename, cache_dir) ): orig_unlink(filename) orig_unlink(filename) From 2fbfa635da7688728d28523d1f5533db452cb8a0 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 17:28:18 +0800 Subject: [PATCH 15/33] Deprecate pypy38 --- .github/workflows/ci.yml | 3 ++- tox.ini | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1781ddc..9b64af78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,8 @@ jobs: ["3.10", "py310"], ["3.11", "py311"], ["3.12", "py312"], - ["pypy3.8", "pypy3"], + ["pypy3.9", "pypy39"], + ["pypy3.10", "pypy310"], ["3.8", "codestyle"], ["3.8", "lint"], ["3.8", "typecheck"], diff --git a/tox.ini b/tox.ini index 89b8802c..08363214 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311,py3},codestyle,lint,typecheck +envlist = py{38,39,310,311,312,py39,py310},codestyle,lint,typecheck [testenv] commands = pytest {posargs} From c7f537022412ca9bcf025d5db002f2cd1c2aca30 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 17:34:54 +0800 Subject: [PATCH 16/33] Add type annotations --- tests/test_parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index b3047ab6..03a98f22 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -44,7 +44,7 @@ def test_cache_cleared_by_other_process( extract("google.com") orig_unlink = os.unlink - def is_relative_to(path, other_path): + def is_relative_to(path: Path, other_path: str | Path) -> bool: """Return True if path is relative to other_path or False. Taken from standard library (Python 3.9+ required). From d0098039f0adae28dcad14cb332e0e1c87cea32d Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 17:54:56 +0800 Subject: [PATCH 17/33] Restore pypy38 --- .github/workflows/ci.yml | 3 ++- tox.ini | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b64af78..3082f2c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] jobs: - test-and-coverage: + test: strategy: fail-fast: false matrix: @@ -13,6 +13,7 @@ jobs: ["3.10", "py310"], ["3.11", "py311"], ["3.12", "py312"], + ["pypy3.8", "pypy38"], ["pypy3.9", "pypy39"], ["pypy3.10", "pypy310"], ["3.8", "codestyle"], diff --git a/tox.ini b/tox.ini index 08363214..21251d48 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311,312,py39,py310},codestyle,lint,typecheck +envlist = py{38,39,310,311,312,py38,py39,py310},codestyle,lint,typecheck [testenv] commands = pytest {posargs} From 9dd4dccec9d73c23be865f6dd8cf72b14db75c00 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 18:22:17 +0800 Subject: [PATCH 18/33] disable logging --- tests/cli_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cli_test.py b/tests/cli_test.py index 92e61ec3..300b3aee 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -1,6 +1,7 @@ """tldextract integration tests.""" import json +import logging import sys import pytest @@ -34,7 +35,7 @@ def test_cli_posargs( monkeypatch.setattr( sys, "argv", ["tldextract", "example.com", "bbc.co.uk", "forums.bbc.co.uk"] ) - + logging.getLogger("requests").setLevel(logging.WARNING) main() stdout, stderr = capsys.readouterr() From 2549b126ceed4a1a14ece941aaf4f9d2aff7eeaa Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 18:42:08 +0800 Subject: [PATCH 19/33] Try flushing --- tests/cli_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/cli_test.py b/tests/cli_test.py index 300b3aee..c55c013a 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -1,7 +1,6 @@ """tldextract integration tests.""" import json -import logging import sys import pytest @@ -35,7 +34,7 @@ def test_cli_posargs( monkeypatch.setattr( sys, "argv", ["tldextract", "example.com", "bbc.co.uk", "forums.bbc.co.uk"] ) - logging.getLogger("requests").setLevel(logging.WARNING) + capsys.readouterr() main() stdout, stderr = capsys.readouterr() @@ -59,7 +58,7 @@ def test_cli_namedargs( "forums.bbc.co.uk", ], ) - + capsys.readouterr() main() stdout, stderr = capsys.readouterr() @@ -72,7 +71,7 @@ def test_cli_json_output( ) -> None: """Test CLI with --json option.""" monkeypatch.setattr(sys, "argv", ["tldextract", "--json", "www.bbc.co.uk"]) - + capsys.readouterr() main() stdout, stderr = capsys.readouterr() From 616049d44e46cd25453bfbcf54eaf80c565274f3 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Thu, 9 Nov 2023 18:49:42 +0800 Subject: [PATCH 20/33] Forget about Freeman --- .github/workflows/ci.yml | 2 -- tests/cli_test.py | 6 +++--- tox.ini | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3082f2c1..ec3234cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,6 @@ jobs: ["3.11", "py311"], ["3.12", "py312"], ["pypy3.8", "pypy38"], - ["pypy3.9", "pypy39"], - ["pypy3.10", "pypy310"], ["3.8", "codestyle"], ["3.8", "lint"], ["3.8", "typecheck"], diff --git a/tests/cli_test.py b/tests/cli_test.py index c55c013a..92e61ec3 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -34,7 +34,7 @@ def test_cli_posargs( monkeypatch.setattr( sys, "argv", ["tldextract", "example.com", "bbc.co.uk", "forums.bbc.co.uk"] ) - capsys.readouterr() + main() stdout, stderr = capsys.readouterr() @@ -58,7 +58,7 @@ def test_cli_namedargs( "forums.bbc.co.uk", ], ) - capsys.readouterr() + main() stdout, stderr = capsys.readouterr() @@ -71,7 +71,7 @@ def test_cli_json_output( ) -> None: """Test CLI with --json option.""" monkeypatch.setattr(sys, "argv", ["tldextract", "--json", "www.bbc.co.uk"]) - capsys.readouterr() + main() stdout, stderr = capsys.readouterr() diff --git a/tox.ini b/tox.ini index 21251d48..0e4ea2e7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311,312,py38,py39,py310},codestyle,lint,typecheck +envlist = py{38,39,310,311,312,py38},codestyle,lint,typecheck [testenv] commands = pytest {posargs} From 18aad21dd9bd5546ca8a185d399883b9cf1c0b5e Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 00:00:42 +0800 Subject: [PATCH 21/33] Remove redundant os.path.join --- tests/custom_suffix_test.py | 13 ++++++------- tldextract/cli.py | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/custom_suffix_test.py b/tests/custom_suffix_test.py index 1add1ff6..771531e6 100644 --- a/tests/custom_suffix_test.py +++ b/tests/custom_suffix_test.py @@ -7,13 +7,12 @@ import tldextract from tldextract.tldextract import ExtractResult -FAKE_SUFFIX_LIST_URL = os.path.join( - Path( - os.path.dirname(os.path.abspath(__file__)), - "fixtures", - "fake_suffix_list_fixture.dat", - ).as_uri(), -) +FAKE_SUFFIX_LIST_URL = Path( + os.path.dirname(os.path.abspath(__file__)), + "fixtures", + "fake_suffix_list_fixture.dat", +).as_uri() + EXTRA_SUFFIXES = ["foo1", "bar1", "baz1"] extract_using_fake_suffix_list = tldextract.TLDExtract( diff --git a/tldextract/cli.py b/tldextract/cli.py index e9aa6ce6..c20c176f 100644 --- a/tldextract/cli.py +++ b/tldextract/cli.py @@ -5,8 +5,8 @@ import json import logging import os.path -import pathlib import sys +from pathlib import Path from ._version import version as __version__ from .tldextract import TLDExtract @@ -80,7 +80,7 @@ def main() -> None: suffix_list_urls = [] for source in args.suffix_list_url: if os.path.isfile(source): - as_path_uri = pathlib.Path(os.path.abspath(source)).as_uri() + as_path_uri = Path(os.path.abspath(source)).as_uri() suffix_list_urls.append(as_path_uri) else: suffix_list_urls.append(source) From e97c6a3f27d389ba0d918c6e5156b9fe4c28e70e Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 00:13:06 +0800 Subject: [PATCH 22/33] Maybe we do not need this one --- tldextract/cache.py | 15 +++++++-------- tldextract/cli.py | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tldextract/cache.py b/tldextract/cache.py index 16f55ee9..faeb5140 100644 --- a/tldextract/cache.py +++ b/tldextract/cache.py @@ -9,7 +9,6 @@ import os.path import sys from collections.abc import Callable, Hashable, Iterable -from pathlib import Path from typing import ( TypeVar, cast, @@ -80,15 +79,15 @@ def get_cache_dir() -> str: if xdg_cache_home is None: user_home = os.getenv("HOME", None) if user_home: - xdg_cache_home = str(Path(user_home, ".cache")) + xdg_cache_home = os.path.join(user_home, ".cache") if xdg_cache_home is not None: - return str( - Path(xdg_cache_home, "python-tldextract", get_pkg_unique_identifier()) + return os.path.join( + xdg_cache_home, "python-tldextract", get_pkg_unique_identifier() ) # fallback to trying to use package directory itself - return str(Path(os.path.dirname(__file__), ".suffix_cache")) + return os.path.join(os.path.dirname(__file__), ".suffix_cache") class DiskCache: @@ -154,7 +153,7 @@ def clear(self) -> None: self.file_ext + ".lock" ): try: - os.unlink(str(Path(root, filename))) + os.unlink(os.path.join(root, filename)) except FileNotFoundError: pass except OSError as exc: @@ -166,10 +165,10 @@ def clear(self) -> None: def _key_to_cachefile_path( self, namespace: str, key: str | dict[str, Hashable] ) -> str: - namespace_path = str(Path(self.cache_dir, namespace)) + namespace_path = os.path.join(self.cache_dir, namespace) hashed_key = _make_cache_key(key) - cache_path = str(Path(namespace_path, hashed_key + self.file_ext)) + cache_path = os.path.join(namespace_path, hashed_key + self.file_ext) return cache_path diff --git a/tldextract/cli.py b/tldextract/cli.py index c20c176f..e9aa6ce6 100644 --- a/tldextract/cli.py +++ b/tldextract/cli.py @@ -5,8 +5,8 @@ import json import logging import os.path +import pathlib import sys -from pathlib import Path from ._version import version as __version__ from .tldextract import TLDExtract @@ -80,7 +80,7 @@ def main() -> None: suffix_list_urls = [] for source in args.suffix_list_url: if os.path.isfile(source): - as_path_uri = Path(os.path.abspath(source)).as_uri() + as_path_uri = pathlib.Path(os.path.abspath(source)).as_uri() suffix_list_urls.append(as_path_uri) else: suffix_list_urls.append(source) From 0e6e4ca3482bc3539c1ea679bdac69912cc04002 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 00:18:03 +0800 Subject: [PATCH 23/33] pathlib is needed in cache.py --- tldextract/cache.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tldextract/cache.py b/tldextract/cache.py index faeb5140..d72eaac0 100644 --- a/tldextract/cache.py +++ b/tldextract/cache.py @@ -6,9 +6,9 @@ import json import logging import os -import os.path import sys from collections.abc import Callable, Hashable, Iterable +from pathlib import Path from typing import ( TypeVar, cast, @@ -79,15 +79,15 @@ def get_cache_dir() -> str: if xdg_cache_home is None: user_home = os.getenv("HOME", None) if user_home: - xdg_cache_home = os.path.join(user_home, ".cache") + xdg_cache_home = str(Path(user_home, ".cache")) if xdg_cache_home is not None: - return os.path.join( - xdg_cache_home, "python-tldextract", get_pkg_unique_identifier() + return str( + Path(xdg_cache_home, "python-tldextract", get_pkg_unique_identifier()) ) # fallback to trying to use package directory itself - return os.path.join(os.path.dirname(__file__), ".suffix_cache") + return str(Path(os.path.dirname(__file__), ".suffix_cache")) class DiskCache: @@ -153,7 +153,7 @@ def clear(self) -> None: self.file_ext + ".lock" ): try: - os.unlink(os.path.join(root, filename)) + os.unlink(str(Path(root, filename))) except FileNotFoundError: pass except OSError as exc: @@ -165,10 +165,10 @@ def clear(self) -> None: def _key_to_cachefile_path( self, namespace: str, key: str | dict[str, Hashable] ) -> str: - namespace_path = os.path.join(self.cache_dir, namespace) + namespace_path = str(Path(self.cache_dir, namespace)) hashed_key = _make_cache_key(key) - cache_path = os.path.join(namespace_path, hashed_key + self.file_ext) + cache_path = str(Path(namespace_path, hashed_key + self.file_ext)) return cache_path From 631906db5a37d99f24d49ce3d3c4966127e04865 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 00:37:51 +0800 Subject: [PATCH 24/33] Remove redundant jobs --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec3234cc..ad611173 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,19 @@ jobs: ["3.8", "lint"], ["3.8", "typecheck"], ] + exclude: + - os: "macos-latest" + python-version: ["3.8", "codestyle"] + - os: "macos-latest" + python-version: ["3.8", "lint"] + - os: "macos-latest" + python-version: ["3.8", "typecheck"] + - os: "windows-latest" + python-version: ["3.8", "codestyle"] + - os: "windows-latest" + python-version: ["3.8", "lint"] + - os: "windows-latest" + python-version: ["3.8", "typecheck"] runs-on: ${{ matrix.os }} steps: - name: Check out repository From 53f4d5abe42a4b3a7871ebdd1e9f1498cdbbd2ac Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 13:14:28 +0800 Subject: [PATCH 25/33] exclude --- .github/workflows/ci.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad611173..e78ee4c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,19 +18,10 @@ jobs: ["3.8", "lint"], ["3.8", "typecheck"], ] + non-ubuntu-format: + - ${{ (matrix.os != 'ubuntu-latest') && !startsWith(matrix.python-version[1], 'py') }} exclude: - - os: "macos-latest" - python-version: ["3.8", "codestyle"] - - os: "macos-latest" - python-version: ["3.8", "lint"] - - os: "macos-latest" - python-version: ["3.8", "typecheck"] - - os: "windows-latest" - python-version: ["3.8", "codestyle"] - - os: "windows-latest" - python-version: ["3.8", "lint"] - - os: "windows-latest" - python-version: ["3.8", "typecheck"] + - non-ubuntu-format: true runs-on: ${{ matrix.os }} steps: - name: Check out repository From a70570805231fb6197bbf76c13968630a6d98c3e Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 13:22:20 +0800 Subject: [PATCH 26/33] include --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e78ee4c0..5ab9b51b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,14 @@ jobs: ["3.11", "py311"], ["3.12", "py312"], ["pypy3.8", "pypy38"], - ["3.8", "codestyle"], - ["3.8", "lint"], - ["3.8", "typecheck"], ] - non-ubuntu-format: - - ${{ (matrix.os != 'ubuntu-latest') && !startsWith(matrix.python-version[1], 'py') }} - exclude: - - non-ubuntu-format: true + include: + - os: ubuntu-latest + python-version: ["3.8", "codestyle"] + - os: ubuntu-latest + python-version: ["3.8", "lint"] + - os: ubuntu-latest + python-version: ["3.8", "typecheck"] runs-on: ${{ matrix.os }} steps: - name: Check out repository From dc0e34a8c530227937177ccfd7ba40e251baf937 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 13:33:48 +0800 Subject: [PATCH 27/33] include --- .github/workflows/ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ab9b51b..a27999d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,22 +6,22 @@ jobs: fail-fast: false matrix: os: [macos-latest, windows-latest, ubuntu-latest] - python-version: + language: [ - ["3.8", "py38"], - ["3.9", "py39"], - ["3.10", "py310"], - ["3.11", "py311"], - ["3.12", "py312"], - ["pypy3.8", "pypy38"], + {python-version: "3.8", toxenv: "py38"}, + {python-version: "3.9", toxenv: "py39"}, + {python-version: "3.10", toxenv: "py310"}, + {python-version: "3.11", toxenv: "py311"}, + {python-version: "3.12", toxenv: "py312"}, + {python-version: "pypy3.8", toxenv: "pypy38"}, ] include: - os: ubuntu-latest - python-version: ["3.8", "codestyle"] + language: {python-version: "3.8", toxenv: "codestyle"} - os: ubuntu-latest - python-version: ["3.8", "lint"] + language: {python-version: "3.8", toxenv: "lint"} - os: ubuntu-latest - python-version: ["3.8", "typecheck"] + language: {python-version: "3.8", toxenv: "typecheck"} runs-on: ${{ matrix.os }} steps: - name: Check out repository @@ -29,7 +29,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version[0] }} + python-version: ${{ matrix.language.python-version }} - name: Install Python requirements run: | pip install --upgrade pip @@ -37,4 +37,4 @@ jobs: - name: Test run: tox env: - TOXENV: ${{ matrix.python-version[1] }} + TOXENV: ${{ matrix.language.toxenv }} From 3a297185ed33fe221a1ea7269d283e3cf3ffa898 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 13:54:15 +0800 Subject: [PATCH 28/33] Change workflow name --- .github/workflows/ci.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a27999d3..acb4d300 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: build on: [push, pull_request] jobs: test: diff --git a/README.md b/README.md index 9434d34b..c00e29fd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# tldextract [![PyPI version](https://badge.fury.io/py/tldextract.svg)](https://badge.fury.io/py/tldextract) [![Build Status](https://travis-ci.com/john-kurkowski/tldextract.svg?branch=master)](https://app.travis-ci.com/github/john-kurkowski/tldextract) +# tldextract [![PyPI version](https://badge.fury.io/py/tldextract.svg)](https://badge.fury.io/py/tldextract) [![Build Status](https://github.com/john-kurkowski/tldextract/actions/workflows/ci.yml/badge.svg)](https://github.com/john-kurkowski/tldextract/actions/workflows/ci.yml) `tldextract` accurately separates a URL's subdomain, domain, and public suffix, using [the Public Suffix List (PSL)](https://publicsuffix.org). From ba8d9bdb0bc80a06d01bae29f47ff97b9cee6a4b Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 16:20:24 +0800 Subject: [PATCH 29/33] Will newer deps work? --- .github/workflows/ci.yml | 2 ++ pyproject.toml | 6 +++--- tox.ini | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acb4d300..dc6ff4d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,8 @@ jobs: {python-version: "3.11", toxenv: "py311"}, {python-version: "3.12", toxenv: "py312"}, {python-version: "pypy3.8", toxenv: "pypy38"}, + {python-version: "pypy3.9", toxenv: "pypy39"}, + {python-version: "pypy3.10", toxenv: "pypy310"}, ] include: - os: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index d323aae7..9e295e10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,9 +34,9 @@ readme = "README.md" dependencies = [ "idna", - "requests>=2.1.0", - "requests-file>=1.4", - "filelock>=3.0.8", + "requests>=2.31.0", + "requests-file>=1.5", + "filelock>=3.13.1", ] [project.optional-dependencies] diff --git a/tox.ini b/tox.ini index 0e4ea2e7..21251d48 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311,312,py38},codestyle,lint,typecheck +envlist = py{38,39,310,311,312,py38,py39,py310},codestyle,lint,typecheck [testenv] commands = pytest {posargs} From a6a98fd5753adf1cb92b475feb0c0e74fda71f85 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 16:25:59 +0800 Subject: [PATCH 30/33] Will newer deps work? --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9e295e10..4d6fe45f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,8 @@ tldextract = "tldextract.cli:main" [build-system] requires = [ - "setuptools>=61.2", - "setuptools_scm[toml]>=6.2", + "setuptools>=68.2", + "setuptools_scm[toml]>=8.0", ] build-backend = "setuptools.build_meta" From 9416187864e328fd17a9a1d9ad1db283d88fda06 Mon Sep 17 00:00:00 2001 From: Wu Tingfeng Date: Sat, 11 Nov 2023 16:28:37 +0800 Subject: [PATCH 31/33] Add classifier Python 3.12 --- .github/workflows/ci.yml | 2 -- pyproject.toml | 11 ++++++----- tox.ini | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc6ff4d5..acb4d300 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,6 @@ jobs: {python-version: "3.11", toxenv: "py311"}, {python-version: "3.12", toxenv: "py312"}, {python-version: "pypy3.8", toxenv: "pypy38"}, - {python-version: "pypy3.9", toxenv: "pypy39"}, - {python-version: "pypy3.10", toxenv: "pypy310"}, ] include: - os: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 4d6fe45f..07dcd9da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] requires-python = ">=3.8" dynamic = ["version"] @@ -34,9 +35,9 @@ readme = "README.md" dependencies = [ "idna", - "requests>=2.31.0", - "requests-file>=1.5", - "filelock>=3.13.1", + "requests>=2.1.0", + "requests-file>=1.4", + "filelock>=3.0.8", ] [project.optional-dependencies] @@ -61,8 +62,8 @@ tldextract = "tldextract.cli:main" [build-system] requires = [ - "setuptools>=68.2", - "setuptools_scm[toml]>=8.0", + "setuptools>=61.2", + "setuptools_scm[toml]>=6.2", ] build-backend = "setuptools.build_meta" diff --git a/tox.ini b/tox.ini index 21251d48..0e4ea2e7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311,312,py38,py39,py310},codestyle,lint,typecheck +envlist = py{38,39,310,311,312,py38},codestyle,lint,typecheck [testenv] commands = pytest {posargs} From eb99020cb04c949f60650593830fa13252e29c15 Mon Sep 17 00:00:00 2001 From: John Kurkowski Date: Sat, 11 Nov 2023 19:37:29 -0800 Subject: [PATCH 32/33] Trigger CI From 4930f0e7ea8be87afe1650218277ac1fee70f9dd Mon Sep 17 00:00:00 2001 From: John Kurkowski Date: Mon, 13 Nov 2023 11:59:40 -0800 Subject: [PATCH 33/33] Clarify comment --- tests/test_parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 03a98f22..6ae10dfd 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -47,7 +47,7 @@ def test_cache_cleared_by_other_process( def is_relative_to(path: Path, other_path: str | Path) -> bool: """Return True if path is relative to other_path or False. - Taken from standard library (Python 3.9+ required). + Taken from the Python 3.9 standard library. Reference: https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.is_relative_to """ try: