[Backport maintenance/3.2.x] Add --prefer-stubs=y option (#9646)

Backport ce47a62 from #9632.

doc/user_guide/configuration/all-options.rst

@@ -167,6 +167,13 @@ Standard Checkers
 **Default:**  ``True``
 
 
+--prefer-stubs
+""""""""""""""
+*Resolve imports to .pyi stubs if available. May reduce no-member messages and increase not-an-iterable messages.*
+
+**Default:**  ``False``
+
+
 --py-version
 """"""""""""
 *Minimum Python version to use for version dependent checks. Will default to the version used to run pylint.*
@@ -271,6 +278,8 @@ Standard Checkers
 
    persistent = true
 
+   prefer-stubs = false
+
    py-version = "sys.version_info[:2]"
 
    recursive = false

doc/whatsnew/fragments/9139.internal

@@ -0,0 +1,5 @@
+Update astroid version to 3.2.1. This solves some reports of ``RecursionError``
+and also makes the *prefer .pyi stubs* feature in astroid 3.2.0 *opt-in*
+with the aforementioned ``--prefer-stubs=y`` option.
+
+Refs #9139

doc/whatsnew/fragments/9626.bugfix

@@ -0,0 +1,9 @@
+Add `--prefer-stubs=yes` option to opt-in to the astroid 3.2 feature
+that prefers `.pyi` stubs over same-named `.py` files. This has the
+potential to reduce `no-member` errors but at the cost of more errors
+such as `not-an-iterable` from function bodies appearing as `...`.
+
+Defaults to `no`.
+
+Closes #9626
+Closes #9623

examples/pylintrc

@@ -87,6 +87,10 @@ load-plugins=
 # Pickle collected data for later comparisons.
 persistent=yes
 
+# Resolve imports to .pyi stubs if available. May reduce no-member messages
+# and increase not-an-iterable messages.
+prefer-stubs=no
+
 # Minimum Python version to use for version dependent checks. Will default to
 # the version used to run pylint.
 py-version=3.10

examples/pyproject.toml

@@ -77,6 +77,10 @@ limit-inference-results = 100
 # Pickle collected data for later comparisons.
 persistent = true
 
+# Resolve imports to .pyi stubs if available. May reduce no-member messages
+# and increase not-an-iterable messages.
+prefer-stubs = false
+
 # Minimum Python version to use for version dependent checks. Will default to the
 # version used to run pylint.
 py-version = "3.10"

pylint/lint/base_options.py

@@ -415,6 +415,17 @@ def _make_linter_options(linter: PyLinter) -> Options:
                 "Useful if running pylint in a server-like mode.",
             },
         ),
+        (
+            "prefer-stubs",
+            {
+                "default": False,
+                "type": "yn",
+                "metavar": "<y or n>",
+                "help": "Resolve imports to .pyi stubs if available. May "
+                "reduce no-member messages and increase not-an-iterable "
+                "messages.",
+            },
+        ),
     )
 
 

pylint/lint/pylinter.py

@@ -1074,6 +1074,7 @@ def open(self) -> None:
         MANAGER.max_inferable_values = self.config.limit_inference_results
         MANAGER.extension_package_whitelist.update(self.config.extension_pkg_allow_list)
         MANAGER.module_denylist.update(self.config.ignored_modules)
+        MANAGER.prefer_stubs = self.config.prefer_stubs
         if self.config.extension_pkg_whitelist:
             MANAGER.extension_package_whitelist.update(
                 self.config.extension_pkg_whitelist

pylint/utils/utils.py

@@ -52,6 +52,7 @@
     "suggestion-mode",
     "analyse-fallback-blocks",
     "allow-global-unused-variables",
+    "prefer-stubs",
 ]
 GLOBAL_OPTION_INT = Literal["max-line-length", "docstring-min-length"]
 GLOBAL_OPTION_LIST = Literal["ignored-modules"]

pyproject.toml

@@ -41,7 +41,7 @@ dependencies    = [
     # Also upgrade requirements_test_min.txt.
     # Pinned to dev of second minor update to allow editable installs and fix primer issues,
     # see https://github.com/pylint-dev/astroid/issues/1341
-    "astroid>=3.2.0,<=3.3.0-dev0",
+    "astroid>=3.2.1,<=3.3.0-dev0",
     "isort>=4.2.5,<6,!=5.13.0",
     "mccabe>=0.6,<0.8",
     "tomli>=1.1.0;python_version<'3.11'",

requirements_test_min.txt

@@ -1,6 +1,6 @@
 .[testutils,spelling]
 # astroid dependency is also defined in pyproject.toml
-astroid==3.2.0  # Pinned to a specific version for tests
+astroid==3.2.1  # Pinned to a specific version for tests
 typing-extensions~=4.11
 py~=1.11.0
 pytest~=7.4

tests/lint/test_pylinter.py

@@ -59,3 +59,12 @@ def test_open_pylinter_denied_modules(linter: PyLinter) -> None:
         assert MANAGER.module_denylist == {"mod1", "mod2", "mod3"}
     finally:
         MANAGER.module_denylist = set()
+
+
+def test_open_pylinter_prefer_stubs(linter: PyLinter) -> None:
+    try:
+        linter.config.prefer_stubs = True
+        linter.open()
+        assert MANAGER.prefer_stubs
+    finally:
+        MANAGER.prefer_stubs = False

tests/lint/unittest_lint.py

@@ -1049,7 +1049,7 @@ def test_by_module_statement_value(initialized_linter: PyLinter) -> None:
 
 def test_finds_pyi_file() -> None:
     run = Run(
-        [join(REGRTEST_DATA_DIR, "pyi")],
+        ["--prefer-stubs=y", join(REGRTEST_DATA_DIR, "pyi")],
         exit=False,
     )
     assert run.linter.current_file is not None
@@ -1061,6 +1061,8 @@ def test_recursive_finds_pyi_file() -> None:
         [
             "--recursive",
             "y",
+            "--prefer-stubs",
+            "y",
             join(REGRTEST_DATA_DIR, "pyi"),
         ],
         exit=False,
@@ -1069,6 +1071,20 @@ def test_recursive_finds_pyi_file() -> None:
     assert run.linter.current_file.endswith("foo.pyi")
 
 
+def test_no_false_positive_from_pyi_stub() -> None:
+    run = Run(
+        [
+            "--recursive",
+            "y",
+            "--prefer-stubs",
+            "n",
+            join(REGRTEST_DATA_DIR, "uses_module_with_stub.py"),
+        ],
+        exit=False,
+    )
+    assert not run.linter.stats.by_msg
+
+
 @pytest.mark.parametrize(
     "ignore_parameter,ignore_parameter_value",
     [

tests/regrtest_data/pyi/foo.py

@@ -0,0 +1,2 @@
+def three_item_iterable():
+    return [1, 2, 3]

tests/regrtest_data/pyi/foo.pyi

@@ -1 +1,4 @@
 foo = 1
+
+def three_item_iterable():
+    ...

tests/regrtest_data/uses_module_with_stub.py

@@ -0,0 +1,5 @@
+"""If the stub is preferred over the .py, this might emit not-an-iterable"""
+from pyi.foo import three_item_iterable
+
+for val in three_item_iterable():
+    print(val)

tests/test_functional.py

@@ -51,6 +51,7 @@ def revert_stateful_config_changes(linter: PyLinter) -> Iterator[PyLinter]:
     yield linter
     # Revert any stateful configuration changes.
     MANAGER.brain["module_denylist"] = set()
+    MANAGER.brain["prefer_stubs"] = False
 
 
 @pytest.mark.usefixtures("revert_stateful_config_changes")