From 89153650661b239064ee66c2eda2f273d59e8444 Mon Sep 17 00:00:00 2001 From: Harutaka Kawamura Date: Tue, 20 Feb 2024 19:55:54 +0900 Subject: [PATCH] Remove pylint (#11183) Signed-off-by: harupy <17039389+harupy@users.noreply.github.com> --- .github/workflows/lint.yml | 3 +- .../matchers/{pylint.json => clint.json} | 2 +- .github/workflows/recipe-template.yml | 2 +- .pre-commit-config.yaml | 15 +- dev/clint/README.md | 26 + dev/clint/pyproject.toml | 17 + dev/clint/src/clint/__init__.py | 51 ++ dev/clint/src/clint/__main__.py | 3 + .../clint/src/clint/builtin.py | 42 -- dev/clint/src/clint/config.py | 17 + dev/clint/src/clint/linter.py | 170 +++++++ dev/disallow_rule_ids.py | 48 -- mlflow/pyspark/ml/__init__.py | 4 +- mlflow/utils/import_hooks/__init__.py | 8 +- pylint_plugins/__init__.py | 7 - pylint_plugins/errors.py | 38 -- pylint_plugins/no_rst.py | 75 --- pylintrc | 480 ------------------ pyproject.toml | 11 + requirements/lint-requirements.txt | 2 +- .../langchain/test_langchain_model_export.py | 2 +- tests/utils/test_requirements_utils.py | 2 +- 22 files changed, 310 insertions(+), 715 deletions(-) rename .github/workflows/matchers/{pylint.json => clint.json} (92%) create mode 100644 dev/clint/README.md create mode 100644 dev/clint/pyproject.toml create mode 100644 dev/clint/src/clint/__init__.py create mode 100644 dev/clint/src/clint/__main__.py rename pylint_plugins/import_checker.py => dev/clint/src/clint/builtin.py (69%) create mode 100644 dev/clint/src/clint/config.py create mode 100644 dev/clint/src/clint/linter.py delete mode 100644 dev/disallow_rule_ids.py delete mode 100644 pylint_plugins/__init__.py delete mode 100644 pylint_plugins/errors.py delete mode 100644 pylint_plugins/no_rst.py delete mode 100644 pylintrc diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 12d7909a4737a..50ea3cb703555 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -38,7 +38,7 @@ jobs: id: setup-python - name: Add problem matchers run: | - echo "::add-matcher::.github/workflows/matchers/pylint.json" + echo "::add-matcher::.github/workflows/matchers/clint.json" echo "::add-matcher::.github/workflows/matchers/format.json" echo "::add-matcher::.github/workflows/matchers/ruff.json" - uses: ./.github/actions/cache-pip @@ -46,7 +46,6 @@ jobs: run: | python -m venv .venv source .venv/bin/activate - source ./dev/install-common-deps.sh --ml pip install -r requirements/lint-requirements.txt - uses: ./.github/actions/pipdeptree - name: Install pre-commit hooks diff --git a/.github/workflows/matchers/pylint.json b/.github/workflows/matchers/clint.json similarity index 92% rename from .github/workflows/matchers/pylint.json rename to .github/workflows/matchers/clint.json index 01edc219253ed..8109fa7d684ef 100644 --- a/.github/workflows/matchers/pylint.json +++ b/.github/workflows/matchers/clint.json @@ -1,7 +1,7 @@ { "problemMatcher": [ { - "owner": "pylint", + "owner": "clint", "severity": "error", "pattern": [ { diff --git a/.github/workflows/recipe-template.yml b/.github/workflows/recipe-template.yml index 1a1dc44456255..4e3eebbd6ea91 100644 --- a/.github/workflows/recipe-template.yml +++ b/.github/workflows/recipe-template.yml @@ -49,7 +49,7 @@ jobs: - uses: ./.github/actions/cache-pip - name: Add problem matchers run: | - echo "::add-matcher::.github/workflows/matchers/pylint.json" + echo "::add-matcher::.github/workflows/matchers/clint.json" echo "::add-matcher::.github/workflows/matchers/black.json" echo "::add-matcher::.github/workflows/matchers/ruff.json" - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e50af6a4338d..22df8e1d79b82 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,18 +32,9 @@ repos: stages: [commit] require_serial: true - - id: pylint - name: pylint - entry: pylint - language: system - types: [python] - args: ["--rcfile=pylintrc"] - stages: [commit] - require_serial: true - - - id: disallow-rule-ids - name: disallow-rule-ids - entry: python ./dev/disallow_rule_ids.py + - id: custom-python-lint + name: custom-python-lint + entry: clint language: system types: [python] stages: [commit] diff --git a/dev/clint/README.md b/dev/clint/README.md new file mode 100644 index 0000000000000..fef7dc875a0c6 --- /dev/null +++ b/dev/clint/README.md @@ -0,0 +1,26 @@ +# Clint + +A custom linter for mlflow to enforce rules that ruff doesn't cover. + +## Installation + +``` +pip install -e dev/clint +``` + +## Usage + +```bash +clint file.py ... +``` + +## Integrating with Visual Studio Code + +1. Install [the Pylint extension](https://marketplace.visualstudio.com/items?itemName=ms-python.pylint) +2. Add the following setting in your `settings.json` file: + +```json +{ + "pylint.path": ["${interpreter}", "-m", "clint"] +} +``` diff --git a/dev/clint/pyproject.toml b/dev/clint/pyproject.toml new file mode 100644 index 0000000000000..9fc0fd271e6b4 --- /dev/null +++ b/dev/clint/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "clint" +version = "0.1.0" +description = "A custom linter for mlflow to enforce rules that ruff doesn't cover." +readme = "README.md" +authors = [{ name = "mlflow", email = "mlflow@mlflow.com" }] +dependencies = ["tomli"] + +[tool.setuptools] +package-dir = { "" = "src" } + +[project.scripts] +clint = "clint:main" diff --git a/dev/clint/src/clint/__init__.py b/dev/clint/src/clint/__init__.py new file mode 100644 index 0000000000000..f9a36b3f20b15 --- /dev/null +++ b/dev/clint/src/clint/__init__.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import argparse +import itertools +import json +import os +import re +import sys +from concurrent.futures import ProcessPoolExecutor, as_completed +from dataclasses import dataclass +from typing import Literal + +from clint.config import Config +from clint.linter import lint_file + + +@dataclass +class Args: + files: list[str] + output_format: Literal["text", "json"] + + @classmethod + def parse(cls) -> Args: + parser = argparse.ArgumentParser(description="Custom linter for mlflow.") + parser.add_argument("files", nargs="+", help="Files to lint.") + parser.add_argument("--output-format", default="text") + args, _ = parser.parse_known_args() + return cls(files=args.files, output_format=args.output_format) + + +def main(): + config = Config.load() + EXCLUDE_REGEX = re.compile("|".join(map(re.escape, config.exclude))) + args = Args.parse() + with ProcessPoolExecutor() as pool: + futures = [ + pool.submit(lint_file, f) + for f in args.files + if not EXCLUDE_REGEX.match(f) and os.path.exists(f) + ] + violations_iter = itertools.chain.from_iterable(f.result() for f in as_completed(futures)) + if violations := list(violations_iter): + if args.output_format == "json": + sys.stdout.write(json.dumps([v.json() for v in violations])) + elif args.output_format == "text": + sys.stderr.write("\n".join(map(str, violations)) + "\n") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/dev/clint/src/clint/__main__.py b/dev/clint/src/clint/__main__.py new file mode 100644 index 0000000000000..ca94cecfc1bb0 --- /dev/null +++ b/dev/clint/src/clint/__main__.py @@ -0,0 +1,3 @@ +from clint import main + +main() diff --git a/pylint_plugins/import_checker.py b/dev/clint/src/clint/builtin.py similarity index 69% rename from pylint_plugins/import_checker.py rename to dev/clint/src/clint/builtin.py index c3df7781f486f..c6b28d5961f6f 100644 --- a/pylint_plugins/import_checker.py +++ b/dev/clint/src/clint/builtin.py @@ -1,10 +1,3 @@ -import astroid -from pylint.checkers import BaseChecker -from pylint.interfaces import IAstroidChecker -from pylint.lint import PyLinter - -from pylint_plugins.errors import LAZY_BUILTIN_IMPORT, to_msgs - # https://github.com/PyCQA/isort/blob/b818cec889657cb786beafe94a6641f8fc0f0e64/isort/stdlibs/py311.py BUILTIN_MODULES = { "_ast", @@ -221,38 +214,3 @@ "zlib", "zoneinfo", } - - -class ImportChecker(BaseChecker): - __implements__ = IAstroidChecker - - name = "import-checker" - msgs = to_msgs(LAZY_BUILTIN_IMPORT) - priority = -1 - - def __init__(self, linter: PyLinter) -> None: - super().__init__(linter) - self.stack = [] - - def in_function(self): - return bool(self.stack) - - def is_builtin_module(self, name): - return name.split(".", 1)[0] in BUILTIN_MODULES - - def visit_functiondef(self, node: astroid.FunctionDef): - self.stack.append(node) - - def leave_functiondef(self, node: astroid.FunctionDef): # pylint: disable=unused-argument - self.stack.pop() - - def visit_importfrom(self, node: astroid.ImportFrom): - if self.in_function() and self.is_builtin_module(node.modname): - self.add_message(LAZY_BUILTIN_IMPORT.name, node=node, args=(node.modname,)) - - def visit_import(self, node: astroid.Import): - if self.in_function(): - if builtin_modules := [name for name, _ in node.names if self.is_builtin_module(name)]: - self.add_message( - LAZY_BUILTIN_IMPORT.name, node=node, args=(", ".join(builtin_modules),) - ) diff --git a/dev/clint/src/clint/config.py b/dev/clint/src/clint/config.py new file mode 100644 index 0000000000000..4fd88a1f0bd34 --- /dev/null +++ b/dev/clint/src/clint/config.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import tomli + + +@dataclass +class Config: + exclude: list[str] + + @classmethod + def load(cls) -> Config: + with open("pyproject.toml", "rb") as f: + data = tomli.load(f) + exclude = data["tool"]["clint"]["exclude"] + return cls(exclude) diff --git a/dev/clint/src/clint/linter.py b/dev/clint/src/clint/linter.py new file mode 100644 index 0000000000000..43ad4d07ded34 --- /dev/null +++ b/dev/clint/src/clint/linter.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import ast +import re +import tokenize +from dataclasses import dataclass + +from clint.builtin import BUILTIN_MODULES + +PARAM_REGEX = re.compile(r"\s+:param\s+\w+:", re.MULTILINE) +RETURN_REGEX = re.compile(r"\s+:returns?:", re.MULTILINE) +DISABLE_COMMENT_REGEX = re.compile(r"clint:\s*disable=([a-z0-9-]+)") + + +def ignore_map(code: str) -> dict[str, set[int]]: + """ + Creates a mapping of rule name to line numbers to ignore. + + { + "": {, ...}, + ... + } + """ + mapping: dict[str, set[int]] = {} + readline = iter(code.splitlines(True)).__next__ + for tok in tokenize.generate_tokens(readline): + if tok.type != tokenize.COMMENT: + continue + if m := DISABLE_COMMENT_REGEX.search(tok.string): + mapping.setdefault(m.group(1), set()).add(tok.start[0]) + return mapping + + +@dataclass +class Rule: + id: str + name: str + message: str + + +@dataclass +class Violation: + rule: Rule + path: str + lineno: int + col_offset: int + + def __str__(self): + return f"{self.path}:{self.lineno}:{self.col_offset}: {self.rule.id}: {self.rule.message}" + + def json(self) -> dict[str, str | int | None]: + return { + "type": "error", + "module": None, + "obj": None, + "line": self.lineno, + "column": self.col_offset, + "endLine": self.lineno, + "endColumn": self.col_offset, + "path": self.path, + "symbol": self.rule.name, + "message": self.rule.message, + "message-id": self.rule.id, + } + + +NO_RST = Rule( + "Z0001", + "no-rst", + "Do not use RST style. Use Google style instead.", +) +LAZY_BUILTIN_IMPORT = Rule( + "Z0002", + "lazy-builtin-import", + "Builtin modules must be imported at the top level.", +) + +# TODO: Remove this once we convert all docstrings to Google style. +NO_RST_IGNORE = { + "mlflow/gateway/client.py", + "mlflow/gateway/providers/utils.py", + "mlflow/keras/callback.py", + "mlflow/metrics/base.py", + "mlflow/metrics/genai/base.py", + "mlflow/models/utils.py", + "mlflow/projects/databricks.py", + "mlflow/projects/kubernetes.py", + "mlflow/store/_unity_catalog/registry/rest_store.py", + "mlflow/store/artifact/azure_data_lake_artifact_repo.py", + "mlflow/store/artifact/gcs_artifact_repo.py", + "mlflow/store/model_registry/rest_store.py", + "mlflow/store/tracking/rest_store.py", + "mlflow/utils/docstring_utils.py", + "mlflow/utils/rest_utils.py", + "tests/utils/test_docstring_utils.py", +} + + +class Linter(ast.NodeVisitor): + def __init__(self, path: str, ignore: dict[str, set[int]]): + self.stack: list[ast.FunctionDef | ast.AsyncFunctionDef] = [] + self.path = path + self.ignore = ignore + self.violations: list[Violation] = [] + + def _check(self, node: ast.AST, rule: Rule) -> None: + if (lines := self.ignore.get(rule.name)) and node.lineno in lines: + return + self.violations.append(Violation(rule, self.path, node.lineno, node.col_offset)) + + def _docstring( + self, node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef + ) -> ast.Constant | None: + if ( + isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant) + and isinstance(node.body[0].value.s, str) + ): + return node.body[0].value + return None + + def _no_rst(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + if self.path in NO_RST_IGNORE: + return + + if (nd := self._docstring(node)) and ( + PARAM_REGEX.search(nd.s) or RETURN_REGEX.search(nd.s) + ): + self._check(nd, NO_RST) + + def _is_in_function(self) -> bool: + return bool(self.stack) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self.stack.append(node) + self._no_rst(node) + self.generic_visit(node) + self.stack.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self.stack.append(node) + self._no_rst(node) + self.generic_visit(node) + self.stack.pop() + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self.stack.append(node) + self._no_rst(node) + self.generic_visit(node) + self.stack.pop() + + def visit_Import(self, node: ast.Import) -> None: + if self._is_in_function(): + for alias in node.names: + if alias.name.split(".", 1)[0] in BUILTIN_MODULES: + self._check(node, LAZY_BUILTIN_IMPORT) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + if self._is_in_function() and node.module.split(".", 1)[0] in BUILTIN_MODULES: + self._check(node, LAZY_BUILTIN_IMPORT) + self.generic_visit(node) + + +def lint_file(path: str) -> list[Violation]: + with open(path) as f: + code = f.read() + linter = Linter(path, ignore_map(code)) + linter.visit(ast.parse(code)) + return linter.violations diff --git a/dev/disallow_rule_ids.py b/dev/disallow_rule_ids.py deleted file mode 100644 index e281dd3ca9b43..0000000000000 --- a/dev/disallow_rule_ids.py +++ /dev/null @@ -1,48 +0,0 @@ -import argparse -import re -import sys -import tokenize - - -def iter_comments(f): - for tok_type, tok, start, _, _ in tokenize.generate_tokens(f.readline): - if tok_type == tokenize.COMMENT: - yield tok, start - - -RULE_ID_REGEX = re.compile(r"[A-Z][0-9]*") - - -def is_rule_id(s): - return bool(RULE_ID_REGEX.fullmatch(s)) - - -DISABLE_COMMENT_REGEX = re.compile(r"pylint:\s+disable=([\w\s\-,]+)") -DELIMITER_REGEX = re.compile(r"\s*,\s*") - - -def extract_codes(comment): - if m := DISABLE_COMMENT_REGEX.search(comment): - return DELIMITER_REGEX.split(m.group(1)) - return None - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("files", nargs="*") - args = parser.parse_args() - - exit_code = 0 - msg = "Use rule name (e.g. unused-variable) instead of rule ID (e.g. W0612)" - for path in args.files: - with open(path) as f: - for comment, (row, col) in iter_comments(f): - if codes := extract_codes(comment): - if code := next(filter(is_rule_id, codes), None): - print(f"{path}:{row}:{col}: {code}: {msg}") - exit_code = 1 - sys.exit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/mlflow/pyspark/ml/__init__.py b/mlflow/pyspark/ml/__init__.py index 4093c5b191599..2a66ae93296bd 100644 --- a/mlflow/pyspark/ml/__init__.py +++ b/mlflow/pyspark/ml/__init__.py @@ -97,12 +97,12 @@ def _read_log_model_allowlist(): # New in 3.9: https://docs.python.org/3/library/importlib.resources.html#importlib.resources.files if sys.version_info.major > 2 and sys.version_info.minor > 8: - from importlib.resources import as_file, files # pylint: disable=lazy-builtin-import + from importlib.resources import as_file, files # clint: disable=lazy-builtin-import with as_file(files(__name__).joinpath("log_model_allowlist.txt")) as file: builtin_allowlist_file = file.as_posix() else: - from importlib.resources import path # pylint: disable=lazy-builtin-import + from importlib.resources import path # clint: disable=lazy-builtin-import with path(__name__, "log_model_allowlist.txt") as file: builtin_allowlist_file = file.as_posix() diff --git a/mlflow/utils/import_hooks/__init__.py b/mlflow/utils/import_hooks/__init__.py index 1fc266c5584d8..f47d3283c40e4 100644 --- a/mlflow/utils/import_hooks/__init__.py +++ b/mlflow/utils/import_hooks/__init__.py @@ -184,7 +184,7 @@ def import_hook(module): def discover_post_import_hooks(group): # New in 3.9: https://docs.python.org/3/library/importlib.resources.html#importlib.resources.files if sys.version_info.major > 2 and sys.version_info.minor > 8: - from importlib.resources import files # pylint: disable=lazy-builtin-import + from importlib.resources import files # clint: disable=lazy-builtin-import for entrypoint in ( resource.name for resource in files(group).iterdir() if resource.is_file() @@ -192,7 +192,7 @@ def discover_post_import_hooks(group): callback = _create_import_hook_from_entrypoint(entrypoint) register_post_import_hook(callback, entrypoint.name) else: - from importlib.resources import contents # pylint: disable=lazy-builtin-import + from importlib.resources import contents # clint: disable=lazy-builtin-import for entrypoint in contents(group): callback = _create_import_hook_from_entrypoint(entrypoint) @@ -286,7 +286,7 @@ def find_module(self, fullname, path=None): # real loader to import the module and invoke the # post import hooks. try: - import importlib.util # pylint: disable=lazy-builtin-import + import importlib.util # clint: disable=lazy-builtin-import loader = importlib.util.find_spec(fullname).loader # If an ImportError (or AttributeError) is encountered while finding the module, @@ -324,7 +324,7 @@ def find_spec(self, fullname, path, target=None): # pylint: disable=unused-argu # Now call back into the import system again. try: - import importlib.util # pylint: disable=lazy-builtin-import + import importlib.util # clint: disable=lazy-builtin-import spec = importlib.util.find_spec(fullname) # Replace the module spec's loader with a wrapped version that executes import diff --git a/pylint_plugins/__init__.py b/pylint_plugins/__init__.py deleted file mode 100644 index eb14cb1a1a65e..0000000000000 --- a/pylint_plugins/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from pylint_plugins.import_checker import ImportChecker -from pylint_plugins.no_rst import NoRstChecker - - -def register(linter): - linter.register_checker(ImportChecker(linter)) - linter.register_checker(NoRstChecker(linter)) diff --git a/pylint_plugins/errors.py b/pylint_plugins/errors.py deleted file mode 100644 index a8246f1c46836..0000000000000 --- a/pylint_plugins/errors.py +++ /dev/null @@ -1,38 +0,0 @@ -from functools import reduce -from typing import Dict, NamedTuple, Tuple - - -class Message(NamedTuple): - id: str - name: str - message: str - reason: str - - def to_dict(self) -> Dict[str, Tuple[str, str, str]]: - return {self.id: (self.message, self.name, self.reason)} - - -def to_msgs(*messages: Message) -> Dict[str, Tuple[str, str, str]]: - return reduce(lambda x, y: {**x, **y.to_dict()}, messages, {}) - - -LAZY_BUILTIN_IMPORT = Message( - id="W0007", - name="lazy-builtin-import", - message="Import built-in module(s) (%s) at the top of the file.", - reason="There is no reason they should be imported inside a function.", -) - -USELESS_ASSIGNMENT = Message( - id="W0008", - name="useless-assignment", - message="Useless assignment. Use immediate return instead.", - reason="For simplicity and readability", -) - -NO_RST = Message( - id="W0009", - name="no-rst", - message="Do not use RST style. Use Google style instead.", - reason="For readability", -) diff --git a/pylint_plugins/no_rst.py b/pylint_plugins/no_rst.py deleted file mode 100644 index 4190d4b35321f..0000000000000 --- a/pylint_plugins/no_rst.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -import re -from pathlib import Path - -from astroid import nodes -from pylint import checkers -from pylint.interfaces import IAstroidChecker - -from pylint_plugins.errors import NO_RST, to_msgs - -PARAM_REGEX = re.compile(r"\s+:param\s+\w+:", re.MULTILINE) -RETURN_REGEX = re.compile(r"\s+:returns?:", re.MULTILINE) - -# TODO: Remove this once all docstrings are updated -IGNORE = { - str(Path(p).resolve()) - for p in [ - "dev/set-mlflow-spark-scala-version.py", - "mlflow/gateway/client.py", - "mlflow/gateway/providers/utils.py", - "mlflow/keras/callback.py", - "mlflow/legacy_databricks_cli/configure/provider.py", - "mlflow/metrics/base.py", - "mlflow/metrics/genai/base.py", - "mlflow/mleap/__init__.py", - "mlflow/models/evaluation/default_evaluator.py", - "mlflow/projects/databricks.py", - "mlflow/projects/kubernetes.py", - "mlflow/store/_unity_catalog/registry/rest_store.py", - "mlflow/store/artifact/azure_data_lake_artifact_repo.py", - "mlflow/store/artifact/gcs_artifact_repo.py", - "mlflow/store/model_registry/rest_store.py", - "mlflow/store/tracking/rest_store.py", - "mlflow/utils/docstring_utils.py", - "mlflow/utils/rest_utils.py", - "mlflow/models/utils.py", - "tests/conftest.py", - "tests/evaluate/test_validation.py", - "tests/helper_functions.py", - "tests/projects/test_project_spec.py", - "tests/resources/data/dataset.py", - "tests/resources/mlflow-test-plugin/mlflow_test_plugin/dummy_dataset.py", - "tests/sagemaker/mock/__init__.py", - "tests/store/artifact/test_cli.py", - "tests/tracking/fluent/test_fluent.py", - "tests/transformers/helper.py", - "tests/utils/test_docstring_utils.py", - ] -} - - -class NoRstChecker(checkers.BaseChecker): - __implements__ = IAstroidChecker - - name = "no-rst" - msgs = to_msgs(NO_RST) - priority = -1 - - def visit_classdef(self, node: nodes.ClassDef) -> None: - self._check_docstring(node) - - def visit_functiondef(self, node: nodes.FunctionDef) -> None: - self._check_docstring(node) - - visit_asyncfunctiondef = visit_functiondef - - def _check_docstring(self, node: nodes.Module | nodes.ClassDef | nodes.FunctionDef) -> None: - if ( - node.root().file not in IGNORE - and node.doc_node - and (doc := node.doc_node.value) - and (PARAM_REGEX.search(doc) or RETURN_REGEX.search(doc)) - ): - self.add_message(NO_RST.name, node=node.doc_node) diff --git a/pylintrc b/pylintrc deleted file mode 100644 index c11c4046b43a9..0000000000000 --- a/pylintrc +++ /dev/null @@ -1,480 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-allow-list= - -# Add files or directories to the denylist. They should be base names, not -# paths. -ignore= - -# Add files or directories matching the regex patterns to the denylist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Add files or directories matching the regex patterns to the ignore-list. -# The regex matches against paths. -ignore-paths=setup.py, - docs, - examples, - mlflow/protos, - mlflow/ml_package_versions.py, - mlflow/server/graphql/autogenerated_graphql_schema.py, - mlflow/server/js, - mlflow/store/db_migrations, - mlflow/temporary_db_migrations_for_pre_1_users, - tests/protos - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=0 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins=pylint_plugins - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=all - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=signature-differs, - unused-arguments, - consider-using-dict-items, - # Built-in rules - # -------------- - # Custom rules - lazy-builtin-import, - no-rst, - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=optparse.Values,sys.exit - - -[BASIC] - -# Naming style matching correct argument names -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style -#argument-rgx= - -# Naming style matching correct attribute names -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style -#class-attribute-rgx= - -# Naming style matching correct class names -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming-style -#class-rgx= - -# Naming style matching correct constant names -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma -good-names=i, - j, - k, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming style matching correct inline iteration names -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style -#inlinevar-rgx= - -# Naming style matching correct method names -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style -#method-rgx= - -# Naming style matching correct module names -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style -#variable-rgx= - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=https?://\S+ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules=distutils,tensorflow.keras - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=yes - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/pyproject.toml b/pyproject.toml index 789ccc1cd7f32..adac0a6a632dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,3 +112,14 @@ forced-separate = ["tests"] [tool.ruff.lint.flake8-tidy-imports.banned-api] "pkg_resources".msg = "We're migrating away from pkg_resources. Please use importlib.resources or importlib.metadata instead." + +[tool.clint] +exclude = [ + "docs", + "examples", + "mlflow/protos", + "mlflow/ml_package_versions.py", + "mlflow/server/js", + "mlflow/store/db_migrations", + "tests/protos", +] diff --git a/requirements/lint-requirements.txt b/requirements/lint-requirements.txt index 70b360e95d08c..a4ba225d94b6b 100644 --- a/requirements/lint-requirements.txt +++ b/requirements/lint-requirements.txt @@ -1,6 +1,6 @@ -pylint==2.14.4 ruff==0.2.1 rstcheck==6.1.1 black[jupyter]==23.7.0 blacken-docs==1.16.0 pre-commit==2.20.0 +-e ./dev/clint diff --git a/tests/langchain/test_langchain_model_export.py b/tests/langchain/test_langchain_model_export.py index 0fed5b297a811..6894c586ea380 100644 --- a/tests/langchain/test_langchain_model_export.py +++ b/tests/langchain/test_langchain_model_export.py @@ -576,7 +576,7 @@ def test_log_and_load_retriever_chain(tmp_path): # Define the loader_fn def load_retriever(persist_directory): - from typing import List # pylint: disable=lazy-builtin-import + from typing import List # clint: disable=lazy-builtin-import import numpy as np from langchain.embeddings.base import Embeddings diff --git a/tests/utils/test_requirements_utils.py b/tests/utils/test_requirements_utils.py index 7a660dd2d961f..aafe6512ed40c 100644 --- a/tests/utils/test_requirements_utils.py +++ b/tests/utils/test_requirements_utils.py @@ -210,7 +210,7 @@ def test_capture_imported_modules(): from mlflow.utils._capture_modules import _CaptureImportedModules with _CaptureImportedModules() as cap: - import math # pylint: disable=lazy-builtin-import # noqa: F401 + import math # clint: disable=lazy-builtin-import # noqa: F401 __import__("pandas") importlib.import_module("numpy")