Switch from black and flake8 to ruff for Python files (#24590)

* Switch from black and flake8 to ruff for Python files

* fix

* Update .pre-commit-config.yaml

* Update pyproject.toml

* address

* fix fmt

* future

* noqa

* fix comment

.pre-commit-config.yaml

@@ -7,23 +7,12 @@ default_stages:
   - pre-commit
 
 repos:
-- repo: https://github.com/pycqa/flake8
-  rev: c6e0d27593a45342ffa96a18bba708a5aab32fdf # 3.9.2 should match major Python version
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: 0ccbb5b7942d83fbcf7cb5e0fd99633efd2351d7 # v0.3.5
   hooks:
-    - id: flake8
-      additional_dependencies:
-        - flake8-bugbear==21.11.28
-        - flake8-comprehensions==3.7.0
-        - flake8-unused-arguments==0.0.6
-        - flake8-use-fstring==1.3.0
-- repo: https://github.com/psf/black
-  rev: ae2c0758c9e61a385df9700dc9c231bf54887041 # 22.3.0
-  hooks:
-    - id: black
-- repo: https://github.com/timothycrosley/isort/
-  rev: e44834b7b294701f596c9118d6c370f86671a50d # 5.12.0
-  hooks:
-    - id: isort
+    - id: ruff
+      args: [ --fix ]
+    - id: ruff-format
 - repo: https://github.com/jendrikseipp/vulture
   rev: 44aed44e226ec0e5660851462f764ec5d5da957c # v2.3
   hooks:

cmd/agent/dist/checks/libs/wmi/sampler.py

@@ -24,4 +24,5 @@
 Original discussion thread: https://github.com/DataDog/dd-agent/issues/1952
 Credits to @TheCloudlessSky (https://github.com/TheCloudlessSky)
 """
+
 from datadog_checks.base.checks.win.wmi.sampler import WMISampler  # noqa: F401

devenv/tasks/__init__.py

@@ -1,6 +1,7 @@
 """
 Invoke entrypoint, import here all the tasks we want to make available
 """
+
 import os
 
 from invoke import Collection

devenv/tasks/packer.py

@@ -1,7 +1,6 @@
 """
 Packer namespaced tasks
 """
-from __future__ import print_function
 
 from invoke import task
 from invoke.exceptions import Exit

docs/dev/linters.md

@@ -23,9 +23,7 @@ To run the linters locally, run `inv linter.go`.
 ## Python
 
 For Python, we're using ([see invoke task](https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/tasks/linter_tasks.py/#L17-L33)):
-- [flake8](https://flake8.pycqa.org/en/latest), a style linter.
-- [black](https://black.readthedocs.io/en/stable/), a code formatter.
-- [isort](https://pycqa.github.io/isort/), to sort the imports.
+- [ruff](https://github.com/astral-sh/ruff), a style linter and a code formatter.
 - [vulture](https://github.com/jendrikseipp/vulture), to find unused code.
 
 Their configuration is defined in both the [setup.cfg](https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/setup.cfg) and the [pyproject.toml](https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/pyproject.toml) files.

pkg/config/legacy/tests/config.py

@@ -191,7 +191,7 @@ def get_histogram_aggregates(configstr=None):
         for val in vals:
             val = val.strip()
             if val not in valid_values:
-                log.warning(f"Ignored histogram aggregate {val}, invalid")
+                log.warning("Ignored histogram aggregate %s, invalid", val)
                 continue
             else:
                 result.append(val)
@@ -216,10 +216,10 @@ def get_histogram_percentiles(configstr=None):
                 if floatval <= 0 or floatval >= 1:
                     raise ValueError
                 if len(val) > 4:
-                    log.warning(f"Histogram percentiles are rounded to 2 digits: {floatval} rounded")
+                    log.warning("Histogram percentiles are rounded to 2 digits: %s rounded", floatval)
                 result.append(float(val[0:4]))
             except ValueError:
-                log.warning(f"Bad histogram percentile value {val}, must be float in ]0;1[, skipping")
+                log.warning("Bad histogram percentile value %s, must be float in ]0;1[, skipping", val)
     except Exception:
         log.exception("Error when parsing histogram percentiles, skipping")
         return None
@@ -287,11 +287,11 @@ def get_config(options=None):
         # Core config
         # ap
         if not config.has_option('Main', 'api_key'):
-            log.warning(u"No API key was found. Aborting.")
+            log.warning("No API key was found. Aborting.")
             sys.exit(2)
 
         if not config.has_option('Main', 'dd_url'):
-            log.warning(u"No dd_url was found. Aborting.")
+            log.warning("No dd_url was found. Aborting.")
             sys.exit(2)
 
         # Endpoints
@@ -531,13 +531,13 @@ def extract_agent_config(config):
         conf_backend = config.get('Main', 'sd_config_backend')
 
     if backend not in SD_BACKENDS:
-        log.error(f"The backend {backend} is not supported. Service discovery won't be enabled.")
+        log.error("The backend %s is not supported. Service discovery won't be enabled.", backend)
         agentConfig['service_discovery'] = False
 
     if conf_backend is None:
         log.warning('No configuration backend provided for service discovery. Only auto config templates will be used.')
     elif conf_backend not in SD_CONFIG_BACKENDS:
-        log.error(f"The config backend {conf_backend} is not supported. Only auto config templates will be used.")
+        log.error("The config backend %s is not supported. Only auto config templates will be used.", conf_backend)
         conf_backend = None
     agentConfig['sd_config_backend'] = conf_backend
 

pkg/gohai/cpu/from-lscpu-arm.py

@@ -10,7 +10,7 @@
 
 
 def main():
-    lines = iter(open(sys.argv[1], "r"))
+    lines = iter(open(sys.argv[1], "r"))  # noqa: UP015
 
     # part_lists are the lists of part numbers, keyed by name
     part_lists = {}

pyproject.toml

@@ -3,28 +3,74 @@
 # verbose regular expressions by Black. Use [ ] to denote a significant space
 # character.
 
-[tool.black]
-include = '\.pyi?$'
+[tool.ruff]
+# TODO: Some images still use python3.8 in the CI, such as datadog-agent-buildimages/docker_x64 or circleci
+target-version = "py38"
+exclude = [
+    ".git",
+    ".github",
+    ".circleci",
+    "chocolatey",
+    "Dockerfiles",
+    "docs",
+    "google-marketplace",
+    "omnibus",
+    "pkg-config",
+    "releasenotes",
+    "vendor",
+    "venv",
+    "dev",
+]
 line-length = 120
-skip-string-normalization = true
-exclude = '''
-(
-  /(
-    | \.git
-    | docs
-    | releasenotes
-    | releasenotes-installscript
-    | vendor
-    | dev
-  )/
-)
-'''
 
-[tool.isort]
-default_section = 'THIRDPARTY'
-force_grid_wrap = 0
-include_trailing_comma = true
-known_first_party = 'datadog_checks'
-line_length = 120
-multi_line_output = 3
-use_parentheses = true
+[tool.ruff.lint]
+# Rules were ported over from the legacy flake8 settings for parity
+# All the rules can be found here: https://beta.ruff.rs/docs/rules/
+select = [
+    "B",
+    "C",
+    "E",
+    "F",
+    "G",
+    "U",
+    "W",
+    "B003",
+    "B006",
+    "B007",
+]
+ignore = [
+    # From legacy flake8 settings
+    # Ignore:
+    # - black-incompatible options: E203
+    # - bugbear overlap: E722
+    # - style options: W2,W3,W50,E111,E114,E117,E2,E3,E5,E74
+    # - Unnecessary dict call: C408
+    # - complex-structure : C901
+    # - raise-without-from-inside-except: B904
+    "E203",
+    "W2",
+    "W3",
+    "W50",
+    "E111",
+    "E114",
+    "E117",
+    "E2",
+    "E3",
+    "E5",
+    "E74",
+    "E722",
+    "C408",
+    "C901",
+    "B904",
+]
+unfixable = [
+    # Don't touch unused imports
+    "F401",
+]
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.format]
+# Enable preview style formatting.
+quote-style = "preserve"

rtloader/demo/main.py

@@ -2,7 +2,7 @@
 # under the Apache License Version 2.0.
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2019-present Datadog, Inc.
-from __future__ import print_function
+from __future__ import print_function  # noqa fmt: off
 
 import aggregator
 import tagger

rtloader/test/python/datadog_checks/base/checks/__init__.py

@@ -1,5 +1,5 @@
 # AgentCheck stubs for testing
-class AgentCheck(object):
+class AgentCheck(object):  # noqa: UP004
     def __init__(self, *args, **kwargs):  # noqa: U100
         pass
 

rtloader/test/python/fake_check/__init__.py

@@ -2,6 +2,7 @@
 
 was_canceled = False
 
+
 # Fake check for testing purposes
 class FakeCheck(AgentCheck):
     def cancel(self):

tasks/agent.py

@@ -634,7 +634,7 @@ def check_supports_python_version(_, check_dir, python):
     project_file = os.path.join(check_dir, 'pyproject.toml')
     setup_file = os.path.join(check_dir, 'setup.py')
     if os.path.isfile(project_file):
-        with open(project_file, 'r') as f:
+        with open(project_file) as f:
             data = toml.loads(f.read())
 
         project_metadata = data['project']
@@ -651,7 +651,7 @@ def check_supports_python_version(_, check_dir, python):
         else:
             print('False', end='')
     elif os.path.isfile(setup_file):
-        with open(setup_file, 'r') as f:
+        with open(setup_file) as f:
             tree = ast.parse(f.read(), filename=setup_file)
 
         prefix = f'Programming Language :: Python :: {python}'

tasks/bench.py

@@ -2,7 +2,6 @@
 Benchmarking tasks
 """
 
-
 import os
 
 from invoke import task

tasks/build_tags.py

@@ -1,12 +1,13 @@
 """
 Utilities to manage build tags
 """
+
 # TODO: check if we really need the typing import.
 # Recent versions of Python should be able to use dict and list directly in type hints,
 # so we only need to check that we don't run this code with old Python versions.
+from __future__ import annotations
 
 import sys
-from typing import List
 
 from invoke import task
 
@@ -230,8 +231,8 @@
 def compute_build_tags_for_flavor(
     build: str,
     arch: str,
-    build_include: List[str],
-    build_exclude: List[str],
+    build_include: list[str],
+    build_exclude: list[str],
     flavor: AgentFlavor = AgentFlavor.base,
     include_sds: bool = False,
 ):

tasks/components.py

@@ -1,6 +1,7 @@
 """
 Invoke entrypoint, import here all the tasks we want to make available
 """
+
 import os
 import pathlib
 from collections import namedtuple
@@ -264,7 +265,7 @@ def lint_components(_, fix=False):
         with open(filename, "w") as f:
             f.write(components_md)
     else:
-        with open(filename, "r") as f:
+        with open(filename) as f:
             current = f.read()
             if current != components_md:
                 print(f"** {filename} differs")
@@ -273,7 +274,7 @@ def lint_components(_, fix=False):
 
     # Check .github/CODEOWNERS
     filename = ".github/CODEOWNERS"
-    with open(filename, "r") as f:
+    with open(filename) as f:
         current = f.read()
     codeowners = '\n'.join(make_codeowners(current.splitlines(), bundles, components_without_bundle))
     if fix:
@@ -419,7 +420,7 @@ def read_file_content(template_path):
     """
     Read all lines in files and return them as a single string.
     """
-    with open(template_path, "r") as file:
+    with open(template_path) as file:
         return file.read()
 
 

tasks/devcontainer.py

@@ -3,6 +3,7 @@
 
 Helpers for getting vscode set up nicely
 """
+
 import json
 import os
 from collections import OrderedDict
@@ -53,7 +54,7 @@ def setup(
     devcontainer = {}
     fullpath = os.path.join(DEVCONTAINER_DIR, DEVCONTAINER_FILE)
     if os.path.exists(fullpath):
-        with open(fullpath, "r") as sf:
+        with open(fullpath) as sf:
             devcontainer = json.load(sf, object_pairs_hook=OrderedDict)
 
     local_build_tags = ",".join(use_tags)
@@ -104,9 +105,9 @@ def setup(
             "extensions": ["golang.Go"],
         }
     }
-    devcontainer[
-        "postStartCommand"
-    ] = "git config --global --add safe.directory /workspaces/datadog-agent && invoke install-tools && invoke deps"
+    devcontainer["postStartCommand"] = (
+        "git config --global --add safe.directory /workspaces/datadog-agent && invoke install-tools && invoke deps"
+    )
 
     with open(fullpath, "w") as sf:
         json.dump(devcontainer, sf, indent=4, sort_keys=False, separators=(',', ': '))

tasks/docker_tasks.py

@@ -2,7 +2,6 @@
 Docker related tasks
 """
 
-
 import os
 import shutil
 import sys
@@ -121,7 +120,7 @@ def pull_base_images(ctx, dockerfile, signed_pull=True):
     images = set()
     stages = set()
 
-    with open(dockerfile, "r") as f:
+    with open(dockerfile) as f:
         for line in f:
             words = line.split()
             # Get source images

tasks/ebpf.py

@@ -141,7 +141,7 @@ def print_verification_stats(
             write_verifier_stats(verifier_stats, f, jsonfmt)
         return
 
-    with open(base, 'r') as f:
+    with open(base) as f:
         base_verifier_stats = json.load(f)
 
     stats_diff = dict()