Use ruff as a linter (#2031)

### Changes
Add ruff as a step to the `pre-commit` pipeline and adjust the codebase
to conform.

### Reason for changes
Ruff seems to do a good job at basic linting, works much faster than
pylint and doesn't even seem to need a virtual environment to function
properly. We might want to replace pylint with this entirely to lessen
the load on CI.

### Related tickets
N/A

### Tests
precommit

.pre-commit-config.yaml

@@ -15,6 +15,12 @@ repos:
       - id: isort
         name: isort (python)
 
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.282
+    hooks:
+      - id: ruff
+
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.33.0
     hooks:

.pylintrc

@@ -22,7 +22,8 @@ disable = arguments-differ,
           duplicate-code,
           consider-using-f-string,
           logging-fstring-interpolation,
-          cyclic-import
+          cyclic-import,
+          useless-import-alias
 
 max-line-length = 120
 ignore-docstrings = yes

docs/api/source/conf.py

@@ -67,7 +67,7 @@ def collect_api_entities() -> APIInfo:
     """
     retval = APIInfo()
     modules = {}
-    skipped_modules = {}  # type: Dict[str, str]
+    skipped_modules: Dict[str, str] = {}
     import nncf
 
     for _, modname, _ in pkgutil.walk_packages(path=nncf.__path__, prefix=nncf.__name__ + ".", onerror=lambda x: None):
@@ -135,6 +135,7 @@ def collect_api_entities() -> APIInfo:
     "onnxruntime",
     "openvino",
     "tensorflow",
+    "keras",
     "tensorflow_addons",
     # Need add backend implementation functions to avoid endless loops on registered functions by mock module,
     "nncf.experimental.tensor.torch_functions",

examples/tensorflow/object_detection/preprocessing/yolo_v4_preprocessing.py

@@ -152,13 +152,19 @@ def _preprocess_true_boxes(
         true_boxes[..., 2:4] = boxes_wh / input_shape[::-1]
 
         batch_size = true_boxes.shape[0]
-        grid_shapes = [input_shape // {0: 32, 1: 16, 2: 8}[l] for l in range(num_layers)]
+        grid_shapes = [input_shape // {0: 32, 1: 16, 2: 8}[layer_idx] for layer_idx in range(num_layers)]
         y_true = [
             np.zeros(
-                (batch_size, grid_shapes[l][0], grid_shapes[l][1], len(anchor_mask[l]), 5 + num_classes),
+                (
+                    batch_size,
+                    grid_shapes[layer_idx][0],
+                    grid_shapes[layer_idx][1],
+                    len(anchor_mask[layer_idx]),
+                    5 + num_classes,
+                ),
                 dtype="float32",
             )
-            for l in range(num_layers)
+            for layer_idx in range(num_layers)
         ]
 
         # Expand dim to apply broadcasting.
@@ -196,22 +202,22 @@ def _preprocess_true_boxes(
                 best_anchors = np.expand_dims(best_anchors, -1)
 
             for t, row in enumerate(best_anchors):
-                for l in range(num_layers):
+                for layer_idx in range(num_layers):
                     for n in row:
                         # use different matching policy for single & multi anchor assign
                         if multi_anchor_assign:
-                            matching_rule = iou[t, n] > iou_thresh and n in anchor_mask[l]
+                            matching_rule = iou[t, n] > iou_thresh and n in anchor_mask[layer_idx]
                         else:
-                            matching_rule = n in anchor_mask[l]
+                            matching_rule = n in anchor_mask[layer_idx]
 
                         if matching_rule:
-                            i = np.floor(true_boxes[b, t, 0] * grid_shapes[l][1]).astype("int32")
-                            j = np.floor(true_boxes[b, t, 1] * grid_shapes[l][0]).astype("int32")
-                            k = anchor_mask[l].index(n)
+                            i = np.floor(true_boxes[b, t, 0] * grid_shapes[layer_idx][1]).astype("int32")
+                            j = np.floor(true_boxes[b, t, 1] * grid_shapes[layer_idx][0]).astype("int32")
+                            k = anchor_mask[layer_idx].index(n)
                             c = true_boxes[b, t, 4].astype("int32")
-                            y_true[l][b, j, i, k, 0:4] = true_boxes[b, t, 0:4]
-                            y_true[l][b, j, i, k, 4] = 1
-                            y_true[l][b, j, i, k, 5 + c] = 1
+                            y_true[layer_idx][b, j, i, k, 0:4] = true_boxes[b, t, 0:4]
+                            y_true[layer_idx][b, j, i, k, 4] = 1
+                            y_true[layer_idx][b, j, i, k, 5 + c] = 1
         return y_true
 
     def _preprocess2(self, image_data, box_data):

examples/torch/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2023 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

examples/torch/common/model_loader.py

@@ -92,7 +92,7 @@ def is_url(uri):
     try:
         parsed_url = urllib.parse.urlparse(uri)
         return parsed_url.scheme and parsed_url.netloc
-    except:  # pylint: disable=bare-except
+    except:  # pylint: disable=bare-except  # noqa: E722
         return False
 
 

examples/torch/object_detection/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2023 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

examples/torch/object_detection/datasets/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2023 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

examples/torch/object_detection/models/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2023 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

examples/torch/object_detection/utils/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2023 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nncf/__init__.py

@@ -12,29 +12,29 @@
 Neural Network Compression Framework (NNCF) for enhanced OpenVINO™ inference.
 """
 
-from nncf.common.logging import nncf_logger
-from nncf.common.logging.logger import disable_logging
-from nncf.common.logging.logger import set_log_level
-from nncf.common.strip import strip
-from nncf.config import NNCFConfig
-from nncf.data import Dataset
-from nncf.parameters import CompressWeightsMode
-from nncf.parameters import DropType
-from nncf.parameters import ModelType
-from nncf.parameters import TargetDevice
-from nncf.quantization import QuantizationPreset
-from nncf.quantization import compress_weights
-from nncf.quantization import quantize
-from nncf.quantization import quantize_with_accuracy_control
-from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters
-from nncf.scopes import IgnoredScope
-from nncf.version import __version__
+from nncf.common.logging import nncf_logger as nncf_logger
+from nncf.common.logging.logger import disable_logging as disable_logging
+from nncf.common.logging.logger import set_log_level as set_log_level
+from nncf.common.strip import strip as strip
+from nncf.config import NNCFConfig as NNCFConfig
+from nncf.data import Dataset as Dataset
+from nncf.parameters import CompressWeightsMode as CompressWeightsMode
+from nncf.parameters import DropType as DropType
+from nncf.parameters import ModelType as ModelType
+from nncf.parameters import TargetDevice as TargetDevice
+from nncf.quantization import QuantizationPreset as QuantizationPreset
+from nncf.quantization import compress_weights as compress_weights
+from nncf.quantization import quantize as quantize
+from nncf.quantization import quantize_with_accuracy_control as quantize_with_accuracy_control
+from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters as AdvancedQuantizationParameters
+from nncf.scopes import IgnoredScope as IgnoredScope
+from nncf.version import __version__ as __version__
 
 _SUPPORTED_FRAMEWORKS = ["torch", "tensorflow", "onnx", "openvino"]
 
 
-from importlib.util import find_spec as _find_spec  # pylint:disable=wrong-import-position
-from pathlib import Path as _Path  # pylint:disable=wrong-import-position
+from importlib.util import find_spec as _find_spec  # noqa: E402 # pylint:disable=wrong-import-position
+from pathlib import Path as _Path  # noqa: E402 # pylint:disable=wrong-import-position
 
 _AVAILABLE_FRAMEWORKS = {}
 

nncf/common/accuracy_aware_training/__init__.py

@@ -12,5 +12,7 @@
 Accuracy Aware Training functionality.
 """
 
-from nncf.common.accuracy_aware_training.training_loop import AccuracyAwareTrainingMode
-from nncf.common.accuracy_aware_training.training_loop import create_accuracy_aware_training_loop
+from nncf.common.accuracy_aware_training.training_loop import AccuracyAwareTrainingMode as AccuracyAwareTrainingMode
+from nncf.common.accuracy_aware_training.training_loop import (
+    create_accuracy_aware_training_loop as create_accuracy_aware_training_loop,
+)

nncf/common/accuracy_aware_training/training_loop.py

@@ -21,6 +21,7 @@
 from scipy.interpolate import interp1d
 
 from nncf.api.compression import CompressionAlgorithmController
+from nncf.common.accuracy_aware_training.runner import BaseAccuracyAwareTrainingRunner
 from nncf.common.accuracy_aware_training.runner_factory import AdaptiveCompressionLevelTrainingRunnerCreator
 from nncf.common.accuracy_aware_training.runner_factory import EarlyExitTrainingRunnerCreator
 from nncf.common.accuracy_aware_training.statistics import TrainingLoopStatistics
@@ -89,7 +90,7 @@ class BaseEarlyExitCompressionTrainingLoop(TrainingLoop, ABC):
     """
 
     def __init__(self, compression_controller: CompressionAlgorithmController):
-        self.runner = None  # type: BaseAccuracyAwareTrainingRunner
+        self.runner: BaseAccuracyAwareTrainingRunner = None
         self.compression_controller = compression_controller
         self._current_compression_rate = None
 

nncf/common/graph/__init__.py

@@ -9,4 +9,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nncf.common.graph.graph import *
+# Make these names available from nncf.common.graph directly
+from nncf.common.graph.graph import LayerName as LayerName
+from nncf.common.graph.graph import NNCFGraph as NNCFGraph
+from nncf.common.graph.graph import NNCFGraphEdge as NNCFGraphEdge
+from nncf.common.graph.graph import NNCFGraphPatternIO as NNCFGraphPatternIO
+from nncf.common.graph.graph import NNCFNode as NNCFNode
+from nncf.common.graph.graph import NNCFNodeName as NNCFNodeName

nncf/common/graph/graph.py

@@ -194,11 +194,11 @@ def __init__(self):
         self._nx_graph = nx.DiGraph()
         self._node_id_to_key_dict = {}
         self._nodes: Dict[str, NNCFNode] = {}
-        self._input_nncf_nodes = {}  # type: Dict[int, NNCFNode]
-        self._output_nncf_nodes = {}  # type: Dict[int, NNCFNode]
+        self._input_nncf_nodes: Dict[int, NNCFNode] = {}
+        self._output_nncf_nodes: Dict[int, NNCFNode] = {}
 
-        self._node_ids_vs_layer_names = {}  # type: Dict[int, LayerName]
-        self._layer_name_vs_shared_nodes = defaultdict(list)  # type: Dict[LayerName, List[NNCFNode]]
+        self._node_ids_vs_layer_names: Dict[int, LayerName] = {}
+        self._layer_name_vs_shared_nodes: Dict[LayerName, List[NNCFNode]] = defaultdict(list)
 
     @property
     def nodes(self) -> Dict[str, NNCFNode]:

nncf/common/graph/patterns/__init__.py

@@ -9,8 +9,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nncf.common.graph.patterns.patterns import GraphPattern
-from nncf.common.graph.patterns.patterns import HWFusedPatternNames
-from nncf.common.graph.patterns.patterns import IgnoredPatternNames
-from nncf.common.graph.patterns.patterns import Patterns
-from nncf.common.graph.patterns.patterns import merge_two_types_of_operations
+from nncf.common.graph.patterns.patterns import GraphPattern as GraphPattern
+from nncf.common.graph.patterns.patterns import HWFusedPatternNames as HWFusedPatternNames
+from nncf.common.graph.patterns.patterns import IgnoredPatternNames as IgnoredPatternNames
+from nncf.common.graph.patterns.patterns import Patterns as Patterns
+from nncf.common.graph.patterns.patterns import merge_two_types_of_operations as merge_two_types_of_operations

nncf/common/insertion_point_graph.py

@@ -16,11 +16,11 @@
 
 import networkx as nx
 
-from nncf.common.graph import Dtype
 from nncf.common.graph import NNCFGraph
 from nncf.common.graph import NNCFNodeName
 from nncf.common.graph.graph import NNCFNode
 from nncf.common.graph.graph_matching import find_subgraphs_matching_pattern
+from nncf.common.graph.layer_attributes import Dtype
 from nncf.common.graph.operator_metatypes import INPUT_NOOP_METATYPES
 from nncf.common.graph.patterns import GraphPattern
 from nncf.common.logging import nncf_logger
@@ -107,11 +107,11 @@ def __init__(
             # Post-hook all nodes if an exact list is not specified
             allowed_post_hook_insertion_points = self._get_default_post_hook_ip_list(nncf_graph)
 
-        target_node_name_vs_pre_hook_ips = defaultdict(set)  # type: Dict[NNCFNodeName, Set[PreHookInsertionPoint]]
+        target_node_name_vs_pre_hook_ips: Dict[NNCFNodeName, Set[PreHookInsertionPoint]] = defaultdict(set)
         for pre_hook_ip in allowed_pre_hook_insertion_points:
             target_node_name_vs_pre_hook_ips[pre_hook_ip.target_node_name].add(pre_hook_ip)
 
-        target_node_name_vs_post_hook_ips = defaultdict(set)  # type: Dict[NNCFNodeName, Set[PostHookInsertionPoint]]
+        target_node_name_vs_post_hook_ips: Dict[NNCFNodeName, Set[PostHookInsertionPoint]] = defaultdict(set)
         for post_hook_ip in allowed_post_hook_insertion_points:
             target_node_name_vs_post_hook_ips[post_hook_ip.target_node_name].add(post_hook_ip)
 

nncf/common/logging/__init__.py

@@ -9,4 +9,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nncf.common.logging.logger import nncf_logger
+from nncf.common.logging.logger import nncf_logger as nncf_logger

nncf/common/pruning/clusterization.py

@@ -40,8 +40,8 @@ class Clusterization(Generic[T]):
     """
 
     def __init__(self, id_fn: Callable[[T], Hashable] = None):
-        self.clusters = {}  # type: Dict[int, Cluster[T]]
-        self._element_to_cluster = {}  # type: Dict[Hashable, int]
+        self.clusters: Dict[int, Cluster[T]] = {}
+        self._element_to_cluster: Dict[Hashable, int] = {}
         if id_fn is None:
             self._id_fn = lambda x: x.id
         else:

nncf/common/pruning/mask_propagation.py

@@ -109,7 +109,7 @@ def symbolic_mask_propagation(
                 input_masks = get_input_masks(node, self._graph)
                 if any(input_masks):
                     assert len(input_masks) == 1
-                    input_mask = input_masks[0]  # type: SymbolicMask
+                    input_mask: SymbolicMask = input_masks[0]
 
                     for producer in input_mask.mask_producers:
                         previously_dims_equal = (

nncf/common/pruning/operations.py

@@ -227,7 +227,7 @@ def generate_output_mask(
             input_mask[node.node_name] if isinstance(input_mask, dict) else input_mask for input_mask in input_masks
         ]
 
-        not_empty_masks = [mask for mask in input_masks if mask is not None]  # type: List[NNCFTensor]
+        not_empty_masks: List[NNCFTensor] = [mask for mask in input_masks if mask is not None]
         if not not_empty_masks:
             return None
 

nncf/common/pruning/utils.py

@@ -295,9 +295,9 @@ def __init__(
         self.decision = decision
         if not isinstance(possible_reasons, list):
             possible_reasons = [possible_reasons]
-        self._reasons = (
+        self._reasons: Optional[List[PruningAnalysisReason]] = (
             possible_reasons if not decision and possible_reasons else None
-        )  # type: Optional[List[PruningAnalysisReason]]
+        )
 
     def __repr__(self) -> str:
         representation = f"Prunable: {self.decision}"
@@ -380,7 +380,7 @@ def get_input_channels(node: NNCFNode) -> int:
     :param node: Given prunable node.
     :return: Count of input channels of the given node.
     """
-    layer_attrs = node.layer_attributes  # type: Union[ConvolutionLayerAttributes, LinearLayerAttributes]
+    layer_attrs: Union[ConvolutionLayerAttributes, LinearLayerAttributes] = node.layer_attributes
     if isinstance(layer_attrs, ConvolutionLayerAttributes):
         return layer_attrs.in_channels
     if isinstance(layer_attrs, LinearLayerAttributes):
@@ -395,7 +395,7 @@ def get_output_channels(node: NNCFNode) -> int:
     :param node: Given prunable node.
     :return: Count of output channels of the given node.
     """
-    layer_attrs = node.layer_attributes  # type: Union[ConvolutionLayerAttributes, LinearLayerAttributes]
+    layer_attrs: Union[ConvolutionLayerAttributes, LinearLayerAttributes] = node.layer_attributes
     if isinstance(layer_attrs, ConvolutionLayerAttributes):
         return layer_attrs.out_channels
     if isinstance(layer_attrs, LinearLayerAttributes):

nncf/common/quantization/config_assignment.py

@@ -72,7 +72,7 @@ def assign_qconfig_lists_to_modules(
     :return: A dict of each weighted node vs. the list of quantizer configs allowed for quantizing the associated
       weights
     """
-    retval = {}  # type: Dict[NNCFNode, List[QuantizerConfig]]
+    retval: Dict[NNCFNode, List[QuantizerConfig]] = {}
     default_qconfig = deepcopy(default_weight_qconfig)
     if global_weight_constraints is not None:
         default_qconfig = global_weight_constraints.apply_constraints_to(default_qconfig)