WIP: Naive cythonization for performance

.gitignore

@@ -13,3 +13,5 @@ astroid.egg-info/
 .cache/
 .eggs/
 .pytest_cache/
+*.c
+*.so

astroid/__pkginfo__.py

@@ -31,6 +31,7 @@
     "wrapt",
     'typing;python_version<"3.5"',
     'typed-ast>=1.3.0;implementation_name== "cpython"',
+    'Cython'
 ]
 
 # pylint: disable=redefined-builtin; why license is a builtin anyway?

astroid/inference.py → astroid/inference.pyx

@@ -35,14 +35,15 @@
 from astroid.interpreter import dunder_lookup
 from astroid import protocols
 from astroid import util
+cimport cython
 
 
 MANAGER = manager.AstroidManager()
 
 
 # .infer method ###############################################################
 
-
+@cython.binding(True)
 def infer_end(self, context=None):
     """inference's end for node such as Module, ClassDef, FunctionDef,
     Const...
@@ -77,6 +78,7 @@ def _infer_sequence_helper(node, context=None):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def infer_sequence(self, context=None):
     if not any(isinstance(e, nodes.Starred) for e in self.elts):
         yield self
@@ -95,6 +97,7 @@ def infer_sequence(self, context=None):
 nodes.Set._infer = infer_sequence
 
 
+@cython.binding(True)
 def infer_map(self, context=None):
     if not any(isinstance(k, nodes.DictUnpack) for k, _ in self.items):
         yield self
@@ -171,7 +174,7 @@ def _higher_function_scope(node):
         return current.parent
     return None
 
-
+@cython.binding(True)
 def infer_name(self, context=None):
     """infer a Name: use name lookup rules"""
     frame, stmts = self.lookup(self.name)
@@ -200,6 +203,7 @@ def infer_name(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_call(self, context=None):
     """infer a Call node by trying to guess what the function returns"""
     callcontext = contextmod.copy_context(context)
@@ -227,6 +231,7 @@ def infer_call(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_import(self, context=None, asname=True):
     """infer an Import node: return the imported module/object"""
     name = context.lookupname
@@ -247,6 +252,7 @@ def infer_import(self, context=None, asname=True):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_import_from(self, context=None, asname=True):
     """infer a ImportFrom node: return the imported module/object"""
     name = context.lookupname
@@ -274,6 +280,7 @@ def infer_import_from(self, context=None, asname=True):
 nodes.ImportFrom._infer = infer_import_from
 
 
+@decorators.raise_if_nothing_inferred
 def infer_attribute(self, context=None):
     """infer an Attribute node by using getattr on the associated object"""
     for owner in self.expr.infer(context):
@@ -319,6 +326,7 @@ def infer_attribute(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_global(self, context=None):
     if context.lookupname is None:
         raise exceptions.InferenceError(node=self, context=context)
@@ -337,6 +345,7 @@ def infer_global(self, context=None):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def infer_subscript(self, context=None):
     """Inference for subscripts
 
@@ -399,6 +408,7 @@ def infer_subscript(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def _infer_boolop(self, context=None):
     """Infer a boolean operation (and / or / not).
 
@@ -455,7 +465,7 @@ def _infer_boolop(self, context=None):
 
 # UnaryOp, BinOp and AugAssign inferences
 
-
+@cython.binding(True)
 def _filter_operation_errors(self, infer_callable, context, error):
     for result in infer_callable(self, context):
         if isinstance(result, error):
@@ -467,6 +477,7 @@ def _filter_operation_errors(self, infer_callable, context, error):
             yield result
 
 
+@cython.binding(True)
 def _infer_unaryop(self, context=None):
     """Infer what an UnaryOp should return when evaluated."""
     for operand in self.operand.infer(context):
@@ -523,6 +534,7 @@ def _infer_unaryop(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_unaryop(self, context=None):
     """Infer what an UnaryOp should return when evaluated."""
     yield from _filter_operation_errors(
@@ -535,9 +547,9 @@ def infer_unaryop(self, context=None):
 nodes.UnaryOp._infer = infer_unaryop
 
 
-def _is_not_implemented(const):
+def _is_not_implemented(const_node):
     """Check if the given const node is NotImplemented."""
-    return isinstance(const, nodes.Const) and const.value is NotImplemented
+    return isinstance(const_node, nodes.Const) and const_node.value is NotImplemented
 
 
 def _invoke_binop_inference(instance, opnode, op, other, context, method_name):
@@ -730,6 +742,7 @@ def _infer_binary_operation(left, right, binary_opnode, context, flow_factory):
     yield util.BadBinaryOperationMessage(left_type, binary_opnode.op, right_type)
 
 
+@cython.binding(True)
 def _infer_binop(self, context):
     """Binary operation inference logic."""
     left = self.left
@@ -757,6 +770,7 @@ def _infer_binop(self, context):
 
 @decorators.yes_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_binop(self, context=None):
     return _filter_operation_errors(
         self, _infer_binop, context, util.BadBinaryOperationMessage
@@ -767,6 +781,7 @@ def infer_binop(self, context=None):
 nodes.BinOp._infer = infer_binop
 
 
+@cython.binding(True)
 def _infer_augassign(self, context=None):
     """Inference logic for augmented binary operations."""
     if context is None:
@@ -796,6 +811,7 @@ def _infer_augassign(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_augassign(self, context=None):
     return _filter_operation_errors(
         self, _infer_augassign, context, util.BadBinaryOperationMessage
@@ -809,6 +825,7 @@ def infer_augassign(self, context=None):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def infer_arguments(self, context=None):
     name = context.lookupname
     if name is None:
@@ -821,6 +838,7 @@ def infer_arguments(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_assign(self, context=None):
     """infer a AssignName/AssignAttr: need to inspect the RHS part of the
     assign node
@@ -839,6 +857,7 @@ def infer_assign(self, context=None):
 
 @decorators.raise_if_nothing_inferred
 @decorators.path_wrapper
+@cython.binding(True)
 def infer_empty_node(self, context=None):
     if not self.has_underlying_object():
         yield util.Uninferable
@@ -853,6 +872,7 @@ def infer_empty_node(self, context=None):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def infer_index(self, context=None):
     return self.value.infer(context)
 
@@ -861,6 +881,7 @@ def infer_index(self, context=None):
 
 # TODO: move directly into bases.Instance when the dependency hell
 # will be solved.
+@cython.binding(True)
 def instance_getitem(self, index, context=None):
     # Rewrap index to Const for this case
     new_context = contextmod.bind_context_to_node(context, self)

astroid/protocols.py → astroid/protocols.pyx

@@ -23,6 +23,7 @@
 import collections
 import operator as operator_mod
 import sys
+cimport cython
 
 import itertools
 
@@ -124,6 +125,7 @@ def _infer_unary_op(obj, op):
 
 
 @decorators.yes_if_nothing_inferred
+@cython.binding(True)
 def const_infer_binary_op(self, opnode, operator, other, context, _):
     not_implemented = nodes.Const(NotImplemented)
     if isinstance(other, nodes.Const):
@@ -148,6 +150,7 @@ def const_infer_binary_op(self, opnode, operator, other, context, _):
 nodes.Const.infer_binary_op = const_infer_binary_op
 
 
+@cython.binding(True)
 def _multiply_seq_by_int(self, opnode, other, context):
     node = self.__class__(parent=opnode)
     filtered_elts = (
@@ -172,6 +175,7 @@ def _filter_uninferable_nodes(elts, context):
 
 
 @decorators.yes_if_nothing_inferred
+@cython.binding(True)
 def tl_infer_binary_op(self, opnode, operator, other, context, method):
     not_implemented = nodes.Const(NotImplemented)
     if isinstance(other, self.__class__) and operator == "+":
@@ -204,6 +208,7 @@ def tl_infer_binary_op(self, opnode, operator, other, context, method):
 
 
 @decorators.yes_if_nothing_inferred
+@cython.binding(True)
 def instance_class_infer_binary_op(self, opnode, operator, other, context, method):
     return method.infer_call_result(self, context)
 
@@ -267,6 +272,7 @@ def _resolve_looppart(parts, assign_path, context):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def for_assigned_stmts(self, node=None, context=None, assign_path=None):
     if isinstance(self, nodes.AsyncFor) or getattr(self, "is_async", False):
         # Skip inferring of async code for now
@@ -284,6 +290,7 @@ def for_assigned_stmts(self, node=None, context=None, assign_path=None):
 nodes.Comprehension.assigned_stmts = for_assigned_stmts
 
 
+@cython.binding(True)
 def sequence_assigned_stmts(self, node=None, context=None, assign_path=None):
     if assign_path is None:
         assign_path = []
@@ -306,7 +313,7 @@ def sequence_assigned_stmts(self, node=None, context=None, assign_path=None):
 nodes.Tuple.assigned_stmts = sequence_assigned_stmts
 nodes.List.assigned_stmts = sequence_assigned_stmts
 
-
+@cython.binding(True)
 def assend_assigned_stmts(self, node=None, context=None, assign_path=None):
     return self.parent.assigned_stmts(node=self, context=context)
 
@@ -315,6 +322,7 @@ def assend_assigned_stmts(self, node=None, context=None, assign_path=None):
 nodes.AssignAttr.assigned_stmts = assend_assigned_stmts
 
 
+@cython.binding(True)
 def _arguments_infer_argname(self, name, context):
     # arguments information may be missing, in which case we can't do anything
     # more
@@ -360,6 +368,7 @@ def _arguments_infer_argname(self, name, context):
         yield util.Uninferable
 
 
+@cython.binding(True)
 def arguments_assigned_stmts(self, node=None, context=None, assign_path=None):
     if context.callcontext:
         # reset call context/name
@@ -375,6 +384,7 @@ def arguments_assigned_stmts(self, node=None, context=None, assign_path=None):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def assign_assigned_stmts(self, node=None, context=None, assign_path=None):
     if not assign_path:
         yield self.value
@@ -386,6 +396,7 @@ def assign_assigned_stmts(self, node=None, context=None, assign_path=None):
     return dict(node=self, unknown=node, assign_path=assign_path, context=context)
 
 
+@cython.binding(True)
 def assign_annassigned_stmts(self, node=None, context=None, assign_path=None):
     for inferred in assign_assigned_stmts(self, node, context, assign_path):
         if inferred is None:
@@ -440,6 +451,7 @@ def _resolve_assignment_parts(parts, assign_path, context):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def excepthandler_assigned_stmts(self, node=None, context=None, assign_path=None):
     for assigned in node_classes.unpack_infer(self.type):
         if isinstance(assigned, nodes.ClassDef):
@@ -452,6 +464,7 @@ def excepthandler_assigned_stmts(self, node=None, context=None, assign_path=None
 nodes.ExceptHandler.assigned_stmts = excepthandler_assigned_stmts
 
 
+@cython.binding(True)
 def _infer_context_manager(self, mgr, context):
     inferred = next(mgr.infer(context=context))
     if isinstance(inferred, bases.Generator):
@@ -500,6 +513,7 @@ def _infer_context_manager(self, mgr, context):
 
 
 @decorators.raise_if_nothing_inferred
+@cython.binding(True)
 def with_assigned_stmts(self, node=None, context=None, assign_path=None):
     """Infer names and other nodes from a *with* statement.
 
@@ -573,6 +587,7 @@ def __enter__(self):
 
 
 @decorators.yes_if_nothing_inferred
+@cython.binding(True)
 def starred_assigned_stmts(self, node=None, context=None, assign_path=None):
     """
     Arguments:

astroid/raw_building.py → astroid/raw_building.pyx

@@ -22,6 +22,7 @@
 import os
 import sys
 import types
+cimport cython
 
 from astroid import bases
 from astroid import manager
@@ -74,7 +75,7 @@ def attach_dummy_node(node, name, runtime_object=_marker):
     enode.object = runtime_object
     _attach_local_node(node, enode, name)
 
-
+@cython.binding(True)
 def _has_underlying_object(self):
     return self.object is not None and self.object is not _marker
 
@@ -377,8 +378,8 @@ def imported_member(self, node, member, name):
 _CONST_PROXY = {}
 
 # TODO : find a nicer way to handle this situation;
-def _set_proxied(const):
-    return _CONST_PROXY[const.value.__class__]
+def _set_proxied(_const):
+    return _CONST_PROXY[_const.value.__class__]
 
 
 def _astroid_bootstrapping():
@@ -440,5 +441,4 @@ def _astroid_bootstrapping():
             builder.object_build(cls, _type)
             astroid_builtin[_type.__name__] = cls
 
-
 _astroid_bootstrapping()

setup.py

@@ -15,6 +15,7 @@
 from setuptools import setup, find_packages
 from setuptools.command import easy_install
 from setuptools.command import install_lib
+from Cython.Build import cythonize
 
 
 real_path = os.path.realpath(__file__)
@@ -45,6 +46,7 @@ def install():
                  setup_requires=['pytest-runner'],
                  test_suite='test',
                  tests_require=['pytest'],
+                 ext_modules=cythonize(["astroid/*.pyx", "astroid/interpreter/*.pyx", "astroid/interpreter/_import/*.pyx", "astroid/brain/*.pyx"], compiler_directives={'language_level': '3'})
                  )