Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix col_offset attribute for nodes involving with on PyPy #1520

Merged
merged 3 commits into from
Apr 22, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ChangeLog
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ What's New in astroid 2.11.4?
=============================
Release date: TBA

* Fix ``col_offset`` attribute for nodes involving ``with`` on ``PyPy``.


What's New in astroid 2.11.3?
Expand Down
23 changes: 17 additions & 6 deletions astroid/rebuilder.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
order to get a single Astroid representation
"""

import ast
import sys
import token
import tokenize
from io import StringIO
from tokenize import TokenInfo, generate_tokens
from typing import (
TYPE_CHECKING,
Callable,
Dict,
Generator,
Expand All @@ -39,9 +39,6 @@
else:
from typing_extensions import Final

if TYPE_CHECKING:
import ast


REDIRECT: Final[Dict[str, str]] = {
"arguments": "Arguments",
Expand Down Expand Up @@ -1185,9 +1182,14 @@ def _visit_for(
self, cls: Type[T_For], node: Union["ast.For", "ast.AsyncFor"], parent: NodeNG
) -> T_For:
"""visit a For node by returning a fresh instance of it"""
col_offset = node.col_offset
if IS_PYPY and not PY39_PLUS and isinstance(node, ast.AsyncFor) and self._data:
# pylint: disable-next=unsubscriptable-object
col_offset = self._data[node.lineno - 1].index("async")

newnode = cls(
lineno=node.lineno,
col_offset=node.col_offset,
col_offset=col_offset,
# end_lineno and end_col_offset added in 3.8
end_lineno=getattr(node, "end_lineno", None),
end_col_offset=getattr(node, "end_col_offset", None),
Expand Down Expand Up @@ -1292,6 +1294,10 @@ def _visit_functiondef(
position=self._get_position_info(node, newnode),
doc_node=self.visit(doc_ast_node, newnode),
)
if IS_PYPY and PY36 and newnode.position:
# PyPy: col_offset in Python 3.6 doesn't include 'async',
# use position.col_offset instead.
newnode.col_offset = newnode.position.col_offset
self._fix_doc_node_position(newnode)
self._global_names.pop()
return newnode
Expand Down Expand Up @@ -1903,9 +1909,14 @@ def _visit_with(
node: Union["ast.With", "ast.AsyncWith"],
parent: NodeNG,
) -> T_With:
col_offset = node.col_offset
if IS_PYPY and not PY39_PLUS and isinstance(node, ast.AsyncWith) and self._data:
# pylint: disable-next=unsubscriptable-object
col_offset = self._data[node.lineno - 1].index("async")

newnode = cls(
lineno=node.lineno,
col_offset=node.col_offset,
col_offset=col_offset,
# end_lineno and end_col_offset added in 3.8
end_lineno=getattr(node, "end_lineno", None),
end_col_offset=getattr(node, "end_col_offset", None),
Expand Down
44 changes: 38 additions & 6 deletions tests/unittest_nodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -1074,20 +1074,52 @@ def test(a): return a

class Python35AsyncTest(unittest.TestCase):
def test_async_await_keywords(self) -> None:
async_def, async_for, async_with, await_node = builder.extract_node(
(
async_def,
async_for,
async_with,
async_for2,
async_with2,
await_node,
) = builder.extract_node(
"""
async def func(): #@
async for i in range(10): #@
f = __(await i)
async with test(): #@
pass
async for i \
in range(10): #@
pass
async with test(), \
test2(): #@
pass
"""
)
self.assertIsInstance(async_def, nodes.AsyncFunctionDef)
self.assertIsInstance(async_for, nodes.AsyncFor)
self.assertIsInstance(async_with, nodes.AsyncWith)
self.assertIsInstance(await_node, nodes.Await)
self.assertIsInstance(await_node.value, nodes.Name)
assert isinstance(async_def, nodes.AsyncFunctionDef)
assert async_def.lineno == 2
assert async_def.col_offset == 0

assert isinstance(async_for, nodes.AsyncFor)
assert async_for.lineno == 3
assert async_for.col_offset == 4

assert isinstance(async_with, nodes.AsyncWith)
assert async_with.lineno == 5
assert async_with.col_offset == 4

assert isinstance(async_for2, nodes.AsyncFor)
assert async_for2.lineno == 7
assert async_for2.col_offset == 4

assert isinstance(async_with2, nodes.AsyncWith)
assert async_with2.lineno == 9
assert async_with2.col_offset == 4

assert isinstance(await_node, nodes.Await)
assert isinstance(await_node.value, nodes.Name)
assert await_node.lineno == 4
assert await_node.col_offset == 15

def _test_await_async_as_string(self, code: str) -> None:
ast_node = parse(code)
Expand Down