Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CodeStyle][Ruff][BUAA][F-[28-35]] Fix ruff FURB diagnostic for 8 files in tools/ #67277

Merged
merged 3 commits into from
Aug 10, 2024
Merged

[CodeStyle][Ruff][BUAA][F-[28-35]] Fix ruff FURB diagnostic for 8 files in tools/ #67277

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4c6d9e5
update 8 files in PURB
Whsjrczr Aug 9, 2024
1591461
Revert a change
SigureMo Aug 9, 2024
3d1ff1a
use listcomp to avoid lazy access file when closed
SigureMo Aug 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tools/diff_use_default_grad_op_maker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def generate_spec(filename):

def read_spec(filename):
with open(filename, 'r') as f:
return {line.strip() for line in f.readlines()}
return {line.strip() for line in f}


def get_spec_diff(dev_filename, pr_filename):
Expand Down
96 changes: 58 additions & 38 deletions tools/externalError/spider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,19 @@ def parsing(externalErrorDesc):
ssl._create_default_https_context = ssl._create_unverified_context
html = urllib.request.urlopen(url).read().decode('utf-8')
res_div = r'<div class="section">.*?<p>CUDA error types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
m_div = re.findall(res_div, html, re.S | re.M)[0]
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]

res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
m_dt = re.findall(res_dt, m_div, re.S | re.M)
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
for error in m_dt:
res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
m_type = re.findall(res_type, error[0], re.S | re.M)[0]
m_type = re.findall(res_type, error[0], re.DOTALL | re.MULTILINE)[0]
m_message = error[1]
m_message = m_message.replace('\n', '')
res_a = r'(<a class=.*?</a>)'
res_shape = r'<a class=.*?>(.*?)</a>'
list_a = re.findall(res_a, m_message, re.S | re.M)
list_shape = re.findall(res_shape, m_message, re.S | re.M)
list_a = re.findall(res_a, m_message, re.DOTALL | re.MULTILINE)
list_shape = re.findall(res_shape, m_message, re.DOTALL | re.MULTILINE)
assert len(list_a) == len(list_shape)
for idx in range(len(list_a)):
m_message = m_message.replace(list_a[idx], list_shape[idx])
Expand All @@ -57,16 +57,20 @@ def parsing(externalErrorDesc):

res_span = r'(<span class=.*?</span>)'
res_span_detail = r'<span class=.*?>(.*?)</span>'
list_span = re.findall(res_span, m_message, re.S | re.M)
list_span_detail = re.findall(res_span_detail, m_message, re.S | re.M)
list_span = re.findall(res_span, m_message, re.DOTALL | re.MULTILINE)
list_span_detail = re.findall(
res_span_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_span) == len(list_span_detail)
for idx in range(len(list_span)):
m_message = m_message.replace(list_span[idx], list_span_detail[idx])

res_p = r'(<p>.*?</p>)'
res_p_detail = r'<p>(.*?)</p>'
list_p = re.findall(res_p, m_message, re.S | re.M)
list_p_detail = re.findall(res_p_detail, m_message, re.S | re.M)
list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
list_p_detail = re.findall(
res_p_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_p) == len(list_p_detail)
for idx in range(len(list_p)):
m_message = m_message.replace(list_p[idx], list_p_detail[idx])
Expand Down Expand Up @@ -94,13 +98,13 @@ def parsing(externalErrorDesc):
html = urllib.request.urlopen(url).read().decode('utf-8')

res_div = r'<div class="section">.*?<p>CURAND function call status types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
m_div = re.findall(res_div, html, re.S | re.M)[0]
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]

res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
m_dt = re.findall(res_dt, m_div, re.S | re.M)
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
for error in m_dt:
res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
m_type = re.findall(res_type, error[0], re.S | re.M)[0]
m_type = re.findall(res_type, error[0], re.DOTALL | re.MULTILINE)[0]
m_message = error[1]

_Messages = allMessageDesc.messages.add()
Expand Down Expand Up @@ -144,17 +148,19 @@ def parsing(externalErrorDesc):
f.write(html)

res_div = r'<div class="section" id="cudnnStatus_t__section_lmp_dgr_2jb"><a name="cudnnStatus_t__section_lmp_dgr_2jb" shape="rect">(.*?)</div>'
m_div = re.findall(res_div, html, re.S | re.M)[0]
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]

res_dt = r'<dt class="dt dlterm"><samp class="ph codeph">(.*?)</samp></dt>.*?<dd class="dd">(.*?)</dd>'
m_dt = re.findall(res_dt, m_div, re.S | re.M)
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
for error in m_dt:
m_message = error[1]

res_class = r'<p class="p">.*?</p>'
res_class_detail = r'<p class="p">(.*?)</p>'
list_class = re.findall(res_class, m_message, re.S | re.M)
list_class_detail = re.findall(res_class_detail, m_message, re.S | re.M)
list_class = re.findall(res_class, m_message, re.DOTALL | re.MULTILINE)
list_class_detail = re.findall(
res_class_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_class) == len(list_class_detail)
for idx in range(len(list_class)):
m_message = m_message.replace(
Expand All @@ -163,24 +169,28 @@ def parsing(externalErrorDesc):

res_a = r'(<a class="xref".*?</a>)'
res_shape = r'<a class="xref".*?>(.*?)</a>'
list_a = re.findall(res_a, m_message, re.S | re.M)
list_shape = re.findall(res_shape, m_message, re.S | re.M)
list_a = re.findall(res_a, m_message, re.DOTALL | re.MULTILINE)
list_shape = re.findall(res_shape, m_message, re.DOTALL | re.MULTILINE)
assert len(list_a) == len(list_shape)
for idx in range(len(list_a)):
m_message = m_message.replace(list_a[idx], list_shape[idx])

res_span = r'(<span class="ph">.*?</span>)'
res_span_detail = r'<span class="ph">(.*?)</span>'
list_span = re.findall(res_span, m_message, re.S | re.M)
list_span_detail = re.findall(res_span_detail, m_message, re.S | re.M)
list_span = re.findall(res_span, m_message, re.DOTALL | re.MULTILINE)
list_span_detail = re.findall(
res_span_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_span) == len(list_span_detail)
for idx in range(len(list_span)):
m_message = m_message.replace(list_span[idx], list_span_detail[idx])

res_samp = r'(<samp class="ph codeph">.*?</samp>)'
res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
list_samp = re.findall(res_samp, m_message, re.S | re.M)
list_samp_detail = re.findall(res_samp_detail, m_message, re.S | re.M)
list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
list_samp_detail = re.findall(
res_samp_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_samp) == len(list_samp_detail)
for idx in range(len(list_samp)):
m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
Expand Down Expand Up @@ -216,27 +226,31 @@ def parsing(externalErrorDesc):
html = urllib.request.urlopen(url).read().decode('utf-8')

res_div = r'<p class="p">The type is used for function status returns. All cuBLAS library.*?<div class="tablenoborder">(.*?)</div>'
m_div = re.findall(res_div, html, re.S | re.M)[0]
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]

res_dt = r'<p class="p"><samp class="ph codeph">(.*?)</samp></p>.*?colspan="1">(.*?)</td>'
m_dt = re.findall(res_dt, m_div, re.S | re.M)
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)

for error in m_dt:
m_message = error[1]
m_message = re.sub(r'\n +', ' ', m_message)

res_p = r'<p class="p">.*?</p>'
res_p_detail = r'<p class="p">(.*?)</p>'
list_p = re.findall(res_p, m_message, re.S | re.M)
list_p_detail = re.findall(res_p_detail, m_message, re.S | re.M)
list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
list_p_detail = re.findall(
res_p_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_p) == len(list_p_detail)
for idx in range(len(list_p)):
m_message = m_message.replace(list_p[idx], list_p_detail[idx])

res_samp = r'<samp class="ph codeph">.*?</samp>'
res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
list_samp = re.findall(res_samp, m_message, re.S | re.M)
list_samp_detail = re.findall(res_samp_detail, m_message, re.S | re.M)
list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
list_samp_detail = re.findall(
res_samp_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_samp) == len(list_samp_detail)
for idx in range(len(list_samp)):
m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
Expand Down Expand Up @@ -278,12 +292,12 @@ def parsing(externalErrorDesc):
html = urllib.request.urlopen(url).read().decode('utf-8')

res_div = r'This is a status type returned by the library functions and.*?<div class="tablenoborder">(.*?)</div>'
m_div = re.findall(res_div, html, re.S | re.M)[0]
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]

res_dt = (
r'<samp class="ph codeph">(.*?)</samp></td>.*?colspan="1">(.*?)</td>'
)
m_dt = re.findall(res_dt, m_div, re.S | re.M)
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)

for error in m_dt:
m_message = error[1]
Expand All @@ -292,25 +306,31 @@ def parsing(externalErrorDesc):

res_p = r'<p class="p">.*?</p>'
res_p_detail = r'<p class="p">(.*?)</p>'
list_p = re.findall(res_p, m_message, re.S | re.M)
list_p_detail = re.findall(res_p_detail, m_message, re.S | re.M)
list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
list_p_detail = re.findall(
res_p_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_p) == len(list_p_detail)
for idx in range(len(list_p)):
m_message = m_message.replace(list_p[idx], list_p_detail[idx])

res_samp = r'<samp class="ph codeph">.*?</samp>'
res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
list_samp = re.findall(res_samp, m_message, re.S | re.M)
list_samp_detail = re.findall(res_samp_detail, m_message, re.S | re.M)
list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
list_samp_detail = re.findall(
res_samp_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_samp) == len(list_samp_detail)
for idx in range(len(list_samp)):
m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])

res_strong = r'<strong class="ph b">.*?</strong>'
res_strong_detail = r'<strong class="ph b">(.*?)</strong>'
list_strong = re.findall(res_strong, m_message, re.S | re.M)
list_strong = re.findall(
res_strong, m_message, re.DOTALL | re.MULTILINE
)
list_strong_detail = re.findall(
res_strong_detail, m_message, re.S | re.M
res_strong_detail, m_message, re.DOTALL | re.MULTILINE
)
assert len(list_strong) == len(list_strong_detail)
for idx in range(len(list_strong)):
Expand All @@ -331,10 +351,10 @@ def parsing(externalErrorDesc):
allMessageDesc.type = external_error_pb2.NCCL
html = urllib.request.urlopen(url).read().decode('utf-8')
res_div = r'<code class="descname">ncclResult_t</code>(.*?)</div>'
m_div = re.findall(res_div, html, re.S | re.M)[0]
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]

res_dt = r'<code class="descname">(.*?)</code>.*?<span class="pre">(.*?)</span></code>\)(.*?)</p>\n</dd></dl>'
m_dt = re.findall(res_dt, m_div, re.S | re.M)
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
for error in m_dt:
m_message = re.sub(r'\n', '', error[2])
_Messages = allMessageDesc.messages.add()
Expand Down
2 changes: 1 addition & 1 deletion tools/gen_ut_cmakelists.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ def parse_assigned_dist_ut_ports(self, current_work_dir, depth=0):
)
) as csv_file:
found = False
for line in csv_file.readlines():
for line in csv_file:
(
name,
_,
Expand Down
2 changes: 1 addition & 1 deletion tools/get_single_test_cov.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def analysisFNDAFile(rootPath, test):
matchObj = re.match(
r'(.*)Maker(.*)|(.*)Touch(.*)Regist(.*)|(.*)Touch(.*)JitKernel(.*)|(.*)converterC2Ev(.*)',
fn,
re.I,
re.IGNORECASE,
)
if matchObj is None:
OP_REGIST = False
Expand Down
6 changes: 3 additions & 3 deletions tools/get_ut_file_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def handle_ut_file_map(rootPath):
all_ut = f'{rootPath}/build/all_uts_paddle'
with open(all_ut, 'r') as f:
all_ut_list = []
for ut in f.readlines():
for ut in f:
ut = ut.replace('\n', '')
all_ut_list.append(ut.strip())
f.close()
Expand Down Expand Up @@ -190,7 +190,7 @@ def ut_file_map_supplement(rootPath):

with open(all_uts_paddle, 'r') as f:
all_uts_paddle_list = []
for ut in f.readlines():
for ut in f:
all_uts_paddle_list.append(ut.strip())
f.close()

Expand All @@ -204,7 +204,7 @@ def ut_file_map_supplement(rootPath):
prec_delta_new = f"{rootPath}/build/prec_delta"
with open(prec_delta_new, 'r') as f:
prec_delta_new_list = []
for ut in f.readlines():
for ut in f:
prec_delta_new_list.append(ut.strip())
f.close()
prec_delta_new_list.append(
Expand Down
6 changes: 3 additions & 3 deletions tools/parse_kernel_info.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ def parse_paddle_kernels(lib="phi", kernel_type="function", print_detail=False):
print(
f"{value.op_type.ljust(max_op_type_lengths + 4)} : {value.supported_dtypes}"
)
print("")
print()
return stats


Expand All @@ -155,7 +155,7 @@ def main(lib):
print(f"phi function kernels : {phi_function_kernels_stats}")
print(f"phi structure kernels : {phi_structure_kernels_stats}")
print(f"phi all kernels : {phi_all_kernels_stats}")
print("")
print()
else:
fluid_ops_stats = parse_paddle_kernels(lib, "fluid", print_detail=False)
phi_ops_stats = parse_paddle_kernels(lib, "phi", print_detail=False)
Expand All @@ -168,7 +168,7 @@ def main(lib):
print(f"fluid operators : {fluid_ops_stats}")
print(f"phi operators : {phi_ops_stats}")
print(f"all operators : {all_ops_stats}")
print("")
print()


main(lib="fluid")
Expand Down
16 changes: 8 additions & 8 deletions tools/sampcd_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def _patch_tensor_place():
(.*?) # Place=(XXX)
(\,.*?\))
""",
re.X | re.S,
re.VERBOSE | re.DOTALL,
)

_check_output = checker.check_output
Expand Down Expand Up @@ -128,7 +128,7 @@ def _patch_float_precision(digits):
)?
)
""",
re.X | re.S,
re.VERBOSE | re.DOTALL,
)

_check_output = checker.check_output
Expand Down Expand Up @@ -201,7 +201,7 @@ class TimeoutDirective(Directive):
)
)
""",
re.X | re.S,
re.VERBOSE | re.DOTALL,
)

def __init__(self, timeout):
Expand Down Expand Up @@ -239,7 +239,7 @@ class SingleProcessDirective(Directive):
\s
)
""",
re.X | re.S,
re.VERBOSE | re.DOTALL,
)

def parse_directive(self, docstring):
Expand Down Expand Up @@ -276,7 +276,7 @@ class Fluid(BadStatement):
.*
(\bfluid\b)
""",
re.X,
re.VERBOSE,
)

def check(self, docstring):
Expand All @@ -300,7 +300,7 @@ class SkipNoReason(BadStatement):
[+]SKIP
(?P<reason>.*)
""",
re.X,
re.VERBOSE,
)

def check(self, docstring):
Expand All @@ -324,7 +324,7 @@ class DeprecatedRequired(BadStatement):
(?P<directive>require[sd]?\s*:)
(?P<env>.+)
""",
re.X,
re.VERBOSE,
)

def check(self, docstring):
Expand Down Expand Up @@ -398,7 +398,7 @@ def __init__(
(doctest) # directive prefix, which should be replaced
(?=(:\s*.*\n)) # positive lookahead, directive content
""",
re.X,
re.VERBOSE,
)

self.directive_prefix = 'xdoctest'
Expand Down
Loading