Skip to content

Commit

Permalink
Directly getting API.spec for tools/sampcd_processor.py (PaddlePaddle…
Browse files Browse the repository at this point in the history 
…#22728)
  • Loading branch information
@lelelelelez
lelelelelez authored Feb 25, 2020
1 parent 1c06534 commit 882e7f7
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 48 deletions.
2 changes: 1 addition & 1 deletion paddle/scripts/paddle_build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1138,9 +1138,9 @@ function main() {
generate_upstream_develop_api_spec ${PYTHON_ABI:-""} ${parallel_number}
cmake_gen ${PYTHON_ABI:-""}
build ${parallel_number}
example
check_sequence_op_unittest
generate_api_spec ${PYTHON_ABI:-""} "PR"
example
assert_api_spec_approvals
;;
build)
Expand Down
186 changes: 139 additions & 47 deletions tools/sampcd_processor.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -18,6 +18,9 @@
import multiprocessing
import math
import platform
import inspect
import paddle
import paddle.fluid
"""
please make sure to run in the tools path
usage: python sample_test.py {arg1}
Expand Down Expand Up @@ -197,7 +200,8 @@ def sampcd_header_print(name, sampcd, htype, hname):
print(msg)
result = False
# msg is the returned code execution report
os.remove("samplecode_temp/" + tfname)
#os.remove("samplecode_temp/" + tfname)

return result


Expand Down Expand Up @@ -271,20 +275,24 @@ def srccoms_extract(srcfile, wlist):
wlist(list): white list
Returns:
result: True or False
result: True or False
"""

process_result = True
srcc = srcfile.read()

# 2. get defs and classes header line number
# set file pointer to its beginning
srcfile.seek(0, 0)
srcls = srcfile.readlines() # source lines

# 1. fetch__all__ list
allidx = srcc.find("__all__")

srcfile_new = srcfile.name
srcfile_new = srcfile_new.replace('.py', '')
srcfile_list = srcfile_new.split('/')
srcfile_str = ''
for i in range(4, len(srcfile_list)):
srcfile_str = srcfile_str + srcfile_list[i] + '.'
if allidx != -1:
alllist = []
# get all list for layers/ops.py
Expand Down Expand Up @@ -312,7 +320,6 @@ def srccoms_extract(srcfile, wlist):
api_alllist_count = len(alllist)
api_count = 0
handled = []

# get src contents in layers/ops.py
if srcfile.name.find("ops.py") != -1:
for i in range(0, len(srcls)):
Expand All @@ -329,19 +336,18 @@ def srccoms_extract(srcfile, wlist):
opcom += srcls[j]
if srcls[j].find("\"\"\"") != -1:
break
process_result = sampcd_extract_and_run(opcom, opname,
"def", opname)
api_count += 1
handled.append(
opname) # ops.py also has normal formatted functions
# use list 'handled' to mark the functions have been handled here
# which will be ignored in the following step

for i in range(0, len(srcls)):
if srcls[i].startswith(
'def '): # a function header is detected in line i
f_header = srcls[i].replace(" ", '')
fn = f_header[len('def'):f_header.find('(')] # function name
if "%s%s" % (srcfile_str, fn) not in methods:
continue
if fn in handled:
continue
if fn in alllist:
Expand All @@ -357,9 +363,12 @@ def srccoms_extract(srcfile, wlist):
else:
if not sampcd_extract_and_run(fcombody, fn, "def", fn):
process_result = False

if srcls[i].startswith('class '):
c_header = srcls[i].replace(" ", '')
cn = c_header[len('class'):c_header.find('(')] # class name
if '%s%s' % (srcfile_str, cn) not in methods:
continue
if cn in handled:
continue
if cn in alllist:
Expand All @@ -371,6 +380,7 @@ def srccoms_extract(srcfile, wlist):
if classcom != "":
if not sampcd_extract_and_run(classcom, cn, "class",
cn):

process_result = False
else:
print("WARNING: no comments in class itself ", cn,
Expand All @@ -392,6 +402,10 @@ def srccoms_extract(srcfile, wlist):
mn = thisl[indent + len('def '):thisl.find(
'(')] # method name
name = cn + "." + mn # full name
if '%s%s' % (
srcfile_str, name
) not in methods: # class method not in api.spec
continue
if mn.startswith('_'):
continue
if name in wlist or name + "@" + srcfile.name in wlist:
Expand All @@ -418,6 +432,7 @@ def srccoms_extract(srcfile, wlist):
if not sampcd_extract_and_run(
thismtdcom, name, "method", name):
process_result = False

return process_result


Expand All @@ -430,50 +445,71 @@ def test(file_list):
return process_result


def get_filenames(path):
'''
Given a path ``path``, this function will
get the modules that pending for check.
Args:
path(path): the path of API.spec
Returns:
list: the modules pending for check .
'''
filenames = []
global methods
methods = []
API_spec = '%s/%s' % (os.path.abspath(os.path.join(os.getcwd(), "..")),
path)
with open(API_spec) as f:
for line in f.readlines():
api = line.split(' ', 1)[0]
try:
module = eval(api).__module__
except AttributeError:
continue
if len(module.split('.')) > 2:
filename = '../python/'
module_py = '%s.py' % module.split('.')[-1]
for i in range(0, len(module.split('.')) - 1):
filename = filename + '%s/' % module.split('.')[i]
filename = filename + module_py
else:
print("\n----Exception in get api filename----\n")
print("\n" + api + 'module is ' + module + "\n")
if filename not in filenames:
filenames.append(filename)
# get all methods
method = ''
if inspect.isclass(eval(api)):
name = api.split('.')[-1]
elif inspect.isfunction(eval(api)):
name = api.split('.')[-1]
elif inspect.ismethod(eval(api)):
name = '%s.%s' % (api.split('.')[-2], api.split('.')[-1])
else:
name = ''
print("\n----Exception in get api methods----\n")
print("\n" + line + "\n")
print("\n" + api + ' method is None!!!' + "\n")
for j in range(2, len(module.split('.'))):
method = method + '%s.' % module.split('.')[j]
method = method + name
if method not in methods:
methods.append(method)
return filenames


'''
Important constant lists:
filenames : the modules pending for check .
wlist : a list of API that should not trigger the example check .
It is composed of wlist_temp + wlist_inneed + wlist_ignore.
srcfile: the source .py code file
'''

filenames = [
"../python/paddle/fluid/layers/control_flow.py",
"../python/paddle/fluid/layers/io.py",
"../python/paddle/fluid/layers/nn.py",
"../python/paddle/fluid/layers/ops.py",
"../python/paddle/fluid/layers/tensor.py",
"../python/paddle/fluid/layers/learning_rate_scheduler.py",
"../python/paddle/fluid/layers/detection.py",
"../python/paddle/fluid/layers/metric_op.py"
]
filenames += [
"../python/paddle/fluid/dygraph/layers.py",
"../python/paddle/fluid/dygraph/base.py",
"../python/paddle/fluid/dygraph/nn.py",
"../python/paddle/fluid/dygraph/tracer.py",
"../python/paddle/fluid/dygraph/profiler.py",
"../python/paddle/fluid/dygraph/parallel.py",
"../python/paddle/fluid/dygraph/checkpoint.py",
"../python/paddle/fluid/dygraph/learning_rate_scheduler.py",
"../python/paddle/fluid/dygraph/backward_strategy.py"
]
filenames += [
"../python/paddle/fluid/data_feeder.py",
"../python/paddle/fluid/dataset.py", "../python/paddle/fluid/clip.py",
"../python/paddle/fluid/metrics.py", "../python/paddle/fluid/executor.py",
"../python/paddle/fluid/initializer.py", "../python/paddle/fluid/io.py",
"../python/paddle/fluid/nets.py", "../python/paddle/fluid/optimizer.py",
"../python/paddle/fluid/profiler.py",
"../python/paddle/fluid/regularizer.py",
"../python/paddle/fluid/backward.py", "../python/paddle/fluid/average.py",
"../python/paddle/fluid/unique_name.py",
"../python/paddle/fluid/framework.py",
"../python/paddle/fluid/evaluator.py",
"../python/paddle/fluid/param_attr.py"
]
wlist_inneed = [
"append_LARS", "BuildStrategy.debug_graphviz_path",
"BuildStrategy.enable_sequential_execution",
Expand Down Expand Up @@ -505,6 +541,58 @@ def test(file_list):
'StaticRNN.output', "cuda_places", "CUDAPinnedPlace", "CUDAPlace",
"Program.parse_from_string"
]

wlist_nosample = [
'Compressor', 'Compressor.config', 'Compressor.run', 'run_check',
'HDFSClient.upload', 'HDFSClient.download', 'HDFSClient.is_exist',
'HDFSClient.is_dir', 'HDFSClient.delete', 'HDFSClient.rename',
'HDFSClient.makedirs', 'HDFSClient.ls', 'HDFSClient.lsr', 'multi_download',
'multi_upload', 'TrainingDecoder.block',
'QuantizeTranspiler.training_transpile',
'QuantizeTranspiler.freeze_program', 'AutoMixedPrecisionLists',
'Uniform.sample', 'Uniform.log_prob', 'Uniform.entropy',
'Categorical.kl_divergence', 'Categorical.entropy',
'MultivariateNormalDiag.entropy', 'MultivariateNormalDiag.kl_divergence',
'RNNCell', 'RNNCell.call', 'RNNCell.get_initial_states', 'GRUCell.call',
'LSTMCell.call', 'Decoder', 'Decoder.initialize', 'Decoder.step',
'Decoder.finalize', 'fused_elemwise_activation', 'search_pyramid_hash',
'convert_dist_to_sparse_program', 'load_persistables_for_increment',
'load_persistables_for_inference', 'cache', 'buffered', 'xmap_readers'
]

wlist_no_op_pass = ['gelu', 'erf']

wlist_ci_nopass = [
'DecodeHelper', 'DecodeHelper.initialize', 'DecodeHelper.sample',
'DecodeHelper.next_inputs', 'TrainingHelper.initialize',
'TrainingHelper.sample', 'TrainingHelper.next_inputs',
'GreedyEmbeddingHelper.initialize', 'GreedyEmbeddingHelper.sample',
'GreedyEmbeddingHelper.next_inputs', 'LayerList.append', 'HDFSClient',
'InitState', 'TracedLayer', 'SampleEmbeddingHelper.sample',
'BasicDecoder.initialize', 'BasicDecoder.step', 'ParameterList.append',
'GreedyEmbeddingHelper', 'SampleEmbeddingHelper', 'BasicDecoder', 'lstm',
'partial_sum'
]

wlist_nopass = [
'StateCell', 'StateCell.compute_state', 'TrainingDecoder',
'TrainingDecoder.step_input', 'TrainingDecoder.static_input',
'TrainingDecoder.output', 'BeamSearchDecoder', 'GradClipByValue',
'GradClipByNorm', 'Variable.detach', 'Variable.numpy', 'Variable.set_value',
'Variable.gradient', 'BeamSearchDecoder.decode',
'BeamSearchDecoder.read_array', 'CompiledProgram',
'CompiledProgram.with_data_parallel', 'append_backward', 'guard',
'to_variable', 'op_freq_statistic', 'save_dygraph', 'load_dygraph',
'ParallelExecutor', 'ParallelExecutor.run',
'ParallelExecutor.drop_local_exe_scopes', 'GradClipByGlobalNorm',
'extend_with_decoupled_weight_decay', 'switch', 'Normal', 'memory_usage',
'decorate', 'PiecewiseDecay', 'InverseTimeDecay', 'PolynomialDecay',
'NoamDecay', 'start_profiler', 'profiler', 'tree_conv', 'multiclass_nms2',
'DataFeedDesc', 'Conv2D', 'Conv3D', 'Conv3DTranspose', 'Embedding', 'NCE',
'PRelu', 'BilinearTensorProduct', 'GroupNorm', 'SpectralNorm', 'TreeConv',
'prroi_pool'
]

wlist_temp = [
'ChunkEvaluator',
'EditDistance',
Expand Down Expand Up @@ -609,7 +697,8 @@ def test(file_list):
"deformable_conv", "cuda_places", "CUDAPinnedPlace", "CUDAPlace",
"cuda_profiler", 'DGCMomentumOptimizer'
]
wlist = wlist_temp + wlist_inneed + wlist_ignore

wlist = wlist_temp + wlist_inneed + wlist_ignore + wlist_nosample + wlist_nopass + wlist_no_op_pass + wlist_ci_nopass

if len(sys.argv) < 2:
print("Error: inadequate number of arguments")
Expand All @@ -630,17 +719,20 @@ def test(file_list):
print("sample_test running under python", platform.python_version())
if not os.path.isdir("./samplecode_temp"):
os.mkdir("./samplecode_temp")

cpus = multiprocessing.cpu_count()
filenames = get_filenames('paddle/fluid/API_PR.spec')
filenames.remove('../python/paddle/fluid/core_avx.py')
one_part_filenum = int(math.ceil(len(filenames) / cpus))
divided_file_list = [
filenames[i:i + one_part_filenum]
for i in range(0, len(filenames), one_part_filenum)
]

po = multiprocessing.Pool()
results = po.map_async(test, divided_file_list)
po.close()
po.join()

result = results.get()

# delete temp files
Expand Down

0 comments on commit 882e7f7

Please sign in to comment.