From b3a409fe8d937c6beb0647758f51e5181a1bc12f Mon Sep 17 00:00:00 2001 From: kaiyaointel Date: Mon, 11 Jul 2022 16:22:35 +0800 Subject: [PATCH] add neural_coder for neural_coder INC integration in INC 1.13 rls (#1052) --- neural_coder/README.md | 78 ++ neural_coder/__init__.py | 17 + neural_coder/coders/__init__.py | 15 + neural_coder/coders/pytorch/__init__.py | 15 + neural_coder/coders/pytorch/amp.py | 136 +++ neural_coder/coders/pytorch/batch_size.py | 87 ++ neural_coder/coders/pytorch/benchmark.py | 219 ++++ neural_coder/coders/pytorch/channels_last.py | 124 +++ neural_coder/coders/pytorch/cuda_to_cpu.py | 59 ++ .../coders/pytorch/dummy_dataloader.py | 139 +++ .../intel_extension_for_pytorch/__init__.py | 15 + .../intel_extension_for_pytorch/ipex.py | 203 ++++ neural_coder/coders/pytorch/jit.py | 272 +++++ neural_coder/coders/pytorch/lightning.py | 86 ++ .../pytorch/neural_compressor/__init__.py | 15 + .../neural_compressor/dynamic_quant.py | 120 +++ .../pytorch/neural_compressor/static_quant.py | 161 +++ neural_coder/coders/pytorch/torchdynamo.py | 338 +++++++ neural_coder/coders/tensorflow/__init__.py | 14 + neural_coder/coders/tensorflow/amp.py | 64 ++ neural_coder/coders/transform.py | 85 ++ neural_coder/examples/nlp/distilbert.py | 45 + neural_coder/examples/vision/alexnet.py | 23 + neural_coder/examples/vision/resnet18.py | 23 + neural_coder/examples/vision/resnet50.py | 23 + neural_coder/globals.py | 83 ++ neural_coder/graphers/__init__.py | 14 + neural_coder/graphers/code_line.py | 271 +++++ neural_coder/graphers/function.py | 189 ++++ neural_coder/graphers/model.py | 250 +++++ neural_coder/interface.py | 947 ++++++++++++++++++ neural_coder/numa_launcher.py | 807 +++++++++++++++ neural_coder/utils/__init__.py | 15 + neural_coder/utils/common.py | 20 + neural_coder/utils/cpu_info.py | 41 + neural_coder/utils/handle_user_input.py | 140 +++ neural_coder/utils/line_operation.py | 111 ++ 37 files changed, 5264 insertions(+) create mode 100644 neural_coder/README.md create mode 100644 neural_coder/__init__.py create mode 100644 neural_coder/coders/__init__.py create mode 100644 neural_coder/coders/pytorch/__init__.py create mode 100644 neural_coder/coders/pytorch/amp.py create mode 100644 neural_coder/coders/pytorch/batch_size.py create mode 100644 neural_coder/coders/pytorch/benchmark.py create mode 100644 neural_coder/coders/pytorch/channels_last.py create mode 100644 neural_coder/coders/pytorch/cuda_to_cpu.py create mode 100644 neural_coder/coders/pytorch/dummy_dataloader.py create mode 100644 neural_coder/coders/pytorch/intel_extension_for_pytorch/__init__.py create mode 100644 neural_coder/coders/pytorch/intel_extension_for_pytorch/ipex.py create mode 100644 neural_coder/coders/pytorch/jit.py create mode 100644 neural_coder/coders/pytorch/lightning.py create mode 100644 neural_coder/coders/pytorch/neural_compressor/__init__.py create mode 100644 neural_coder/coders/pytorch/neural_compressor/dynamic_quant.py create mode 100644 neural_coder/coders/pytorch/neural_compressor/static_quant.py create mode 100644 neural_coder/coders/pytorch/torchdynamo.py create mode 100644 neural_coder/coders/tensorflow/__init__.py create mode 100644 neural_coder/coders/tensorflow/amp.py create mode 100644 neural_coder/coders/transform.py create mode 100644 neural_coder/examples/nlp/distilbert.py create mode 100644 neural_coder/examples/vision/alexnet.py create mode 100644 neural_coder/examples/vision/resnet18.py create mode 100644 neural_coder/examples/vision/resnet50.py create mode 100644 neural_coder/globals.py create mode 100644 neural_coder/graphers/__init__.py create mode 100644 neural_coder/graphers/code_line.py create mode 100644 neural_coder/graphers/function.py create mode 100644 neural_coder/graphers/model.py create mode 100644 neural_coder/interface.py create mode 100644 neural_coder/numa_launcher.py create mode 100644 neural_coder/utils/__init__.py create mode 100644 neural_coder/utils/common.py create mode 100644 neural_coder/utils/cpu_info.py create mode 100644 neural_coder/utils/handle_user_input.py create mode 100644 neural_coder/utils/line_operation.py diff --git a/neural_coder/README.md b/neural_coder/README.md new file mode 100644 index 00000000000..fc0cb44ff15 --- /dev/null +++ b/neural_coder/README.md @@ -0,0 +1,78 @@ +Neural Coder +=========================== +## What do we offer? + +Neural Coder is a novel deployment toolkit for one-click acceleration on Deep Learning scripts via performing automated code insertions of CUDA to CPU platform conversions and Deep Learning optimization APIs. Subsequently, Neural Coder can perform automated benchmark on all applicable optimization sets acquired from the automated enabling, and evaluate for the best out-of-box performance. + +Neural Coder leverages static program analysis techniques and heuristic optimization rules to simplify the usage of various Deep Learning optimization APIs for increasing computation efficiency of AI models and improving user experience for general AI customers. We demonstrate great improvement of developer productivity and aim to facilitate enhanced Deep Learning acceleration adoption via this toolkit. + +Neural Coder helps you code Deep Learning optimizations automatically into your scripts. For example, to apply +- Automatic Mixed Precision (torch.cpu.amp.autocast) +- JIT Script computation graph transformation (torch.jit.script) +- Channels Last memory format transformation (torch.channels_last) + +simultaneously on below PyTorch evaluation code, we generate the optimized code in one-click by detecting the correct position to insert the correct API code lines: +```diff + import torch + import torchvision.models as models + my_model = models.resnet50(pretrained=True) ++ import torch ++ with torch.no_grad(): ++ my_model = my_model.to(memory_format=torch.channels_last) ++ import torch ++ with torch.no_grad(): ++ my_model.eval() ++ my_model = torch.jit.script(my_model) ++ my_model = torch.jit.freeze(my_model) + my_model.eval() + batch_size = 112 + input = torch.rand(batch_size, 3, 224, 224) + with torch.no_grad(): ++ import torch ++ with torch.cpu.amp.autocast(enabled=True, dtype=torch.bfloat16): + my_model(input) +``` + +## Getting Started! +We currently provide 3 user-facing APIs: enable, bench and superbench. +#### Enable +Users can use ```enable()``` to enable specific features into DL scripts: +``` +from neural_coder import enable +enable(code="examples/vision/resnet50.py", + features=["pytorch_jit_script", "pytorch_channels_last"]) +``` +To run benchmark directly on the optimization together with the enabling: +``` +from neural_coder import enable +enable(code="examples/vision/resnet50.py", + features=["pytorch_jit_script", "pytorch_channels_last"], + run_bench=True, + mode="throughput") +``` +#### Bench +To run benchmark on your code with an existing patch: +``` +from neural_coder import bench +bench(code="examples/vision/resnet50.py", + patch_path="${your_patch_path}", + mode="throughput") +``` +#### SuperBench +To sweep on optimization sets with a fixed benchmark configuration: +``` +from neural_coder import superbench +superbench(code="examples/vision/resnet50.py", + sweep_objective="feature", + mode="throughput") +``` +To sweep on benchmark configurations for a fixed optimization set: +``` +from neural_coder import superbench +superbench(code="examples/vision/resnet50.py", + sweep_objective="bench_config", + bench_feature=["pytorch_jit_script"，"pytorch_channels_last"]) +``` + +## Contact +Please contact us at [kai.yao@intel.com](mailto:kai.yao@intel.com) for any Neural Coder related question. diff --git a/neural_coder/__init__.py b/neural_coder/__init__.py new file mode 100644 index 00000000000..46dd8eee68c --- /dev/null +++ b/neural_coder/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .interface import enable +from .interface import bench +from .interface import superbench \ No newline at end of file diff --git a/neural_coder/coders/__init__.py b/neural_coder/coders/__init__.py new file mode 100644 index 00000000000..162b389a776 --- /dev/null +++ b/neural_coder/coders/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/neural_coder/coders/pytorch/__init__.py b/neural_coder/coders/pytorch/__init__.py new file mode 100644 index 00000000000..162b389a776 --- /dev/null +++ b/neural_coder/coders/pytorch/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/neural_coder/coders/pytorch/amp.py b/neural_coder/coders/pytorch/amp.py new file mode 100644 index 00000000000..56cd2bf1e0c --- /dev/null +++ b/neural_coder/coders/pytorch/amp.py @@ -0,0 +1,136 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ... import globals +from ...utils.line_operation import get_line_indent_level, is_eval_func_model_name + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class PTAMP(object): + def __init__(self, mode): + self.mode = mode + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + for file_path in globals.list_code_path: + code = open(file_path, 'r').read() + lines = code.split('\n') + line_idx = 0 + for i in range(len(lines)): + line = lines[i] + for model_name in globals.list_model_name: + if is_eval_func_model_name(model_name, line) and "# Neural Coder appended" not in line: + indent_level = get_line_indent_level(line) + + # 1. indenting + # indenting can have multiple location, so is a list of numbers + trans_indenting_location = [] + trans_indenting_level = [] + + if ")" in line: # e.g. model(xxx) + trans_indenting_location.append(line_idx) + trans_indenting_level.append(1) + else: # e.g. model(xxx， + # xxx， + # xxx + # ) + trans_indenting_location.append(line_idx) + trans_indenting_level.append(1) + do_search = True + i_search = 1 + while do_search: + trans_indenting_location.append( + line_idx + i_search) + trans_indenting_level.append(1) + following_line = lines[line_idx + i_search] + if ")" in following_line: + do_search = False + i_search += 1 + + # 2. insert + trans_insert_location = line_idx # insert only has 1 location, so is a number + + lines_to_insert = "" + if self.mode == "cpu": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + \ + "with torch.cpu.amp.autocast(enabled=True, dtype=torch.bfloat16):" + if self.mode == "cuda": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + \ + "with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):" + + # 1. indenting: transform "model(input)" to " model(input)" + if file_path not in globals.list_trans_indenting_modified_file: + globals.list_trans_indenting_modified_file.append( + file_path) + globals.list_trans_indenting_location_idxs.append( + trans_indenting_location) + globals.list_trans_indenting_level.append( + trans_indenting_level) + else: + idx = globals.list_trans_indenting_modified_file.index( + file_path) + for i in trans_indenting_location: + globals.list_trans_indenting_location_idxs[idx].append( + i) + for i in trans_indenting_level: + globals.list_trans_indenting_level[idx].append( + i) + + # 2. insert: add "with autocast()" line + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + logger.debug( + f"globals.list_trans_indenting_modified_file: {globals.list_trans_indenting_modified_file}") + logger.debug( + f"globals.list_trans_indenting_location_idxs: {globals.list_trans_indenting_location_idxs}") + logger.debug( + f"globals.list_trans_indenting_level: {globals.list_trans_indenting_level}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/batch_size.py b/neural_coder/coders/pytorch/batch_size.py new file mode 100644 index 00000000000..82fbbeae460 --- /dev/null +++ b/neural_coder/coders/pytorch/batch_size.py @@ -0,0 +1,87 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ... import globals + + +class BatchSizeCoder(object): + def __init__(self, file) -> None: + self.file = file + self.result = [] + + def transform(self): + lines = self.file.split('\n') + for line in lines: + if self.not_modify(line): + new_line = self.modify(line) + self.result.append(new_line) + else: + if line == '' and self.result[-1] == '': + continue + self.result.append(line) + for index, line in enumerate(self.result): + if index != len(self.result)-1: + self.result[index] += '\n' + return ''.join(self.result) + + def not_modify(self, s): + if 'batch_size' in s and '=' in s: + return True + return False + + def modify(self, s): + idx = s.find('batch_size') + s_right = s[idx:] + if ' = ' in s_right: + index = s.find(' = ') + s_left = s[:index] + if 'batch_size' in s_left: + if ',' in s_left: + index1 = s_left.find(',') + index2 = s_left.find('batch_size') + if index1 > index2: + slice1 = s_left[:index1] + else: + s_left1 = s_left[:index2] + s_right = s_left[index2:] + index3 = s_left1.rfind(',') + if ',' in s_right: + index4 = s_right.find(',') + len(s_left1) + slice1 = s_left[index3+2:index4] + else: + slice1 = s_left[index3+2:index] + s1 = slice1 + ' = ' + globals.target_batch_size + s = s[:] + '\n' + s1 + else: + s_right = s[index+3:] + s_right = s_right.replace( + s_right, globals.target_batch_size) + s = s_left + ' = ' + s_right + elif 'batch_size=' in s: + idx = s.find('batch_size=') + s_right = s[idx:] + idx2 = s_right.find('batch_size') + if ',' in s_right: + index2 = s_right.find(',') + old = s_right[idx2:index2] + s = s.replace(old, "batch_size=" + globals.target_batch_size) + elif ')' in s_right: + index2 = s_right.find(')') + old = s_right[idx2:index2] + s = s.replace(old, "batch_size=" + globals.target_batch_size) + else: + old = s_right[idx2:] + s = s.replace(old, "batch_size=" + globals.target_batch_size) + return s diff --git a/neural_coder/coders/pytorch/benchmark.py b/neural_coder/coders/pytorch/benchmark.py new file mode 100644 index 00000000000..6acbb861c2a --- /dev/null +++ b/neural_coder/coders/pytorch/benchmark.py @@ -0,0 +1,219 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ... import globals +from ...utils.line_operation import get_line_indent_level, is_eval_func_model_name + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class Benchmark(object): + def __init__(self): + pass + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + for file_path in globals.list_code_path: + try: + code = open(file_path, 'r').read() + lines = code.split('\n') + line_idx = 0 + for i in range(len(lines)): + line = lines[i] + for model_name in globals.list_model_name: + if is_eval_func_model_name(model_name, line) and "# Neural Coder appended" not in line: + indent_level = get_line_indent_level(line) + + # 1. indenting + # indenting can have multiple location, so is a list of numbers + trans_indenting_location = [] + trans_indenting_level = [] + + if ")" in line: # e.g. model(xxx) + trans_indenting_location.append(line_idx) + trans_indenting_level.append(1) + else: # e.g. model(xxx， + # xxx， + # xxx + # ) + trans_indenting_location.append(line_idx) + trans_indenting_level.append(1) + do_search = True + i_search = 1 + while do_search: + trans_indenting_location.append( + line_idx + i_search) + trans_indenting_level.append(1) + following_line = lines[line_idx + i_search] + if ")" in following_line: + do_search = False + i_search += 1 + + # 1. register indenting: transform "model(input)" to " model(input)" + if file_path not in globals.list_trans_indenting_modified_file: + globals.list_trans_indenting_modified_file.append( + file_path) + globals.list_trans_indenting_location_idxs.append( + trans_indenting_location) + globals.list_trans_indenting_level.append( + trans_indenting_level) + else: + idx = globals.list_trans_indenting_modified_file.index( + file_path) + for i in trans_indenting_location: + globals.list_trans_indenting_location_idxs[idx].append( + i) + for i in trans_indenting_level: + globals.list_trans_indenting_level[idx].append( + i) + + # 2-1. insert (before model(input)) + trans_insert_location = [] # insert only has 1 location, so is a number + + trans_insert_location = line_idx # insert before + + lines_to_insert = "" + lines_to_insert += " " * indent_level + "import time" + "\n" + lines_to_insert += " " * indent_level + "count_iter_ = 0" + "\n" + lines_to_insert += " " * indent_level + "total_time_ = 0" + "\n" + lines_to_insert += " " * indent_level + "num_iter_ = " + \ + globals.num_benchmark_iteration + "\n" + lines_to_insert += " " * indent_level + "num_warmup_iter_ = 10" + "\n" + lines_to_insert += " " * indent_level + "list_batch_time_ = []" + "\n" + lines_to_insert += " " * indent_level + \ + "for i_ in range(num_iter_):" + "\n" + lines_to_insert += " " * indent_level + " " * \ + 4 + "count_iter_ = count_iter_ + 1" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + "if count_iter_ > num_warmup_iter_:" + "\n" + lines_to_insert += " " * indent_level + " " * 8 + "t1_ = time.time()" + + # 2-1. register insert (before model(input)) + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + # 2-2. insert (after model(input)) + trans_insert_location = [] # insert only has 1 location, so is a number + + if ")" in line: # e.g. model() + trans_insert_location = line_idx + 1 # insert after + else: # e.g. model(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + trans_insert_location = line_idx + i_search # insert after + + lines_to_insert = "" + lines_to_insert += " " * indent_level + " " * 4 + \ + "if count_iter_ > num_warmup_iter_:" + "\n" + lines_to_insert += " " * indent_level + " " * 8 + "t2_ = time.time()" + "\n" + lines_to_insert += " " * indent_level + \ + " " * 8 + "batch_time_ = t2_ - t1_" + "\n" + lines_to_insert += " " * indent_level + " " * 8 + \ + "list_batch_time_.append(batch_time_)" + "\n" + lines_to_insert += " " * indent_level + " " * 8 + \ + "total_time_ = total_time_ + batch_time_" + "\n" + lines_to_insert += " " * indent_level + \ + 'print("Neural_Coder_Bench_IPS: ",' \ + 'round((num_iter_ - num_warmup_iter_) / total_time_, 3))' + "\n" + lines_to_insert += " " * indent_level + \ + 'print("Neural_Coder_Bench_MSPI: ",' \ + 'round(total_time_ / (num_iter_ - num_warmup_iter_) * 1000, 3))' + "\n" + lines_to_insert += " " * indent_level + "list_batch_time_.sort()" + "\n" + lines_to_insert += " " * indent_level + \ + "p50_latency_ = list_batch_time_[int(len(list_batch_time_) * 0.50) - 1] * 1000" \ + + "\n" + lines_to_insert += " " * indent_level + \ + "p90_latency_ = list_batch_time_[int(len(list_batch_time_) * 0.90) - 1] * 1000" \ + + "\n" + lines_to_insert += " " * indent_level + \ + "p99_latency_ = list_batch_time_[int(len(list_batch_time_) * 0.99) - 1] * 1000" \ + + "\n" + lines_to_insert += " " * indent_level + \ + 'print("Neural_Coder_Bench_P50: ", round(p50_latency_, 3))' + "\n" + lines_to_insert += " " * indent_level + \ + 'print("Neural_Coder_Bench_P90: ", round(p90_latency_, 3))' + "\n" + lines_to_insert += " " * indent_level + \ + 'print("Neural_Coder_Bench_P99: ", round(p99_latency_, 3))' + "\n" + + # 2-2. register insert (after model(input)) + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + except: + logger.debug( + f"This file has skipped patching due to unrecognizable code format: {file_path}") + + logger.debug( + f"globals.list_trans_indenting_modified_file: {globals.list_trans_indenting_modified_file}") + logger.debug( + f"globals.list_trans_indenting_location_idxs: {globals.list_trans_indenting_location_idxs}") + logger.debug( + f"globals.list_trans_indenting_level: {globals.list_trans_indenting_level}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/channels_last.py b/neural_coder/coders/pytorch/channels_last.py new file mode 100644 index 00000000000..b36d2ac70c3 --- /dev/null +++ b/neural_coder/coders/pytorch/channels_last.py @@ -0,0 +1,124 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ... import globals +from ...utils.line_operation import get_line_indent_level + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class ChannelsLast(object): + def __init__(self, list_model_def_instance): + self.list_model_def_instance = list_model_def_instance + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + try: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + + # search for features to put below them + put_below_feature_end_idx = 0 + for i in range(len(lines)): + line = lines[i] + if " # NeuralCoder: pytorch_inc_static_quant [end line]" in line or \ + " # NeuralCoder: pytorch_inc_dynamic_quant [end line]" in line: + put_below_feature_end_idx = i + + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + " = " + \ + model_name + \ + ".to(memory_format=torch.channels_last)" + + if ")" in line: # e.g. model = Net(xxx) + trans_insert_location = max( + line_idx + 1, put_below_feature_end_idx + 1) + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + trans_insert_location = max( + line_idx + i_search, put_below_feature_end_idx + 1) + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + except: + logger.debug( + f"This file has skipped patching due to unrecognizable code format: {file_path}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/cuda_to_cpu.py b/neural_coder/coders/pytorch/cuda_to_cpu.py new file mode 100644 index 00000000000..e8fe9fc44f5 --- /dev/null +++ b/neural_coder/coders/pytorch/cuda_to_cpu.py @@ -0,0 +1,59 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class CudaToCpu(object): + def __init__(self, file) -> None: + self.file = file + self.result = [] + + def transform(self): + # import pdb + # pdb.set_trace() + lines = self.file.split('\n') + for line in lines: + if self.is_delete(line): + pass + elif self.is_modify(line): + new_line = self.modify(line) + self.result.append(new_line) + else: + if line == '' and self.result[-1] == '': + continue + self.result.append(line) + for index, line in enumerate(self.result): + if index != len(self.result)-1: + self.result[index] += '\n' + return ''.join(self.result) + + def is_delete(self, s): + if 'cuda.' in s and '=' not in s: + return True + else: + return False + + def is_modify(self, s): + if '\'cuda\'' in s or '\'cuda:0\'' in s or 'cuda()' in s: + return True + else: + return False + + def modify(self, s): + if '\'cuda\'' in s or '\'cuda:0\'' in s: + old = '\'cuda\'' if '\'cuda\'' in s else '\'cuda:0\'' + s = s.replace(old, '\'cpu\'') + elif 'cuda()' in s: + old = 'cuda' + s = s.replace(old, 'cpu') + return s diff --git a/neural_coder/coders/pytorch/dummy_dataloader.py b/neural_coder/coders/pytorch/dummy_dataloader.py new file mode 100644 index 00000000000..b2a7c0dad32 --- /dev/null +++ b/neural_coder/coders/pytorch/dummy_dataloader.py @@ -0,0 +1,139 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ... import globals +from ...utils.line_operation import get_line_indent_level, is_eval_func_model_name, get_line_lhs + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class DummyDataLoader(object): + def __init__(self, list_model_def_instance): + self.list_model_def_instance = list_model_def_instance + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + + # search DataLoader + dataloader_name = "" + for i in range(len(lines)): # each item is a str of this code line + line = lines[i] + if "DataLoader(" in line and "=" in line and line.find("=") < line.find("DataLoader"): + dataloader_name = get_line_lhs(line) + dataloader_def_line_idx = i + + if dataloader_name != "": + return + else: + input_dimension_str = "3, 224, 224)" + for i in range(len(lines)): + line = lines[i] + if ("input" in line and "=" in line and line.find("=") > line.find("input")) \ + or ("image" in line and "=" in line and line.find("=") > line.find("image")): + input_dimension_str = line[line.find(",")+2:] + + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + \ + "from torch.utils.data import Dataset" + "\n" + lines_to_insert += " " * indent_level + \ + "class DummyDataset(Dataset):" + "\n" + lines_to_insert += " " * indent_level + \ + " def __init__(self, *shapes, num_samples: int = 10000):" + "\n" + lines_to_insert += " " * indent_level + " super().__init__()" + "\n" + lines_to_insert += " " * indent_level + " self.shapes = shapes" + "\n" + lines_to_insert += " " * indent_level + \ + " self.num_samples = num_samples" + "\n" + lines_to_insert += " " * indent_level + \ + " def __len__(self):" + "\n" + lines_to_insert += " " * indent_level + " return self.num_samples" + "\n" + lines_to_insert += " " * indent_level + \ + " def __getitem__(self, idx: int):" + "\n" + lines_to_insert += " " * indent_level + " sample = []" + "\n" + lines_to_insert += " " * indent_level + \ + " for shape in self.shapes:" + "\n" + lines_to_insert += " " * indent_level + \ + " spl = torch.rand(*shape)" + "\n" + lines_to_insert += " " * indent_level + \ + " sample.append(spl)" + "\n" + lines_to_insert += " " * indent_level + " return sample" + "\n" + lines_to_insert += " " * indent_level + \ + "from torch.utils.data import DataLoader" + "\n" + lines_to_insert += " " * indent_level + \ + "my_dataset = DummyDataset((" + \ + input_dimension_str + ", (1, ))" + "\n" + lines_to_insert += " " * indent_level + \ + "my_dataloader = DataLoader(my_dataset, batch_size=1)" + + trans_insert_location = 0 + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/intel_extension_for_pytorch/__init__.py b/neural_coder/coders/pytorch/intel_extension_for_pytorch/__init__.py new file mode 100644 index 00000000000..162b389a776 --- /dev/null +++ b/neural_coder/coders/pytorch/intel_extension_for_pytorch/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/neural_coder/coders/pytorch/intel_extension_for_pytorch/ipex.py b/neural_coder/coders/pytorch/intel_extension_for_pytorch/ipex.py new file mode 100644 index 00000000000..695ff354e2b --- /dev/null +++ b/neural_coder/coders/pytorch/intel_extension_for_pytorch/ipex.py @@ -0,0 +1,203 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .... import globals +from ....utils.line_operation import get_line_indent_level, is_eval_func_model_name, get_line_lhs + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class IPEX(object): + def __init__(self, list_model_def_instance, mode): + self.list_model_def_instance = list_model_def_instance + self.mode = mode + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + try: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + + # search eval func and inputs + eval_func_line = "" + for i in range(len(lines)): + line = lines[i] + if is_eval_func_model_name(model_name, line): + eval_func_line = line + inputs = eval_func_line[eval_func_line.find( + "(")+1:eval_func_line.find(")")] + + # search dataloader or inputs definition line + # (for getting line_idx to insert after it, because IPEX INT8 API contains the referral to inputs) + dataloader_inputs_def_line_idx = 0 + for i in range(len(lines)): + line = lines[i] + if "DataLoader(" in line and "=" in line and line.find("=") < line.find("DataLoader"): + dataloader_inputs_def_line_idx = i + if inputs.replace("*", "") in line \ + and "=" in line and line.find("=") > line.find(inputs.replace("*", "")): + dataloader_inputs_def_line_idx = i + if dataloader_inputs_def_line_idx == 0: + logger.warning( + f"You must define an inputs/dataloader and have an evaluate funcion \ + (e.g. 'output = model(input)') in this file: {file_path}") + continue + + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + + if self.mode == "fp32": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + \ + "import intel_extension_for_pytorch as ipex" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + ".eval()" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + \ + " = ipex.optimize(" + model_name + \ + ", dtype=torch.float32)" + if self.mode == "bf16": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + \ + "import intel_extension_for_pytorch as ipex" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + ".eval()" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + \ + " = ipex.optimize(" + model_name + \ + ", dtype=torch.bfloat16)" + if self.mode == "int8_static_quant": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + \ + "import intel_extension_for_pytorch as ipex" + "\n" + lines_to_insert += " " * indent_level + \ + "qconfig = ipex.quantization.default_static_qconfig" + "\n" + lines_to_insert += " " * indent_level + model_name + \ + " = ipex.quantization.prepare(" + model_name + \ + ", qconfig, example_inputs=" + inputs.replace( + "*", "") + ", inplace=False)" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + \ + " " * 4 + "for i in range(10):" + "\n" + lines_to_insert += " " * indent_level + " " * 8 + \ + eval_func_line.lstrip() + " # Neural Coder appended" + "\n" + lines_to_insert += " " * indent_level + model_name + \ + " = ipex.quantization.convert(" + \ + model_name + ")" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + eval_func_line.lstrip() + " # Neural Coder appended" + "\n" + if self.mode == "int8_dynamic_quant": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + \ + "import intel_extension_for_pytorch as ipex" + "\n" + lines_to_insert += " " * indent_level + \ + "qconfig = ipex.quantization.default_dynamic_qconfig" + "\n" + lines_to_insert += " " * indent_level + model_name + \ + " = ipex.quantization.prepare(" + model_name + \ + ", qconfig, example_inputs=" + inputs.replace( + "*", "") + ", inplace=False)" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + \ + " " * 4 + "for i in range(10):" + "\n" + lines_to_insert += " " * indent_level + " " * 8 + \ + eval_func_line.lstrip() + " # Neural Coder appended" + "\n" + lines_to_insert += " " * indent_level + model_name + \ + " = ipex.quantization.convert(" + \ + model_name + ")" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + eval_func_line.lstrip() + " # Neural Coder appended" + "\n" + + if ")" in line: # e.g. model = Net(xxx) + if "int8" in self.mode: + trans_insert_location = max( + line_idx + 1, dataloader_inputs_def_line_idx + 1) + else: + trans_insert_location = line_idx + 1 + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + if "int8" in self.mode: + trans_insert_location = max( + line_idx + i_search, dataloader_inputs_def_line_idx + 1) + else: + trans_insert_location = line_idx + i_search + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + except: + logger.debug( + f"This file has skipped patching due to unrecognizable code format: {file_path}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/jit.py b/neural_coder/coders/pytorch/jit.py new file mode 100644 index 00000000000..92070e57177 --- /dev/null +++ b/neural_coder/coders/pytorch/jit.py @@ -0,0 +1,272 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ... import globals +from ...utils.line_operation import get_line_indent_level, is_eval_func_model_name + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class JITScript(object): + def __init__(self, list_model_def_instance, mode): + self.list_model_def_instance = list_model_def_instance + self.mode = mode + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + try: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + + # search for features to put below them + put_below_feature_end_idx = 0 + for i in range(len(lines)): + line = lines[i] + if " # NeuralCoder: pytorch_inc_static_quant [end line]" in line or \ + " # NeuralCoder: pytorch_inc_dynamic_quant [end line]" in line: + put_below_feature_end_idx = i + + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + + if self.mode == "plain": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + ".eval()" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + \ + " = torch.jit.script(" + \ + model_name + ")" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + model_name + \ + " = torch.jit.freeze(" + model_name + ")" + if self.mode == "ofi": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + ".eval()" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + \ + " = torch.jit.optimize_for_inference(torch.jit.script(" + \ + model_name + "))" + + if ")" in line: # e.g. model = Net(xxx) + trans_insert_location = max( + line_idx + 1, put_below_feature_end_idx + 1) + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + trans_insert_location = max( + line_idx + i_search, put_below_feature_end_idx + 1) + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + except: + logger.debug( + f"This file has skipped patching due to unrecognizable code format: {file_path}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") + + +class JITTrace(object): + def __init__(self, list_model_def_instance, mode): + self.list_model_def_instance = list_model_def_instance + self.mode = mode + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + try: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + + # search eval func and inputs + eval_func_line = "" + for i in range(len(lines)): + line = lines[i] + if is_eval_func_model_name(model_name, line): + eval_func_line = line + inputs = eval_func_line[eval_func_line.find( + "(")+1:eval_func_line.find(")")] + + # search dataloader or inputs definition line (for getting line_idx to insert after it, + # because IPEX INT8 API contains the referral to inputs) + dataloader_inputs_def_line_idx = 0 + for i in range(len(lines)): + line = lines[i] + # if "DataLoader(" in line and "=" in line and line.find("=") < line.find("DataLoader"): + # dataloader_inputs_def_line_idx = i + if inputs.replace("*", "") in line and "=" in line \ + and line.find("=") > line.find(inputs.replace("*", "")): + dataloader_inputs_def_line_idx = i + + # search for features to put below them + put_below_feature_end_idx = 0 + for i in range(len(lines)): + line = lines[i] + if " # NeuralCoder: pytorch_inc_static_quant [end line]" in line \ + or " # NeuralCoder: pytorch_inc_dynamic_quant [end line]" in line: + put_below_feature_end_idx = i + + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + + if self.mode == "plain": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + ".eval()" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + \ + " = torch.jit.trace(" + model_name + ", " + inputs.replace( + "*", "") + ", check_trace=False, strict=False)" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + model_name + \ + " = torch.jit.freeze(" + model_name + ")" + if self.mode == "ofi": + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "with torch.no_grad():" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + ".eval()" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + model_name + \ + " = torch.jit.optimize_for_inference(torch.jit.trace(" + model_name + \ + ", " + inputs.replace( + "*", "") + ", check_trace=False, strict=False))" + + if ")" in line: # e.g. model = Net(xxx) + trans_insert_location = max( + line_idx + 1, dataloader_inputs_def_line_idx + 1, put_below_feature_end_idx + 1) + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + trans_insert_location = max( + line_idx + i_search, dataloader_inputs_def_line_idx + 1, + put_below_feature_end_idx + 1) + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + except: + logger.debug( + f"This file has skipped patching due to unrecognizable code format: {file_path}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/lightning.py b/neural_coder/coders/pytorch/lightning.py new file mode 100644 index 00000000000..ed1f440ee80 --- /dev/null +++ b/neural_coder/coders/pytorch/lightning.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Lightning(object): + def __init__(self, file) -> None: + self.file = file + self.result = [] + + def transform(self): + lines = self.file.split('\n') + for line in lines: + if self.not_add_accelerator(line) or self.not_add_precision(line): + new_line = self.add(line) + if self.not_modify(new_line): + new_line = self.modify(new_line) + self.result.append(new_line) + elif self.not_modify(line): + new_line = self.modify(line) + self.result.append(new_line) + if not self.not_add_accelerator(line) and not self.not_add_precision(line) and not self.not_modify(line): + if line == '' and self.result[-1] == '': + continue + self.result.append(line) + + for index, line in enumerate(self.result): + if index != len(self.result)-1: + self.result[index] += '\n' + return ''.join(self.result) + + def not_add_precision(self, s): + if 'Trainer' in s: + if 'precision' not in s: + return True + else: + return False + return False + + def not_add_accelerator(self, s): + if 'Trainer' in s: + if 'accelerator' not in s: + return True + else: + return False + return False + + def add(self, s): + if 'Trainer' in s: + if 'precision' not in s: + s_index = s.find(')') + s = s[:s_index] + ', precision=\"bf16\"' + s[s_index:] + if 'accelerator' not in s: + s_index = s.find(')') + s = s[:s_index] + ', accelerator=\"cpu\"' + s[s_index:] + return s + + def not_modify(self, s): + if 'bf16' in s and 'cpu' in s: + return False + return True + + def modify(self, s): + if '16' in s: + old = '16' + s = s.replace(old, '\"bf16\"') + if '32' in s: + old = '32' + s = s.replace(old, '\"bf16\"') + if '\"gpu\"' in s: + old = '\"gpu\"' + s = s.replace(old, '\"cpu\"') + if '\"tpu\"' in s: + old = '\"tpu\"' + s = s.replace(old, '\"cpu\"') + return s diff --git a/neural_coder/coders/pytorch/neural_compressor/__init__.py b/neural_coder/coders/pytorch/neural_compressor/__init__.py new file mode 100644 index 00000000000..162b389a776 --- /dev/null +++ b/neural_coder/coders/pytorch/neural_compressor/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/neural_coder/coders/pytorch/neural_compressor/dynamic_quant.py b/neural_coder/coders/pytorch/neural_compressor/dynamic_quant.py new file mode 100644 index 00000000000..0a61808dc41 --- /dev/null +++ b/neural_coder/coders/pytorch/neural_compressor/dynamic_quant.py @@ -0,0 +1,120 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .... import globals +from ....utils.line_operation import get_line_indent_level + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class DynamicQuant(object): + def __init__(self, list_model_def_instance): + self.list_model_def_instance = list_model_def_instance + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + lines_to_insert += " " * indent_level + "from neural_compressor.conf.config import QuantConf" + \ + " # NeuralCoder: pytorch_inc_dynamic_quant [start line]" + "\n" + lines_to_insert += " " * indent_level + \ + "from neural_compressor.experimental import Quantization, common" + "\n" + lines_to_insert += " " * indent_level + "quant_config = QuantConf()" + "\n" + lines_to_insert += " " * indent_level + \ + 'quant_config.usr_cfg.quantization.approach = "post_training_dynamic_quant"' + "\n" + lines_to_insert += " " * indent_level + \ + 'quant_config.usr_cfg.model.framework = "pytorch"' + "\n" + lines_to_insert += " " * indent_level + \ + "quantizer = Quantization(quant_config)" + "\n" + lines_to_insert += " " * indent_level + \ + "quantizer.model = common.Model(" + \ + model_name + ")" + "\n" + lines_to_insert += " " * indent_level + model_name + " = quantizer()" + "\n" + lines_to_insert += " " * indent_level + model_name + " = " + model_name + \ + ".model" + \ + " # NeuralCoder: pytorch_inc_dynamic_quant [end line]" + + if ")" in line: # e.g. model = Net(xxx) + trans_insert_location = line_idx + 1 + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + trans_insert_location = line_idx + i_search + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/neural_compressor/static_quant.py b/neural_coder/coders/pytorch/neural_compressor/static_quant.py new file mode 100644 index 00000000000..c99af2f0dd5 --- /dev/null +++ b/neural_coder/coders/pytorch/neural_compressor/static_quant.py @@ -0,0 +1,161 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .... import globals +from ....utils.line_operation import get_line_indent_level, is_eval_func_model_name, get_line_lhs + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class StaticQuant(object): + def __init__(self, list_model_def_instance): + self.list_model_def_instance = list_model_def_instance + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + + # search DataLoader + dataloader_name = "" + for i in range(len(lines)): + line = lines[i] + if "DataLoader(" in line and "=" in line and line.find("=") < line.find("DataLoader"): + dataloader_name = get_line_lhs(line) + dataloader_def_line_idx = i + + # search eval func + eval_func_line = "" + for i in range(len(lines)): + line = lines[i] + if is_eval_func_model_name(model_name, line): + eval_func_line = line + + # for scripts that does not have "dataloader" defined but does have "input" defined + if dataloader_name == "": + dataloader_name = "my_dataloader" + dataloader_def_line_idx = 0 + for i in range(len(lines)): + line = lines[i] + if ("input" in line and "=" in line and line.find("=") > line.find("input")) \ + or ("image" in line and "=" in line and line.find("=") > line.find("image")): + # for adding the static quant after input/image definition line + dataloader_def_line_idx = i + + if eval_func_line != "": + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + lines_to_insert += " " * indent_level + \ + "def eval_func(" + model_name + "):" + \ + " # NeuralCoder: pytorch_inc_static_quant [start line]" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + eval_func_line.lstrip() + " # Neural Coder appended" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + "return 1" + "\n" + lines_to_insert += " " * indent_level + \ + "from neural_compressor.conf.config import QuantConf" + "\n" + lines_to_insert += " " * indent_level + \ + "from neural_compressor.experimental import Quantization, common" + "\n" + lines_to_insert += " " * indent_level + "quant_config = QuantConf()" + "\n" + lines_to_insert += " " * indent_level + \ + 'quant_config.usr_cfg.model.framework = "pytorch_fx"' + "\n" + lines_to_insert += " " * indent_level + \ + "quantizer = Quantization(quant_config)" + "\n" + lines_to_insert += " " * indent_level + \ + "quantizer.model = common.Model(" + \ + model_name + ")" + "\n" + lines_to_insert += " " * indent_level + \ + "quantizer.calib_dataloader = " + dataloader_name + "\n" + lines_to_insert += " " * indent_level + "quantizer.eval_func = eval_func" + "\n" + lines_to_insert += " " * indent_level + model_name + " = quantizer()" + "\n" + lines_to_insert += " " * indent_level + model_name + " = " + model_name + \ + ".model" + \ + " # NeuralCoder: pytorch_inc_static_quant [end line]" + + if ")" in line: # e.g. model = Net(xxx) + # in case dataloader is defined under model, has to insert after dataloader definition + # because static quant API refers to the dataloader_name + trans_insert_location = max( + line_idx + 1, dataloader_def_line_idx + 1) + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line: + do_search = False + i_search += 1 + # in case dataloader is defined under model, has to insert after dataloader definition + # because static quant API refers to the dataloader_name + trans_insert_location = max( + line_idx + i_search, dataloader_def_line_idx + 1) + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/pytorch/torchdynamo.py b/neural_coder/coders/pytorch/torchdynamo.py new file mode 100644 index 00000000000..44c1fd7fcf3 --- /dev/null +++ b/neural_coder/coders/pytorch/torchdynamo.py @@ -0,0 +1,338 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +from ... import globals +from ...utils.line_operation import get_line_indent_level, is_eval_func_model_name + +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class TorchDynamo(object): + def __init__(self): + pass + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + for file_path in globals.list_code_path: + code = open(file_path, 'r').read() + lines = code.split('\n') + line_idx = 0 + for i in range(len(lines)): + line = lines[i] + for model_name in globals.list_model_name: + if is_eval_func_model_name(model_name, line) and "# Neural Coder appended" not in line: + indent_level = get_line_indent_level(line) + + # 1. indenting + # indenting can have multiple location, so is a list of numbers + trans_indenting_location = [] + trans_indenting_level = [] + + if ")" in line: # e.g. model(xxx) + trans_indenting_location.append(line_idx) + trans_indenting_level.append(1) + else: # e.g. model(xxx， + # xxx， + # xxx + # ) + trans_indenting_location.append(line_idx) + trans_indenting_level.append(1) + do_search = True + i_search = 1 + while do_search: + trans_indenting_location.append( + line_idx + i_search) + trans_indenting_level.append(1) + following_line = lines[line_idx + i_search] + if ")" in following_line: + do_search = False + i_search += 1 + + # 2. insert + trans_insert_location = line_idx # insert only has 1 location, so is a number + + lines_to_insert = "" + lines_to_insert += " " * indent_level + "import torchdynamo" + "\n" + lines_to_insert += " " * indent_level + \ + "with torchdynamo.optimize(dynamo_backend):" + + # 1. indenting: transform "model(input)" to " model(input)" + if file_path not in globals.list_trans_indenting_modified_file: + globals.list_trans_indenting_modified_file.append( + file_path) + globals.list_trans_indenting_location_idxs.append( + trans_indenting_location) + globals.list_trans_indenting_level.append( + trans_indenting_level) + else: + idx = globals.list_trans_indenting_modified_file.index( + file_path) + for i in trans_indenting_location: + globals.list_trans_indenting_location_idxs[idx].append( + i) + for i in trans_indenting_level: + globals.list_trans_indenting_level[idx].append( + i) + + # 2. insert: add "with autocast()" line + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + logger.debug( + f"globals.list_trans_indenting_modified_file: {globals.list_trans_indenting_modified_file}") + logger.debug( + f"globals.list_trans_indenting_location_idxs: {globals.list_trans_indenting_location_idxs}") + logger.debug( + f"globals.list_trans_indenting_level: {globals.list_trans_indenting_level}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") + + +class TorchDynamoJITScript(object): + def __init__(self, list_model_def_instance, mode): + self.list_model_def_instance = list_model_def_instance + self.mode = mode + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + try: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + + if self.mode == "plain": + lines_to_insert += " " * indent_level + "from typing import List" + "\n" + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "import torchdynamo" + "\n" + lines_to_insert += " " * indent_level + \ + "def dynamo_backend(gm: torch.fx.GraphModule," \ + "example_inputs: List[torch.Tensor]):" + "\n" + lines_to_insert += " " * indent_level + \ + " " * 4 + "return torch.jit.script(gm)" + if self.mode == "ofi": + lines_to_insert += " " * indent_level + "from typing import List" + "\n" + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "import torchdynamo" + "\n" + lines_to_insert += " " * indent_level + \ + "def dynamo_backend(gm: torch.fx.GraphModule," \ + "example_inputs: List[torch.Tensor]):" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + "return torch.jit.optimize_for_inference(torch.jit.script(gm))" + + if ")" in line: # e.g. model = Net(xxx) + trans_insert_location = line_idx + 1 + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + trans_insert_location = line_idx + i_search + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + except: + logger.debug( + f"This file has skipped patching due to unrecognizable code format: {file_path}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") + + +class TorchDynamoJITTrace(object): + def __init__(self, list_model_def_instance, mode): + self.list_model_def_instance = list_model_def_instance + self.mode = mode + + def print_info(self): + for i in self.list_model_def_instance: + logger.debug(f"i.print_info(): {i.print_info()}") + + # collect file transformation info and register (store) in globals + # (i.e. which file to add which lines at which location) + def register_transformation(self): + list_code = [] + for i in globals.list_code_path: + list_code.append(open(i, 'r').read()) + + for ins in self.list_model_def_instance: + try: + model_name = ins.model_name + file_path = ins.file_path + model_def_line_idx = ins.model_def_line_idx + function_def_line_idx = ins.function_def_line_idx + class_name = ins.class_name + + # transformation + file_path_idx = globals.list_code_path.index(file_path) + lines = list_code[file_path_idx].split('\n') + line_idx = 0 + for i in range(len(lines)): + line = lines[i] + if line_idx == model_def_line_idx: + indent_level = get_line_indent_level(line) + lines_to_insert = "" + + if self.mode == "plain": + lines_to_insert += " " * indent_level + "from typing import List" + "\n" + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "import torchdynamo" + "\n" + lines_to_insert += " " * indent_level + \ + "def dynamo_backend(gm: torch.fx.GraphModule, \ + example_inputs: List[torch.Tensor]):" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + "return torch.jit.trace(gm, example_inputs)" + if self.mode == "ofi": + lines_to_insert += " " * indent_level + "from typing import List" + "\n" + lines_to_insert += " " * indent_level + "import torch" + "\n" + lines_to_insert += " " * indent_level + "import torchdynamo" + "\n" + lines_to_insert += " " * indent_level + \ + "def dynamo_backend(gm: torch.fx.GraphModule, \ + example_inputs: List[torch.Tensor]):" + "\n" + lines_to_insert += " " * indent_level + " " * 4 + \ + "return torch.jit.optimize_for_inference(torch.jit.trace(gm, example_inputs))" + + if ")" in line: # e.g. model = Net(xxx) + trans_insert_location = line_idx + 1 + else: # e.g. model = Net(xxx， + # xxx， + # xxx + # ) + do_search = True + i_search = 1 + while do_search: + following_line = lines[line_idx + i_search] + if ")" in following_line and following_line[indent_level] == ")": + do_search = False + i_search += 1 + trans_insert_location = line_idx + i_search + + if file_path not in globals.list_trans_insert_modified_file: + globals.list_trans_insert_modified_file.append( + file_path) + globals.list_trans_insert_location_idxs.append( + [trans_insert_location]) + globals.list_trans_insert_number_insert_lines.append( + [lines_to_insert.count("\n") + 1]) + globals.list_trans_insert_lines_to_insert.append( + [lines_to_insert]) + else: + idx = globals.list_trans_insert_modified_file.index( + file_path) + globals.list_trans_insert_location_idxs[idx].append( + trans_insert_location) + globals.list_trans_insert_number_insert_lines[idx].append( + lines_to_insert.count("\n") + 1) + globals.list_trans_insert_lines_to_insert[idx].append( + lines_to_insert) + + line_idx += 1 + + except: + logger.debug( + f"This file has skipped patching due to unrecognizable code format: {file_path}") + + logger.debug( + f"globals.list_trans_insert_modified_file: {globals.list_trans_insert_modified_file}") + logger.debug( + f"globals.list_trans_insert_location_idxs: {globals.list_trans_insert_location_idxs}") + logger.debug( + f"globals.list_trans_insert_number_insert_lines: {globals.list_trans_insert_number_insert_lines}") + logger.debug( + f"globals.list_trans_insert_lines_to_insert: {globals.list_trans_insert_lines_to_insert}") diff --git a/neural_coder/coders/tensorflow/__init__.py b/neural_coder/coders/tensorflow/__init__.py new file mode 100644 index 00000000000..8566d97d35f --- /dev/null +++ b/neural_coder/coders/tensorflow/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/neural_coder/coders/tensorflow/amp.py b/neural_coder/coders/tensorflow/amp.py new file mode 100644 index 00000000000..e26cd65d6c2 --- /dev/null +++ b/neural_coder/coders/tensorflow/amp.py @@ -0,0 +1,64 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ...utils.line_operation import get_line_lhs + + +class TensorFlowKerasAMP(object): + def __init__(self, file) -> None: + self.file = file + self.result = [] + self.keras_edited_flag = False + + def transform(self): + # import pdb + # pdb.set_trace() + lines = self.file.split('\n') + for line in lines: + if self.is_modify(line): + if '.ConfigProto()' in line: # TF AMP + config_name = get_line_lhs(line) + new_line_1 = "from tensorflow.core.protobuf import rewriter_config_pb2" + new_line_2 = config_name + \ + ".graph_options.rewrite_options.auto_mixed_precision_mkl = \ + rewriter_config_pb2.RewriterConfig.ON" + self.result.append(line) + self.result.append(new_line_1) + self.result.append(new_line_2) + elif 'keras' in line and 'import' in line: # Keras AMP + if not self.keras_edited_flag: + new_line_1 = "from tensorflow.keras.mixed_precision import experimental as mixed_precision" + new_line_2 = "policy = mixed_precision.Policy('mixed_bfloat16')" + new_line_3 = "mixed_precision.set_policy(policy)" + self.result.append(line) + self.result.append(new_line_1) + self.result.append(new_line_2) + self.result.append(new_line_3) + self.keras_edited_flag = True + else: + self.result.append(line) + else: + + self.result.append(line) + for index, line in enumerate(self.result): + if index != len(self.result)-1: + self.result[index] += '\n' + return ''.join(self.result) + + def is_modify(self, s): + if '.ConfigProto()' in s or ('keras' in s and 'import' in s): + return True + else: + return False diff --git a/neural_coder/coders/transform.py b/neural_coder/coders/transform.py new file mode 100644 index 00000000000..919b1c9cfa9 --- /dev/null +++ b/neural_coder/coders/transform.py @@ -0,0 +1,85 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import globals + +# [insert] some code lines into file + + +def execute_insert_transformation(list_transformed_code): + for file_path in globals.list_trans_insert_modified_file: + trans_location_idxs = globals.list_trans_insert_location_idxs[globals.list_trans_insert_modified_file.index( + file_path)] + trans_number_insert_lines = \ + globals.list_trans_insert_number_insert_lines[globals.list_trans_insert_modified_file.index( + file_path)] + trans_lines_to_insert = \ + globals.list_trans_insert_lines_to_insert[globals.list_trans_insert_modified_file.index( + file_path)] + + file_path_idx = globals.list_code_path.index(file_path) + lines_transformed = list_transformed_code[file_path_idx].split('\n') + + # this part is for "insert" kind of transformation only (math) + t = [0] + u = 0 + for n in trans_number_insert_lines: + u = u + n + t.append(u) + t = t[:-1] + + trans_location_idxs = [sum(i) for i in zip(trans_location_idxs, t)] + + for idx in trans_location_idxs: # actual transformation (insertion) + additions = trans_lines_to_insert[trans_location_idxs.index( + idx)].split("\n") + additions = additions[::-1] # reverse + for i in range(len(additions)): + lines_transformed.insert(idx, additions[i]) + + # transfer lines_transformed to code format ("\n" save write) + code_transformed = "".join([i + "\n" for i in lines_transformed])[0:-1] + + list_transformed_code[file_path_idx] = code_transformed + + return list_transformed_code + +# [indenting] some code lines with " " into file + + +def execute_indenting_transformation(list_transformed_code): + for file_path in globals.list_trans_indenting_modified_file: + trans_location_idxs = \ + globals.list_trans_indenting_location_idxs[globals.list_trans_indenting_modified_file.index( + file_path)] + trans_indenting_level = \ + globals.list_trans_indenting_level[globals.list_trans_indenting_modified_file.index( + file_path)] + + file_path_idx = globals.list_code_path.index(file_path) + lines_transformed = list_transformed_code[file_path_idx].split('\n') + + for idx in trans_location_idxs: # actual transformation (indenting) + this_indenting_level = trans_indenting_level[trans_location_idxs.index( + idx)] + lines_transformed[idx] = " " * 4 * \ + this_indenting_level + lines_transformed[idx] + + # transfer lines_transformed to code format ("\n" save write) + code_transformed = "".join([i + "\n" for i in lines_transformed])[0:-1] + + list_transformed_code[file_path_idx] = code_transformed + + return list_transformed_code diff --git a/neural_coder/examples/nlp/distilbert.py b/neural_coder/examples/nlp/distilbert.py new file mode 100644 index 00000000000..eab5513a51a --- /dev/null +++ b/neural_coder/examples/nlp/distilbert.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer +) + +finetuned_model = "distilbert-base-uncased-finetuned-sst-2-english" + + +class MyDataLoader(object): + def __init__(self): + self.tokenizer = AutoTokenizer.from_pretrained(finetuned_model) + self.sequence = "Shanghai is a beautiful city!" + self.encoded_input = self.tokenizer( + self.sequence, + return_tensors='pt' + ) + self.label = 1 # negative sentence: 0; positive sentence: 1 + self.batch_size = 1 + + def __iter__(self): + yield self.encoded_input, self.label + + +my_nlp_model = AutoModelForSequenceClassification.from_pretrained( + finetuned_model, +) + +my_nlp_dataloader = MyDataLoader() + +output = my_nlp_model(**my_nlp_dataloader.encoded_input) diff --git a/neural_coder/examples/vision/alexnet.py b/neural_coder/examples/vision/alexnet.py new file mode 100644 index 00000000000..10ac6d8b080 --- /dev/null +++ b/neural_coder/examples/vision/alexnet.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +import torchvision.models as models +model = models.alexnet(pretrained=True) +model.eval() +batch_size = 1 +input = torch.rand(batch_size, 3, 224, 224) +with torch.no_grad(): + model(input) diff --git a/neural_coder/examples/vision/resnet18.py b/neural_coder/examples/vision/resnet18.py new file mode 100644 index 00000000000..a7fadf9c70b --- /dev/null +++ b/neural_coder/examples/vision/resnet18.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +import torchvision.models as models +model = models.resnet18(pretrained=True) +model.eval() +batch_size = 1 +input = torch.rand(batch_size, 3, 224, 224) +with torch.no_grad(): + model(input) diff --git a/neural_coder/examples/vision/resnet50.py b/neural_coder/examples/vision/resnet50.py new file mode 100644 index 00000000000..c7091e2e7bc --- /dev/null +++ b/neural_coder/examples/vision/resnet50.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +import torchvision.models as models +model = models.resnet50(pretrained=True) +model.eval() +batch_size = 1 +input = torch.rand(batch_size, 3, 224, 224) +with torch.no_grad(): + model(input) diff --git a/neural_coder/globals.py b/neural_coder/globals.py new file mode 100644 index 00000000000..4557a4a33f4 --- /dev/null +++ b/neural_coder/globals.py @@ -0,0 +1,83 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# whethre to consider all import modules and perform transformation based on all codes plus all import modules +import logging +consider_imports = True + +# target batch size for feature of changing PyTorch batch size +target_batch_size = 1 + +# number of benchmark iteration for feature of PyTorch benchmark +num_benchmark_iteration = 30 + +# print info for debugging purpose +logging_level = logging.INFO + + +def reset_globals(): + global list_code_path + + global list_code_line_instance + + global list_class_def_instance + global list_class_name + global list_parent_class_name + + global list_model_def_instance + global list_model_name + + global list_trans_insert_modified_file + global list_trans_insert_location_idxs + global list_trans_insert_number_insert_lines + global list_trans_insert_lines_to_insert + + global list_trans_indenting_modified_file + global list_trans_indenting_location_idxs + global list_trans_indenting_level + + global list_all_function_name + global list_all_function_return_item + + global list_wrapper_base_function_name + global list_wrapper_children_function_name + global list_wrapper_all_function_name + + list_code_path = [] + list_code_line_instance = [] # list of CodeLine instances + + list_class_def_instance = [] # list of ClassDefinition instances + list_class_name = [] # list of class names + list_parent_class_name = [] + + list_model_def_instance = [] + list_model_name = [] + + # for code transformation recording + list_trans_insert_modified_file = [] + list_trans_insert_location_idxs = [] + list_trans_insert_number_insert_lines = [] + list_trans_insert_lines_to_insert = [] + + list_trans_indenting_modified_file = [] + list_trans_indenting_location_idxs = [] + list_trans_indenting_level = [] + + list_all_function_name = [] + list_all_function_return_item = [] + + list_wrapper_base_function_name = [] + list_wrapper_children_function_name = [] + list_wrapper_all_function_name = [] diff --git a/neural_coder/graphers/__init__.py b/neural_coder/graphers/__init__.py new file mode 100644 index 00000000000..8566d97d35f --- /dev/null +++ b/neural_coder/graphers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/neural_coder/graphers/code_line.py b/neural_coder/graphers/code_line.py new file mode 100644 index 00000000000..d51e60ca0dd --- /dev/null +++ b/neural_coder/graphers/code_line.py @@ -0,0 +1,271 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List +from .. import globals +from ..utils.line_operation import get_line_indent_level +from ..utils.line_operation import multi_line_comment_detection +from ..utils.line_operation import single_line_comment_or_empty_line_detection +import pprint + + +class CodeLine: + def __init__(self): + self.file_path = None + self.line_idx = None + self.line_content = None + self.indent_level = None + self.is_multi_line_comment = None + self.is_single_line_comment_or_empty = None + self.is_class_def_line = None + self.is_in_class = None + self.class_name = None + self.parent_class_name = None + self.class_def_line_idx = None + self.class_end_line_idx = None + self.is_func_def_line = None + self.is_in_func = None + self.func_name = None + self.func_return_idx = None + self.return_item = None + self.func_def_line_idx = None + self.func_end_line_idx = None + + def print_info(self): + pp = pprint.PrettyPrinter() + pp.pprint(self.__dict__) + + +def register_code_line(): + print_class_related_info = False + print_func_related_info = False + print("{:<100} {:<10} {:<20} {:<20} {:<20} {:<40} {:<20} {:<20}".format('line', + 'line_idx', + 'is_class_def_line', + 'is_in_class', + 'class_name', + 'parent_class_name', + 'class_def_line_idx', + 'class_end_line_idx')) \ + if print_class_related_info else 0 + print("{:<100} {:<10} {:<20} {:<20} {:<20} {:<20} {:<20} {:<20} {:<20}".format('line', + 'line_idx', + 'is_func_def_line', + 'is_in_func', + 'func_name', + 'func_return_idx', + 'return_item', + 'func_def_line_idx', + 'func_end_line_idx')) \ + if print_func_related_info else 0 + for path in globals.list_code_path: + code = open(path, 'r').read() + lines = code.split('\n') + + line_idx = 0 + is_multi_line_comment = False + end_multi_line_comment_flag = False + + is_class_def_line = False + is_in_class = False + class_name = "" + parent_class_name = [] + class_def_line_idx = -1 + class_end_line_idx = -1 + + is_func_def_line = False + is_in_func = False + func_name = "" + func_return_idx = -1 + return_item = "" + func_def_line_idx = -1 + func_end_line_idx = -1 + + for line in lines: + CL = CodeLine() + CL.file_path = path + CL.line_idx = line_idx + CL.line_content = line + CL.indent_level = get_line_indent_level(line) + + is_multi_line_comment, end_multi_line_comment_flag = multi_line_comment_detection( + line, is_multi_line_comment, end_multi_line_comment_flag) + CL.is_multi_line_comment = is_multi_line_comment + + is_single_line_comment_or_empty = single_line_comment_or_empty_line_detection( + line) + CL.is_single_line_comment_or_empty = is_single_line_comment_or_empty + + # class + is_class_def_line = False + if "class " in line and line.lstrip()[0:5] == "class": + is_in_class = True + is_class_def_line = True + line_ls = line.lstrip() + if "(" in line_ls: # "class A(B):" + class_name = line_ls[line_ls.find(" ")+1:line_ls.find("(")] + parent_content = line_ls[line_ls.find( + "(")+1:line_ls.find(")")] + if "," in parent_content: # "class A(B, C):" + parent_class_name = [] + parent_content_items = parent_content.split(", ") + for parent_content_item in parent_content_items: + parent_class_name.append(parent_content_item) + else: # "class A(B):" + parent_class_name = [parent_content] + else: # "class A:" + class_name = line_ls[line_ls.find(" ")+1:line_ls.find(":")] + parent_class_name = [] + + # search for class end line + class_def_indent_level = get_line_indent_level(line) + class_def_line_idx = line_idx + search_idx = line_idx + 1 + search_following_lines = True + _is_multi_line_comment = False + _end_multi_line_comment_flag = False + while search_following_lines: + try: + following_line = lines[search_idx] + except: # end of file situation + class_end_line_idx = search_idx + break + following_indent_level = get_line_indent_level( + following_line) + + _is_multi_line_comment, _end_multi_line_comment_flag = multi_line_comment_detection( + following_line, _is_multi_line_comment, _end_multi_line_comment_flag) + _is_single_line_comment_or_empty = single_line_comment_or_empty_line_detection( + following_line) + + # judge_1: indent is equal to def indent + judge_1 = following_indent_level <= class_def_indent_level + # judge_2: not starting with")" + judge_2 = True if ( + following_line != "" and following_line[following_indent_level] != ")") else False + # judge_3: is not a comment or empty line + judge_3 = True if ( + not _is_multi_line_comment and not _is_single_line_comment_or_empty) else False + + if judge_1 and judge_2 and judge_3: + search_following_lines = False + class_end_line_idx = search_idx + + search_idx += 1 + + if is_in_class and line_idx == class_end_line_idx: + is_in_class = False + class_name = "" + parent_class_name = [] + class_def_line_idx = -1 + class_end_line_idx = -1 + + # function + if is_in_func and line_idx == func_end_line_idx: + is_in_func = False + func_return_idx = -1 + return_item = "" + func_name = "" + func_def_line_idx = -1 + func_end_line_idx = -1 + + is_func_def_line = False + # only consider outermost function, not consider def(def()) + if not is_in_func and "def " in line: + is_in_func = True + is_func_def_line = True + func_name = line[line.find("def")+4:line.find("(")] + + # search for func end line + func_def_indent_level = get_line_indent_level(line) + func_def_line_idx = line_idx + search_idx = line_idx + 1 + search_following_lines = True + _is_multi_line_comment = False + _end_multi_line_comment_flag = False + while search_following_lines: + try: + following_line = lines[search_idx] + except: # end of file situation + func_end_line_idx = search_idx + break + following_indent_level = get_line_indent_level( + following_line) + + if "return" in following_line: + func_return_idx = search_idx + return_item = following_line[following_line.find( + "return")+7:].strip() + + _is_multi_line_comment, _end_multi_line_comment_flag = multi_line_comment_detection( + following_line, _is_multi_line_comment, _end_multi_line_comment_flag) + _is_single_line_comment_or_empty = single_line_comment_or_empty_line_detection( + following_line) + + # judge_1: indent is equal to def indent + judge_1 = following_indent_level <= func_def_indent_level + # judge_2: not starting with")" + judge_2 = True if ( + following_line != "" and following_line[following_indent_level] != ")") else False + # judge_3: is not a comment or empty line + judge_3 = True if ( + not _is_multi_line_comment and not _is_single_line_comment_or_empty) else False + + if judge_1 and judge_2 and judge_3: + search_following_lines = False + func_end_line_idx = search_idx + + search_idx += 1 + + CL.is_class_def_line = is_class_def_line + CL.is_in_class = is_in_class + CL.class_name = class_name + CL.parent_class_name = parent_class_name + CL.class_def_line_idx = class_def_line_idx + CL.class_end_line_idx = class_end_line_idx + print("{:<100} {:<10} {:<20} {:<20} {:<20} {:<40} {:<20} {:<20}".format(line[0:100], + line_idx, + is_class_def_line, + is_in_class, + class_name, + str( + parent_class_name), + class_def_line_idx, + class_end_line_idx)) if print_class_related_info else 0 + + CL.is_func_def_line = is_func_def_line + CL.is_in_func = is_in_func + CL.func_name = func_name + CL.func_return_idx = func_return_idx + CL.return_item = return_item + CL.func_def_line_idx = func_def_line_idx + CL.func_end_line_idx = func_end_line_idx + print("{:<100} {:<10} {:<20} {:<20} \ + {:<20} {:<20} {:<20} {:<20} {:<20}".format(line[0:100], + line_idx, + is_func_def_line, + is_in_func, + func_name, + func_return_idx, + return_item[0:20], + func_def_line_idx, + func_end_line_idx)) \ + if print_func_related_info else 0 + + globals.list_code_line_instance.append(CL) + line_idx += 1 + + # for i in globals.list_code_line_instance: + # i.print_info() diff --git a/neural_coder/graphers/function.py b/neural_coder/graphers/function.py new file mode 100644 index 00000000000..7aae0e53729 --- /dev/null +++ b/neural_coder/graphers/function.py @@ -0,0 +1,189 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List +from ..utils.line_operation import get_line_indent_level +from .. import globals +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +# register all relationships of ( [function name] : [return_item] ) pair of the list of code path provided +# but only for "return xxx()" (return a function w/o class prefix) or "return xxx" (return an instance) +# e.g. +# def a1(): +# return b1() +# def b1(): +# return x +# def c(): +# return T.q() +# INPUT: ["example.py"] (above code snippet) +# OUTPUT: +# globals.list_all_function_return_item = ["b1", "x"] +# globals.list_all_function_name = ["a1", "b1"] +def register_func_wrap_pair(): + logger.info( + f"Analyzing function wrapping relationship for call graph analysis...") + for path in globals.list_code_path: + code = open(path, 'r').read() + lines = code.split('\n') + line_idx = 0 + is_in_function = False + func_end_line_idx = -1 + function_def_line_idx = -1 + for line in lines: + indent_level = get_line_indent_level(line) + + # handle function's end line + if is_in_function and line_idx == func_end_line_idx: + is_in_function = False + + # handle function's defnition line, to initiate a function + if not is_in_function and "def " in line: # only deal with outermost def + function_name = line[line.find("def")+4:line.find("(")] + + def_indent_level = get_line_indent_level(line) + function_def_line_idx = line_idx + + is_in_function = True + + # search for function end line + search_idx = line_idx + 1 + search_following_lines = True + multi_comment_flag = False + while search_following_lines: + try: + following_line = lines[search_idx] + except: # end of file + func_end_line_idx = search_idx + break + following_indent_level = get_line_indent_level( + following_line) + + # judge_1: indent is equal to def indent + judge_1 = following_indent_level <= def_indent_level + # judge_2: not starting with")" + judge_2 = True if ( + following_line != "" and following_line[following_indent_level] != ")") else False + # judge_3: is not a comment + c1 = False + c2 = False + if multi_comment_flag: + c1 = True # multi-line comment + if len(line) > 0 and len(line.lstrip()) > 0 and line.lstrip()[0] == "#": + c2 = True # single-line comment + if '"""' in following_line: + multi_comment_flag = not multi_comment_flag + + judge_3 = True if (not c1 and not c2) else False + + if judge_1 and judge_2 and judge_3: + search_following_lines = False + func_end_line_idx = search_idx + + search_idx += 1 + + line_idx += 1 + continue + + # handle inside a function + if is_in_function and line_idx < func_end_line_idx: + # handle return + if "return" in line: + line_s = line[line.find("return")+7:].strip() + # line_s common case: 1. "" 2. "xxx" 3. "xxx, xxx" 3. "xxx()" 4. "xxx(xxx)" 5. "xxx(xxx, xxx)" + if line_s == "": # case 1 + pass + elif line.strip()[0:6] != "return": + pass + elif 'f"' in line or "#" in line or "if" in line or "." in line or '""' in line or "+" in line: + pass + elif "(" in line_s: # case 4 or case 5 + return_item = line_s[:line_s.find("(")] + globals.list_all_function_return_item.append( + return_item) + globals.list_all_function_name.append(function_name) + elif ", " in line_s: # case 3 + ls = line_s.split(", ") + for return_item in ls: + globals.list_all_function_return_item.append( + return_item) + globals.list_all_function_name.append( + function_name) + else: # case 2 + return_item = line_s + globals.list_all_function_return_item.append( + return_item) + globals.list_all_function_name.append(function_name) + + line_idx += 1 + continue + + logger.debug( + f"globals.list_all_function_name: {globals.list_all_function_name}") + logger.debug( + f"globals.list_all_function_return_item: {globals.list_all_function_return_item}") + +# get all wrapper children names of the base function name +# e.g. +# class Net(nn.Module): +# xxx +# +# def _resnet(): +# model = Net() +# return model +# +# def resnet34(): +# xxx +# return _resnet() +# +# def resnet18(): +# xxx +# return _resnet() +# +# def resnet18_large(): +# xxx +# return resnet18() +# +# INPUT: "_resnet" +# OUTPUT: ["resnet18", "resnet34", "resnet18_large"] + + +def get_all_wrap_children(base_function_name: str) -> List: + length = range(len(globals.list_all_function_return_item)) + base_function_name = [base_function_name] + do_search = True + list_child_all = [] + + while do_search: + current_count = len(list_child_all) + for this_base in base_function_name: + this_list_child = [] + for i in length: + if globals.list_all_function_return_item[i] == this_base: + this_list_child.append(globals.list_all_function_name[i]) + base_function_name = this_list_child + + list_child_all += this_list_child + list_child_all = list(set(list_child_all)) + + if len(list_child_all) == current_count: + do_search = False + + return list_child_all diff --git a/neural_coder/graphers/model.py b/neural_coder/graphers/model.py new file mode 100644 index 00000000000..f84a965eb35 --- /dev/null +++ b/neural_coder/graphers/model.py @@ -0,0 +1,250 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# FOR PYTORCH ONLY + +from .function import get_all_wrap_children +import pprint +import re +from ..utils.line_operation import get_line_indent_level, of_definition_format +from typing import List +from .. import globals +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +class ClassDefinition: + # (class_name + file_path) is the unique determination + def __init__(self, class_name, file_path, class_def_line_idx, parent_class_name): + self.class_name = class_name + self.file_path = file_path + self.class_def_line_idx = class_def_line_idx + self.parent_class_name = parent_class_name + + def print_info(self): + logger.debug(f"ClassDefinitionprint_info(): {self.__dict__}") + + +class ModelDefinition: + def __init__(self, + model_name, + class_name, + file_path, + model_def_line_idx, + function_def_line_idx, + function_name + ): + self.model_name = model_name + self.class_name = class_name + self.file_path = file_path + self.model_def_line_idx = model_def_line_idx + self.function_def_line_idx = function_def_line_idx + self.function_name = function_name + + def print_info(self): + logger.debug(f"ModelDefinition.print_info(): {self.__dict__}") + + +# search nnModule classes +def register_nnModule_class(): + logger.info(f"Analyzing nn.Module class definitions in all files ...") + # search raw nnModule class (e.g. class ClassName(nn.Module):) + for cl in globals.list_code_line_instance: + parent_class_has_nnModule = list(set(cl.parent_class_name) & set( + ["nn.Module", "torch.nn.Module", "nn.Sequential", "torch.Sequential", "_BaseAutoModelClass"])) != [] + if cl.is_class_def_line and parent_class_has_nnModule: + CD = ClassDefinition(class_name=cl.class_name, + file_path=cl.file_path, + class_def_line_idx=cl.class_def_line_idx, + parent_class_name=cl.parent_class_name) + CD.print_info() + globals.list_class_name.append(cl.class_name) + globals.list_class_def_instance.append(CD) + + # search child class of nnModule class (recursively), (e.g. class ClassName(ClassNameFather):) + # this is to complete the nnModule class list + # e.g. A(nn.Module), B(A), C(B), D(C) + search_scope = globals.list_class_name + do_search = True + while do_search: + list_child_class_name = [] + for cl in globals.list_code_line_instance: + parent_class_has_nnModule = list( + set(cl.parent_class_name) & set(search_scope)) != [] + if cl.is_class_def_line and parent_class_has_nnModule: + CD = ClassDefinition(class_name=cl.class_name, + file_path=cl.file_path, + class_def_line_idx=cl.class_def_line_idx, + parent_class_name=cl.parent_class_name) + CD.print_info() + globals.list_class_name.append(cl.class_name) + globals.list_class_def_instance.append(CD) + list_child_class_name.append(cl.class_name) + search_scope = list_child_class_name + if len(search_scope) == 0: + do_search = False + + # unique + globals.list_class_name = list(set(globals.list_class_name)) + + logger.debug(f"class name count: {len(globals.list_class_name)}") + logger.debug(f"class name list : {globals.list_class_name}") + + +# search nnModule instance definition +def register_nnModule_instance_definition(): + logger.info( + f"Analyzing nn.Module instance (model instance) definitions in all files ...") + # search model definition lines like "model_name = ClassName(xxx)" + def_cl = [] + for cl in globals.list_code_line_instance: + if not cl.is_multi_line_comment and not cl.is_single_line_comment_or_empty: + is_def, lhs, rhs = of_definition_format(cl.line_content) + if is_def and \ + rhs in globals.list_class_name + ["Module", "Sequential"] and \ + cl.class_name not in globals.list_class_name and \ + "(" not in cl.return_item: + def_cl.append(cl) + + list_lhs = [] + list_rhs = [] + list_is_in_func = [] + list_func_name = [] + list_return_item = [] + list_file_path = [] + list_line_idx = [] + list_func_def_line_idx = [] + for cl in def_cl: + is_def, lhs, rhs = of_definition_format(cl.line_content) + list_lhs.append(lhs) + list_rhs.append(rhs) + list_is_in_func.append(cl.is_in_func) + list_func_name.append(cl.func_name) + list_return_item.append(cl.return_item) + list_file_path.append(cl.file_path) + list_line_idx.append(cl.line_idx) + list_func_def_line_idx.append(cl.func_def_line_idx) + + # register qualified model's name of lines like "model_name = ClassName(xxx)" + globals.list_wrapper_base_function_name = [] + for i in range(len(list_lhs)): + # situation 1: "model = Net()" outside any function + if not list_is_in_func[i] and "tokenizer" not in list_lhs[i]: + # register this model + globals.list_model_name.append(list_lhs[i]) + MD = ModelDefinition(model_name=list_lhs[i], + class_name=list_rhs[i], + file_path=list_file_path[i], + model_def_line_idx=list_line_idx[i], + function_def_line_idx=-1, + function_name="null") # this MD is for all models defined outside a function + MD.print_info() + globals.list_model_def_instance.append(MD) + elif list_is_in_func[i]: # situation 2: "model = Net()" is inside a function + # situation 2-1: the function does not return another model's name, and is not __init__ + if list_return_item[i] not in list_lhs and \ + list_func_name[i] != "__init__" and "tokenizer" not in list_lhs[i]: + # register this model + globals.list_model_name.append(list_lhs[i]) + MD = ModelDefinition(model_name=list_lhs[i], + class_name=list_rhs[i], + file_path=list_file_path[i], + model_def_line_idx=list_line_idx[i], + function_def_line_idx=list_func_def_line_idx[i], + function_name=list_func_name[i]) \ + # this MD is for all models defined outside a function + MD.print_info() + globals.list_model_def_instance.append(MD) + # situation 2-2: the function returns another model's name + elif list_return_item[i] in list_lhs: + globals.list_wrapper_base_function_name.append( + list_func_name[i]) + + # register function_name like "xxx" in "def xxx() ... return NNModuleClass()" + for cl in globals.list_code_line_instance: + if cl.is_in_func and cl.line_idx == cl.func_return_idx \ + and cl.return_item[:cl.return_item.find("(")] in globals.list_class_name: + globals.list_wrapper_base_function_name.append(cl.func_name) + + # for all base function_name (that returns nnModule instance), + # find all wrapper function_name of the base wrapper function_name + globals.list_wrapper_base_function_name = list( + set(globals.list_wrapper_base_function_name)) + globals.list_wrapper_children_function_name = [] + for i in globals.list_wrapper_base_function_name: + globals.list_wrapper_children_function_name += get_all_wrap_children(i) + globals.list_wrapper_all_function_name = globals.list_wrapper_base_function_name + \ + globals.list_wrapper_children_function_name + globals.list_wrapper_all_function_name = list( + set(globals.list_wrapper_all_function_name)) + + # register function_name like "xxx" in "def xxx() ... model = some_wrapper_function() ... return model" + for cl in globals.list_code_line_instance: + if cl.is_in_func and not cl.is_multi_line_comment and not cl.is_single_line_comment_or_empty: + is_def, lhs, rhs = of_definition_format(cl.line_content) + if is_def and \ + rhs in globals.list_wrapper_all_function_name and \ + cl.class_name not in globals.list_class_name and \ + cl.return_item == lhs: + globals.list_wrapper_base_function_name.append(cl.func_name) + + # (again) + # for all base function_name (that returns nnModule instance), + # find all wrapper function_name of the base wrapper function_name + globals.list_wrapper_base_function_name = list( + set(globals.list_wrapper_base_function_name)) + for i in globals.list_wrapper_base_function_name: + globals.list_wrapper_children_function_name += get_all_wrap_children(i) + globals.list_wrapper_all_function_name += globals.list_wrapper_base_function_name + \ + globals.list_wrapper_children_function_name + globals.list_wrapper_all_function_name = list( + set(globals.list_wrapper_all_function_name)) + + # print all wrapper function names for debug purpose + logger.debug( + f"globals.list_wrapper_all_function_name: {globals.list_wrapper_all_function_name}") + + for cl in globals.list_code_line_instance: + if not cl.is_multi_line_comment and not cl.is_single_line_comment_or_empty and cl.func_name != "__init__": + is_def, lhs, rhs = of_definition_format(cl.line_content) + if is_def and \ + rhs in globals.list_wrapper_all_function_name and \ + rhs not in ["self.model", "model", "self.call", "call"] and \ + "forward" not in rhs and \ + "config" not in lhs and "congfig" not in lhs and "," not in lhs and \ + "inference" not in lhs and "tokenizer" not in lhs and \ + cl.class_name not in globals.list_class_name and \ + cl.func_name not in globals.list_wrapper_all_function_name: + # register this model + globals.list_model_name.append(lhs) + MD = ModelDefinition(model_name=lhs, + class_name="(note: this is a func-defined model)"+rhs, + file_path=cl.file_path, + model_def_line_idx=cl.line_idx, + function_def_line_idx=cl.func_def_line_idx, + function_name=cl.func_name) + # this MD is for all models defined by a wrapper function (e.g. model = models.resnet18()) + # model def can be outside any function, or in a function that is not itself a wrapper function + MD.print_info() + globals.list_model_def_instance.append(MD) + + globals.list_model_name = list(set(globals.list_model_name)) + + # print all model names for debug purpose + logger.debug(f"model name list: {globals.list_model_name}") diff --git a/neural_coder/interface.py b/neural_coder/interface.py new file mode 100644 index 00000000000..f1584cd12e4 --- /dev/null +++ b/neural_coder/interface.py @@ -0,0 +1,947 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import subprocess +import logging +import time + +from . import globals + +if not os.path.exists("neural_coder_workspace"): + os.makedirs("neural_coder_workspace") + + +def enable( + code, + features, + target_batch_size=1, # effective for feature "pytorch_change_batch_size" + num_benchmark_iteration=30, # effective for feature "pytorch_benchmark" + generate_patch=True, + overwrite=False, + # TO-ADD: return a folder with user-input code and artificial pip folders + brutal_mode=False, + save_patch_path="", + patch_suffix=".diff", + remove_copy=True, + consider_imports=True, + patch_imports=False, + logging_level="info", + run_bench=False, + entry_code="", + entry_code_args="", + mode="throughput", + cpu_set_env=True, + ncore_per_instance=-1, # only for "self_defined" mode + ninstances=-1, # only for "self_defined" mode + bench_batch_size=-1, # only for "self_defined" mode +): + + # Preparation + + # set up workspace + ws_path = "neural_coder_workspace/" + \ + "enable" + str(int(time.time())) + "/" + os.makedirs(ws_path) + + # user parameters + globals.consider_imports = consider_imports + logging_var = "logging." + logging_level.upper() + globals.logging_level = eval(logging_var) + + # set up logging + logger = logging.getLogger(ws_path) + logger.setLevel(globals.logging_level) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s: - %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + fh = logging.FileHandler(ws_path+'enable.log') + fh.setLevel(globals.logging_level) + fh.setFormatter(formatter) + ch = logging.StreamHandler() + ch.setLevel(globals.logging_level) + ch.setFormatter(formatter) + + logger.addHandler(fh) + logger.addHandler(ch) + + # print key inputs + logger.info(f"Enabling started ...") + logger.info(f"code: {code}") + logger.info(f"features: {features}") + + # feature list for reference + ''' + feature_list = ["pytorch_jit_script", + "pytorch_jit_script_ofi", + "pytorch_inc_dynamic_quant", + "pytorch_inc_static_quant", + "pytorch_ipex_fp32", + "pytorch_ipex_bf16", + "pytorch_ipex_int8_static_quant", + "pytorch_ipex_int8_dynamic_quant", + "pytorch_channels_last", + "pytorch_mixed_precision_cpu", + "pytorch_mixed_precision_cuda", + "pytorch_torchdynamo_jit_script", + "pytorch_torchdynamo_jit_script_ofi", + "pytorch_torchdynamo_jit_trace", + "pytorch_torchdynamo_jit_trace_ofi", + "pytorch_benchmark", + "pytorch_change_batch_size", + "pytorch_cuda_to_cpu", + "pytorch_lightning_bf16_cpu", + "tensorflow_amp", + "keras_amp",] + ''' + + # Benchmark + if run_bench: + # add "pytorch_change_batch_size" to features + from .utils.cpu_info import get_num_cpu_cores + ncores = get_num_cpu_cores() + if mode == "throughput": + target_batch_size = 2 * ncores + elif mode == "multi_instance": + target_batch_size = 1 + elif mode == "latency": + target_batch_size = 1 + elif mode == "self_defined": + target_batch_size = bench_batch_size + + if "pytorch_change_batch_size" not in features: + features.append("pytorch_change_batch_size") + + # add "pytorch_benchmark" to features + if "pytorch_benchmark" not in features: + features.append("pytorch_benchmark") + + logger.info( + f"Will perform benchmark on [{mode}] mode with batch size [{target_batch_size}] ...") + + # Rearrange Feature Order (due to certain regulations: e.g. channels_last is before IPEX, and IPEX is before JIT) + from .utils.common import move_element_to_front + for feature in ["pytorch_benchmark", + "pytorch_channels_last", + "pytorch_ipex_fp32", + "pytorch_ipex_bf16", + "pytorch_ipex_int8_static_quant", + "pytorch_ipex_int8_dynamic_quant", + "pytorch_jit_script", + "pytorch_jit_script_ofi", + "pytorch_torchdynamo_jit_script", + "pytorch_torchdynamo_jit_script_ofi", + "pytorch_torchdynamo_jit_trace", + "pytorch_torchdynamo_jit_trace_ofi", + "pytorch_inc_static_quant", + "pytorch_inc_dynamic_quant", + "pytorch_mixed_precision_cpu", + "pytorch_mixed_precision_cuda", ]: + features = move_element_to_front(features, feature) + + # Enabling + + transformed_list_code_path = [] + + for feature in features: + + # reset globals + globals.reset_globals() + + from .utils import handle_user_input + globals.list_code_path, num_user_code_path = handle_user_input.get_all_code_path( + code) + if len(transformed_list_code_path) > 0: + globals.list_code_path = transformed_list_code_path + + # common for all features (transformations), + list_transformed_code = [] + # in this list, each item stores the transformed code of the corresponding original code + # by the order in code_path + + # global behaviors + logger.info( + f"Performing code transformation for feature: [{feature}] ...") + + from .graphers.code_line import register_code_line + from .graphers.model import register_nnModule_class, register_nnModule_instance_definition + from .graphers.function import register_func_wrap_pair + from .coders.transform import execute_insert_transformation, execute_indenting_transformation + + register_code_line() + register_func_wrap_pair() + register_nnModule_class() + register_nnModule_instance_definition() + + # AMP + if "pytorch_mixed_precision_cpu" == feature: + from .coders.pytorch.amp import PTAMP + opt = PTAMP("cpu") + opt.register_transformation() + elif "pytorch_mixed_precision_cuda" == feature: + from .coders.pytorch.amp import PTAMP + opt = PTAMP("cuda") + opt.register_transformation() + + # TorchDynamo + if "pytorch_torchdynamo_jit_script" == feature: + from .coders.pytorch.torchdynamo import TorchDynamo + opt = TorchDynamo() + opt.register_transformation() + from .coders.pytorch.torchdynamo import TorchDynamoJITScript + opt = TorchDynamoJITScript( + globals.list_model_def_instance, "plain") + opt.register_transformation() + elif "pytorch_torchdynamo_jit_script_ofi" == feature: + from .coders.pytorch.torchdynamo import TorchDynamo + opt = TorchDynamo() + opt.register_transformation() + from .coders.pytorch.torchdynamo import TorchDynamoJITScript + opt = TorchDynamoJITScript(globals.list_model_def_instance, "ofi") + opt.register_transformation() + elif "pytorch_torchdynamo_jit_trace" == feature: + from .coders.pytorch.torchdynamo import TorchDynamo + opt = TorchDynamo() + opt.register_transformation() + from .coders.pytorch.torchdynamo import TorchDynamoJITTrace + opt = TorchDynamoJITTrace(globals.list_model_def_instance, "plain") + opt.register_transformation() + elif "pytorch_torchdynamo_jit_trace_ofi" == feature: + from .coders.pytorch.torchdynamo import TorchDynamo + opt = TorchDynamo() + opt.register_transformation() + from .coders.pytorch.torchdynamo import TorchDynamoJITTrace + opt = TorchDynamoJITTrace(globals.list_model_def_instance, "ofi") + opt.register_transformation() + + # Channels Last + if "pytorch_channels_last" == feature: + from .coders.pytorch.channels_last import ChannelsLast + opt = ChannelsLast(globals.list_model_def_instance) + opt.register_transformation() + + # JIT + if "pytorch_jit_script" == feature: + from .coders.pytorch.jit import JITScript + opt = JITScript(globals.list_model_def_instance, "plain") + opt.register_transformation() + + elif "pytorch_jit_script_ofi" == feature: + from .coders.pytorch.jit import JITScript + opt = JITScript(globals.list_model_def_instance, "ofi") + opt.register_transformation() + + # IPEX + if "pytorch_ipex_fp32" == feature: + from .coders.pytorch.intel_extension_for_pytorch.ipex import IPEX + opt = IPEX(globals.list_model_def_instance, "fp32") + opt.register_transformation() + + elif "pytorch_ipex_bf16" == feature: + from .coders.pytorch.intel_extension_for_pytorch.ipex import IPEX + opt = IPEX(globals.list_model_def_instance, "bf16") + opt.register_transformation() + from .coders.pytorch.amp import PTAMP # enable amp together + opt = PTAMP("cpu") + opt.register_transformation() + + elif "pytorch_ipex_int8_static_quant" == feature: + from .coders.pytorch.intel_extension_for_pytorch.ipex import IPEX + opt = IPEX(globals.list_model_def_instance, "int8_static_quant") + opt.register_transformation() + + elif "pytorch_ipex_int8_dynamic_quant" == feature: + from .coders.pytorch.intel_extension_for_pytorch.ipex import IPEX + opt = IPEX(globals.list_model_def_instance, "int8_dynamic_quant") + opt.register_transformation() + + # INC + if "pytorch_inc_dynamic_quant" == feature: + from .coders.pytorch.neural_compressor.dynamic_quant import DynamicQuant + opt = DynamicQuant(globals.list_model_def_instance) + opt.register_transformation() + + elif "pytorch_inc_static_quant" == feature: + from .coders.pytorch.dummy_dataloader import DummyDataLoader # detect dataloader first + opt = DummyDataLoader(globals.list_model_def_instance) + opt.register_transformation() + from .coders.pytorch.neural_compressor.static_quant import StaticQuant + opt = StaticQuant(globals.list_model_def_instance) + opt.register_transformation() + + # Benchmark + if "pytorch_benchmark" == feature: + from .coders.pytorch.benchmark import Benchmark + globals.num_benchmark_iteration = str(num_benchmark_iteration) + opt = Benchmark() + opt.register_transformation() + + # transformation execution + for i in globals.list_code_path: + list_transformed_code.append(open(i, 'r').read()) + list_transformed_code = execute_indenting_transformation( + list_transformed_code) + list_transformed_code = execute_insert_transformation( + list_transformed_code) + + # other features (which use direct line transform instead of register-and-execute transform, + # these features will be transformed here) + for i in range(len(list_transformed_code)): + # Batch Size + if "pytorch_change_batch_size" == feature: + from .coders.pytorch.batch_size import BatchSizeCoder + globals.target_batch_size = str(target_batch_size) + list_transformed_code[i] = BatchSizeCoder( + list_transformed_code[i]).transform() + + # CUDA to CPU + if "pytorch_cuda_to_cpu" == feature: + from .coders.pytorch.cuda_to_cpu import CudaToCpu + list_transformed_code[i] = CudaToCpu( + list_transformed_code[i]).transform() + + # Lightning + if "pytorch_lightning_bf16_cpu" == feature: + from .coders.pytorch.lightning import Lightning + list_transformed_code[i] = Lightning( + list_transformed_code[i]).transform() + + # TF & Keras AMP + if "tensorflow_mixed_precision" == feature: + from .coders.tensorflow.amp import TensorFlowKerasAMP + list_transformed_code[i] = TensorFlowKerasAMP( + list_transformed_code[i]).transform() + + logger.info(f"Code transformation for feature: [{feature}] finished.") + + for path in globals.list_code_path: + if path[-14:] == "_nc_enabled.py": + path_transformed = path + else: + path_transformed = path[:-3] + "_nc_enabled.py" + open(path_transformed, "w").write( + list_transformed_code[globals.list_code_path.index(path)]) + globals.list_code_path[globals.list_code_path.index( + path)] = path_transformed + transformed_list_code_path = globals.list_code_path + + # Output of Enabling + + globals.list_code_path, num_user_code_path = handle_user_input.get_all_code_path( + code) + + if save_patch_path == "": + save_patch_path = ws_path + + if generate_patch: + whole_patch_user_code = "" + for path in globals.list_code_path[0:num_user_code_path]: + path_transformed = path[:-3] + "_nc_enabled.py" + cmd_gen_patch = "diff -up " + path + " " + path_transformed + sp_gen_patch = subprocess.Popen( + cmd_gen_patch, env=os.environ, shell=True, stdout=subprocess.PIPE) # nosec + sp_gen_patch.wait() + this_patch, _ = sp_gen_patch.communicate() + this_patch = str(this_patch)[2:-1] + whole_patch_user_code += this_patch + open(save_patch_path + "neural_coder_patch" + patch_suffix, "w").write( + whole_patch_user_code.replace(r'\n', '\n').replace(r'\t', '\t').replace(r"\'", "\'")) + abs_patch_path = os.path.abspath( + save_patch_path + "neural_coder_patch" + patch_suffix) + logger.info(f"The patch is saved to: [{abs_patch_path}]") + + if overwrite: + sp_overwrite = subprocess.Popen( + "patch -d/ -p0 < " + abs_patch_path, env=os.environ, shell=True, stdout=subprocess.PIPE) # nosec + sp_overwrite.wait() + os.remove(abs_patch_path) # remove patch after overwrite + + if patch_imports: + whole_patch_import_modules = "" + for path in globals.list_code_path[num_user_code_path:]: + path_transformed = path[:-3] + "_nc_enabled.py" + cmd_gen_patch = "diff -up " + path + " " + path_transformed + sp_gen_patch = subprocess.Popen( + cmd_gen_patch, env=os.environ, shell=True, stdout=subprocess.PIPE) # nosec + sp_gen_patch.wait() + this_patch, _ = sp_gen_patch.communicate() + this_patch = str(this_patch)[2:-1] + whole_patch_import_modules += this_patch + open(save_patch_path + "neural_coder_patch_import_modules" + patch_suffix, "w").write( + whole_patch_import_modules.replace(r'\n', '\n').replace(r'\t', '\t').replace(r"\'", "\'")) + abs_patch_path = os.path.abspath( + save_patch_path + "neural_coder_patch_import_modules" + patch_suffix) + logger.info( + f"The patch for imported modules is saved to: [{abs_patch_path}]") + + # remove copy for imports + if remove_copy: + for path in globals.list_code_path: + try: + path_transformed = path[:-3] + "_nc_enabled.py" + os.remove(path_transformed) + except: + pass + + # Benchmark + if run_bench: + bench_performance, bench_mode, bench_ws_path = bench( + code=code, + entry_code=entry_code, + entry_code_args=entry_code_args, + patch_path=abs_patch_path, + mode=mode, + cpu_set_env=cpu_set_env, + ncore_per_instance=ncore_per_instance, # only for "self_defined" mode + ninstances=ninstances, # only for "self_defined" mode + bench_batch_size=bench_batch_size, # only for "self_defined" mode + ) + + return bench_performance, bench_mode, bench_ws_path + + +''' +bench API works on either "optimized code", or "patch" + "original code" +it does not enable benchmark, or enable change of batch size, all the enabling is done in enable API +which means the "optimized code" should already have "pytorch_benchmark" and "pytorch_change_batch_size" enabled +or the "patch" should already have the code modification for "pytorch_benchmark" and +"pytorch_change_batch_size" in it +''' + + +def bench( + code, + entry_code="", + entry_code_args="", + patch_path="", + mode="throughput", # throughput, latency, multi_instance or self_defined + logging_level="info", + cpu_set_env=True, + ncore_per_instance=-1, # only for "self_defined" mode + ninstances=-1, # only for "self_defined" mode + bench_batch_size=-1, # only for "self_defined" mode +): + + # set up workspace + ws_path = "neural_coder_workspace/" + "bench" + str(int(time.time())) + "/" + os.makedirs(ws_path) + + # set up logging + logging_var = "logging." + logging_level.upper() + globals.logging_level = eval(logging_var) + + logger = logging.getLogger(ws_path) + logger.setLevel(globals.logging_level) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s: - %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + fh = logging.FileHandler(ws_path+'bench.log') + fh.setLevel(globals.logging_level) + fh.setFormatter(formatter) + ch = logging.StreamHandler() + ch.setLevel(globals.logging_level) + ch.setFormatter(formatter) + + logger.addHandler(ch) + logger.addHandler(fh) + + # print key inputs + logger.info(f"Benchmarking started ...") + logger.info(f"code: {code}") + logger.info(f"mode: {mode}") + + # entry code + if entry_code == "": + # if not specify entry_code, then code has to be a list of one element, + # or a single string of single path, otherwise quit + if type(code) == list and len(code) == 1: + entry_code = code[0] + elif type(code) == str: + entry_code = code + else: + logger.error( + f"You have to specify an entry_code of your code: [{code}]") + quit() + + # patch + if patch_path != "": + sp_patch = subprocess.Popen("patch -d/ -p0 < " + patch_path, + env=os.environ, shell=True, stdout=subprocess.PIPE) # nosec + sp_patch.wait() + + # if mode is "self_defined", user must specify ncpi, nins and bs + if mode == "self_defined": + if ncore_per_instance == -1 or ninstances == -1 or bench_batch_size == -1: + logger.error( + f"You have to specify ncore_per_instance, ninstances and bench_batch_size for \ + self-defined benchmark mode.") + quit() + + # numactl + from . import numa_launcher + + from .utils.cpu_info import get_num_cpu_cores + ncores = get_num_cpu_cores() + + # numactl setup for different modes + if mode == "throughput": + ncore_per_instance = ncores + ninstances = 1 + bench_batch_size = 2 * ncores + elif mode == "multi_instance": + ncore_per_instance = 4 + ninstances = int(ncores / ncore_per_instance) + bench_batch_size = 1 + elif mode == "latency": + ncore_per_instance = 1 + ninstances = ncores + bench_batch_size = 1 + elif mode == "self_defined": + ncore_per_instance = ncore_per_instance + ninstances = ninstances + bench_batch_size = bench_batch_size + + # set cpu env variables + if cpu_set_env: + cmd_env = '' + cmd_env += 'export LD_PRELOAD=${CONDA_PREFIX}/lib/libjemalloc.so' + cmd_env += ' && ' + cmd_env += 'export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libiomp5.so' + cmd_env += ' && ' + cmd_env += 'export MALLOC_CONF="oversize_threshold:1,background_thread:true,metadata_thp:auto,\ + dirty_decay_ms:9000000000,muzzy_decay_ms:9000000000"' + cmd_env += ' && ' + cmd_env += 'export KMP_AFFINITY="granularity=fine,compact,1,0"' + cmd_env += ' && ' + cmd_env += 'export KMP_BLOCKTIME=1' + cmd_env += ' && ' + cmd_env += 'export DNNL_PRIMITIVE_CACHE_CAPACITY=1024' + cmd_env += ' && ' + cmd_env += 'export KMP_SETTINGS=1' + + sp_set_env = subprocess.Popen( + cmd_env, env=os.environ, shell=True, stdout=subprocess.PIPE) # nosec + sp_set_env.wait() + + # benchmark + logger.info(f"Start benchmark on the code ...") + + bench_log_path = ws_path + "performance.log" + os.remove(bench_log_path) if os.path.exists(bench_log_path) else 0 + + entry_code_args = [entry_code_args] + numa_launcher.exec_launcher( + ncore_per_instance, ninstances, entry_code, entry_code_args, bench_log_path) + + # get performance (throughput and latency) + bench_log = open(bench_log_path, "r").read().split('\n') + IPS = 0 + MSPI = 0 + count_MSPI = 0 + P50 = 0 + count_P50 = 0 + P90 = 0 + count_P90 = 0 + P99 = 0 + count_P99 = 0 + for line in bench_log: + if "Neural_Coder_Bench_IPS" in line: + try: + IPS += float(line[line.find(":")+3:]) + except ValueError as ve: + pass + if "Neural_Coder_Bench_MSPI" in line: + try: + MSPI += float(line[line.find(":")+3:]) + count_MSPI += 1 + except ValueError as ve: + pass + if "Neural_Coder_Bench_P50" in line: + try: + P50 += float(line[line.find(":")+3:]) + count_P50 += 1 + except ValueError as ve: + pass + if "Neural_Coder_Bench_P90" in line: + try: + P90 += float(line[line.find(":")+3:]) + count_P90 += 1 + except ValueError as ve: + pass + if "Neural_Coder_Bench_P99" in line: + try: + P99 += float(line[line.find(":")+3:]) + count_P99 += 1 + except ValueError as ve: + pass + + FPS = round(IPS * bench_batch_size, 3) + try: + MSPI = round(MSPI / count_MSPI, 3) + except: + MSPI = 0 + try: + P50 = round(P50 / count_P50, 3) + except: + MSPI = 0 + try: + P90 = round(P90 / count_P90, 3) + except: + MSPI = 0 + try: + P99 = round(P99 / count_P99, 3) + except: + MSPI = 0 + + logger.info(f"Collected throughput on the code is: [{FPS}] (fps)") + logger.info(f"Collected latency on the code is: [{MSPI}] (mspi)") + logger.info(f"Collected latency_p50 on the code is: [{P50}] (mspi)") + logger.info(f"Collected latency_p90 on the code is: [{P90}] (mspi)") + logger.info(f"Collected latency_p99 on the code is: [{P99}] (mspi)") + + # unpatch + if patch_path != "": + sp_unpatch = subprocess.Popen( + "patch -R -d/ -p0 < " + patch_path, env=os.environ, shell=True, stdout=subprocess.PIPE) # nosec + sp_unpatch.wait() + + return [FPS, MSPI, P50, P90, P99], mode, os.path.abspath(ws_path) + + +def superbench( + code, + entry_code="", + entry_code_args="", + sweep_objective="feature", + bench_feature=[], + mode="throughput", + num_benchmark_iteration=30, + logging_level="info", + cpu_conversion=True, + cpu_set_env=True, + ncore_per_instance=-1, # only for "self_defined" mode + ninstances=-1, # only for "self_defined" mode + bench_batch_size=-1, # only for "self_defined" mode +): + + # set up workspace + ws_path = "neural_coder_workspace/" + \ + "superbench" + str(int(time.time())) + "/" + os.makedirs(ws_path) + + # set up logging + logging_var = "logging." + logging_level.upper() + globals.logging_level = eval(logging_var) + + logger = logging.getLogger(ws_path) + logger.setLevel(globals.logging_level) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s: - %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + fh = logging.FileHandler(ws_path+'superbench.log') + fh.setLevel(globals.logging_level) + fh.setFormatter(formatter) + ch = logging.StreamHandler() + ch.setLevel(globals.logging_level) + ch.setFormatter(formatter) + + logger.addHandler(ch) + logger.addHandler(fh) + + # print key inputs + logger.info(f"Superbench started ...") + logger.info(f"code: {code}") + logger.info(f"mode: {mode}") + logger.info(f"sweep_objective: {sweep_objective}") + logger.info(f"num_benchmark_iteration: {num_benchmark_iteration}") + + # entry code + if entry_code == "": + # if not specify entry_code, then code has to be a list of one element, + # or a single string of single path, otherwise quit + if type(code) == list and len(code) == 1: + entry_code = code[0] + elif type(code) == str: + entry_code = code + else: + logger.error( + f"You have to specify an entry_code of your code: [{code}]") + quit() + + if sweep_objective == "feature": + list_FPS = [] + list_features = [] + list_mode = [] + list_ws_path = [] + result = [] + + # features that is a "backend": + backends = ["", + "pytorch_ipex_fp32", + "pytorch_ipex_bf16", + "pytorch_ipex_int8_static_quant", + "pytorch_ipex_int8_dynamic_quant", + "pytorch_inc_static_quant", + "pytorch_inc_dynamic_quant", + ] + + # features that can be standalone (either use alone or use with "backend"): + standalones_pool = ["pytorch_channels_last", + "pytorch_mixed_precision_cpu", + "pytorch_jit_script", + "pytorch_jit_script_ofi", + "pytorch_torchdynamo_jit_script", + "pytorch_torchdynamo_jit_script_ofi", + "pytorch_torchdynamo_jit_trace", + "pytorch_torchdynamo_jit_trace_ofi", + ] + + if logging_level == "debug": + # features that is a "backend": + backends = ["", + "pytorch_ipex_fp32", + "pytorch_inc_static_quant", + ] + + # features that can be standalone (either use alone or use with "backend"): + standalones_pool = ["pytorch_channels_last", + ] + + standalones = [] + standalones.append("") + from itertools import combinations + for num_items in range(len(standalones_pool)): + list_comb = list(combinations(standalones_pool, num_items + 1)) + for item in list_comb: + jit_feature_count = 0 + for i in list(item): + if "jit" in i: + jit_feature_count += 1 + if jit_feature_count <= 1: + standalones.append(list(item)) # only appends one JIT item + + for backend in backends: + for standalone in standalones: + features = [] + features.append(backend) + features += standalone + + # exclude conflict features (like jit and jit_ofi) + if "pytorch_ipex_fp32" in features and "pytorch_mixed_precision_cpu" in features: + continue + if "pytorch_ipex_bf16" in features and "pytorch_mixed_precision_cpu" in features: + continue + if "pytorch_ipex_int8_static_quant" in features and "pytorch_mixed_precision_cpu" in features: + continue + if "pytorch_ipex_int8_dynamic_quant" in features and "pytorch_mixed_precision_cpu" in features: + continue + if "pytorch_inc_static_quant" in features and "pytorch_mixed_precision_cpu" in features: + continue + if "pytorch_inc_dynamic_quant" in features and "pytorch_mixed_precision_cpu" in features: + continue + + if cpu_conversion: + features.append("pytorch_cuda_to_cpu") + + if features[0] == "" and len(features) > 1: + features = features[1:] # remove "" + + bench_performance, bench_mode, bench_ws_path = enable( + code=code, + entry_code=entry_code, + entry_code_args=entry_code_args, + features=features, + mode=mode, + run_bench=True, + num_benchmark_iteration=num_benchmark_iteration, + cpu_set_env=cpu_set_env, + ncore_per_instance=ncore_per_instance, + ninstances=ninstances, + bench_batch_size=bench_batch_size, + ) + + def remove_if_have(list, element): + if element in list: + list.remove(element) + return list + + features = remove_if_have(features, "pytorch_benchmark") + features = remove_if_have( + features, "pytorch_change_batch_size") + features = remove_if_have(features, "pytorch_cuda_to_cpu") + + logger.info( + f"Benchmark result of acceleration set [{features}] is [{bench_performance[0]}] (FPS)") + + d = {} # initialize dict + d["features"] = features + d["FPS"] = bench_performance[0] + d["mode"] = bench_mode + d["workspace_path"] = bench_ws_path + result.append(d) + + list_FPS.append(bench_performance[0]) + list_features.append(features) + list_mode.append(bench_mode) + list_ws_path.append(bench_ws_path) + + # print result + logger.info( + f"Superbench result of sweeping [{sweep_objective}] printed below with sorted FPS: ") + print("{:<20} {:<20} {:<120}".format( + 'Numactl Mode', 'Performance (FPS)', 'Features Applied')) + sort_index = sorted(range(len(list_FPS)), + key=lambda k: list_FPS[k], reverse=True) + for i in sort_index: + if list_FPS[i] != 0: + print("{:<20} {:<20} {:<120}".format( + str(list_mode[i]), str(list_FPS[i]), str(list_features[i]))) + + # for superbench report generation + list_optimization_set_top3 = [] + list_performance_top3 = [] + count_top3 = 0 + for i in sort_index: + if list_FPS[i] != 0: + list_performance_top3.append(list_FPS[i]) + list_optimization_set_top3.append(list_features[i]) + count_top3 += 1 + if count_top3 == 3: + break + + original_model_performance = 0 + original_model_ranking = 0 + for i in sort_index: + if list_FPS[i] != 0: + original_model_ranking += 1 + if list_features[i] == []: + original_model_performance = list_FPS[i] + break + + return list_optimization_set_top3, \ + list_performance_top3, original_model_ranking, original_model_performance + + elif sweep_objective == "bench_config": + result_ncpi = [] + result_nins = [] + result_bs = [] + result_regular_thp = [] + result_p50_thp = [] + result_p90_thp = [] + result_p99_thp = [] + if bench_feature == []: + logger.error( + f'You must specify a feature (optimization set) for benchmark when "sweep_objective" \ + is "bench_config"') + quit() + else: + from .utils.cpu_info import get_num_cpu_cores + ncores = get_num_cpu_cores() + list_ncpi = [1, 2, 4, 8] + for i in [1, 2, 4, 8]: + list_ncpi.append(int(ncores / i)) + list_ncpi = list(set(list_ncpi)) + list_ncpi.sort() + logger.debug(f"list_ncpi = {list_ncpi}") + + for this_ncpi in list_ncpi: + ncore_per_instance = this_ncpi + ninstances = int(ncores / this_ncpi) + list_bs = [1, 2, 4, 8, this_ncpi * 1, this_ncpi * 2, this_ncpi * + 4, this_ncpi * 8, this_ncpi * 16, this_ncpi * 32, this_ncpi * 64] + list_bs = list(set(list_bs)) + list_bs.sort() + if logging_level == "debug": + list_bs = [list_bs[-5]] + logger.debug(f"this_ncpi = {this_ncpi}") + logger.debug(f"list_bs = {list_bs}") + for this_bs in list_bs: + bench_batch_size = this_bs + try: + bench_performance, bench_mode, bench_ws_path = enable( + code=code, + entry_code=entry_code, + entry_code_args=entry_code_args, + features=bench_feature, + mode="self_defined", # sweep bench_config, so mode set to "self_defined" + run_bench=True, + num_benchmark_iteration=num_benchmark_iteration, + cpu_set_env=cpu_set_env, + ncore_per_instance=ncore_per_instance, + ninstances=ninstances, + bench_batch_size=bench_batch_size, + ) + + socket_regular_thp = bench_performance[0] + socket_p50_thp = round( + 1000 / bench_performance[2] * ninstances * bench_batch_size, 3) + socket_p90_thp = round( + 1000 / bench_performance[3] * ninstances * bench_batch_size, 3) + socket_p99_thp = round( + 1000 / bench_performance[4] * ninstances * bench_batch_size, 3) + + result_ncpi.append(ncore_per_instance) + result_nins.append(ninstances) + result_bs.append(bench_batch_size) + result_regular_thp.append(socket_regular_thp) + result_p50_thp.append(socket_p50_thp) + result_p90_thp.append(socket_p90_thp) + result_p99_thp.append(socket_p99_thp) + + logger.info( + f"ncpi: {ncore_per_instance}, nins: {ninstances}, bs: {bench_batch_size}, \ + regular_thp: {socket_regular_thp}, p50_thp: {socket_p50_thp}, \ + p90_thp: {socket_p90_thp}, p99_thp: {socket_p99_thp}") + + except: + logger.warning( + f"ncpi: {ncore_per_instance}, nins: {ninstances}, bs: {bench_batch_size}, Benchmark \ + failed. It might be due to HW limitation such as CPU load limit.") + continue + + # print result + for item in [result_regular_thp, result_p50_thp, result_p90_thp, result_p99_thp]: + if item is result_regular_thp: + display_item_name = "Throughput" + elif item is result_p50_thp: + display_item_name = "Throughput based on P50-Latency" + elif item is result_p90_thp: + display_item_name = "Throughput based on P90-Latency" + elif item is result_p99_thp: + display_item_name = "Throughput based on P99-Latency" + + print("{:<30} {:<30} {:<30} {:<30}".format( + 'Num Cores Per Instance', 'Num of Instances', 'Batch Size', display_item_name)) + sort_index = sorted( + range(len(item)), key=lambda k: item[k], reverse=True) + for i in sort_index: + print("{:<30} {:<30} {:<30} {:<30}".format(str(result_ncpi[i]), str( + result_nins[i]), str(result_bs[i]), str(item[i]))) + + list_config_best_ncpi = [] + list_config_best_nins = [] + list_config_best_bs = [] + list_config_best_performance = [] + for item in [result_regular_thp, result_p50_thp, result_p90_thp, result_p99_thp]: + sort_index = sorted( + range(len(item)), key=lambda k: item[k], reverse=True) + for i in sort_index: + list_config_best_ncpi.append(result_ncpi[i]) + list_config_best_nins.append(result_nins[i]) + list_config_best_bs.append(result_bs[i]) + list_config_best_performance.append(item[i]) + break # only fetch the top result + + return list_config_best_ncpi, list_config_best_nins, list_config_best_bs, list_config_best_performance diff --git a/neural_coder/numa_launcher.py b/neural_coder/numa_launcher.py new file mode 100644 index 00000000000..6d4450a0f2c --- /dev/null +++ b/neural_coder/numa_launcher.py @@ -0,0 +1,807 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import absolute_import, division, print_function, unicode_literals +import sys +import platform +import subprocess +import os +from os.path import expanduser +import re +import glob +import numpy as np +from argparse import ArgumentParser, REMAINDER +from argparse import RawTextHelpFormatter +import logging +import psutil +from datetime import datetime + +format_str = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +logging.basicConfig(level=logging.INFO, format=format_str) +logger = logging.getLogger(__name__) + + +class CPUinfo(): + ''' + Get CPU inforamation, such as cores list and NUMA information. + ''' + + def __init__(self): + + self.cpuinfo = [] + if platform.system() == "Windows": + raise RuntimeError("Windows platform is not supported!!!") + elif platform.system() == "Linux": + args = ["lscpu", "--parse=CPU,Core,Socket,Node"] + lscpu_info = subprocess.check_output( + args, universal_newlines=True).split("\n") + + # Get information about cpu, core, socket and node + for line in lscpu_info: + pattern = r"^([\d]+,[\d]+,[\d]+,[\d]?)" + regex_out = re.search(pattern, line) + if regex_out: + self.cpuinfo.append(regex_out.group(1).strip().split(",")) + self.get_socket_info() + + def get_socket_info(self): + self.sockets = int(max([line[2] for line in self.cpuinfo])) + 1 + self.socket_physical_cores = [] # socket_id is index + self.socket_logical_cores = [] # socket_id is index + self.physical_core_socket_map = {} # phyical core to numa node id + self.logical_core_socket_map = {} # logical core to numa node id + + self.nodes = int(max([line[3] for line in self.cpuinfo])) + 1 + self.node_physical_cores = [] # node_id is index + self.node_logical_cores = [] # node_id is index + self.physical_core_node_map = {} # phyical core to numa node id + self.logical_core_node_map = {} # logical core to numa node id + + for socket_id in range(self.sockets): + cur_socket_physical_core = [] + cur_socket_logical_core = [] + for line in self.cpuinfo: + if socket_id == int(line[2]): + if int(line[1]) not in cur_socket_physical_core: + cur_socket_physical_core.append(int(line[1])) + self.physical_core_socket_map[int( + line[1])] = int(socket_id) + cur_socket_logical_core.append(int(line[0])) + self.logical_core_socket_map[int(line[0])] = int(socket_id) + self.socket_physical_cores.append(cur_socket_physical_core) + self.socket_logical_cores.append(cur_socket_logical_core) + + for node_id in range(self.nodes): + cur_node_physical_core = [] + cur_node_logical_core = [] + for line in self.cpuinfo: + nid = line[3] if line[3] != '' else '0' + if node_id == int(nid): + if int(line[1]) not in cur_node_physical_core: + cur_node_physical_core.append(int(line[1])) + self.physical_core_node_map[int( + line[1])] = int(node_id) + cur_node_logical_core.append(int(line[0])) + self.logical_core_node_map[int(line[0])] = int(node_id) + self.node_physical_cores.append(cur_node_physical_core) + self.node_logical_cores.append(cur_node_logical_core) + + def socket_nums(self): + return self.sockets + + def node_nums(self): + return self.nodes + + def physical_core_nums(self): + return len(self.node_physical_cores) * len(self.node_physical_cores[0]) + + def logical_core_nums(self): + return len(self.node_logical_cores) * len(self.node_logical_cores[0]) + + def get_node_physical_cores(self, node_id): + if node_id < 0 or node_id > self.nodes - 1: + logger.error("Invalid node id") + return self.node_physical_cores[node_id] + + def get_node_logical_cores(self, node_id): + if node_id < 0 or node_id > self.nodes - 1: + logger.error("Invalid node id") + return self.node_logical_cores[node_id] + + def get_all_physical_cores(self): + return np.array(self.node_physical_cores).flatten().tolist() + + def get_all_logical_cores(self): + return np.array(self.node_logical_cores).flatten().tolist() + + def numa_aware_check(self, core_list): + ''' + Check whether all cores in core_list are in the same NUMA node. cross NUMA will reduce perforamnce. + We strongly advice to not use cores on different nodes. + ''' + cores_numa_map = self.logical_core_node_map + if len(core_list) < 1: + return True + numa_ids = [] + for core in core_list: + numa_id = cores_numa_map[core] + if not numa_id in numa_ids: + numa_ids.append(numa_id) + if len(numa_ids) > 1: + logger.warning("Numa Aware: cores:{} on different NUMA nodes:{}".format( + str(core_list), str(numa_ids))) + return numa_ids + + +class Launcher(): + r""" + Base class for launcher + """ + + def __init__(self): + self.cpuinfo = CPUinfo() + + def launch(self, args): + pass + + def add_lib_preload(self, lib_type=None): + ''' + Enale TCMalloc/JeMalloc/intel OpenMP + ''' + library_paths = [] + if "CONDA_PREFIX" in os.environ: + library_paths.append(os.environ["CONDA_PREFIX"] + "/lib/") + if "VIRTUAL_ENV" in os.environ: + library_paths.append(os.environ["VIRTUAL_ENV"] + "/lib/") + + library_paths += ["{}/.local/lib/".format(expanduser("~")), "/usr/local/lib/", + "/usr/local/lib64/", "/usr/lib/", "/usr/lib64/"] + + lib_find = False + lib_set = False + for item in os.getenv("LD_PRELOAD", "").split(":"): + if item.endswith('lib{}.so'.format(lib_type)): + lib_set = True + break + if not lib_set: + for lib_path in library_paths: + library_file = lib_path + "lib" + lib_type + ".so" + matches = glob.glob(library_file) + if len(matches) > 0: + if "LD_PRELOAD" in os.environ: + os.environ["LD_PRELOAD"] = matches[0] + \ + ":" + os.environ["LD_PRELOAD"] + else: + os.environ["LD_PRELOAD"] = matches[0] + lib_find = True + break + return lib_set or lib_find + + def set_memory_allocator(self, enable_tcmalloc=True, enable_jemalloc=False, use_default_allocator=False): + ''' + Enable TCMalloc/JeMalloc with LD_PRELOAD and set configuration for JeMalloc. + By default, PTMalloc will be used for PyTorch, but TCMalloc and JeMalloc can get better + memory resue and reduce page fault to improve performance. + ''' + if enable_tcmalloc and enable_jemalloc: + logger.error( + "Unable to enable TCMalloc and JEMalloc at the same time") + exit(-1) + + if enable_tcmalloc: + find_tc = self.add_lib_preload(lib_type="tcmalloc") + if not find_tc: + logger.warning("Unable to find the {} library file lib{}.so in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib" + " or /.local/lib/ or /usr/local/lib/ or \ + /usr/local/lib64/ or /usr/lib or /usr/lib64 or " + "{}/.local/lib/ so the LD_PRELOAD environment variable will not be set." + "you can use 'conda install -c conda-forge gperftools' to install tcmalloc" + .format("TCmalloc", "tcmalloc", expanduser("~"))) + else: + logger.info("Use TCMalloc memory allocator") + + elif enable_jemalloc: + find_je = self.add_lib_preload(lib_type="jemalloc") + if not find_je: + logger.warning("Unable to find the {} library file lib{}.so in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib" + " or /.local/lib/ or /usr/local/lib/ or \ + /usr/local/lib64/ or /usr/lib or /usr/lib64 or " + "{}/.local/lib/ so the LD_PRELOAD environment variable will not be set." + "you can use 'conda install -c conda-forge jemalloc' to install jemalloc" + .format("JeMalloc", "jemalloc", expanduser("~"))) + else: + logger.info("Use JeMalloc memory allocator") + self.set_env( + 'MALLOC_CONF', "oversize_threshold:1,background_thread:true,metadata_thp:auto") + + elif use_default_allocator: + pass + + else: + find_tc = self.add_lib_preload(lib_type="tcmalloc") + if find_tc: + logger.info("Use TCMalloc memory allocator") + return + find_je = self.add_lib_preload(lib_type="jemalloc") + if find_je: + logger.info("Use JeMalloc memory allocator") + return + logger.warning("Neither TCMalloc nor JeMalloc is found in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib" + " or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or " + "{}/.local/lib/ so the LD_PRELOAD environment variable will not be set. \ + This may drop the performance" + .format(expanduser("~"))) + + def logger_env(self, env_name=""): + if env_name in os.environ: + logger.info("{}={}".format(env_name, os.environ[env_name])) + + def set_env(self, env_name, env_value=None): + if not env_value: + logger.warning("{} is None".format(env_name)) + if env_name not in os.environ: + os.environ[env_name] = env_value + elif os.environ[env_name] != env_value: + logger.warning("{} in environment variable is {} while the value you set is {}".format( + env_name, os.environ[env_name], env_value)) + logger.warning("Resetting {} to {}".format(env_name, env_value)) + os.environ[env_name] = env_value + self.logger_env(env_name) + + # set_kmp_affinity is used to control whether to set KMP_AFFINITY or not. + # In scenario that use all cores on all nodes, including logical cores, + # setting KMP_AFFINITY disables logical cores. In this case, KMP_AFFINITY should not be set. + def set_multi_thread_and_allocator(self, + ncore_per_instance, + disable_iomp=False, + set_kmp_affinity=True, + enable_tcmalloc=True, + enable_jemalloc=False, + use_default_allocator=False): + ''' + Set multi-thread configuration and enable Intel openMP and TCMalloc/JeMalloc. + By default, GNU openMP and PTMalloc are used in PyTorch. + but Intel openMP and TCMalloc/JeMalloc are better alternatives + to get performance benifit. + ''' + self.set_memory_allocator( + enable_tcmalloc, enable_jemalloc, use_default_allocator) + self.set_env("OMP_NUM_THREADS", str(ncore_per_instance)) + if not disable_iomp: + find_iomp = self.add_lib_preload(lib_type="iomp5") + if not find_iomp: + logger.warning("Unable to find the {} library file lib{}.so in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib" + " or /.local/lib/ or /usr/local/lib/ or \ + /usr/local/lib64/ or /usr/lib or /usr/lib64 or " + "{}/.local/lib/ so the LD_PRELOAD environment variable will not be set." + "you can use 'conda install intel-openm' to install intel openMP" + .format("iomp", "iomp5", expanduser("~"))) + else: + logger.info("Using Intel OpenMP") + if set_kmp_affinity: + self.set_env("KMP_AFFINITY", + "granularity=fine,compact,1,0") + self.set_env("KMP_BLOCKTIME", "1") + self.logger_env("LD_PRELOAD") + + +class MultiInstanceLauncher(Launcher): + r""" + Launcher for single instance and multi-instance + """ + + def launch(self, args): + processes = [] + cores = [] + set_kmp_affinity = True + if args.core_list: # user specify what cores will be used by params + cores = [int(x) for x in args.core_list.split(",")] + if args.ncore_per_instance == -1: + logger.error( + "please specify the '--ncore_per_instance' if you have pass the --core_list params") + exit(-1) + elif args.ninstances > 1 and args.ncore_per_instance * args.ninstances < len(cores): + logger.warning("only first {} cores will be used, but you specify {} cores in core_list".format( + args.ncore_per_instance * args.ninstances, len(cores))) + else: + args.ninstances = len(cores) // args.ncore_per_instance + + else: + if args.use_logical_core: + if args.node_id != -1: + cores = self.cpuinfo.get_node_logical_cores(args.node_id) + else: + cores = self.cpuinfo.get_all_logical_cores() + # When using all cores on all nodes, including logical cores, + # setting KMP_AFFINITY disables logical cores. Thus, KMP_AFFINITY should not be set. + set_kmp_affinity = False + else: + if args.node_id != -1: + cores = self.cpuinfo.get_node_physical_cores(args.node_id) + else: + cores = self.cpuinfo.get_all_physical_cores() + if not args.multi_instance and args.ninstances == -1 and args.ncore_per_instance == -1: + args.ninstances = 1 + args.ncore_per_instance = len(cores) + elif args.multi_instance and args.ninstances == -1 and args.ncore_per_instance == -1: + args.throughput_mode = True + elif args.ncore_per_instance == -1 and args.ninstances != -1: + if args.ninstances > len(cores): + logger.error("there are {} total cores but you specify {} ninstances; \ + please make sure ninstances <= total_cores)".format( + len(cores), args.ninstances)) + exit(-1) + else: + args.ncore_per_instance = len(cores) // args.ninstances + elif args.ncore_per_instance != -1 and args.ninstances == -1: + args.ninstances = len(cores) // args.ncore_per_instance + else: + if args.ninstances * args.ncore_per_instance > len(cores): + logger.error( + "Please make sure ninstances * ncore_per_instance <= total_cores") + exit(-1) + if args.latency_mode: + logger.warning( + '--latency_mode is exclusive to --ninstances, \ + --ncore_per_instance, --node_id and --use_logical_core. \ + They won\'t take effect even they are set explicitly.') + args.ncore_per_instance = 4 + cores = self.cpuinfo.get_all_physical_cores() + args.ninstances = len(cores) // args.ncore_per_instance + + if args.throughput_mode: + logger.warning( + '--throughput_mode is exclusive to --ninstances, \ + --ncore_per_instance, --node_id and --use_logical_core. \ + They won\'t take effect even they are set explicitly.') + args.ninstances = self.cpuinfo.node_nums() + cores = self.cpuinfo.get_all_physical_cores() + args.ncore_per_instance = len(cores) // args.ninstances + + if args.ninstances > 1 and args.instance_idx != -1: + logger.info("assigning {} cores for instance {}".format( + args.ncore_per_instance, args.instance_idx)) + + self.set_multi_thread_and_allocator(args.ncore_per_instance, + args.disable_iomp, + set_kmp_affinity, + args.enable_tcmalloc, + args.enable_jemalloc, + args.use_default_allocator) + os.environ["LAUNCH_CMD"] = "#" + for i in range(args.ninstances): + cmd = [] + cur_process_cores = "" + if not args.disable_numactl: + cmd = ["numactl"] + cores = sorted(cores) + if args.instance_idx == -1: # sequentially assign ncores_per_instance to ninstances + core_list = cores[i * args.ncore_per_instance: ( + i + 1) * args.ncore_per_instance] + else: # assign ncores_per_instance from instance_idx + core_list = cores[args.instance_idx * args.ncore_per_instance: ( + args.instance_idx + 1) * args.ncore_per_instance] + + core_ranges = [] + for core in core_list: + if len(core_ranges) == 0: + range_elem = {'start': core, 'end': core} + core_ranges.append(range_elem) + else: + if core - core_ranges[-1]['end'] == 1: + core_ranges[-1]['end'] = core + else: + range_elem = {'start': core, 'end': core} + core_ranges.append(range_elem) + for r in core_ranges: + cur_process_cores = cur_process_cores + \ + "{}-{},".format(r['start'], r['end']) + cur_process_cores = cur_process_cores[:-1] + numa_params = "-C {} ".format(cur_process_cores) + numa_params += "-m {}".format(",".join( + [str(numa_id) for numa_id in self.cpuinfo.numa_aware_check(core_list)])) + cmd.extend(numa_params.split()) + with_python = not args.no_python + if with_python: + cmd.append(sys.executable) + cmd.append("-u") + if args.module: + cmd.append("-m") + cmd.append(args.program) + log_name = args.log_file_prefix + \ + "_instance_{}_cores_".format( + i) + cur_process_cores.replace(',', '_') + ".log" + log_name = os.path.join(args.log_path, log_name) + cmd.extend(args.program_args) + os.environ["LAUNCH_CMD"] += " ".join(cmd) + ",#" + cmd_s = " ".join(cmd) + cmd_s = "{} 2>&1 | tee -a {}".format(cmd_s, args.log_path) + logger.info(cmd_s) + process = subprocess.Popen( + cmd_s, env=os.environ, shell=True) # nosec + processes.append(process) + + if args.instance_idx != -1: # launches single instance, instance_idx, only + break + + os.environ["LAUNCH_CMD"] = os.environ["LAUNCH_CMD"][:-2] + for process in processes: + process.wait() + if process.returncode != 0: + raise subprocess.CalledProcessError( + returncode=process.returncode, cmd=cmd_s) + + +class DistributedTrainingLauncher(Launcher): + r""" + Launcher for distributed traning with MPI launcher + """ + + def get_mpi_pin_domain(self, nproc_per_node, ccl_worker_count, total_cores): + ''' + I_MPI_PIN_DOMAIN specify the cores used for every MPI process. + The first ccl_worker_count cores of every rank for ccl communication + and the other cores will be used to do computation. + For example: on CascadeLake 8280 CPU, 2 ranks on one node. ccl_worker_count=4 + CCL_WORKER_COUNT=4 + CCL_WORKER_AFFINITY="0,1,2,3,28,29,30,31" + I_MPI_PIN_DOMAIN=[0xffffff0,0xffffff0000000] + ''' + ppn = nproc_per_node + cores_per_rank = total_cores // ppn + pin_domain = "[" + for proc in range(ppn): + domain_binary = 0 + begin = proc * cores_per_rank + ccl_worker_count + end = proc * cores_per_rank + cores_per_rank - 1 + for i in range(begin, end + 1): + domain_binary |= (1 << i) + pin_domain += hex(domain_binary) + "," + pin_domain += "]" + return pin_domain + + def get_ccl_worker_affinity(self, nproc_per_node, ccl_worker_count, total_cores): + ''' + Computation and communication use different cores when using oneCCL + backend for distributed training. we use first ccl_worker_count cores of + every rank for ccl communication + ''' + ppn = nproc_per_node + cores_per_rank = total_cores // ppn + affinity = '' + for proc in range(ppn): + for ccl_worker in range(ccl_worker_count): + affinity += str(proc * cores_per_rank + ccl_worker) + "," + affinity = affinity[:-1] + return affinity + + def launch(self, args): + ''' + Set ENVs and launch MPI process for distributed training. + ''' + if args.nnodes > 1 and not os.path.exists(args.hostfile): + raise ValueError("hostfile is necessary when you use \ + multi-node distributed training," + "Please create hostfile which include the ip list you used for distributed running") + elif args.nnodes > 1: + ipv4_addr_pattern = \ + r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$" + ip_list = [] + with open(args.hostfile) as f: + for line in f: + line = line.strip().strip("\n") + # is_valid = re.match(ipv4_addr_pattern, line) + # if not is_valid: + # logger.error("{} is not valid IPV4 address".format(line)) + # exit(-1) + # else: + # ip_list.append(line) + ip_list.append(line) + if len(ip_list) < args.nnodes: + logger.error("The number of IP {} should greater than nnodes parameters {}".format( + len(ip_list), args.nnodes)) + exit(-1) + master_check = False + dic = psutil.net_if_addrs() + for adapter in dic: + snicList = dic[adapter] + for snic in snicList: + if snic.address == ip_list[0]: + master_check = True + if not master_check: + logger.error( + "MASTER_ADDR is incorrect. Please make sure the first line {} \ + in your hostfile is ip address of the current node".format(ip_list[0])) + exit(-1) + + logger.info("Begin to validate the ip connect") + args.master_addr = ip_list[0] + for ip in ip_list[1:]: + completed_process = subprocess.run( + "ssh -o PasswordAuthentication=no {} ':'".format(ip), shell=True) # nosec + if completed_process.returncode != 0: + logger.error( + "Passwordless SSH login to {} failed, please make sure you have setup SSH public key right") + exit(-1) + else: + logger.info("connection from master node {} to slave node {} is OK".format( + args.master_addr, ip)) + + total_cores_per_node = self.cpuinfo.physical_core_nums() + if args.use_logical_core: + total_cores_per_node = self.cpuinfo.logical_core_nums() + + # set distributed related environmental variables + self.set_env("MASTER_ADDR", args.master_addr) + self.set_env("MASTER_PORT", str(args.master_port)) + mpi_pin_domain = self.get_mpi_pin_domain( + args.nproc_per_node, args.ccl_worker_count, total_cores_per_node) + self.set_env("I_MPI_PIN_DOMAIN", mpi_pin_domain) + + ppn = args.nproc_per_node + cores_per_rank = total_cores_per_node // ppn + + opm_num_threads = cores_per_rank - args.ccl_worker_count + self.set_multi_thread_and_allocator(opm_num_threads, + args.disable_iomp, + True, + args.enable_tcmalloc, + args.enable_jemalloc, + args.use_default_allocator) + + self.set_env("CCL_WORKER_COUNT", str(args.ccl_worker_count)) + ccl_affinity = self.get_ccl_worker_affinity( + args.nproc_per_node, args.ccl_worker_count, total_cores_per_node) + self.set_env("CCL_WORKER_AFFINITY", ccl_affinity) + + os.environ["LAUNCH_CMD"] = "#" + cmd = ['mpiexec.hydra'] + mpi_config = "-l -np {} -ppn {} -genv I_MPI_PIN_DOMAIN={} -genv OMP_NUM_THREADS={} ".format( + args.nnodes * args.nproc_per_node, args.nproc_per_node, mpi_pin_domain, opm_num_threads) + mpi_config += args.more_mpi_params + if args.nnodes > 1: + mpi_config += " -hostfile {}".format(args.hostfile) + cmd.extend(mpi_config.split()) + with_python = not args.no_python + if with_python: + cmd.append(sys.executable) + cmd.append("-u") + if args.module: + cmd.append("-m") + cmd.append(args.program) + cmd.extend(args.program_args) + logger.info(cmd) + process = subprocess.Popen(cmd, env=os.environ) + process.wait() + os.environ["LAUNCH_CMD"] += " ".join(cmd) + ",#" + os.environ["LAUNCH_CMD"] = os.environ["LAUNCH_CMD"][:-2] + + +def add_distributed_training_params(parser): + + cpuinfo = CPUinfo() + node_nums = cpuinfo.node_nums() + + group = parser.add_argument_group( + "Distributed Training Parameters With oneCCL backend") + group.add_argument("--nnodes", metavar='\b', type=int, default=1, + help="The number of nodes to use for distributed " + "training") + group.add_argument("--nproc_per_node", metavar='\b', type=int, default=node_nums, + help="The number of processes to launch on each node") + # ccl control + group.add_argument("--ccl_worker_count", metavar='\b', default=4, type=int, + help="Core numbers per rank used for ccl communication") + # mpi control + group.add_argument("--master_addr", metavar='\b', default="127.0.0.1", type=str, + help="Master node (rank 0)'s address, should be either " + "the IP address or the hostname of node 0, for " + "single node multi-proc training, the " + "--master_addr can simply be 127.0.0.1") + group.add_argument("--master_port", metavar='\b', default=29500, type=int, + help="Master node (rank 0)'s free port that needs to " + "be used for communication during distributed " + "training") + group.add_argument("--hostfile", metavar='\b', default="hostfile", type=str, + help="Hostfile is necessary for multi-node multi-proc " + "training. hostfile includes the node address list " + "node address which should be either the IP address" + "or the hostname.") + group.add_argument("--more_mpi_params", metavar='\b', default="", type=str, + help="User can pass more parameters for mpiexec.hydra " + "except for -np -ppn -hostfile and -genv I_MPI_PIN_DOMAIN") + + +def add_memory_allocator_params(parser): + + group = parser.add_argument_group("Memory Allocator Parameters") + # allocator control + group.add_argument("--enable_tcmalloc", action='store_true', default=False, + help="Enable tcmalloc allocator") + group.add_argument("--enable_jemalloc", action='store_true', default=False, + help="Enable jemalloc allocator") + group.add_argument("--use_default_allocator", action='store_true', default=False, + help="Use default memory allocator") + + +def add_multi_instance_params(parser): + + group = parser.add_argument_group("Multi-instance Parameters") + # multi-instance control + group.add_argument("--ncore_per_instance", metavar='\b', default=-1, type=int, + help="Cores per instance") + group.add_argument("--ninstances", metavar='\b', default=-1, type=int, + help="For multi-instance, you should give the cores number you used for per instance.") + group.add_argument("--instance_idx", metavar='\b', default="-1", type=int, + help="Specify instance index to assign ncores_per_instance for instance_idx; \ + otherwise ncore_per_instance will be assigned sequentially to ninstances. \ + Please refer to \ + https://github.com/intel/intel-extension-for-pytorch/\ + blob/master/docs/tutorials/performance_tuning/launch_script.md") + group.add_argument("--latency_mode", action='store_true', default=False, + help="By detault 4 core per instance and use all physical cores") + group.add_argument("--throughput_mode", action='store_true', default=False, + help="By default one instance per node and use all physical cores") + group.add_argument("--node_id", metavar='\b', default=-1, type=int, + help="node id for multi-instance, by default all nodes will be used") + group.add_argument("--use_logical_core", action='store_true', default=False, + help="Whether only use physical cores") + group.add_argument("--disable_numactl", action='store_true', default=False, + help="Disable numactl") + group.add_argument("--core_list", metavar='\b', default=None, type=str, + help="Specify the core list as 'core_id, core_id, ....', otherwise, \ + all the cores will be used.") + group.add_argument("--log_path", metavar='\b', default="", type=str, + help="The log file directory. Default path is '', which means disable logging to files.") + group.add_argument("--log_file_prefix", metavar='\b', default="run", type=str, + help="log file prefix") + + +def add_kmp_iomp_params(parser): + + group = parser.add_argument_group("IOMP Parameters") + group.add_argument("--disable_iomp", action='store_true', default=False, + help="By default, we use Intel OpenMP and libiomp5.so will be add to LD_PRELOAD") + + +def parse_args(): + """ + Helper function parsing the command line options + @retval ArgumentParser + """ + parser = ArgumentParser( + description="This is a script for launching PyTorch training and inference on Intel Xeon CPU " + "with optimal configurations. Now, single instance inference/training, multi-instance " + "inference/training and distributed training with oneCCL backend is enabled. " + "To get the peak performance on Intel Xeon CPU, the script optimizes the configuration " + "of thread and memory management. For thread management, the script configures thread " + "affinity and the preload of Intel OMP library. For memory management, it configures " + "NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc) " + "\n################################# Basic usage ############################# \n" + "\n 1. single instance\n" + "\n >>> python -m intel_extension_for_pytorch.cpu.launch python_script args \n" + "\n2. multi-instance \n" + "\n >>> python -m intel_extension_for_pytorch.cpu.launch --ninstances xxx \ + --ncore_per_instance xx python_script args\n" + "\n3. Single-Node multi-process distributed training\n" + "\n >>> python -m intel_extension_for_pytorch.cpu.launch --distributed python_script args\n" + "\n4. Multi-Node multi-process distributed training: (e.g. two nodes)\n" + "\n rank 0: *(IP: 192.168.10.10, and has a free port: 295000)*\n" + "\n >>> python -m intel_extension_for_pytorch.cpu.launch --distributed --nproc_per_node=2\n" + "\n --nnodes=2 --hostfile hostfile python_script args\n" + "\n############################################################################# \n", + formatter_class=RawTextHelpFormatter) + + parser.add_argument("--multi_instance", action='store_true', default=False, + help="Enable multi-instance, by default one instance per node") + + parser.add_argument('--distributed', action='store_true', default=False, + help='Enable distributed training.') + parser.add_argument("-m", "--module", default=False, action="store_true", + help="Changes each process to interpret the launch script " + "as a python module, executing with the same behavior as" + "'python -m'.") + + parser.add_argument("--no_python", default=False, action="store_true", + help="Do not prepend the --program script with \"python\" - just exec " + "it directly. Useful when the script is not a Python script.") + + add_memory_allocator_params(parser) + add_kmp_iomp_params(parser) + + add_distributed_training_params(parser) + add_multi_instance_params(parser) + # positional + parser.add_argument("program", type=str, + help="The full path to the proram/script to be launched. " + "followed by all the arguments for the script") + + # rest from the training program + parser.add_argument('program_args', nargs=REMAINDER) + return parser.parse_args() + + +def exec_launcher(ncore_per_instance, ninstances, program, program_args, log_path): + env_before = set(os.environ.keys()) + if platform.system() == "Windows": + raise RuntimeError("Windows platform is not supported!!!") + + # args = parse_args() + import argparse + args = argparse.Namespace( + multi_instance=True, distributed=False, module=False, no_python=False, enable_tcmalloc=False, + enable_jemalloc=False, use_default_allocator=False, disable_iomp=False, nnodes=1, nproc_per_node=2, + ccl_worker_count=4, master_addr='127.0.0.1', master_port=29500, hostfile='hostfile', more_mpi_params='', + ncore_per_instance=ncore_per_instance, ninstances=ninstances, instance_idx=-1, latency_mode=False, + throughput_mode=False, node_id=-1, + use_logical_core=False, disable_numactl=False, core_list=None, log_path=log_path, log_file_prefix='run', + program=program, program_args=program_args) + + # if args.log_path: + # path = os.path.dirname(args.log_path if args.log_path.endswith('/') else args.log_path + '/') + # if not os.path.exists(path): + # os.makedirs(path) + # args.log_path = path + + # args.log_file_prefix = '{}_{}'.format(args.log_file_prefix, datetime.now().strftime("%Y%m%d%H%M%S")) + # fileHandler = logging.FileHandler("{0}/{1}_instances.log".format(args.log_path, args.log_file_prefix)) + # logFormatter = logging.Formatter(format_str) + # fileHandler.setFormatter(logFormatter) + # logger.addHandler(fileHandler) + + if args.distributed and args.multi_instance: + raise RuntimeError( + "Either args.distributed or args.multi_instance should be set") + + if args.latency_mode and args.throughput_mode: + raise RuntimeError( + "Either args.latency_mode or args.throughput_mode should be set") + + if args.nnodes > 1: + args.distributed = True + + if not args.no_python and not args.program.endswith(".py"): + logger.error( + "For non Python script, you should use '--no_python' parameter.") + exit() + + # Verify LD_PRELOAD + if "LD_PRELOAD" in os.environ: + lst_valid = [] + tmp_ldpreload = os.environ["LD_PRELOAD"] + for item in tmp_ldpreload.split(":"): + matches = glob.glob(item) + if len(matches) > 0: + lst_valid.append(item) + else: + logger.warning( + "{} doesn't exist. Removing it from LD_PRELOAD.".format(item)) + if len(lst_valid) > 0: + os.environ["LD_PRELOAD"] = ":".join(lst_valid) + else: + os.environ["LD_PRELOAD"] = "" + + launcher = None + if args.distributed: + launcher = DistributedTrainingLauncher() + else: + launcher = MultiInstanceLauncher() + + launcher.launch(args) + for x in sorted(set(os.environ.keys()) - env_before): + logger.debug('{0}={1}'.format(x, os.environ[x])) + +# if __name__ == "__main__": +# main() diff --git a/neural_coder/utils/__init__.py b/neural_coder/utils/__init__.py new file mode 100644 index 00000000000..162b389a776 --- /dev/null +++ b/neural_coder/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/neural_coder/utils/common.py b/neural_coder/utils/common.py new file mode 100644 index 00000000000..f9d2a70929a --- /dev/null +++ b/neural_coder/utils/common.py @@ -0,0 +1,20 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def move_element_to_front(list, element): + if element in list: + idx = list.index(element) + list.insert(0, list.pop(idx)) + return list diff --git a/neural_coder/utils/cpu_info.py b/neural_coder/utils/cpu_info.py new file mode 100644 index 00000000000..13f2a2bd52b --- /dev/null +++ b/neural_coder/utils/cpu_info.py @@ -0,0 +1,41 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import subprocess +import os + + +def get_num_cpu_cores() -> int: + + cmd_cpu_info = '' + cmd_cpu_info += "sockets_num=$(lscpu |grep 'Socket(s):' |sed 's/[^0-9]//g')" + cmd_cpu_info += ' && ' + cmd_cpu_info += "cores_per_socket=$(lscpu |grep 'Core(s) per socket:' |sed 's/[^0-9]//g')" + cmd_cpu_info += ' && ' + cmd_cpu_info += 'phsical_cores_num=$( echo "${sockets_num} * ${cores_per_socket}" |bc )' + cmd_cpu_info += ' && ' + cmd_cpu_info += "numa_nodes_num=$(lscpu |grep 'NUMA node(s):' |sed 's/[^0-9]//g')" + cmd_cpu_info += ' && ' + cmd_cpu_info += 'cores_per_node=$( echo "${phsical_cores_num} / ${numa_nodes_num}" |bc )' + cmd_cpu_info += ' && ' + cmd_cpu_info += "echo ${cores_per_node}" + sp_grep_cpu_info = subprocess.Popen( + cmd_cpu_info, env=os.environ, shell=True, stdout=subprocess.PIPE) # nosec + sp_grep_cpu_info.wait() + + log_cpu_info, _ = sp_grep_cpu_info.communicate() + ncores = int(str(log_cpu_info)[2:-3]) + + return ncores diff --git a/neural_coder/utils/handle_user_input.py b/neural_coder/utils/handle_user_input.py new file mode 100644 index 00000000000..74f6ed4971d --- /dev/null +++ b/neural_coder/utils/handle_user_input.py @@ -0,0 +1,140 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from typing import List +from .. import globals +import logging + +logging.basicConfig(level=globals.logging_level, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S +0000') +logger = logging.getLogger(__name__) + + +def get_all_code_path(user_input: str) -> List: + user_code_path = get_user_code_path(user_input) + if globals.consider_imports: + import_path = get_imports_path(user_code_path) + else: + import_path = [] + all_code_path = user_code_path + import_path + logger.debug(f"Number of code files to analyze: {len(all_code_path)}") + + num_user_code_path = len(user_code_path) + + return all_code_path, num_user_code_path + + +def get_user_code_path(user_input: str) -> List: + list_path = [] + + # detect whether (a list of files)/file/folder/url + global user_input_type + if type(user_input) == list: + user_input_type = "a list of files" + elif ".py" in user_input: + user_input_type = "file" + elif "github.com" in user_input: + user_input_type = "url" + else: + user_input_type = "folder" + logger.debug(f"user input code type: {user_input_type}") + # get list of file path + if user_input_type == "url": + from git import Repo + Repo.clone_from(user_input, "./cloned_github_repo") + dir_input = "./cloned_github_repo" + if user_input_type == "folder": + dir_input = user_input + + if user_input_type == "file": + list_path.append(os.path.abspath(user_input)) + elif user_input_type == "a list of files": + list_path += [os.path.abspath(i) for i in user_input] + else: + for path, dir_list, file_list in os.walk(dir_input): + for file_name in file_list: + file_path = os.path.join(path, file_name) + if file_path[-3:] == ".py" and file_path[-11:] != "__init__.py" and file_path[-8:] != "setup.py": + list_path.append(os.path.abspath(file_path)) + + return list_path + + +def get_imports_path(user_code_path: List) -> List: + + pip_name_exceptions = ["torch", "numpy", "os", "time", "logging", "inspect", "random", "math", + "sys", "timeit", "tqdm", "importlib", "json", "warnings", "argparse", + "collections", "threading", "subprocess", "shutil", "h5py", "PIL", "ast", + "imageio", "glob", "traceback", "requests", "pandas", "unittest", "tempfile", "linecache"] + + list_pip_path = [] + list_pip_name = [] + + # get list of pip name + for path in user_code_path: + lines = open(path, 'r').read().split('\n') + for line in lines: + is_import_line = False + if line[0:6] == "import" and line[0:8] != "import .": + is_import_line = True + start = 7 + elif line[0:4] == "from" and line[0:6] != "from .": + is_import_line = True + start = 5 + if is_import_line: + space_idx = line[start:].find(" ") + dot_idx = line[start:].find(".") + if space_idx == -1 and dot_idx == -1: + pip_name = line[start:] + elif space_idx > 0 and dot_idx == -1: + pip_name = line[start: start + space_idx] + elif space_idx == -1 and dot_idx > 0: + pip_name = line[start: start + dot_idx] + elif space_idx > 0 and dot_idx > 0: + pip_name = line[start: start + min(space_idx, dot_idx)] + list_pip_name.append(pip_name) + list_pip_name = list( + set(list_pip_name).difference(set(pip_name_exceptions))) + logger.debug(f"list pip name: {list_pip_name}") + + # get list of pip path + cmd_import = " ".join(["import " + i + ";" for i in list_pip_name]) + try: + exec(cmd_import) + except ModuleNotFoundError as mnfe: + logger.error( + f"Please install all required pip modules defined in your Python scripts \ + before running Neural Coder: {mnfe}") + quit() + + import inspect + for i in list_pip_name: + try: + pip_dir_path = inspect.getsourcefile(eval(i)) + pip_dir_path = pip_dir_path[0:pip_dir_path.rfind("/")] + for path, dir_list, file_list in os.walk(pip_dir_path): + for file_name in file_list: + file_path = os.path.join(path, file_name) + if file_path[-3:] == ".py" and file_path[-11:] != "__init__.py" \ + and file_path[-8:] != "setup.py": + list_pip_path.append(os.path.abspath(file_path)) + except TypeError as te: + logger.error( + f"Please reinstall certain pip modules as its detected \ + as a built-in module and the installation path cannot be retrieved: {te}") + + return list_pip_path diff --git a/neural_coder/utils/line_operation.py b/neural_coder/utils/line_operation.py new file mode 100644 index 00000000000..965e3d467ee --- /dev/null +++ b/neural_coder/utils/line_operation.py @@ -0,0 +1,111 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# get line's indent level +def get_line_indent_level(line: str) -> int: + if list(set(line)) == [" "]: + return 0 + else: + return len(line) - len(line.lstrip()) + +# determine if line is multi-line comment + + +def multi_line_comment_detection(line: str, previous_line_is_multi_line_comment: bool, end_ml_comment_flag: bool): + this_line_is_multi_line_comment = previous_line_is_multi_line_comment + + if previous_line_is_multi_line_comment: + if end_ml_comment_flag: + this_line_is_multi_line_comment = False + else: + this_line_is_multi_line_comment = True + if '"""' in line: # end of multi line comment + end_ml_comment_flag = True + else: + end_ml_comment_flag = False + else: + if '"""' in line and line.count('"') == 3: # start of multi line comment, e.g. [ """] + this_line_is_multi_line_comment = True + end_ml_comment_flag = False + else: + this_line_is_multi_line_comment = False + end_ml_comment_flag = False + + if not this_line_is_multi_line_comment: + if '"""' in line: + this_line_is_multi_line_comment = True + end_ml_comment_flag = False + + if '"""' in line and line.count('"') == 6: + this_line_is_multi_line_comment = True + end_ml_comment_flag = True + + return this_line_is_multi_line_comment, end_ml_comment_flag + +# determine if line is single-line comment or empty + + +def single_line_comment_or_empty_line_detection(line: str) -> bool: + this_line_is_single_line_comment_or_empty_line = False + + if len(line) == 0 or line.isspace(): # empty line or all spaces + this_line_is_single_line_comment_or_empty_line = True + elif '"""' in line and line.count('"') == 6: # e.g. [ """ some single-line comment """] + this_line_is_single_line_comment_or_empty_line = True + # e.g. [ # some single-line comment] + elif len(line) > 0 and len(line.lstrip()) > 0 and line.lstrip()[0] == "#": + this_line_is_single_line_comment_or_empty_line = True + + return this_line_is_single_line_comment_or_empty_line + +# determine if line is a eval func of model_name, like "xxx = model_name(yyy)" or "model_name(yyy)" + + +def is_eval_func_model_name(model_name: str, line: str) -> str: + line_ = line.replace(' ', '') + judge_1 = line_.find(model_name + "(") > -1 + judge_2 = (line_.find("=") > 0 and + line_.find("=") < line_.find(model_name) and + line_[line_.find("=")+1:line_.find("(")] == model_name) or line_.find(model_name) == 0 + if judge_1 and judge_2: + return True + else: + return False + +# get lhs of line of format "xxx = yyy" + + +def get_line_lhs(line: str) -> str: + line_ = line.replace(' ', '') + lhs = line_[:line_.find("=")] + return lhs + +# determine if line is for format "xxx = yyy(zzz)" and get lhs and rhs of "=" + + +def of_definition_format(line: str): + line_ = line.replace(' ', '') + is_def = False + lhs = "" + rhs = "" + if "=" in line_ and "(" in line_ and line_.find("=") < line_.find("("): + is_def = True + lhs = line_[:line_.find("=")] + rhs = line_[line_.find("=")+1:line_.find("(")] + if "." not in rhs: + pass + else: + rhs = rhs[rhs.find(".")+1:] + return is_def, lhs, rhs