Skip to content

Commit

Permalink
Fix conformance & e2e tests (#2540)
Browse files Browse the repository at this point in the history
### Changes

- Changed `CPU_THREADS_NUM` to `INFERENCE_NUM_THREADS`;
- Renamed `accuracy_checker.py` to `ac_wrapper.py` due to import issues;

### Reason for changes

- Bugfix;

### Related tickets

- 132634

(cherry picked from commit 52b2bac)

### Tests

- post_training_quantization/309 - passed
- e2e_onnx_model_zoo/612- running
  • Loading branch information
KodiaqQ authored Mar 1, 2024
1 parent c5b3df8 commit 533e320
Show file tree
Hide file tree
Showing 5 changed files with 13 additions and 13 deletions.
File renamed without changes.
8 changes: 4 additions & 4 deletions tests/onnx/test_e2e_ptq.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ def is_ov(request):
def _read_accuracy_checker_result(root_dir: Path, key: str) -> pd.DataFrame:
dfs = []
for task in TASKS:
csv_fp = str(root_dir / task / f"accuracy_checker-{key}.csv")
csv_fp = str(root_dir / task / f"ac_wrapper-{key}.csv")
dfs += [pd.read_csv(csv_fp)]
df = pd.concat(dfs, axis=0)
df = df[["model", "metric_value", "metric_name", "tags"]]
Expand Down Expand Up @@ -402,7 +402,7 @@ def test_reference_model_accuracy(
anno_dir,
output_dir,
eval_size,
program="accuracy_checker.py",
program="ac_wrapper.py",
is_quantized=False,
is_ov_ep=False,
is_cpu_ep=True,
Expand Down Expand Up @@ -445,7 +445,7 @@ def test_onnx_rt_quantized_model_accuracy(
anno_dir,
output_dir,
eval_size,
program="accuracy_checker.py",
program="ac_wrapper.py",
is_quantized=True,
is_ov_ep=is_ov_ep,
is_cpu_ep=is_cpu_ep,
Expand Down Expand Up @@ -474,7 +474,7 @@ def test_ov_quantized_model_accuracy(
anno_dir,
output_dir,
eval_size,
program="accuracy_checker.py",
program="ac_wrapper.py",
is_quantized=True,
)
run_command(command)
Expand Down
6 changes: 3 additions & 3 deletions tests/post_training/pipelines/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,9 +336,9 @@ def compress(self) -> None:
print("Quantization...")

if self.backend in PT_BACKENDS:
cpu_threads_num = os.environ.get("CPU_THREADS_NUM")
if cpu_threads_num is not None:
torch.set_num_threads(int(cpu_threads_num))
inference_num_threads = os.environ.get("INFERENCE_NUM_THREADS")
if inference_num_threads is not None:
torch.set_num_threads(int(inference_num_threads))

start_time = time.perf_counter()
self.run_info.compression_memory_usage = memory_usage(self._compress, max_usage=True)
Expand Down
6 changes: 3 additions & 3 deletions tests/post_training/pipelines/image_classification_timm.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,10 @@ def _validate(self):

core = ov.Core()

if os.environ.get("CPU_THREADS_NUM"):
if os.environ.get("INFERENCE_NUM_THREADS"):
# Set CPU_THREADS_NUM for OpenVINO inference
cpu_threads_num = os.environ.get("CPU_THREADS_NUM")
core.set_property("CPU", properties={"CPU_THREADS_NUM": str(cpu_threads_num)})
inference_num_threads = os.environ.get("INFERENCE_NUM_THREADS")
core.set_property("CPU", properties={"INFERENCE_NUM_THREADS": str(inference_num_threads)})

ov_model = core.read_model(self.path_compressed_ir)
compiled_model = core.compile_model(ov_model)
Expand Down
6 changes: 3 additions & 3 deletions tests/post_training/pipelines/lm_weight_compression.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,10 +188,10 @@ def _validate(self):
is_stateful = self.params.get("is_stateful", False)
core = ov.Core()

if os.environ.get("CPU_THREADS_NUM"):
if os.environ.get("INFERENCE_NUM_THREADS"):
# Set CPU_THREADS_NUM for OpenVINO inference
cpu_threads_num = os.environ.get("CPU_THREADS_NUM")
core.set_property("CPU", properties={"CPU_THREADS_NUM": str(cpu_threads_num)})
inference_num_threads = os.environ.get("INFERENCE_NUM_THREADS")
core.set_property("CPU", properties={"INFERENCE_NUM_THREADS": str(inference_num_threads)})

gt_data_path = TEST_ROOT / "post_training" / "data" / "wwb_ref_answers" / self.fp32_model_name / "ref_qa.csv"
gt_data_path.parent.mkdir(parents=True, exist_ok=True)
Expand Down

0 comments on commit 533e320

Please sign in to comment.