Skip to content

Commit

Permalink
Update Jupyter notebooks (#63)
Browse files Browse the repository at this point in the history 
* Update inference-pytorch-export-libtorch.ipynb

* Update export-onnx-inference-onnxruntime.ipynb

* Add inference time consumed (Closes #54)
  • Loading branch information
@zhiqwang
zhiqwang authored Feb 16, 2021
1 parent c526565 commit 33f4dc0
Show file tree
Hide file tree
Showing 3 changed files with 283 additions and 218 deletions.
190 changes: 143 additions & 47 deletions notebooks/export-onnx-inference-onnxruntime.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,29 +6,15 @@
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"import numpy as np\n",
"from numpy import random\n",
"\n",
"from pathlib import Path\n",
"\n",
"import shutil\n",
"\n",
"import cv2\n",
"from IPython import display\n",
"\n",
"import torch\n",
"from torch import nn\n",
"import torch.nn.functional as F\n",
"\n",
"import onnx\n",
"import onnxruntime\n",
"\n",
"from utils.image_utils import cv2_imshow\n",
"from models import yolov5_onnx\n",
"from yolort.models import yolov5_onnx\n",
"\n",
"from detect import read_image, load_names, overlay_boxes"
"from yolort.utils.image_utils import read_image"
]
},
{
Expand Down Expand Up @@ -58,7 +44,7 @@
"metadata": {},
"outputs": [],
"source": [
"model = yolov5_onnx(pretrained=True, min_size=320, max_size=416, score_thresh=0.5)\n",
"model = yolov5_onnx(pretrained=True, score_thresh=0.45)\n",
"\n",
"model = model.eval()\n",
"model = model.to(device)"
Expand All @@ -77,16 +63,15 @@
"metadata": {},
"outputs": [],
"source": [
"path0 = './test/assets/bus.jpg'\n",
"path1 = './test/assets/zidane.jpg'\n",
"\n",
"img_test0 = read_image(path0, is_half=False)\n",
"img_test0 = img_test0.to(device)\n",
"img_one = cv2.imread('test/assets/bus.jpg')\n",
"img_one = read_image(img_one, is_half=False)\n",
"img_one = img_one.to(device)\n",
"\n",
"img_test1 = read_image(path1, is_half=False)\n",
"img_test1 = img_test1.to(device)\n",
"img_two = cv2.imread('test/assets/zidane.jpg')\n",
"img_two = read_image(img_two, is_half=False)\n",
"img_two = img_two.to(device)\n",
"\n",
"images = [img_test0, img_test1]"
"images = [img_one, img_two]"
]
},
{
Expand All @@ -110,6 +95,26 @@
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 20 ms, sys: 0 ns, total: 20 ms\n",
"Wall time: 20.1 ms\n"
]
}
],
"source": [
"%%time\n",
"with torch.no_grad():\n",
" model_out = model(images)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
Expand All @@ -120,7 +125,7 @@
" [675.6570, 409.5675, 812.7283, 868.2495]], device='cuda:0')"
]
},
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -131,7 +136,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 8,
"metadata": {},
"outputs": [
{
Expand All @@ -140,7 +145,7 @@
"tensor([0.8941, 0.8636, 0.8621, 0.7490], device='cuda:0')"
]
},
"execution_count": 7,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -151,7 +156,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 9,
"metadata": {},
"outputs": [
{
Expand All @@ -160,7 +165,7 @@
"tensor([0, 0, 5, 0], device='cuda:0')"
]
},
"execution_count": 8,
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -173,31 +178,31 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Export to ONNX model"
"## Export to `ONNX` model"
]
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Starting ONNX export with onnx 1.8.0, onnxruntime 1.5.2...\n"
"Starting ONNX export with onnx 1.8.1, onnxruntime 1.6.0...\n"
]
}
],
"source": [
"# TorchScript export\n",
"print(f'Starting ONNX export with onnx {onnx.__version__}, onnxruntime {onnxruntime.__version__}...')\n",
"export_onnx_name = './checkpoints/yolov5/yolov5s.onnx'"
"export_onnx_name = 'checkpoints/yolov5/yolov5s.onnx'"
]
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -206,7 +211,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 12,
"metadata": {},
"outputs": [
{
Expand All @@ -219,13 +224,17 @@
" 'Automatically generated names will be applied to each dynamic axes of input {}'.format(key))\n",
"/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:3123: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
" dtype=torch.float32)).float())) for i in range(dim)]\n",
"/mnt/yolov5-rt-stack/models/anchor_utils.py:26: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
"/data/wangzq/yolov5-rt-stack/yolort/models/anchor_utils.py:31: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
" stride = torch.as_tensor([stride], dtype=dtype, device=device)\n",
"/mnt/yolov5-rt-stack/models/anchor_utils.py:40: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
"/data/wangzq/yolov5-rt-stack/yolort/models/anchor_utils.py:50: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
" anchor_grid = torch.as_tensor(anchor_grid, dtype=dtype, device=device)\n",
"/mnt/yolov5-rt-stack/models/anchor_utils.py:63: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
"/data/wangzq/yolov5-rt-stack/yolort/models/anchor_utils.py:77: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
" shifts = shifts - torch.tensor(0.5, dtype=shifts.dtype, device=device)\n",
"/usr/local/lib/python3.6/dist-packages/torchvision/models/detection/transform.py:271: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
"/data/wangzq/yolov5-rt-stack/yolort/models/box_head.py:344: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
" for idx in range(batch_size): # image idx, image inference\n",
"/data/wangzq/yolov5-rt-stack/yolort/models/transform.py:287: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
" for s, s_orig in zip(new_size, original_size)\n",
"/data/wangzq/yolov5-rt-stack/yolort/models/transform.py:287: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
" for s, s_orig in zip(new_size, original_size)\n",
"/usr/local/lib/python3.6/dist-packages/torch/onnx/symbolic_opset9.py:2378: UserWarning: Exporting aten::index operator of advanced indexing in opset 11 is achieved by combination of multiple ONNX operators, including Reshape, Transpose, Concat, and Gather. If indices include negative values, the exported graph will produce incorrect results.\n",
" \"If indices include negative values, the exported graph will produce incorrect results.\")\n",
Expand All @@ -248,6 +257,64 @@
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Simplifier exported `ONNX` model"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Starting simplifing with onnxsim 0.3.1\n"
]
}
],
"source": [
"import onnxsim\n",
"\n",
"# onnx-simplifier version\n",
"print(f'Starting simplifing with onnxsim {onnxsim.__version__}')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"onnx_simp_name = 'checkpoints/yolov5/yolov5s.simp.onnx'"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"# load your predefined ONNX model\n",
"onnx_model = onnx.load(export_onnx_name)\n",
"\n",
"# convert model\n",
"model_simp, check = onnxsim.simplify(\n",
" onnx_model,\n",
" input_shapes={\"images_tensors\": [3, 640, 640]},\n",
" dynamic_input_shape=True,\n",
")\n",
"\n",
"assert check, \"Simplified ONNX model could not be validated\"\n",
"\n",
"# use model_simp as a standard ONNX model object\n",
"onnx.save(model_simp, onnx_simp_name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -257,7 +324,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -267,7 +334,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -280,7 +347,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -290,19 +357,48 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"ort_session = onnxruntime.InferenceSession(onnx_simp_name)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"ort_session = onnxruntime.InferenceSession(export_onnx_name)\n",
"# compute onnxruntime output prediction\n",
"ort_inputs = dict((ort_session.get_inputs()[i].name, inpt) for i, inpt in enumerate(inputs))\n",
"ort_outs = ort_session.run(None, ort_inputs)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 16 ms, sys: 8 ms, total: 24 ms\n",
"Wall time: 22.4 ms\n"
]
}
],
"source": [
"%%time\n",
"# compute onnxruntime output prediction\n",
"ort_inputs = dict((ort_session.get_inputs()[i].name, inpt) for i, inpt in enumerate(inputs))\n",
"ort_outs = ort_session.run(None, ort_inputs)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
Expand All @@ -315,7 +411,7 @@
],
"source": [
"for i in range(0, len(outputs)):\n",
" torch.testing.assert_allclose(outputs[i], ort_outs[i], rtol=1e-03, atol=1e-05)\n",
" torch.testing.assert_allclose(outputs[i], ort_outs[i], rtol=1e-05, atol=1e-07)\n",
"\n",
"print(\"Exported model has been tested with ONNXRuntime, and the result looks good!\")"
]
Expand Down
Loading

0 comments on commit 33f4dc0

Please sign in to comment.