diff --git a/configs/mmseg/segmentation_ncnn_static.py b/configs/mmseg/segmentation_ncnn_static.py deleted file mode 100644 index 985542b5bf..0000000000 --- a/configs/mmseg/segmentation_ncnn_static.py +++ /dev/null @@ -1,3 +0,0 @@ -_base_ = ['./segmentation_static.py', '../_base_/backends/ncnn.py'] - -onnx_config = dict(input_shape=None) diff --git a/configs/mmseg/segmentation_onnxruntime_static-1024x2048.py b/configs/mmseg/segmentation_onnxruntime_static-1024x2048.py new file mode 100644 index 0000000000..2587a015f4 --- /dev/null +++ b/configs/mmseg/segmentation_onnxruntime_static-1024x2048.py @@ -0,0 +1,3 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/onnxruntime.py'] + +onnx_config = dict(input_shape=[2048, 1024]) diff --git a/configs/mmseg/segmentation_onnxruntime_static.py b/configs/mmseg/segmentation_onnxruntime_static-512x512.py similarity index 63% rename from configs/mmseg/segmentation_onnxruntime_static.py rename to configs/mmseg/segmentation_onnxruntime_static-512x512.py index 802eb08a4d..03d919d7b0 100644 --- a/configs/mmseg/segmentation_onnxruntime_static.py +++ b/configs/mmseg/segmentation_onnxruntime_static-512x512.py @@ -1,3 +1,3 @@ _base_ = ['./segmentation_static.py', '../_base_/backends/onnxruntime.py'] -onnx_config = dict(input_shape=None) +onnx_config = dict(input_shape=[512, 512]) diff --git a/configs/mmseg/segmentation_openvino_static-512x512.py b/configs/mmseg/segmentation_openvino_static-512x512.py new file mode 100644 index 0000000000..29d7582758 --- /dev/null +++ b/configs/mmseg/segmentation_openvino_static-512x512.py @@ -0,0 +1,4 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/openvino.py'] + +backend_config = dict( + model_inputs=[dict(opt_shapes=dict(input=[1, 3, 512, 512]))]) diff --git a/configs/mmseg/segmentation_tensorrt-fp16_static-1024x1024.py b/configs/mmseg/segmentation_tensorrt-fp16_static-1024x1024.py new file mode 100644 index 0000000000..dc887a10d5 --- /dev/null +++ b/configs/mmseg/segmentation_tensorrt-fp16_static-1024x1024.py @@ -0,0 +1,13 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/tensorrt-fp16.py'] + +onnx_config = dict(input_shape=[1024, 1024]) +backend_config = dict( + common_config=dict(max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 1024, 1024], + opt_shape=[1, 3, 1024, 1024], + max_shape=[1, 3, 1024, 1024]))) + ]) diff --git a/configs/mmseg/segmentation_tensorrt-fp16_static-512x512.py b/configs/mmseg/segmentation_tensorrt-fp16_static-512x512.py new file mode 100644 index 0000000000..f6d0d8bb5f --- /dev/null +++ b/configs/mmseg/segmentation_tensorrt-fp16_static-512x512.py @@ -0,0 +1,13 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/tensorrt-fp16.py'] + +onnx_config = dict(input_shape=[512, 512]) +backend_config = dict( + common_config=dict(max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 512, 512], + opt_shape=[1, 3, 512, 512], + max_shape=[1, 3, 512, 512]))) + ]) diff --git a/configs/mmseg/segmentation_tensorrt-int8_static-1024x1024.py b/configs/mmseg/segmentation_tensorrt-int8_static-1024x1024.py new file mode 100644 index 0000000000..b68ac61872 --- /dev/null +++ b/configs/mmseg/segmentation_tensorrt-int8_static-1024x1024.py @@ -0,0 +1,13 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/tensorrt-int8.py'] + +onnx_config = dict(input_shape=[1024, 1024]) +backend_config = dict( + common_config=dict(max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 1024, 1024], + opt_shape=[1, 3, 1024, 1024], + max_shape=[1, 3, 1024, 1024]))) + ]) diff --git a/configs/mmseg/segmentation_tensorrt-int8_static-512x512.py b/configs/mmseg/segmentation_tensorrt-int8_static-512x512.py new file mode 100644 index 0000000000..125c9c1196 --- /dev/null +++ b/configs/mmseg/segmentation_tensorrt-int8_static-512x512.py @@ -0,0 +1,13 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/tensorrt-int8.py'] + +onnx_config = dict(input_shape=[512, 512]) +backend_config = dict( + common_config=dict(max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 512, 512], + opt_shape=[1, 3, 512, 512], + max_shape=[1, 3, 512, 512]))) + ]) diff --git a/configs/mmseg/segmentation_tensorrt_static-1024x1024.py b/configs/mmseg/segmentation_tensorrt_static-1024x1024.py new file mode 100644 index 0000000000..949eec7579 --- /dev/null +++ b/configs/mmseg/segmentation_tensorrt_static-1024x1024.py @@ -0,0 +1,13 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/tensorrt.py'] + +onnx_config = dict(input_shape=[1024, 1024]) +backend_config = dict( + common_config=dict(max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 1024, 1024], + opt_shape=[1, 3, 1024, 1024], + max_shape=[1, 3, 1024, 1024]))) + ]) diff --git a/configs/mmseg/segmentation_tensorrt_static-512x512.py b/configs/mmseg/segmentation_tensorrt_static-512x512.py new file mode 100644 index 0000000000..1fa5ef6695 --- /dev/null +++ b/configs/mmseg/segmentation_tensorrt_static-512x512.py @@ -0,0 +1,13 @@ +_base_ = ['./segmentation_static.py', '../_base_/backends/tensorrt.py'] + +onnx_config = dict(input_shape=[512, 512]) +backend_config = dict( + common_config=dict(max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 512, 512], + opt_shape=[1, 3, 512, 512], + max_shape=[1, 3, 512, 512]))) + ]) diff --git a/docs/en/benchmark.md b/docs/en/benchmark.md index 9aad3987d9..9e66f19b85 100644 --- a/docs/en/benchmark.md +++ b/docs/en/benchmark.md @@ -1480,6 +1480,240 @@ Users can directly test the performance through [how_to_evaluate_a_model.md](tut - $MMSEG_DIR/configs/unet/fcn_unet_s5-d16_4x4_512x1024_160k_cityscapes.py + + ANN + Cityscapes + mIoU + 77.40 + - + - + 77.32 + 77.32 + - + - + $MMSEG_DIR/configs/ann/ann_r50-d8_512x1024_40k_cityscapes.py + + + APCNet + Cityscapes + mIoU + 77.40 + - + - + 77.32 + 77.32 + - + - + $MMSEG_DIR/configs/apcnet/apcnet_r50-d8_512x1024_40k_cityscapes.py + + + BiSeNetV1 + Cityscapes + mIoU + 74.44 + - + - + 74.44 + 74.43 + - + - + $MMSEG_DIR/configs/bisenetv1/bisenetv1_r18-d32_4x4_1024x1024_160k_cityscapes.py + + + BiSeNetV2 + Cityscapes + mIoU + 73.21 + - + - + 73.21 + 73.21 + - + - + $MMSEG_DIR/configs/bisenetv2/bisenetv2_fcn_4x4_1024x1024_160k_cityscapes.py + + + CGNet + Cityscapes + mIoU + 68.25 + - + - + 68.27 + 68.27 + - + - + $MMSEG_DIR/configs/cgnet/cgnet_512x1024_60k_cityscapes.py + + + EMANet + Cityscapes + mIoU + 77.59 + - + - + 77.59 + 77.6 + - + - + $MMSEG_DIR/configs/emanet/emanet_r50-d8_512x1024_80k_cityscapes.py + + + EncNet + Cityscapes + mIoU + 75.67 + - + - + 75.66 + 75.66 + - + - + $MMSEG_DIR/configs/encnet/encnet_r50-d8_512x1024_40k_cityscapes.py + + + ERFNet + Cityscapes + mIoU + 71.08 + - + - + 71.08 + 71.07 + - + - + $MMSEG_DIR/configs/erfnet/erfnet_fcn_4x4_512x1024_160k_cityscapes.py + + + FastFCN + Cityscapes + mIoU + 79.12 + - + - + 79.12 + 79.12 + - + - + $MMSEG_DIR/configs/fastfcn/fastfcn_r50-d32_jpu_aspp_512x1024_80k_cityscapes.py + + + GCNet + Cityscapes + mIoU + 77.69 + - + - + 77.69 + 77.69 + - + - + $MMSEG_DIR/configs/gcnet/gcnet_r50-d8_512x1024_40k_cityscapes.py + + + ICNet + Cityscapes + mIoU + 76.29 + - + - + 76.36 + 76.36 + - + - + $MMSEG_DIR/configs/icnet/icnet_r18-d8_832x832_80k_cityscapes.py + + + ISANet + Cityscapes + mIoU + 78.49 + - + - + 78.49 + 78.49 + - + - + $MMSEG_DIR/configs/isanet/isanet_r50-d8_512x1024_40k_cityscapes.py + + + OCRNet + Cityscapes + mIoU + 74.30 + - + - + 73.66 + 73.67 + - + - + $MMSEG_DIR/configs/ocrnet/ocrnet_hr18s_512x1024_40k_cityscapes.py + + + PointRend + Cityscapes + mIoU + 76.47 + - + - + 76.41 + 76.42 + - + - + $MMSEG_DIR/configs/point_rend/pointrend_r50_512x1024_80k_cityscapes.py + + + Semantic FPN + Cityscapes + mIoU + 74.52 + - + - + 74.52 + 74.52 + - + - + $MMSEG_DIR/configs/sem_fpn/fpn_r50_512x1024_80k_cityscapes.py + + + STDC + Cityscapes + mIoU + 75.10 + - + - + 75.10 + 75.10 + - + - + $MMSEG_DIR/configs/stdc/stdc1_in1k-pre_512x1024_80k_cityscapes.py + + + STDC + Cityscapes + mIoU + 77.17 + - + - + 77.17 + 77.17 + - + - + $MMSEG_DIR/configs/stdc/stdc2_in1k-pre_512x1024_80k_cityscapes.py + + + UPerNet + Cityscapes + mIoU + 77.10 + - + - + 77.19 + 77.18 + - + - + $MMSEG_DIR/configs/upernet/upernet_r50_512x1024_40k_cityscapes.py + diff --git a/docs/en/codebases/mmseg.md b/docs/en/codebases/mmseg.md index 30bbba2082..a2b6859cac 100644 --- a/docs/en/codebases/mmseg.md +++ b/docs/en/codebases/mmseg.md @@ -16,6 +16,33 @@ Please refer to [get_started.md](https://github.com/open-mmlab/mmsegmentation/bl | DeepLabV3+ | Y | Y | Y | Y | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/deeplabv3plus) | | Fast-SCNN[*](#static_shape) | Y | Y | N | Y | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/fastscnn) | | UNet[*](#static_shape) | Y | Y | Y | Y | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/unet) | +| ANN[*](#static_shape) | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/ann) | +| APCNet | Y | Y | Y | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/apcnet) | +| BiSeNetV1 | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/bisenetv1) | +| BiSeNetV2 | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/bisenetv2) | +| CGNet | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/cgnet) | +| DMNet | Y | N | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/dmnet) | +| DNLNet | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/dnlnet) | +| EMANet | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/emanet) | +| EncNet | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/encnet) | +| ERFNet | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/erfnet) | +| FastFCN | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/fastfcn) | +| GCNet | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/gcnet) | +| ICNet[*](#static_shape) | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/icnet) | +| ISANet | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/isanet) | +| NonLocal Net | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/nonlocal_net) | +| OCRNet | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/ocrnet) | +| PointRend | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/point_rend) | +| Semantic FPN | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/sem_fpn) | +| STDC | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/stdc) | +| UPerNet[*](#static_shape) | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/upernet) | +| DANet | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/danet) | +| Segmenter[*](#static_shape) | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/segmenter) | +| SegFormer[*](#static_shape) | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/segformer) | +| SETR | Y | N | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/setr) | +| CCNet | N | N | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/ccnet) | +| PSANet | N | N | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/psanet) | +| DPT | N | N | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/dpt) | ### Reminder diff --git a/docs/en/supported_models.md b/docs/en/supported_models.md index 15f395aaed..dc87318aa1 100644 --- a/docs/en/supported_models.md +++ b/docs/en/supported_models.md @@ -30,6 +30,27 @@ The table below lists the models that are guaranteed to be exportable to other b | DeepLabV3+ | MMSegmentation | Y | Y | Y | Y | Y | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/deeplabv3plus) | | Fast-SCNN[*static](#note) | MMSegmentation | Y | Y | Y | N | Y | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/fastscnn) | | UNet[*static](#note) | MMSegmentation | Y | Y | Y | Y | Y | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/unet) | +| ANN[*](#note) | MMSegmentation | ? | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/ann) | +| APCNet | MMSegmentation | ? | Y | Y | Y | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/apcnet) | +| BiSeNetV1 | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/bisenetv1) | +| BiSeNetV2 | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/bisenetv2) | +| CGNet | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/cgnet) | +| DMNet | MMSegmentation | ? | Y | N | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/dmnet) | +| DNLNet | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/dnlnet) | +| EMANet | MMSegmentation | ? | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/emanet) | +| EncNet | MMSegmentation | ? | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/encnet) | +| ERFNet | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/erfnet) | +| FastFCN | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/fastfcn) | +| GCNet | MMSegmentation | ? | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/gcnet) | +| ICNet[*](#note) | MMSegmentation | ? | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/icnet) | +| ISANet | MMSegmentation | ? | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/isanet) | +| NonLocal Net | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/nonlocal_net) | +| OCRNet | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/ocrnet) | +| PointRend | MMSegmentation | ? | Y | Y | N | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/point_rend) | +| Semantic FPN | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/sem_fpn) | +| STDC | MMSegmentation | ? | Y | Y | Y | N | Y | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/stdc) | +| UPerNet[*](#note) | MMSegmentation | ? | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/upernet) | +| DANet | MMSegmentation | ? | Y | Y | N | N | N | [config](https://github.com/open-mmlab/mmsegmentation/tree/master/configs/danet) | | SRCNN | MMEditing | Y | Y | Y | Y | Y | Y | [config](https://github.com/open-mmlab/mmediting/tree/master/configs/restorers/srcnn) | | ESRGAN | MMEditing | Y | Y | Y | Y | Y | Y | [config](https://github.com/open-mmlab/mmediting/tree/master/configs/restorers/esrgan) | | SRGAN | MMEditing | Y | Y | Y | Y | Y | Y | [config](https://github.com/open-mmlab/mmediting/tree/master/configs/restorers/srresnet_srgan) | diff --git a/docs/zh_cn/benchmark.md b/docs/zh_cn/benchmark.md index a335db488a..3225c44fd8 100644 --- a/docs/zh_cn/benchmark.md +++ b/docs/zh_cn/benchmark.md @@ -1468,6 +1468,240 @@ GPU: ncnn, TensorRT, PPLNN - $MMSEG_DIR/configs/unet/fcn_unet_s5-d16_4x4_512x1024_160k_cityscapes.py + + ANN + Cityscapes + mIoU + 77.40 + - + - + 77.32 + 77.32 + - + - + $MMSEG_DIR/configs/ann/ann_r50-d8_512x1024_40k_cityscapes.py + + + APCNet + Cityscapes + mIoU + 77.40 + - + - + 77.32 + 77.32 + - + - + $MMSEG_DIR/configs/apcnet/apcnet_r50-d8_512x1024_40k_cityscapes.py + + + BiSeNetV1 + Cityscapes + mIoU + 74.44 + - + - + 74.44 + 74.43 + - + - + $MMSEG_DIR/configs/bisenetv1/bisenetv1_r18-d32_4x4_1024x1024_160k_cityscapes.py + + + BiSeNetV2 + Cityscapes + mIoU + 73.21 + - + - + 73.21 + 73.21 + - + - + $MMSEG_DIR/configs/bisenetv2/bisenetv2_fcn_4x4_1024x1024_160k_cityscapes.py + + + CGNet + Cityscapes + mIoU + 68.25 + - + - + 68.27 + 68.27 + - + - + $MMSEG_DIR/configs/cgnet/cgnet_512x1024_60k_cityscapes.py + + + EMANet + Cityscapes + mIoU + 77.59 + - + - + 77.59 + 77.6 + - + - + $MMSEG_DIR/configs/emanet/emanet_r50-d8_512x1024_80k_cityscapes.py + + + EncNet + Cityscapes + mIoU + 75.67 + - + - + 75.66 + 75.66 + - + - + $MMSEG_DIR/configs/encnet/encnet_r50-d8_512x1024_40k_cityscapes.py + + + ERFNet + Cityscapes + mIoU + 71.08 + - + - + 71.08 + 71.07 + - + - + $MMSEG_DIR/configs/erfnet/erfnet_fcn_4x4_512x1024_160k_cityscapes.py + + + FastFCN + Cityscapes + mIoU + 79.12 + - + - + 79.12 + 79.12 + - + - + $MMSEG_DIR/configs/fastfcn/fastfcn_r50-d32_jpu_aspp_512x1024_80k_cityscapes.py + + + GCNet + Cityscapes + mIoU + 77.69 + - + - + 77.69 + 77.69 + - + - + $MMSEG_DIR/configs/gcnet/gcnet_r50-d8_512x1024_40k_cityscapes.py + + + ICNet + Cityscapes + mIoU + 76.29 + - + - + 76.36 + 76.36 + - + - + $MMSEG_DIR/configs/icnet/icnet_r18-d8_832x832_80k_cityscapes.py + + + ISANet + Cityscapes + mIoU + 78.49 + - + - + 78.49 + 78.49 + - + - + $MMSEG_DIR/configs/isanet/isanet_r50-d8_512x1024_40k_cityscapes.py + + + OCRNet + Cityscapes + mIoU + 74.30 + - + - + 73.66 + 73.67 + - + - + $MMSEG_DIR/configs/ocrnet/ocrnet_hr18s_512x1024_40k_cityscapes.py + + + PointRend + Cityscapes + mIoU + 76.47 + - + - + 76.41 + 76.42 + - + - + $MMSEG_DIR/configs/point_rend/pointrend_r50_512x1024_80k_cityscapes.py + + + Semantic FPN + Cityscapes + mIoU + 74.52 + - + - + 74.52 + 74.52 + - + - + $MMSEG_DIR/configs/sem_fpn/fpn_r50_512x1024_80k_cityscapes.py + + + STDC + Cityscapes + mIoU + 75.10 + - + - + 75.10 + 75.10 + - + - + $MMSEG_DIR/configs/stdc/stdc1_in1k-pre_512x1024_80k_cityscapes.py + + + STDC + Cityscapes + mIoU + 77.17 + - + - + 77.17 + 77.17 + - + - + $MMSEG_DIR/configs/stdc/stdc2_in1k-pre_512x1024_80k_cityscapes.py + + + UPerNet + Cityscapes + mIoU + 77.10 + - + - + 77.19 + 77.18 + - + - + $MMSEG_DIR/configs/upernet/upernet_r50_512x1024_40k_cityscapes.py + diff --git a/mmdeploy/codebase/mmseg/models/decode_heads/__init__.py b/mmdeploy/codebase/mmseg/models/decode_heads/__init__.py index 5d505fa8bc..e893f20460 100644 --- a/mmdeploy/codebase/mmseg/models/decode_heads/__init__.py +++ b/mmdeploy/codebase/mmseg/models/decode_heads/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. from .aspp_head import aspp_head__forward +from .ema_head import ema_module__forward from .psp_head import ppm__forward -__all__ = ['aspp_head__forward', 'ppm__forward'] +__all__ = ['aspp_head__forward', 'ppm__forward', 'ema_module__forward'] diff --git a/mmdeploy/codebase/mmseg/models/decode_heads/ema_head.py b/mmdeploy/codebase/mmseg/models/decode_heads/ema_head.py new file mode 100644 index 0000000000..5d839691b7 --- /dev/null +++ b/mmdeploy/codebase/mmseg/models/decode_heads/ema_head.py @@ -0,0 +1,45 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn.functional as F + +from mmdeploy.core import FUNCTION_REWRITER + + +@FUNCTION_REWRITER.register_rewriter( + func_name='mmseg.models.decode_heads.ema_head.EMAModule.forward') +def ema_module__forward(ctx, self, feats): + """Rewrite `forward` for default backend. + + Replace torch.einsum with other operations. + + Args: + ctx (ContextCaller): The context with additional information. + self: The instance of the original class. + feats (Tensor): Input feature. + + Returns: + torch.Tensor: Output feature. + """ + batch_size, channels, height, width = feats.size() + # [batch_size, channels, height*width] + feats = feats.view(batch_size, channels, height * width) + # [batch_size, channels, num_bases] + bases = self.bases.repeat(batch_size, 1, 1) + + with torch.no_grad(): + for i in range(self.num_stages): + # [batch_size, height*width, num_bases] + attention = torch.bmm(feats.transpose(1, 2), bases) + # attention = torch.einsum('bcn,bck->bnk', feats, bases) + attention = F.softmax(attention, dim=2) + # l1 norm + attention_normed = F.normalize(attention, dim=1, p=1) + # [batch_size, channels, num_bases] + bases = torch.bmm(feats, attention_normed) + # bases = torch.einsum('bcn,bnk->bck', feats, attention_normed) + # l2 norm + bases = F.normalize(bases, dim=1, p=2) + feats_recon = torch.bmm(bases, attention.transpose(1, 2)) + # feats_recon = torch.einsum('bck,bnk->bcn', bases, attention) + feats_recon = feats_recon.view(batch_size, channels, height, width) + return feats_recon diff --git a/mmdeploy/codebase/mmseg/models/decode_heads/psp_head.py b/mmdeploy/codebase/mmseg/models/decode_heads/psp_head.py index 81e10ad160..c792237029 100644 --- a/mmdeploy/codebase/mmseg/models/decode_heads/psp_head.py +++ b/mmdeploy/codebase/mmseg/models/decode_heads/psp_head.py @@ -1,4 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. + +import torch.nn as nn from mmseg.ops import resize from mmdeploy.core import FUNCTION_REWRITER @@ -30,6 +32,15 @@ def ppm__forward(ctx, self, x): ppm_outs = [] for ppm in self: + if isinstance(ppm[0], nn.AdaptiveAvgPool2d) and \ + ppm[0].output_size != 1: + assert not is_dynamic_flag, 'AdaptiveAvgPool2d is not \ + supported with dynamic shape in backends' + + # replace AdaptiveAvgPool2d with AvgPool2d explicitly + output_size = 2 * [ppm[0].output_size] + k = [int(size[i] / output_size[i]) for i in range(0, len(size))] + ppm[0] = nn.AvgPool2d(k, stride=k, padding=0, ceil_mode=False) ppm_out = ppm(x) upsampled_ppm_out = resize( ppm_out, diff --git a/mmdeploy/codebase/mmseg/models/segmentors/encoder_decoder.py b/mmdeploy/codebase/mmseg/models/segmentors/encoder_decoder.py index b50778e461..bca614ae86 100644 --- a/mmdeploy/codebase/mmseg/models/segmentors/encoder_decoder.py +++ b/mmdeploy/codebase/mmseg/models/segmentors/encoder_decoder.py @@ -24,8 +24,7 @@ def encoder_decoder__simple_test(ctx, self, img, img_meta, **kwargs): Returns: torch.Tensor: Output segmentation map pf shape [N, 1, H, W]. """ - x = self.extract_feat(img) - seg_logit = self._decode_head_forward_test(x, img_meta) + seg_logit = self.encode_decode(img, img_meta) seg_logit = resize( input=seg_logit, size=img_meta['img_shape'], diff --git a/mmdeploy/pytorch/functions/topk.py b/mmdeploy/pytorch/functions/topk.py index be05fe93e0..64a416b28c 100644 --- a/mmdeploy/pytorch/functions/topk.py +++ b/mmdeploy/pytorch/functions/topk.py @@ -4,6 +4,7 @@ import torch from mmdeploy.core import FUNCTION_REWRITER +from mmdeploy.utils import get_root_logger @FUNCTION_REWRITER.register_rewriter(func_name='torch.topk', backend='default') @@ -47,7 +48,8 @@ def topk__tensorrt(ctx, TensorRT does not support topk with dynamic k. This function cast k to constant integer. """ - + # https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#topKsetup + MAX_TOPK_K = 3840 if dim is None: dim = int(input.ndim - 1) size = input.shape[dim] @@ -55,4 +57,11 @@ def topk__tensorrt(ctx, k = size if not isinstance(k, int): k = int(k) + if k > MAX_TOPK_K: + logger = get_root_logger() + logger.warning( + f'Maximum K of TopK in TensorRT is {MAX_TOPK_K}, but given {k}.' + f' Note that k will be set to {MAX_TOPK_K}.') + k = MAX_TOPK_K + return ctx.origin_func(input, k, dim=dim, largest=largest, sorted=sorted) diff --git a/tests/test_codebase/test_mmseg/test_mmseg_models.py b/tests/test_codebase/test_mmseg/test_mmseg_models.py index 71dbf86187..a29abb898f 100644 --- a/tests/test_codebase/test_mmseg/test_mmseg_models.py +++ b/tests/test_codebase/test_mmseg/test_mmseg_models.py @@ -232,3 +232,32 @@ def test_psphead_forward(backend): rewrite_outputs = rewrite_outputs.to(model_outputs).reshape( model_outputs.shape) assert torch.allclose(rewrite_outputs, model_outputs, rtol=1, atol=1) + + +@pytest.mark.parametrize('backend', [Backend.ONNXRUNTIME]) +def test_emamodule_forward(backend): + check_backend(backend) + from mmseg.models.decode_heads.ema_head import EMAModule + head = EMAModule(8, 2, 2, 1.0).eval() + + deploy_cfg = mmcv.Config( + dict( + backend_config=dict(type=backend.value), + onnx_config=dict( + output_names=['result'], input_shape=(1, 8, 16, 16)), + codebase_config=dict(type='mmseg', task='Segmentation'))) + feats = torch.randn(1, 8, 16, 16) + model_inputs = {'feats': feats} + with torch.no_grad(): + model_outputs = get_model_outputs(head, 'forward', model_inputs) + wrapped_model = WrapModel(head, 'forward') + rewrite_outputs, is_backend_output = get_rewrite_outputs( + wrapped_model=wrapped_model, + model_inputs=model_inputs, + deploy_cfg=deploy_cfg) + if is_backend_output: + rewrite_outputs = rewrite_outputs[0] + rewrite_outputs = rewrite_outputs.to(model_outputs).reshape( + model_outputs.shape) + assert torch.allclose( + rewrite_outputs, model_outputs, rtol=1e-03, atol=1e-05)