From df763703898ddb4195aebf48531ef144b1ac1b4e Mon Sep 17 00:00:00 2001 From: nemonameless Date: Wed, 18 Jan 2023 14:17:11 +0000 Subject: [PATCH 1/3] add ppyoloe architectures --- deploy/python/infer.py | 8 +- deploy/serving/python/web_service.py | 6 +- .../third_engine/demo_onnx_trt/trt_infer.py | 10 +- deploy/third_engine/onnx/infer.py | 6 +- ppdet/engine/export_utils.py | 3 +- ppdet/modeling/architectures/__init__.py | 2 + ppdet/modeling/architectures/ppyoloe.py | 97 +++++++++++++++++++ ppdet/modeling/architectures/yolo.py | 5 + 8 files changed, 121 insertions(+), 16 deletions(-) create mode 100644 ppdet/modeling/architectures/ppyoloe.py diff --git a/deploy/python/infer.py b/deploy/python/infer.py index 6136d0db156..df206f22b59 100644 --- a/deploy/python/infer.py +++ b/deploy/python/infer.py @@ -40,10 +40,10 @@ # Global dictionary SUPPORT_MODELS = { - 'YOLO', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', 'S2ANet', 'JDE', - 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', 'TOOD', 'RetinaNet', - 'StrongBaseline', 'STGCN', 'YOLOX', 'YOLOF', 'PPHGNet', 'PPLCNet', 'DETR', - 'CenterTrack' + 'YOLO', 'PPYOLOE', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', + 'S2ANet', 'JDE', 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', + 'TOOD', 'RetinaNet', 'StrongBaseline', 'STGCN', 'YOLOX', 'YOLOF', 'PPHGNet', + 'PPLCNet', 'DETR', 'CenterTrack' } TUNED_TRT_DYNAMIC_MODELS = {'DETR'} diff --git a/deploy/serving/python/web_service.py b/deploy/serving/python/web_service.py index 05b559a41ca..08be7d2c619 100644 --- a/deploy/serving/python/web_service.py +++ b/deploy/serving/python/web_service.py @@ -30,9 +30,9 @@ # Global dictionary SUPPORT_MODELS = { - 'YOLO', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', 'S2ANet', 'JDE', - 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', 'TOOD', 'RetinaNet', - 'StrongBaseline', 'STGCN', 'YOLOX', 'HRNet' + 'YOLO', 'PPYOLOE', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', + 'S2ANet', 'JDE', 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', + 'TOOD', 'RetinaNet', 'StrongBaseline', 'STGCN', 'YOLOX', 'HRNet' } GLOBAL_VAR = {} diff --git a/deploy/third_engine/demo_onnx_trt/trt_infer.py b/deploy/third_engine/demo_onnx_trt/trt_infer.py index e48f144cde7..2d1381f2553 100644 --- a/deploy/third_engine/demo_onnx_trt/trt_infer.py +++ b/deploy/third_engine/demo_onnx_trt/trt_infer.py @@ -51,9 +51,9 @@ trt.init_libnvinfer_plugins(TRT_LOGGER, namespace="") # Global dictionary SUPPORT_MODELS = { - 'YOLO', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', 'S2ANet', 'JDE', - 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', 'TOOD', 'RetinaNet', - 'StrongBaseline', 'STGCN', 'YOLOX', 'HRNet' + 'YOLO', 'PPYOLOE', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', + 'S2ANet', 'JDE', 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', + 'TOOD', 'RetinaNet', 'StrongBaseline', 'STGCN', 'YOLOX', 'HRNet' } @@ -205,8 +205,8 @@ def create_trt_bindings(engine, context): "is_input": True if engine.binding_is_input(name) else False } if engine.binding_is_input(name): - bindings[name]['cpu_data'] = np.random.randn( - *shape).astype(np.float32) + bindings[name]['cpu_data'] = np.random.randn(*shape).astype( + np.float32) bindings[name]['cuda_ptr'] = cuda.mem_alloc(bindings[name][ 'cpu_data'].nbytes) else: diff --git a/deploy/third_engine/onnx/infer.py b/deploy/third_engine/onnx/infer.py index 9dbe2bde9e0..e916af9d016 100644 --- a/deploy/third_engine/onnx/infer.py +++ b/deploy/third_engine/onnx/infer.py @@ -23,9 +23,9 @@ # Global dictionary SUPPORT_MODELS = { - 'YOLO', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', 'S2ANet', 'JDE', - 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', 'TOOD', 'RetinaNet', - 'StrongBaseline', 'STGCN', 'YOLOX', 'HRNet' + 'YOLO', 'PPYOLOE', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', + 'S2ANet', 'JDE', 'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', + 'TOOD', 'RetinaNet', 'StrongBaseline', 'STGCN', 'YOLOX', 'HRNet' } parser = argparse.ArgumentParser(description=__doc__) diff --git a/ppdet/engine/export_utils.py b/ppdet/engine/export_utils.py index 136bea3411d..eb8e181690b 100644 --- a/ppdet/engine/export_utils.py +++ b/ppdet/engine/export_utils.py @@ -29,6 +29,7 @@ # Global dictionary TRT_MIN_SUBGRAPH = { 'YOLO': 3, + 'PPYOLOE': 3, 'SSD': 60, 'RCNN': 40, 'RetinaNet': 40, @@ -193,7 +194,7 @@ def _dump_infer_config(config, path, image_shape, model): arch_state = True break - if infer_arch in ['YOLOX', 'YOLOF']: + if infer_arch in ['PPYOLOE', 'YOLOX', 'YOLOF']: infer_cfg['arch'] = infer_arch infer_cfg['min_subgraph_size'] = TRT_MIN_SUBGRAPH[infer_arch] arch_state = True diff --git a/ppdet/modeling/architectures/__init__.py b/ppdet/modeling/architectures/__init__.py index 83ab3e0ad84..c89106c4f5f 100644 --- a/ppdet/modeling/architectures/__init__.py +++ b/ppdet/modeling/architectures/__init__.py @@ -16,6 +16,7 @@ from . import faster_rcnn from . import mask_rcnn from . import yolo +from . import ppyoloe from . import cascade_rcnn from . import ssd from . import fcos @@ -44,6 +45,7 @@ from .faster_rcnn import * from .mask_rcnn import * from .yolo import * +from .ppyoloe import * from .cascade_rcnn import * from .ssd import * from .fcos import * diff --git a/ppdet/modeling/architectures/ppyoloe.py b/ppdet/modeling/architectures/ppyoloe.py new file mode 100644 index 00000000000..1b5b109d517 --- /dev/null +++ b/ppdet/modeling/architectures/ppyoloe.py @@ -0,0 +1,97 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from ppdet.core.workspace import register, create +from .meta_arch import BaseArch + +__all__ = ['PPYOLOE'] + + +@register +class PPYOLOE(BaseArch): + __category__ = 'architecture' + __inject__ = ['post_process'] + + def __init__(self, + backbone='CSPResNet', + neck='CustomCSPPAN', + yolo_head='PPYOLOEHead', + post_process='BBoxPostProcess', + for_mot=False): + """ + PPYOLOE network, see https://arxiv.org/abs/2203.16250 + + Args: + backbone (nn.Layer): backbone instance + neck (nn.Layer): neck instance + yolo_head (nn.Layer): anchor_head instance + post_process (object): `BBoxPostProcess` instance + for_mot (bool): whether return other features for multi-object tracking + models, default False in pure object detection models. + """ + super(PPYOLOE, self).__init__() + self.backbone = backbone + self.neck = neck + self.yolo_head = yolo_head + self.post_process = post_process + self.for_mot = for_mot + + @classmethod + def from_config(cls, cfg, *args, **kwargs): + # backbone + backbone = create(cfg['backbone']) + + # fpn + kwargs = {'input_shape': backbone.out_shape} + neck = create(cfg['neck'], **kwargs) + + # head + kwargs = {'input_shape': neck.out_shape} + yolo_head = create(cfg['yolo_head'], **kwargs) + + return { + 'backbone': backbone, + 'neck': neck, + "yolo_head": yolo_head, + } + + def _forward(self): + body_feats = self.backbone(self.inputs) + neck_feats = self.neck(body_feats, self.for_mot) + + if self.training: + yolo_losses = self.yolo_head(neck_feats, self.inputs) + return yolo_losses + else: + yolo_head_outs = self.yolo_head(neck_feats) + if self.post_process is not None: + bbox, bbox_num = self.post_process( + yolo_head_outs, self.yolo_head.mask_anchors, + self.inputs['im_shape'], self.inputs['scale_factor']) + else: + bbox, bbox_num = self.yolo_head.post_process( + yolo_head_outs, self.inputs['scale_factor']) + output = {'bbox': bbox, 'bbox_num': bbox_num} + + return output + + def get_loss(self): + return self._forward() + + def get_pred(self): + return self._forward() diff --git a/ppdet/modeling/architectures/yolo.py b/ppdet/modeling/architectures/yolo.py index f23f0ddd4f7..55174411fc5 100644 --- a/ppdet/modeling/architectures/yolo.py +++ b/ppdet/modeling/architectures/yolo.py @@ -21,6 +21,7 @@ from ..post_process import JDEBBoxPostProcess __all__ = ['YOLOv3'] +# YOLOv3,PP-YOLO,PP-YOLOv2,PP-YOLOE,PP-YOLOE+ use the same architecture as YOLOv3 @register @@ -99,6 +100,7 @@ def _forward(self): yolo_head_outs = self.yolo_head(neck_feats) if self.for_mot: + # the detection part of JDE MOT model boxes_idx, bbox, bbox_num, nms_keep_idx = self.post_process( yolo_head_outs, self.yolo_head.mask_anchors) output = { @@ -110,13 +112,16 @@ def _forward(self): } else: if self.return_idx: + # the detection part of JDE MOT model _, bbox, bbox_num, _ = self.post_process( yolo_head_outs, self.yolo_head.mask_anchors) elif self.post_process is not None: + # anchor based YOLOs: YOLOv3,PP-YOLO,PP-YOLOv2 use mask_anchors bbox, bbox_num = self.post_process( yolo_head_outs, self.yolo_head.mask_anchors, self.inputs['im_shape'], self.inputs['scale_factor']) else: + # anchor free YOLOs: PP-YOLOE, PP-YOLOE+ bbox, bbox_num = self.yolo_head.post_process( yolo_head_outs, self.inputs['scale_factor']) output = {'bbox': bbox, 'bbox_num': bbox_num} From ab38ffd04fdd0e923f6b31ace57ec045e5f94f37 Mon Sep 17 00:00:00 2001 From: nemonameless Date: Sat, 28 Jan 2023 03:31:43 +0000 Subject: [PATCH 2/3] fix deploy ppyoloe arch --- deploy/pptracking/python/det_infer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/pptracking/python/det_infer.py b/deploy/pptracking/python/det_infer.py index c5287945302..cf99f0d70ac 100644 --- a/deploy/pptracking/python/det_infer.py +++ b/deploy/pptracking/python/det_infer.py @@ -39,6 +39,7 @@ # Global dictionary SUPPORT_MODELS = { 'YOLO', + 'PPYOLOE', 'PicoDet', 'JDE', 'FairMOT', From f774a13d6b063ad5662d6f681cd9bcecac15b9a0 Mon Sep 17 00:00:00 2001 From: nemonameless Date: Sat, 28 Jan 2023 04:27:14 +0000 Subject: [PATCH 3/3] add ppyoloe arch coments, test=document_fix --- ppdet/modeling/architectures/ppyoloe.py | 2 ++ ppdet/modeling/architectures/yolo.py | 1 + 2 files changed, 3 insertions(+) diff --git a/ppdet/modeling/architectures/ppyoloe.py b/ppdet/modeling/architectures/ppyoloe.py index 1b5b109d517..0d0e926f49b 100644 --- a/ppdet/modeling/architectures/ppyoloe.py +++ b/ppdet/modeling/architectures/ppyoloe.py @@ -20,6 +20,8 @@ from .meta_arch import BaseArch __all__ = ['PPYOLOE'] +# PP-YOLOE and PP-YOLOE+ are recommended to use this architecture +# PP-YOLOE and PP-YOLOE+ can also use the same architecture of YOLOv3 in yolo.py @register diff --git a/ppdet/modeling/architectures/yolo.py b/ppdet/modeling/architectures/yolo.py index 55174411fc5..78c7654913c 100644 --- a/ppdet/modeling/architectures/yolo.py +++ b/ppdet/modeling/architectures/yolo.py @@ -22,6 +22,7 @@ __all__ = ['YOLOv3'] # YOLOv3,PP-YOLO,PP-YOLOv2,PP-YOLOE,PP-YOLOE+ use the same architecture as YOLOv3 +# PP-YOLOE and PP-YOLOE+ are recommended to use PPYOLOE architecture in ppyoloe.py @register