diff --git a/datumaro/plugins/openvino/README.md b/datumaro/plugins/openvino/README.md new file mode 100644 index 0000000000..a8f37d3ef9 --- /dev/null +++ b/datumaro/plugins/openvino/README.md @@ -0,0 +1,90 @@ +# OpenVINO™ Inference Interpreter +Interpreter samples to parse OpenVINO™ inference outputs. + +## Models supported from interpreter samples +There are detection and image classification examples. + +- Detection (SSD-based) + - Intel Pre-trained Models > Object Detection + - [face-detection-0200](https://docs.openvinotoolkit.org/latest/omz_models_intel_face_detection_0200_description_face_detection_0200.html) + - [face-detection-0202](https://docs.openvinotoolkit.org/latest/omz_models_intel_face_detection_0202_description_face_detection_0202.html) + - [face-detection-0204](https://docs.openvinotoolkit.org/latest/omz_models_intel_face_detection_0204_description_face_detection_0204.html) + - [person-detection-0200](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_detection_0200_description_person_detection_0200.html) + - [person-detection-0201](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_detection_0201_description_person_detection_0201.html) + - [person-detection-0202](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_detection_0202_description_person_detection_0202.html) + - [person-vehicle-bike-detection-2000](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_vehicle_bike_detection_2000_description_person_vehicle_bike_detection_2000.html) + - [person-vehicle-bike-detection-2001](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_vehicle_bike_detection_2001_description_person_vehicle_bike_detection_2001.html) + - [person-vehicle-bike-detection-2002](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_vehicle_bike_detection_2002_description_person_vehicle_bike_detection_2002.html) + - [vehicle-detection-0200](https://docs.openvinotoolkit.org/latest/omz_models_intel_vehicle_detection_0200_description_vehicle_detection_0200.html) + - [vehicle-detection-0201](https://docs.openvinotoolkit.org/latest/omz_models_intel_vehicle_detection_0201_description_vehicle_detection_0201.html) + - [vehicle-detection-0202](https://docs.openvinotoolkit.org/latest/omz_models_intel_vehicle_detection_0202_description_vehicle_detection_0202.html) + + - Public Pre-Trained Models(OMZ) > Object Detection + - [ssd_mobilenet_v1_coco](https://docs.openvinotoolkit.org/latest/omz_models_public_ssd_mobilenet_v1_coco_ssd_mobilenet_v1_coco.html) + - [ssd_mobilenet_v2_coco](https://docs.openvinotoolkit.org/latest/omz_models_public_ssd_mobilenet_v2_coco_ssd_mobilenet_v2_coco.html) + +- Image Classification + - Public Pre-Trained Models(OMZ) > Classification + - [mobilenet-v2-pytorch](https://docs.openvinotoolkit.org/latest/omz_models_public_mobilenet_v2_pytorch_mobilenet_v2_pytorch.html) + +You can find more OpenVINO™ Trained Models [here](https://docs.openvinotoolkit.org/latest/omz_models_intel_index.html) +To run the inference with OpenVINO™, the model format should be Intermediate Representation(IR). +For the Caffe/TensorFlow/MXNet/Kaldi/ONNX models, please see the [Model Conversion Instruction](https://docs.openvinotoolkit.org/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model.html) + +You need to implement your own interpreter samples to support the other OpenVINO™ Trained Models. + +## Model download +- Prerequisites + - OpenVINO™ (To install OpenVINO™, please see the [OpenVINO™ Installation Instruction](https://docs.openvinotoolkit.org/latest/openvino_docs_install_guides_installing_openvino_linux.html)) + - OpenVINO™ models (To download OpenVINO™ models, please see the [Model Downloader Instruction](https://docs.openvinotoolkit.org/latest/omz_tools_downloader_README.html)) + - PASCAL VOC 2012 dataset (To download VOC 2012 dataset, please go [VOC2012 download](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/#devkit)) + + ```bash + # cd /deployment_tools/open_model_zoo/tools/downloader + # ./downloader.py --name + # + # Examples + cd /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader + ./downloader.py --name face-detection-0200 + ``` + +## Model inference +- Prerequisites: + - OpenVINO™ (To install OpenVINO™, please see the [OpenVINO™ Installation Instruction](https://docs.openvinotoolkit.org/latest/openvino_docs_install_guides_installing_openvino_linux.html)) + - Datumaro (To install Datumaro, please see the [User Manual](docs/user_manual.md)) + - OpenVINO™ models (To download OpenVINO™ models, please see the [Model Downloader Instruction](https://docs.openvinotoolkit.org/latest/omz_tools_downloader_README.html)) + - PASCAL VOC 2012 dataset (To download VOC 2012 dataset, please go [VOC2012 download](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/#devkit)) + +- To run the inference with OpenVINO™ models and the interpreter samples, please follow the instructions below. + + ```bash + # source /bin/setupvars.sh + # datum create -o + # datum model add -l -p --copy -- -d -w -i + # datum add path -p -f + # datum model run -p -m model-0 + # + # Examples + # Detection> ssd_mobilenet_v2_coco + source /opt/intel/openvino/bin/setupvars.sh + cd datumaro/plugins/openvino + datum create -o proj_ssd_mobilenet_v2_coco_detection + datum model add -l openvino -p proj_ssd_mobilenet_v2_coco_detection --copy -- \ + --output-layers=do_ExpandDims_conf/sigmoid \ + -d model/ssd_mobilenet_v2_coco.xml \ + -w model/ssd_mobilenet_v2_coco.bin \ + -i samples/ssd_mobilenet_coco_detection_interp.py + datum add path -p proj_ssd_mobilenet_v2_coco_detection -f voc VOCdevkit/ + datum model run -p proj_ssd_mobilenet_v2_coco_detection -m model-0 + + # Classification> mobilenet-v2-pytorch + source /opt/intel/openvino/bin/setupvars.sh + cd datumaro/plugins/openvino + datum create -o proj_mobilenet_v2_classification + datum model add -l openvino -p proj_mobilenet_v2_classification --copy -- \ + -d model/mobilenet-v2-pytorch.xml \ + -w model/mobilenet-v2-pytorch.bin \ + -i samples/mobilenet_v2_pytorch_interp.py + datum add path -p proj_mobilenet_v2_classification -f voc VOCdevkit/ + datum model run -p proj_mobilenet_v2_classification -m model-0 + ``` \ No newline at end of file diff --git a/datumaro/plugins/openvino/__init__.py b/datumaro/plugins/openvino/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/datumaro/plugins/openvino_launcher.py b/datumaro/plugins/openvino/launcher.py similarity index 100% rename from datumaro/plugins/openvino_launcher.py rename to datumaro/plugins/openvino/launcher.py diff --git a/datumaro/plugins/openvino/samples/coco.class b/datumaro/plugins/openvino/samples/coco.class new file mode 100644 index 0000000000..9682e2b88c --- /dev/null +++ b/datumaro/plugins/openvino/samples/coco.class @@ -0,0 +1,91 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +trafficlight +firehydrant +streetsign +stopsign +parkingmeter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +hat +backpack +umbrella +shoe +eyeglasses +handbag +tie +suitcase +frisbee +skis +snowboard +sportsball +kite +baseballbat +baseballglove +skateboard +surfboard +tennisracket +bottle +plate +wineglass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hotdog +pizza +donut +cake +chair +couch +pottedplant +bed +mirror +diningtable +window +desk +toilet +door +tv +laptop +mouse +remote +keyboard +cellphone +microwave +oven +toaster +sink +refrigerator +blender +book +clock +vase +scissors +teddybear +hairdrier +toothbrush +hairbrush \ No newline at end of file diff --git a/datumaro/plugins/openvino/samples/imagenet.class b/datumaro/plugins/openvino/samples/imagenet.class new file mode 100644 index 0000000000..2571d1b729 --- /dev/null +++ b/datumaro/plugins/openvino/samples/imagenet.class @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane2 +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot2 +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue \ No newline at end of file diff --git a/datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py b/datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py new file mode 100644 index 0000000000..43ce43fd21 --- /dev/null +++ b/datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py @@ -0,0 +1,38 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import Label, LabelCategories, AnnotationType +from datumaro.util.annotation_util import softmax + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, output in zip(inputs, outputs): + image_results = [] + output = softmax(output).tolist() + label = output.index(max(output)) + image_results.append(Label(label=label, attributes={"scores": output})) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + + with open("samples/imagenet.class", "r") as file: + for line in file.readlines(): + label = line.strip() + label_categories.add(label) + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_face_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_face_detection_interp.py new file mode 100644 index 0000000000..abb4604f8d --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_face_detection_interp.py @@ -0,0 +1,83 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("face") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py new file mode 100644 index 0000000000..3b3e5de252 --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py @@ -0,0 +1,90 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.3 +model_class_num = 91 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, confs, detections in zip( + inputs, outputs["do_ExpandDims_conf/sigmoid"], outputs["DetectionOutput"] + ): + + input_height, input_width = input.shape[:2] + + confs = confs[0].reshape(-1, model_class_num) + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + + with open("samples/coco.class", "r") as file: + for line in file.readlines(): + label = line.strip() + label_categories.add(label) + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_person_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_person_detection_interp.py new file mode 100644 index 0000000000..3888944df4 --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_person_detection_interp.py @@ -0,0 +1,83 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("person") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py new file mode 100644 index 0000000000..a2de43dd6d --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py @@ -0,0 +1,85 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("vehicle") + label_categories.add("person") + label_categories.add("bike") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py new file mode 100644 index 0000000000..2866133542 --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py @@ -0,0 +1,83 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("vehicle") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/sampler/sampler.py b/datumaro/plugins/sampler/sampler.py index 99791a8b0f..2fefa0e1dc 100644 --- a/datumaro/plugins/sampler/sampler.py +++ b/datumaro/plugins/sampler/sampler.py @@ -123,10 +123,10 @@ def _load_inference_from_subset(extractor, subset_name): raise Exception(f"Item {item.id} does not have annotations") for annotation in item.annotations: - if 'score' not in annotation.attributes: + if 'scores' not in annotation.attributes: raise Exception(f"Item {item.id} - an annotation " - "does not have 'score' attribute") - probs = annotation.attributes['score'] + "does not have 'scores' attribute") + probs = annotation.attributes['scores'] infer_df['ImageID'].append(item.id) diff --git a/docs/user_manual.md b/docs/user_manual.md index 99eb220523..29af84fa99 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -850,6 +850,7 @@ datum model add \ ``` Interpretation script for an OpenVINO detection model (`convert.py`): +You can find OpenVINO™ model interpreter samples in datumaro/plugins/openvino/samples. [Instruction](datumaro/plugins/openvino/README.md) ``` python from datumaro.components.extractor import * diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 31672e1b6d..3f3d316d28 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -42,7 +42,7 @@ def _get_probs(out_range=False): return probs def _generate_classification_dataset(self, config, subset=None, - empty_score=False, out_range=False, no_attr=False, no_img=False): + empty_scores=False, out_range=False, no_attr=False, no_img=False): probs = self._get_probs(out_range) if subset is None: self.subset = ["train", "val", "test"] @@ -56,11 +56,11 @@ def _generate_classification_dataset(self, config, subset=None, num_item = config[label] label_cat.add(label, attributes=None) for _ in range(num_item): - score = probs[idx] + scores = probs[idx] idx += 1 - if empty_score: - score = [] - attr = {"score": score} + if empty_scores: + scores = [] + attr = {"scores": scores} if no_attr: attr = {} img = Image(path=f"test/dataset/{idx}.jpg", size=(90, 90)) @@ -341,7 +341,7 @@ def test_sampler_gives_error(self): data_df["ImagePath"].append(data.image.path) for annotation in data.annotations: - probs = annotation.attributes["score"] + probs = annotation.attributes["scores"] infer_df["ImageID"].append(data.id) for prob_idx, prob in enumerate(probs): @@ -368,7 +368,7 @@ def test_sampler_gives_error(self): data_df["ImagePath"].append(data.image.path) for annotation in data.annotations: - probs = annotation.attributes["score"] + probs = annotation.attributes["scores"] for prob_idx, prob in enumerate(probs): infer_df[f"ClassProbability{prob_idx+1}"].append(prob) @@ -401,14 +401,14 @@ def test_sampler_get_invalid_data(self): result = iter(result) next(result) - with self.subTest("Dataset without Score (Probability)"): + with self.subTest("Dataset without Scores (Probability)"): config = { "label1": 10, "label2": 10, "label3": 10, } - source = self._generate_classification_dataset(config, empty_score=True) + source = self._generate_classification_dataset(config, empty_scores=True) with self.assertRaisesRegex(Exception, "ClassProbability"): result = Sampler( source, @@ -431,7 +431,7 @@ def test_sampler_get_invalid_data(self): } source = self._generate_classification_dataset( - config, empty_score=False, out_range=True + config, empty_scores=False, out_range=True ) with self.assertRaisesRegex(Exception, "Invalid data"): result = Sampler( @@ -447,7 +447,7 @@ def test_sampler_get_invalid_data(self): result = iter(result) next(result) - with self.subTest("No Score Attribute Data"): + with self.subTest("No Scores Attribute Data"): config = { "label1": 10, "label2": 10, @@ -455,7 +455,7 @@ def test_sampler_get_invalid_data(self): } source = self._generate_classification_dataset(config, no_attr=True) - with self.assertRaisesRegex(Exception, "does not have 'score'"): + with self.assertRaisesRegex(Exception, "does not have 'scores'"): result = Sampler( source, algorithm="entropy",