From efb86bbe7f0f54e37427b0681daf7f9483187d3e Mon Sep 17 00:00:00 2001 From: katdimitris Date: Tue, 26 Sep 2023 12:40:30 +0300 Subject: [PATCH 01/15] Added initial version of class filtering --- .../utils/class_filter_wrapper.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py new file mode 100644 index 0000000000..910ad53044 --- /dev/null +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -0,0 +1,29 @@ +from opendr.engine.target import BoundingBox, BoundingBoxList + + +class FilteredLearnerWrapper: + def __init__(self, learner, allowed_classes=None): + self.learner = learner + self.allowed_classes = allowed_classes if allowed_classes is not None else [] + + def infer(self, img, threshold=0.1, keep_size=True): + + boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) + if not self.allowed_classes: + return boxes + else: + obj_det_classes = self.learner.classes + + # Check if all allowed classes exist in the list of object detector's classes + invalid_classes = [cls for cls in self.allowed_classes if cls not in obj_det_classes] + if invalid_classes: + raise ValueError( + f"The following classes are not detected by this detector: {', '.join(invalid_classes)}") + + filtered_boxes = BoundingBoxList( + [box for box in boxes if obj_det_classes[int(box.name)] in self.allowed_classes]) + + return filtered_boxes + + def __getattr__(self, attr): + return getattr(self.learner, attr) From eb5cf240a21f7b692f040523621fcb6c2e5a8eb3 Mon Sep 17 00:00:00 2001 From: katdimitris Date: Thu, 28 Sep 2023 11:34:01 +0300 Subject: [PATCH 02/15] added parameter conditions for all implemented learners --- .../utils/class_filter_wrapper.py | 124 ++++++++++++++++-- 1 file changed, 112 insertions(+), 12 deletions(-) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index 910ad53044..b9b9567758 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -1,4 +1,12 @@ from opendr.engine.target import BoundingBox, BoundingBoxList +from opendr.perception.object_detection_2d.centernet.centernet_learner import CenterNetDetectorLearner +from opendr.perception.object_detection_2d.detr.detr_learner import DetrLearner +from opendr.perception.object_detection_2d.gem.gem_learner import GemLearner +from opendr.perception.object_detection_2d.retinaface.retinaface_learner import RetinaFaceLearner +from opendr.perception.object_detection_2d.ssd.ssd_learner import SingleShotDetectorLearner +from opendr.perception.object_detection_2d.yolov3.yolov3_learner import YOLOv3DetectorLearner +from opendr.perception.object_detection_2d.yolov5.yolov5_learner import YOLOv5DetectorLearner +from opendr.perception.object_detection_2d.nanodet.nanodet_learner import NanodetLearner class FilteredLearnerWrapper: @@ -6,24 +14,116 @@ def __init__(self, learner, allowed_classes=None): self.learner = learner self.allowed_classes = allowed_classes if allowed_classes is not None else [] - def infer(self, img, threshold=0.1, keep_size=True): + if isinstance(self.learner, + (CenterNetDetectorLearner, YOLOv3DetectorLearner, YOLOv5DetectorLearner, NanodetLearner, + RetinaFaceLearner, SingleShotDetectorLearner)): + self.classes = self.learner.classes + if isinstance(self.learner, (DetrLearner, GemLearner)): + coco_classes = [ + "person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", + "trafficlight", "firehydrant", "streetsign", "stopsign", "parkingmeter", "bench", "bird", + "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "hat", + "backpack", "umbrella", "shoe", "eyeglasses", "handbag", "tie", "suitcase", "frisbee", + "skis", "snowboard", "sportsball", "kite", "baseballbat", "baseballglove", "skateboard", + "surfboard", "tennisracket", "bottle", "plate", "wineglass", "cup", "fork", "knife", + "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hotdog", + "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "mirror", "diningtable", + "window", "desk", "toilet", "door", "tvmonitor", "laptop", "mouse", "remote", "keyboard", + "cellphone", "microwave", "oven", "toaster", "sink", "refrigerator", "blender", "book", + "clock", "vase", "scissors", "teddybear", "hairdrier", "toothbrush", "hairbrush" + ] + self.classes = coco_classes - boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) - if not self.allowed_classes: - return boxes - else: - obj_det_classes = self.learner.classes + # Check if all allowed classes exist in the list of object detector's classes + invalid_classes = [cls for cls in self.allowed_classes if cls not in self.classes] + if invalid_classes: + raise ValueError( + f"The following classes are not detected by this detector: {', '.join(invalid_classes)}") - # Check if all allowed classes exist in the list of object detector's classes - invalid_classes = [cls for cls in self.allowed_classes if cls not in obj_det_classes] - if invalid_classes: + def infer(self, img=None, threshold=None, keep_size=None, m1_image=None, m2_image=None, input=None, + conf_threshold=None, iou_threshold=None, nms_max_num=None, nms_threshold=None, scales=None, + mask_thresh=None, size=None, custom_nms=None, nms_thresh=None, nms_topk=None, post_nms=None, + extract_maps=None): + + if isinstance(self.learner, GemLearner): + if m1_image is None or m2_image is None: + raise ValueError( + f"Two image inputs are required. Please provide two valid images.") + boxes = self.learner.infer(m1_image, m2_image) + + elif isinstance(self.learner, NanodetLearner): + if input is None: + raise ValueError( + f"An image input is required. Please provide a valid image.") + if conf_threshold is None: + conf_threshold = 0.35 + if iou_threshold is None: + iou_threshold = 0.6 + if nms_max_num is None: + nms_max_num = 100 + boxes = self.learner.infer(img, conf_threshold, iou_threshold, nms_max_num) + + else: + if img is None: raise ValueError( - f"The following classes are not detected by this detector: {', '.join(invalid_classes)}") + f"An image input is required. Please provide a valid image.") + if isinstance(self.learner, CenterNetDetectorLearner): + if threshold is None: + threshold = 0.2 + if keep_size is None: + keep_size = True + boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) + + elif isinstance(self.learner, YOLOv3DetectorLearner): + if threshold is None: + threshold = 0.1 + if keep_size is None: + keep_size = True + boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) + + elif isinstance(self.learner, YOLOv5DetectorLearner): + if size is None: + size = 640 + boxes = self.learner.infer(img, size) + + elif isinstance(self.learner, DetrLearner): + boxes = self.learner.infer(img) + + elif isinstance(self.learner, RetinaFaceLearner): + if threshold is None: + threshold = 0.8 + if nms_threshold is None: + nms_threshold = 0.4 + if scales is None: + scales = [1024, 1980] + if mask_thresh is None: + mask_thresh = 0.8 + boxes = self.learner.infer(img, threshold, nms_threshold, scales, mask_thresh) + + elif isinstance(self.learner, SingleShotDetectorLearner): + if threshold is None: + threshold = 0.2 + if keep_size is None: + keep_size = False + if nms_thresh is None: + nms_thresh = 0.45 + if nms_topk is None: + nms_topk = 400 + if post_nms is None: + post_nms = 100 + if extract_maps is None: + extract_maps = False + boxes = self.learner.infer(img, threshold, keep_size, custom_nms, + nms_thresh, nms_topk, post_nms, extract_maps) + + if not self.allowed_classes: + return boxes + else: filtered_boxes = BoundingBoxList( - [box for box in boxes if obj_det_classes[int(box.name)] in self.allowed_classes]) + [box for box in boxes if self.classes[int(box.name)] in self.allowed_classes]) - return filtered_boxes + return filtered_boxes def __getattr__(self, attr): return getattr(self.learner, attr) From fecfc31cf0be0bc6195eb9edab1d65fe082f59c6 Mon Sep 17 00:00:00 2001 From: katdimitris Date: Thu, 28 Sep 2023 14:41:55 +0300 Subject: [PATCH 03/15] added parameter conditions for all implemented learners --- .../yolov5/filtered_inference_demo.py | 51 +++++++ .../utils/class_filter_wrapper.py | 134 +++++++++--------- 2 files changed, 118 insertions(+), 67 deletions(-) create mode 100644 projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py diff --git a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py new file mode 100644 index 0000000000..5095104bd1 --- /dev/null +++ b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py @@ -0,0 +1,51 @@ +# Copyright 2020-2023 OpenDR European Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import cv2 +import torch +import numpy as np + +from opendr.engine.data import Image +from opendr.perception.object_detection_2d import YOLOv5DetectorLearner +from opendr.perception.object_detection_2d import draw_bounding_boxes + +from opendr.perception.object_detection_2d.utils.class_filter_wrapper import FilteredLearnerWrapper + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--device", help="Device to use (cpu, cuda)", type=str, default="cuda", choices=["cuda", "cpu"]) + parser.add_argument("--classes", help="Classes of interest to detect", type=str, nargs="+", default=[]) + args = parser.parse_args() + + yolo = YOLOv5DetectorLearner(model_name='yolov5s', device=args.device) + filtered_yolo = FilteredLearnerWrapper(yolo, allowed_classes=args.classes) + + for f in 'zidane.jpg', 'bus.jpg': + torch.hub.download_url_to_file('https://ultralytics.com/images/' + f, f) # download 2 images + im1 = Image.open('zidane.jpg') # OpenDR image + im2 = cv2.imread('bus.jpg') # OpenCV image (BGR to RGB) + + results = yolo.infer(im1) + draw_bounding_boxes(im1.opencv(), results, yolo.classes, show=True, line_thickness=3) + + filtered_results = filtered_yolo.infer(im1) + draw_bounding_boxes(im1.opencv(), filtered_results, filtered_yolo.classes, show=True, line_thickness=3) + + results = yolo.infer(im2) + draw_bounding_boxes(np.copy(im2), results, yolo.classes, show=True, line_thickness=3) + + filtered_results = filtered_yolo.infer(im2) + draw_bounding_boxes(np.copy(im2), filtered_results, filtered_yolo.classes, show=True, line_thickness=3) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index b9b9567758..021ce077ec 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -1,4 +1,4 @@ -from opendr.engine.target import BoundingBox, BoundingBoxList +from opendr.engine.target import BoundingBoxList from opendr.perception.object_detection_2d.centernet.centernet_learner import CenterNetDetectorLearner from opendr.perception.object_detection_2d.detr.detr_learner import DetrLearner from opendr.perception.object_detection_2d.gem.gem_learner import GemLearner @@ -10,15 +10,16 @@ class FilteredLearnerWrapper: - def __init__(self, learner, allowed_classes=None): + def __init__(self, learner, allowed_classes=[]): self.learner = learner - self.allowed_classes = allowed_classes if allowed_classes is not None else [] + self.allowed_classes = allowed_classes if isinstance(self.learner, (CenterNetDetectorLearner, YOLOv3DetectorLearner, YOLOv5DetectorLearner, NanodetLearner, RetinaFaceLearner, SingleShotDetectorLearner)): self.classes = self.learner.classes if isinstance(self.learner, (DetrLearner, GemLearner)): + # added None class for align purposes coco_classes = [ "person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "trafficlight", "firehydrant", "streetsign", "stopsign", "parkingmeter", "bench", "bird", @@ -45,85 +46,84 @@ def infer(self, img=None, threshold=None, keep_size=None, m1_image=None, m2_imag mask_thresh=None, size=None, custom_nms=None, nms_thresh=None, nms_topk=None, post_nms=None, extract_maps=None): - if isinstance(self.learner, GemLearner): - if m1_image is None or m2_image is None: - raise ValueError( - f"Two image inputs are required. Please provide two valid images.") - boxes = self.learner.infer(m1_image, m2_image) + # match variable names + if isinstance(self.learner, NanodetLearner): + img = input + + if img is None: + raise ValueError( + f"An image input is required. Please provide a valid image.") + + if isinstance(self.learner, CenterNetDetectorLearner): + if threshold is None: + threshold = 0.2 + if keep_size is None: + keep_size = True + boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) + + elif isinstance(self.learner, YOLOv3DetectorLearner): + if threshold is None: + threshold = 0.1 + if keep_size is None: + keep_size = True + boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) + + elif isinstance(self.learner, YOLOv5DetectorLearner): + if size is None: + size = 640 + boxes = self.learner.infer(img, size) + + elif isinstance(self.learner, DetrLearner): + boxes = self.learner.infer(img) elif isinstance(self.learner, NanodetLearner): - if input is None: - raise ValueError( - f"An image input is required. Please provide a valid image.") if conf_threshold is None: conf_threshold = 0.35 if iou_threshold is None: iou_threshold = 0.6 if nms_max_num is None: nms_max_num = 100 - boxes = self.learner.infer(img, conf_threshold, iou_threshold, nms_max_num) - - else: - if img is None: - raise ValueError( - f"An image input is required. Please provide a valid image.") - - if isinstance(self.learner, CenterNetDetectorLearner): - if threshold is None: - threshold = 0.2 - if keep_size is None: - keep_size = True - boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) - - elif isinstance(self.learner, YOLOv3DetectorLearner): - if threshold is None: - threshold = 0.1 - if keep_size is None: - keep_size = True - boxes = self.learner.infer(img, threshold=threshold, keep_size=keep_size) - - elif isinstance(self.learner, YOLOv5DetectorLearner): - if size is None: - size = 640 - boxes = self.learner.infer(img, size) - - elif isinstance(self.learner, DetrLearner): - boxes = self.learner.infer(img) - - elif isinstance(self.learner, RetinaFaceLearner): - if threshold is None: - threshold = 0.8 - if nms_threshold is None: - nms_threshold = 0.4 - if scales is None: - scales = [1024, 1980] - if mask_thresh is None: - mask_thresh = 0.8 - boxes = self.learner.infer(img, threshold, nms_threshold, scales, mask_thresh) - - elif isinstance(self.learner, SingleShotDetectorLearner): - if threshold is None: - threshold = 0.2 - if keep_size is None: - keep_size = False - if nms_thresh is None: - nms_thresh = 0.45 - if nms_topk is None: - nms_topk = 400 - if post_nms is None: - post_nms = 100 - if extract_maps is None: - extract_maps = False - boxes = self.learner.infer(img, threshold, keep_size, custom_nms, - nms_thresh, nms_topk, post_nms, extract_maps) + boxes = self.learner.infer(input, conf_threshold, iou_threshold, nms_max_num) + + elif isinstance(self.learner, RetinaFaceLearner): + if threshold is None: + threshold = 0.8 + if nms_threshold is None: + nms_threshold = 0.4 + if scales is None: + scales = [1024, 1980] + if mask_thresh is None: + mask_thresh = 0.8 + boxes = self.learner.infer(img, threshold, nms_threshold, scales, mask_thresh) + + elif isinstance(self.learner, SingleShotDetectorLearner): + if threshold is None: + threshold = 0.2 + if keep_size is None: + keep_size = False + if nms_thresh is None: + nms_thresh = 0.45 + if nms_topk is None: + nms_topk = 400 + if post_nms is None: + post_nms = 100 + if extract_maps is None: + extract_maps = False + boxes = self.learner.infer(img, threshold, keep_size, custom_nms, + nms_thresh, nms_topk, post_nms, extract_maps) if not self.allowed_classes: return boxes + + # Adjust index to align with COCO label numbering + if isinstance(self.learner, DetrLearner): + filtered_boxes = BoundingBoxList( + [box for box in boxes if self.classes[int(box.name) - 1] in self.allowed_classes]) else: filtered_boxes = BoundingBoxList( [box for box in boxes if self.classes[int(box.name)] in self.allowed_classes]) - return filtered_boxes + return filtered_boxes def __getattr__(self, attr): return getattr(self.learner, attr) From 37a0a58cec4880176437e41fd12da74a17643850 Mon Sep 17 00:00:00 2001 From: katdimitris Date: Thu, 28 Sep 2023 15:08:44 +0300 Subject: [PATCH 04/15] final implementation of class filtering for object detectors. added inference demo for yolov5. --- .../yolov5/filtered_inference_demo.py | 8 +++- .../utils/class_filter_wrapper.py | 39 +++++++------------ 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py index 5095104bd1..a3b840f70d 100644 --- a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py +++ b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py @@ -27,10 +27,12 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--device", help="Device to use (cpu, cuda)", type=str, default="cuda", choices=["cuda", "cpu"]) - parser.add_argument("--classes", help="Classes of interest to detect", type=str, nargs="+", default=[]) + parser.add_argument("--classes", help="Classes of interest to detect", type=str, nargs="+", default=['person']) args = parser.parse_args() yolo = YOLOv5DetectorLearner(model_name='yolov5s', device=args.device) + + # By default, only objects of the 'person' class are of interest filtered_yolo = FilteredLearnerWrapper(yolo, allowed_classes=args.classes) for f in 'zidane.jpg', 'bus.jpg': @@ -38,14 +40,18 @@ im1 = Image.open('zidane.jpg') # OpenDR image im2 = cv2.imread('bus.jpg') # OpenCV image (BGR to RGB) + # detection before filtering results = yolo.infer(im1) draw_bounding_boxes(im1.opencv(), results, yolo.classes, show=True, line_thickness=3) + # detection after filtering filtered_results = filtered_yolo.infer(im1) draw_bounding_boxes(im1.opencv(), filtered_results, filtered_yolo.classes, show=True, line_thickness=3) + # detection before filtering results = yolo.infer(im2) draw_bounding_boxes(np.copy(im2), results, yolo.classes, show=True, line_thickness=3) + # detection after filtering filtered_results = filtered_yolo.infer(im2) draw_bounding_boxes(np.copy(im2), filtered_results, filtered_yolo.classes, show=True, line_thickness=3) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index 021ce077ec..a49d44611d 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -1,7 +1,6 @@ from opendr.engine.target import BoundingBoxList from opendr.perception.object_detection_2d.centernet.centernet_learner import CenterNetDetectorLearner from opendr.perception.object_detection_2d.detr.detr_learner import DetrLearner -from opendr.perception.object_detection_2d.gem.gem_learner import GemLearner from opendr.perception.object_detection_2d.retinaface.retinaface_learner import RetinaFaceLearner from opendr.perception.object_detection_2d.ssd.ssd_learner import SingleShotDetectorLearner from opendr.perception.object_detection_2d.yolov3.yolov3_learner import YOLOv3DetectorLearner @@ -13,29 +12,26 @@ class FilteredLearnerWrapper: def __init__(self, learner, allowed_classes=[]): self.learner = learner self.allowed_classes = allowed_classes - if isinstance(self.learner, (CenterNetDetectorLearner, YOLOv3DetectorLearner, YOLOv5DetectorLearner, NanodetLearner, RetinaFaceLearner, SingleShotDetectorLearner)): self.classes = self.learner.classes - if isinstance(self.learner, (DetrLearner, GemLearner)): - # added None class for align purposes + if isinstance(self.learner, DetrLearner): coco_classes = [ - "person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", - "trafficlight", "firehydrant", "streetsign", "stopsign", "parkingmeter", "bench", "bird", - "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "hat", - "backpack", "umbrella", "shoe", "eyeglasses", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sportsball", "kite", "baseballbat", "baseballglove", "skateboard", - "surfboard", "tennisracket", "bottle", "plate", "wineglass", "cup", "fork", "knife", - "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hotdog", - "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "mirror", "diningtable", - "window", "desk", "toilet", "door", "tvmonitor", "laptop", "mouse", "remote", "keyboard", - "cellphone", "microwave", "oven", "toaster", "sink", "refrigerator", "blender", "book", - "clock", "vase", "scissors", "teddybear", "hairdrier", "toothbrush", "hairbrush" + "N/A", "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", + "traffic light", "fire hydrant", "N/A", "stop sign", "parking meter", "bench", "bird", "cat", "dog", + "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "N/A", "backpack", "umbrella", "N/A", + "N/A", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", + "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "N/A", + "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", + "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", + "N/A", "dining table", "N/A", "N/A", "toilet", "N/A", "tv", "laptop", "mouse", "remote", "keyboard", + "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "N/A", "book", "clock", "vase", + "scissors", "teddy bear", "hair drier", "toothbrush", ] self.classes = coco_classes - # Check if all allowed classes exist in the list of object detector's classes + # Verify that allowed classes are in the detector's class list invalid_classes = [cls for cls in self.allowed_classes if cls not in self.classes] if invalid_classes: raise ValueError( @@ -114,15 +110,8 @@ def infer(self, img=None, threshold=None, keep_size=None, m1_image=None, m2_imag if not self.allowed_classes: return boxes - - # Adjust index to align with COCO label numbering - if isinstance(self.learner, DetrLearner): - filtered_boxes = BoundingBoxList( - [box for box in boxes if self.classes[int(box.name) - 1] in self.allowed_classes]) - else: - filtered_boxes = BoundingBoxList( - [box for box in boxes if self.classes[int(box.name)] in self.allowed_classes]) - + filtered_boxes = BoundingBoxList( + [box for box in boxes if self.classes[int(box.name)] in self.allowed_classes]) return filtered_boxes def __getattr__(self, attr): From a8b2def66779b91a7f7504f19a0735d5cc2ebe87 Mon Sep 17 00:00:00 2001 From: katdimitris Date: Thu, 28 Sep 2023 15:59:05 +0300 Subject: [PATCH 05/15] correction inference demo for yolov5. --- .../object_detection_2d/yolov5/filtered_inference_demo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py index a3b840f70d..2457016707 100644 --- a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py +++ b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py @@ -32,14 +32,14 @@ yolo = YOLOv5DetectorLearner(model_name='yolov5s', device=args.device) - # By default, only objects of the 'person' class are of interest - filtered_yolo = FilteredLearnerWrapper(yolo, allowed_classes=args.classes) - for f in 'zidane.jpg', 'bus.jpg': torch.hub.download_url_to_file('https://ultralytics.com/images/' + f, f) # download 2 images im1 = Image.open('zidane.jpg') # OpenDR image im2 = cv2.imread('bus.jpg') # OpenCV image (BGR to RGB) + # By default, only objects of the 'person' class are of interest + filtered_yolo = FilteredLearnerWrapper(yolo, allowed_classes=args.classes) + # detection before filtering results = yolo.infer(im1) draw_bounding_boxes(im1.opencv(), results, yolo.classes, show=True, line_thickness=3) From 3fedaa1ecbebcc7f3f2be34be1391a282431e769 Mon Sep 17 00:00:00 2001 From: katdimitris Date: Thu, 28 Sep 2023 16:32:41 +0300 Subject: [PATCH 06/15] small fixes to pass tests --- .../object_detection_2d/utils/class_filter_wrapper.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index a49d44611d..bc4a8da3bb 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -9,9 +9,10 @@ class FilteredLearnerWrapper: - def __init__(self, learner, allowed_classes=[]): + def __init__(self, learner, allowed_classes=None): self.learner = learner - self.allowed_classes = allowed_classes + self.allowed_classes = allowed_classes if allowed_classes is not None else [] + if isinstance(self.learner, (CenterNetDetectorLearner, YOLOv3DetectorLearner, YOLOv5DetectorLearner, NanodetLearner, RetinaFaceLearner, SingleShotDetectorLearner)): @@ -48,7 +49,7 @@ def infer(self, img=None, threshold=None, keep_size=None, m1_image=None, m2_imag if img is None: raise ValueError( - f"An image input is required. Please provide a valid image.") + "An image input is required. Please provide a valid image.") if isinstance(self.learner, CenterNetDetectorLearner): if threshold is None: @@ -107,6 +108,10 @@ def infer(self, img=None, threshold=None, keep_size=None, m1_image=None, m2_imag extract_maps = False boxes = self.learner.infer(img, threshold, keep_size, custom_nms, nms_thresh, nms_topk, post_nms, extract_maps) + else: + raise ValueError( + "Filtering has not been implemented for the specified detector class." + ) if not self.allowed_classes: return boxes From db0c73df8a5595122e0a6c9de1eef8d5e6be51cc Mon Sep 17 00:00:00 2001 From: katdimitris Date: Thu, 28 Sep 2023 16:38:08 +0300 Subject: [PATCH 07/15] added License Copyright to pass test --- .../utils/class_filter_wrapper.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index bc4a8da3bb..e5898c3443 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -1,3 +1,18 @@ +# Copyright 2020-2023 OpenDR European Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + from opendr.engine.target import BoundingBoxList from opendr.perception.object_detection_2d.centernet.centernet_learner import CenterNetDetectorLearner from opendr.perception.object_detection_2d.detr.detr_learner import DetrLearner @@ -38,10 +53,9 @@ def __init__(self, learner, allowed_classes=None): raise ValueError( f"The following classes are not detected by this detector: {', '.join(invalid_classes)}") - def infer(self, img=None, threshold=None, keep_size=None, m1_image=None, m2_image=None, input=None, - conf_threshold=None, iou_threshold=None, nms_max_num=None, nms_threshold=None, scales=None, - mask_thresh=None, size=None, custom_nms=None, nms_thresh=None, nms_topk=None, post_nms=None, - extract_maps=None): + def infer(self, img=None, threshold=None, keep_size=None, input=None, conf_threshold=None, iou_threshold=None, + nms_max_num=None, nms_threshold=None, scales=None, mask_thresh=None, size=None, custom_nms=None, + nms_thresh=None, nms_topk=None, post_nms=None, extract_maps=None): # match variable names if isinstance(self.learner, NanodetLearner): From 806e08fe48a42d13d24d5921cc57451b15b4ad98 Mon Sep 17 00:00:00 2001 From: katdimitris <56490644+katdimitris@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:04:04 +0300 Subject: [PATCH 08/15] Update src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py Co-authored-by: Kostas Tsampazis <27914645+tsampazk@users.noreply.github.com> --- .../object_detection_2d/utils/class_filter_wrapper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index e5898c3443..1b795cc41b 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -33,7 +33,7 @@ def __init__(self, learner, allowed_classes=None): RetinaFaceLearner, SingleShotDetectorLearner)): self.classes = self.learner.classes if isinstance(self.learner, DetrLearner): - coco_classes = [ + self.classes = [ "N/A", "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "N/A", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "N/A", "backpack", "umbrella", "N/A", @@ -45,7 +45,6 @@ def __init__(self, learner, allowed_classes=None): "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "N/A", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush", ] - self.classes = coco_classes # Verify that allowed classes are in the detector's class list invalid_classes = [cls for cls in self.allowed_classes if cls not in self.classes] From 4d0cc7209e3f2d933e6a9266a968384071b009df Mon Sep 17 00:00:00 2001 From: katdimitris <56490644+katdimitris@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:04:18 +0300 Subject: [PATCH 09/15] Update projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py Co-authored-by: Kostas Tsampazis <27914645+tsampazk@users.noreply.github.com> --- .../object_detection_2d/yolov5/filtered_inference_demo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py index 2457016707..940abecdc1 100644 --- a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py +++ b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py @@ -41,6 +41,7 @@ filtered_yolo = FilteredLearnerWrapper(yolo, allowed_classes=args.classes) # detection before filtering + print("No filtering...") results = yolo.infer(im1) draw_bounding_boxes(im1.opencv(), results, yolo.classes, show=True, line_thickness=3) From a329d6481b86cca3198f828d174a2de386e9f384 Mon Sep 17 00:00:00 2001 From: katdimitris <56490644+katdimitris@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:04:30 +0300 Subject: [PATCH 10/15] Update projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py Co-authored-by: Kostas Tsampazis <27914645+tsampazk@users.noreply.github.com> --- .../object_detection_2d/yolov5/filtered_inference_demo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py index 940abecdc1..a73bf521f5 100644 --- a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py +++ b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py @@ -46,6 +46,7 @@ draw_bounding_boxes(im1.opencv(), results, yolo.classes, show=True, line_thickness=3) # detection after filtering + print("With filtering...") filtered_results = filtered_yolo.infer(im1) draw_bounding_boxes(im1.opencv(), filtered_results, filtered_yolo.classes, show=True, line_thickness=3) From bb0e74dbc1ff16559179c8186df1787aa5fdb46b Mon Sep 17 00:00:00 2001 From: katdimitris <56490644+katdimitris@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:05:09 +0300 Subject: [PATCH 11/15] Update projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py Co-authored-by: Kostas Tsampazis <27914645+tsampazk@users.noreply.github.com> --- .../object_detection_2d/yolov5/filtered_inference_demo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py index a73bf521f5..540ae9da86 100644 --- a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py +++ b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py @@ -51,6 +51,7 @@ draw_bounding_boxes(im1.opencv(), filtered_results, filtered_yolo.classes, show=True, line_thickness=3) # detection before filtering + print("No filtering...") results = yolo.infer(im2) draw_bounding_boxes(np.copy(im2), results, yolo.classes, show=True, line_thickness=3) From 8210e9103f1ebe4b9136cc75599e357ab838ab72 Mon Sep 17 00:00:00 2001 From: katdimitris <56490644+katdimitris@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:06:07 +0300 Subject: [PATCH 12/15] Update projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py Co-authored-by: Kostas Tsampazis <27914645+tsampazk@users.noreply.github.com> --- .../object_detection_2d/yolov5/filtered_inference_demo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py index 540ae9da86..5e79185252 100644 --- a/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py +++ b/projects/python/perception/object_detection_2d/yolov5/filtered_inference_demo.py @@ -56,5 +56,6 @@ draw_bounding_boxes(np.copy(im2), results, yolo.classes, show=True, line_thickness=3) # detection after filtering + print("With filtering...") filtered_results = filtered_yolo.infer(im2) draw_bounding_boxes(np.copy(im2), filtered_results, filtered_yolo.classes, show=True, line_thickness=3) From 6f8b9049516010ac3bf2881a0f8e6aad16cf2641 Mon Sep 17 00:00:00 2001 From: katdimitris <56490644+katdimitris@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:06:13 +0300 Subject: [PATCH 13/15] Update src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py Co-authored-by: Kostas Tsampazis <27914645+tsampazk@users.noreply.github.com> --- .../object_detection_2d/utils/class_filter_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index 1b795cc41b..0014b4d80b 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -57,7 +57,7 @@ def infer(self, img=None, threshold=None, keep_size=None, input=None, conf_thres nms_thresh=None, nms_topk=None, post_nms=None, extract_maps=None): # match variable names - if isinstance(self.learner, NanodetLearner): + if isinstance(self.learner, NanodetLearner) and input is not None: img = input if img is None: From 3e19b7b39aadadbd0ebaa44eff7280afda8c9d43 Mon Sep 17 00:00:00 2001 From: katdimitris <56490644+katdimitris@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:06:19 +0300 Subject: [PATCH 14/15] Update src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py Co-authored-by: Kostas Tsampazis <27914645+tsampazk@users.noreply.github.com> --- .../object_detection_2d/utils/class_filter_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index 0014b4d80b..e89032587c 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -93,7 +93,7 @@ def infer(self, img=None, threshold=None, keep_size=None, input=None, conf_thres iou_threshold = 0.6 if nms_max_num is None: nms_max_num = 100 - boxes = self.learner.infer(input, conf_threshold, iou_threshold, nms_max_num) + boxes = self.learner.infer(img, conf_threshold, iou_threshold, nms_max_num) elif isinstance(self.learner, RetinaFaceLearner): if threshold is None: From 25eb323ccd28eda2d06780d6c6472e5e40f463d1 Mon Sep 17 00:00:00 2001 From: katdimitris Date: Mon, 2 Oct 2023 10:17:10 +0300 Subject: [PATCH 15/15] Removed filtering support for RetinaFace, since it exclusively detects face objects. --- .../utils/class_filter_wrapper.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py index e89032587c..b1f49a13fb 100644 --- a/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py +++ b/src/opendr/perception/object_detection_2d/utils/class_filter_wrapper.py @@ -16,7 +16,6 @@ from opendr.engine.target import BoundingBoxList from opendr.perception.object_detection_2d.centernet.centernet_learner import CenterNetDetectorLearner from opendr.perception.object_detection_2d.detr.detr_learner import DetrLearner -from opendr.perception.object_detection_2d.retinaface.retinaface_learner import RetinaFaceLearner from opendr.perception.object_detection_2d.ssd.ssd_learner import SingleShotDetectorLearner from opendr.perception.object_detection_2d.yolov3.yolov3_learner import YOLOv3DetectorLearner from opendr.perception.object_detection_2d.yolov5.yolov5_learner import YOLOv5DetectorLearner @@ -30,7 +29,7 @@ def __init__(self, learner, allowed_classes=None): if isinstance(self.learner, (CenterNetDetectorLearner, YOLOv3DetectorLearner, YOLOv5DetectorLearner, NanodetLearner, - RetinaFaceLearner, SingleShotDetectorLearner)): + SingleShotDetectorLearner)): self.classes = self.learner.classes if isinstance(self.learner, DetrLearner): self.classes = [ @@ -53,8 +52,8 @@ def __init__(self, learner, allowed_classes=None): f"The following classes are not detected by this detector: {', '.join(invalid_classes)}") def infer(self, img=None, threshold=None, keep_size=None, input=None, conf_threshold=None, iou_threshold=None, - nms_max_num=None, nms_threshold=None, scales=None, mask_thresh=None, size=None, custom_nms=None, - nms_thresh=None, nms_topk=None, post_nms=None, extract_maps=None): + nms_max_num=None, size=None, custom_nms=None, nms_thresh=None, nms_topk=None, post_nms=None, + extract_maps=None): # match variable names if isinstance(self.learner, NanodetLearner) and input is not None: @@ -95,17 +94,6 @@ def infer(self, img=None, threshold=None, keep_size=None, input=None, conf_thres nms_max_num = 100 boxes = self.learner.infer(img, conf_threshold, iou_threshold, nms_max_num) - elif isinstance(self.learner, RetinaFaceLearner): - if threshold is None: - threshold = 0.8 - if nms_threshold is None: - nms_threshold = 0.4 - if scales is None: - scales = [1024, 1980] - if mask_thresh is None: - mask_thresh = 0.8 - boxes = self.learner.infer(img, threshold, nms_threshold, scales, mask_thresh) - elif isinstance(self.learner, SingleShotDetectorLearner): if threshold is None: threshold = 0.2