From 2e33b9a1a8cd6b795941d716b565d2c7ba693ec4 Mon Sep 17 00:00:00 2001 From: RangiLyu Date: Mon, 27 Feb 2023 20:58:05 +0800 Subject: [PATCH 1/5] [Refactor] Refactor LVIS metric with MMEval. --- mmdet/evaluation/metrics/lvis_metric.py | 396 ++++++++---------------- 1 file changed, 123 insertions(+), 273 deletions(-) diff --git a/mmdet/evaluation/metrics/lvis_metric.py b/mmdet/evaluation/metrics/lvis_metric.py index 388c097d5ff..5582416e8ef 100644 --- a/mmdet/evaluation/metrics/lvis_metric.py +++ b/mmdet/evaluation/metrics/lvis_metric.py @@ -1,35 +1,20 @@ # Copyright (c) OpenMMLab. All rights reserved. import itertools import os.path as osp -import tempfile import warnings -from collections import OrderedDict -from typing import Dict, List, Optional, Sequence, Union +from typing import List, Optional, Sequence, Union -import numpy as np -from mmengine.logging import MMLogger +from mmengine.logging import print_log +from mmeval import LVISDetection from terminaltables import AsciiTable +from torch import Tensor from mmdet.registry import METRICS from mmdet.structures.mask import encode_mask_results -from ..functional import eval_recalls -from .coco_metric import CocoMetric - -try: - import lvis - if getattr(lvis, '__version__', '0') >= '10.5.3': - warnings.warn( - 'mmlvis is deprecated, please install official lvis-api by "pip install git+https://github.com/lvis-dataset/lvis-api.git"', # noqa: E501 - UserWarning) - from lvis import LVIS, LVISEval, LVISResults -except ImportError: - lvis = None - LVISEval = None - LVISResults = None @METRICS.register_module() -class LVISMetric(CocoMetric): +class LVISMetric(LVISDetection): """LVIS evaluation metric. Args: @@ -41,8 +26,8 @@ class LVISMetric(CocoMetric): Defaults to 'bbox'. classwise (bool): Whether to evaluate the metric class-wise. Defaults to False. - proposal_nums (Sequence[int]): Numbers of proposals to be evaluated. - Defaults to (100, 300, 1000). + proposal_nums (int): Numbers of proposals to be evaluated. + Defaults to 300. iou_thrs (float | List[float], optional): IoU threshold to compute AP and AR. If not specified, IoUs from 0.5 to 0.95 will be used. Defaults to None. @@ -55,13 +40,16 @@ class LVISMetric(CocoMetric): outfile_prefix (str, optional): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Defaults to None. - collect_device (str): Device name used for collecting results from - different ranks during distributed training. Must be 'cpu' or - 'gpu'. Defaults to 'cpu'. + backend_args (dict, optional): Arguments to instantiate the + preifx of uri corresponding backend. Defaults to None. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. + dist_backend (str | None): The name of the distributed communication + backend. Refer to :class:`mmeval.BaseMetric`. + Defaults to 'torch_cuda'. + **kwargs: Keyword parameters passed to :class:`BaseMetric`. """ default_prefix: Optional[str] = 'lvis' @@ -70,93 +58,35 @@ def __init__(self, ann_file: Optional[str] = None, metric: Union[str, List[str]] = 'bbox', classwise: bool = False, - proposal_nums: Sequence[int] = (100, 300, 1000), + proposal_nums: int = 300, iou_thrs: Optional[Union[float, Sequence[float]]] = None, metric_items: Optional[Sequence[str]] = None, format_only: bool = False, outfile_prefix: Optional[str] = None, - collect_device: str = 'cpu', - prefix: Optional[str] = None) -> None: - if lvis is None: - raise RuntimeError( - 'Package lvis is not installed. Please run "pip install ' - 'git+https://github.com/lvis-dataset/lvis-api.git".') - super().__init__(collect_device=collect_device, prefix=prefix) - # coco evaluation metrics - self.metrics = metric if isinstance(metric, list) else [metric] - allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] - for metric in self.metrics: - if metric not in allowed_metrics: - raise KeyError( - "metric should be one of 'bbox', 'segm', 'proposal', " - f"'proposal_fast', but got {metric}.") - - # do class wise evaluation, default False - self.classwise = classwise - - # proposal_nums used to compute recall or precision. - self.proposal_nums = list(proposal_nums) - - # iou_thrs used to compute recall or precision. - if iou_thrs is None: - iou_thrs = np.linspace( - .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) - self.iou_thrs = iou_thrs - self.metric_items = metric_items - self.format_only = format_only - if self.format_only: - assert outfile_prefix is not None, 'outfile_prefix must be not' - 'None when format_only is True, otherwise the result files will' - 'be saved to a temp directory which will be cleaned up at the end.' - - self.outfile_prefix = outfile_prefix - - # if ann_file is not specified, - # initialize lvis api with the converted dataset - self._lvis_api = LVIS(ann_file) if ann_file else None - - # handle dataset lazy init - self.cat_ids = None - self.img_ids = None - - def fast_eval_recall(self, - results: List[dict], - proposal_nums: Sequence[int], - iou_thrs: Sequence[float], - logger: Optional[MMLogger] = None) -> np.ndarray: - """Evaluate proposal recall with LVIS's fast_eval_recall. - - Args: - results (List[dict]): Results of the dataset. - proposal_nums (Sequence[int]): Proposal numbers used for - evaluation. - iou_thrs (Sequence[float]): IoU thresholds used for evaluation. - logger (MMLogger, optional): Logger used for logging the recall - summary. - Returns: - np.ndarray: Averaged recall results. - """ - gt_bboxes = [] - pred_bboxes = [result['bboxes'] for result in results] - for i in range(len(self.img_ids)): - ann_ids = self._lvis_api.get_ann_ids(img_ids=[self.img_ids[i]]) - ann_info = self._lvis_api.load_anns(ann_ids) - if len(ann_info) == 0: - gt_bboxes.append(np.zeros((0, 4))) - continue - bboxes = [] - for ann in ann_info: - x1, y1, w, h = ann['bbox'] - bboxes.append([x1, y1, x1 + w, y1 + h]) - bboxes = np.array(bboxes, dtype=np.float32) - if bboxes.shape[0] == 0: - bboxes = np.zeros((0, 4)) - gt_bboxes.append(bboxes) - - recalls = eval_recalls( - gt_bboxes, pred_bboxes, proposal_nums, iou_thrs, logger=logger) - ar = recalls.mean(axis=1) - return ar + backend_args: Optional[dict] = None, + prefix: Optional[str] = None, + dist_backend: str = 'torch_cuda', + **kwargs) -> None: + + collect_device = kwargs.pop('collect_device', None) + if collect_device is not None: + warnings.warn( + 'DeprecationWarning: The `collect_device` parameter of ' + '`CocoMetric` is deprecated, use `dist_backend` instead.') + + super().__init__( + ann_file=ann_file, + metric=metric, + iou_thrs=iou_thrs, + classwise=classwise, + proposal_nums=proposal_nums, + metric_items=metric_items, + format_only=format_only, + outfile_prefix=outfile_prefix, + backend_args=backend_args, + dist_backend=dist_backend, + **kwargs) + self.prefix = prefix or self.default_prefix # TODO: data_batch is no longer needed, consider adjusting the # parameter position @@ -170,176 +100,96 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ + predictions, groundtruths = [], [] for data_sample in data_samples: - result = dict() - pred = data_sample['pred_instances'] - result['img_id'] = data_sample['img_id'] - result['bboxes'] = pred['bboxes'].cpu().numpy() - result['scores'] = pred['scores'].cpu().numpy() - result['labels'] = pred['labels'].cpu().numpy() - # encode mask to RLE - if 'masks' in pred: - result['masks'] = encode_mask_results( - pred['masks'].detach().cpu().numpy()) + pred = dict() + pred_instances = data_sample['pred_instances'] + pred['img_id'] = data_sample['img_id'] + pred['bboxes'] = pred_instances['bboxes'].cpu().numpy() + pred['scores'] = pred_instances['scores'].cpu().numpy() + pred['labels'] = pred_instances['labels'].cpu().numpy() + if 'masks' in pred_instances: + pred['masks'] = encode_mask_results( + pred_instances['masks'].detach().cpu().numpy( + )) if isinstance(pred_instances['masks'], + Tensor) else pred_instances['masks'] # some detectors use different scores for bbox and mask - if 'mask_scores' in pred: - result['mask_scores'] = pred['mask_scores'].cpu().numpy() - - # parse gt - gt = dict() - gt['width'] = data_sample['ori_shape'][1] - gt['height'] = data_sample['ori_shape'][0] - gt['img_id'] = data_sample['img_id'] - if self._lvis_api is None: - # TODO: Need to refactor to support LoadAnnotations - assert 'instances' in data_sample, \ - 'ground truth is required for evaluation when ' \ - '`ann_file` is not provided' - gt['anns'] = data_sample['instances'] - # add converted result to the results list - self.results.append((gt, result)) - - def compute_metrics(self, results: list) -> Dict[str, float]: - """Compute the metrics from processed results. - - Args: - results (list): The processed results of each batch. - - Returns: - Dict[str, float]: The computed metrics. The keys are the names of - the metrics, and the values are corresponding results. + if 'mask_scores' in pred_instances: + pred['mask_scores'] = \ + pred_instances['mask_scores'].cpu().numpy() + predictions.append(pred) + # LVIS only supports loading annotation from JSON + ann = dict() # create dummy ann + groundtruths.append(ann) + self.add(predictions, groundtruths) + + def evaluate(self, *args, **kwargs) -> dict: + """Returns metric results and print pretty table of metrics per class. + + This method would be invoked by ``mmengine.Evaluator``. """ - logger: MMLogger = MMLogger.get_current_instance() - - # split gt and prediction list - gts, preds = zip(*results) - - tmp_dir = None - if self.outfile_prefix is None: - tmp_dir = tempfile.TemporaryDirectory() - outfile_prefix = osp.join(tmp_dir.name, 'results') - else: - outfile_prefix = self.outfile_prefix - - if self._lvis_api is None: - # use converted gt json file to initialize coco api - logger.info('Converting ground truth to coco format...') - coco_json_path = self.gt_to_coco_json( - gt_dicts=gts, outfile_prefix=outfile_prefix) - self._lvis_api = LVIS(coco_json_path) - - # handle lazy init - if self.cat_ids is None: - self.cat_ids = self._lvis_api.get_cat_ids() - if self.img_ids is None: - self.img_ids = self._lvis_api.get_img_ids() - - # convert predictions to coco format and dump to json file - result_files = self.results2json(preds, outfile_prefix) - - eval_results = OrderedDict() + metric_results = self.compute(*args, **kwargs) + self.reset() if self.format_only: - logger.info('results are saved in ' - f'{osp.dirname(outfile_prefix)}') - return eval_results - - lvis_gt = self._lvis_api - + print_log( + 'Results are saved in ' + f'{osp.dirname(self.outfile_prefix)}', + logger='current') + return metric_results for metric in self.metrics: - logger.info(f'Evaluating {metric}...') - - # TODO: May refactor fast_eval_recall to an independent metric? - # fast eval recall - if metric == 'proposal_fast': - ar = self.fast_eval_recall( - preds, self.proposal_nums, self.iou_thrs, logger=logger) - log_msg = [] - for i, num in enumerate(self.proposal_nums): - eval_results[f'AR@{num}'] = ar[i] - log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') - log_msg = ''.join(log_msg) - logger.info(log_msg) - continue - - try: - lvis_dt = LVISResults(lvis_gt, result_files[metric]) - except IndexError: - logger.info( - 'The testing results of the whole dataset is empty.') - break - - iou_type = 'bbox' if metric == 'proposal' else metric - lvis_eval = LVISEval(lvis_gt, lvis_dt, iou_type) - lvis_eval.params.imgIds = self.img_ids - metric_items = self.metric_items + result = metric_results.pop(f'{metric}_result') if metric == 'proposal': - lvis_eval.params.useCats = 0 - lvis_eval.params.maxDets = list(self.proposal_nums) - lvis_eval.evaluate() - lvis_eval.accumulate() - lvis_eval.summarize() - if metric_items is None: - metric_items = ['AR@300', 'ARs@300', 'ARm@300', 'ARl@300'] - for k, v in lvis_eval.get_results().items(): - if k in metric_items: - val = float('{:.3f}'.format(float(v))) - eval_results[k] = val - + table_title = ' Recall Results (%)' + if self.metric_items is None: + assert len(result) == 4 + headers = [ + f'AR@{self.proposal_nums}', + f'AR_s@{self.proposal_nums}', + f'AR_m@{self.proposal_nums}', + f'AR_l@{self.proposal_nums}' + ] + else: + assert len(result) == len(self.metric_items) + headers = self.metric_items else: - lvis_eval.evaluate() - lvis_eval.accumulate() - lvis_eval.summarize() - lvis_results = lvis_eval.get_results() - if self.classwise: # Compute per-category AP - # Compute per-category AP - # from https://github.com/facebookresearch/detectron2/ - precisions = lvis_eval.eval['precision'] - # precision: (iou, recall, cls, area range, max dets) - assert len(self.cat_ids) == precisions.shape[2] - - results_per_category = [] - for idx, catId in enumerate(self.cat_ids): - # area range index 0: all area ranges - # max dets index -1: typically 100 per image - # the dimensions of precisions are - # [num_thrs, num_recalls, num_cats, num_area_rngs] - nm = self._lvis_api.load_cats([catId])[0] - precision = precisions[:, :, idx, 0] - precision = precision[precision > -1] - if precision.size: - ap = np.mean(precision) - else: - ap = float('nan') - results_per_category.append( - (f'{nm["name"]}', f'{float(ap):0.3f}')) - eval_results[f'{nm["name"]}_precision'] = round(ap, 3) - - num_columns = min(6, len(results_per_category) * 2) - results_flatten = list( - itertools.chain(*results_per_category)) - headers = ['category', 'AP'] * (num_columns // 2) - results_2d = itertools.zip_longest(*[ - results_flatten[i::num_columns] - for i in range(num_columns) - ]) - table_data = [headers] - table_data += [result for result in results_2d] - table = AsciiTable(table_data) - logger.info('\n' + table.table) - - if metric_items is None: - metric_items = [ - 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'APr', - 'APc', 'APf' + table_title = f' {metric} Results (%)' + if self.metric_items is None: + assert len(result) == 9 + headers = [ + f'{metric}_AP', f'{metric}_AP50', f'{metric}_AP75', + f'{metric}_APs', f'{metric}_APm', f'{metric}_APl', + f'{metric}_APr', f'{metric}_APc', f'{metric}_APf' ] - - for k, v in lvis_results.items(): - if k in metric_items: - key = '{}_{}'.format(metric, k) - val = float('{:.3f}'.format(float(v))) - eval_results[key] = val - - lvis_eval.print_results() - if tmp_dir is not None: - tmp_dir.cleanup() - return eval_results + else: + assert len(result) == len(self.metric_items) + headers = [ + f'{metric}_{item}' for item in self.metric_items + ] + table_data = [headers, result] + table = AsciiTable(table_data, title=table_title) + print_log('\n' + table.table, logger='current') + + if self.classwise and \ + f'{metric}_classwise_result' in metric_results: + print_log( + f'Evaluating {metric} metric of each category...', + logger='current') + classwise_table_title = f' {metric} Classwise Results (%)' + classwise_result = metric_results.pop( + f'{metric}_classwise_result') + + num_columns = min(6, len(classwise_result) * 2) + results_flatten = list(itertools.chain(*classwise_result)) + headers = ['category', f'{metric}_AP'] * (num_columns // 2) + results_2d = itertools.zip_longest(*[ + results_flatten[i::num_columns] for i in range(num_columns) + ]) + table_data = [headers] + table_data += [result for result in results_2d] + table = AsciiTable(table_data, title=classwise_table_title) + print_log('\n' + table.table, logger='current') + evaluate_results = { + f'{self.prefix}/{k}(%)': round(float(v) * 100, 4) + for k, v in metric_results.items() + } + return evaluate_results From 717a28186ae71ae10838a6a7ee81e333e8061af8 Mon Sep 17 00:00:00 2001 From: RangiLyu Date: Tue, 28 Feb 2023 13:55:31 +0800 Subject: [PATCH 2/5] update --- mmdet/evaluation/metrics/lvis_metric.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mmdet/evaluation/metrics/lvis_metric.py b/mmdet/evaluation/metrics/lvis_metric.py index 5582416e8ef..131db485a56 100644 --- a/mmdet/evaluation/metrics/lvis_metric.py +++ b/mmdet/evaluation/metrics/lvis_metric.py @@ -4,7 +4,7 @@ import warnings from typing import List, Optional, Sequence, Union -from mmengine.logging import print_log +from mmengine.logging import MMLogger, print_log from mmeval import LVISDetection from terminaltables import AsciiTable from torch import Tensor @@ -18,9 +18,9 @@ class LVISMetric(LVISDetection): """LVIS evaluation metric. Args: - ann_file (str, optional): Path to the coco format annotation file. + ann_file (str, optional): Path to the COCO LVIS format annotation file. If not specified, ground truth annotations from the dataset will - be converted to coco format. Defaults to None. + be converted to COCO LVIS format. Defaults to None. metric (str | List[str]): Metrics to be evaluated. Valid metrics include 'bbox', 'segm', 'proposal', and 'proposal_fast'. Defaults to 'bbox'. @@ -72,7 +72,8 @@ def __init__(self, if collect_device is not None: warnings.warn( 'DeprecationWarning: The `collect_device` parameter of ' - '`CocoMetric` is deprecated, use `dist_backend` instead.') + '`LVISMetric` is deprecated, use `dist_backend` instead.') + logger = MMLogger.get_current_instance() super().__init__( ann_file=ann_file, @@ -85,6 +86,7 @@ def __init__(self, outfile_prefix=outfile_prefix, backend_args=backend_args, dist_backend=dist_backend, + logger=logger, **kwargs) self.prefix = prefix or self.default_prefix From e13a22b2d47833c767759d377fcbb96972aa4426 Mon Sep 17 00:00:00 2001 From: RangiLyu Date: Fri, 3 Mar 2023 11:49:40 +0800 Subject: [PATCH 3/5] update --- mmdet/evaluation/metrics/lvis_metric.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mmdet/evaluation/metrics/lvis_metric.py b/mmdet/evaluation/metrics/lvis_metric.py index 131db485a56..2a26fd9ef2e 100644 --- a/mmdet/evaluation/metrics/lvis_metric.py +++ b/mmdet/evaluation/metrics/lvis_metric.py @@ -120,10 +120,7 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: pred['mask_scores'] = \ pred_instances['mask_scores'].cpu().numpy() predictions.append(pred) - # LVIS only supports loading annotation from JSON - ann = dict() # create dummy ann - groundtruths.append(ann) - self.add(predictions, groundtruths) + self.add(predictions) def evaluate(self, *args, **kwargs) -> dict: """Returns metric results and print pretty table of metrics per class. From 9361b1e3c5a95b044e88afbc90b332be684e587d Mon Sep 17 00:00:00 2001 From: RangiLyu Date: Fri, 3 Mar 2023 11:54:37 +0800 Subject: [PATCH 4/5] fix ut --- mmdet/evaluation/metrics/lvis_metric.py | 4 +- .../test_metrics/test_lvis_metric.py | 121 +++++++----------- 2 files changed, 47 insertions(+), 78 deletions(-) diff --git a/mmdet/evaluation/metrics/lvis_metric.py b/mmdet/evaluation/metrics/lvis_metric.py index 2a26fd9ef2e..40dc249dae4 100644 --- a/mmdet/evaluation/metrics/lvis_metric.py +++ b/mmdet/evaluation/metrics/lvis_metric.py @@ -136,7 +136,9 @@ def evaluate(self, *args, **kwargs) -> dict: logger='current') return metric_results for metric in self.metrics: - result = metric_results.pop(f'{metric}_result') + result = metric_results.pop(f'{metric}_result', None) + if result is None: # empty results + break if metric == 'proposal': table_title = ' Recall Results (%)' if self.metric_items is None: diff --git a/tests/test_evaluation/test_metrics/test_lvis_metric.py b/tests/test_evaluation/test_metrics/test_lvis_metric.py index 6f4eecc5144..eadd92ccb56 100644 --- a/tests/test_evaluation/test_metrics/test_lvis_metric.py +++ b/tests/test_evaluation/test_metrics/test_lvis_metric.py @@ -137,15 +137,15 @@ def test_evaluate(self): [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { - 'lvis/bbox_AP': 1.0, - 'lvis/bbox_AP50': 1.0, - 'lvis/bbox_AP75': 1.0, - 'lvis/bbox_APs': 1.0, - 'lvis/bbox_APm': 1.0, - 'lvis/bbox_APl': 1.0, - 'lvis/bbox_APr': -1.0, - 'lvis/bbox_APc': 1.0, - 'lvis/bbox_APf': 1.0 + 'lvis/bbox_AP(%)': 100.0, + 'lvis/bbox_AP50(%)': 100.0, + 'lvis/bbox_AP75(%)': 100.0, + 'lvis/bbox_APs(%)': 100.0, + 'lvis/bbox_APm(%)': 100.0, + 'lvis/bbox_APl(%)': 100.0, + 'lvis/bbox_APr(%)': -100.0, + 'lvis/bbox_APc(%)': 100.0, + 'lvis/bbox_APf(%)': 100.0 } self.assertDictEqual(eval_results, target) self.assertTrue( @@ -164,24 +164,24 @@ def test_evaluate(self): [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { - 'lvis/bbox_AP': 1.0, - 'lvis/bbox_AP50': 1.0, - 'lvis/bbox_AP75': 1.0, - 'lvis/bbox_APs': 1.0, - 'lvis/bbox_APm': 1.0, - 'lvis/bbox_APl': 1.0, - 'lvis/bbox_APr': -1.0, - 'lvis/bbox_APc': 1.0, - 'lvis/bbox_APf': 1.0, - 'lvis/segm_AP': 1.0, - 'lvis/segm_AP50': 1.0, - 'lvis/segm_AP75': 1.0, - 'lvis/segm_APs': 1.0, - 'lvis/segm_APm': 1.0, - 'lvis/segm_APl': 1.0, - 'lvis/segm_APr': -1.0, - 'lvis/segm_APc': 1.0, - 'lvis/segm_APf': 1.0 + 'lvis/bbox_AP(%)': 100.0, + 'lvis/bbox_AP50(%)': 100.0, + 'lvis/bbox_AP75(%)': 100.0, + 'lvis/bbox_APs(%)': 100.0, + 'lvis/bbox_APm(%)': 100.0, + 'lvis/bbox_APl(%)': 100.0, + 'lvis/bbox_APr(%)': -100.0, + 'lvis/bbox_APc(%)': 100.0, + 'lvis/bbox_APf(%)': 100.0, + 'lvis/segm_AP(%)': 100.0, + 'lvis/segm_AP50(%)': 100.0, + 'lvis/segm_AP75(%)': 100.0, + 'lvis/segm_APs(%)': 100.0, + 'lvis/segm_APm(%)': 100.0, + 'lvis/segm_APl(%)': 100.0, + 'lvis/segm_APr(%)': -100.0, + 'lvis/segm_APc(%)': 100.0, + 'lvis/segm_APf(%)': 100.0 } self.assertDictEqual(eval_results, target) self.assertTrue( @@ -207,7 +207,7 @@ def test_evaluate(self): [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { - 'lvis/bbox_APm': 1.0, + 'lvis/bbox_APm(%)': 100.0, } self.assertDictEqual(eval_results, target) @@ -228,17 +228,17 @@ def test_classwise_evaluate(self): [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { - 'lvis/bbox_AP': 1.0, - 'lvis/bbox_AP50': 1.0, - 'lvis/bbox_AP75': 1.0, - 'lvis/bbox_APs': 1.0, - 'lvis/bbox_APm': 1.0, - 'lvis/bbox_APl': 1.0, - 'lvis/bbox_APr': -1.0, - 'lvis/bbox_APc': 1.0, - 'lvis/bbox_APf': 1.0, - 'lvis/aerosol_can_precision': 1.0, - 'lvis/air_conditioner_precision': 1.0, + 'lvis/bbox_AP(%)': 100.0, + 'lvis/bbox_AP50(%)': 100.0, + 'lvis/bbox_AP75(%)': 100.0, + 'lvis/bbox_APs(%)': 100.0, + 'lvis/bbox_APm(%)': 100.0, + 'lvis/bbox_APl(%)': 100.0, + 'lvis/bbox_APr(%)': -100.0, + 'lvis/bbox_APc(%)': 100.0, + 'lvis/bbox_APf(%)': 100.0, + 'lvis/bbox_aerosol_can_precision(%)': 100.0, + 'lvis/bbox_air_conditioner_precision(%)': 100.0, } self.assertDictEqual(eval_results, target) @@ -253,40 +253,7 @@ def test_manually_set_iou_thrs(self): ann_file=fake_json_file, metric='bbox', iou_thrs=[0.3, 0.6]) lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) - self.assertEqual(lvis_metric.iou_thrs, [0.3, 0.6]) - - @unittest.skipIf(lvis is None, 'lvis is not installed.') - def test_fast_eval_recall(self): - # create dummy data - fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') - self._create_dummy_lvis_json(fake_json_file) - dummy_pred = self._create_dummy_results() - - # test default proposal nums - lvis_metric = LVISMetric( - ann_file=fake_json_file, metric='proposal_fast') - lvis_metric.dataset_meta = dict( - classes=['aerosol_can', 'air_conditioner']) - lvis_metric.process( - {}, - [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) - eval_results = lvis_metric.evaluate(size=1) - target = {'lvis/AR@100': 1.0, 'lvis/AR@300': 1.0, 'lvis/AR@1000': 1.0} - self.assertDictEqual(eval_results, target) - - # test manually set proposal nums - lvis_metric = LVISMetric( - ann_file=fake_json_file, - metric='proposal_fast', - proposal_nums=(2, 4)) - lvis_metric.dataset_meta = dict( - classes=['aerosol_can', 'air_conditioner']) - lvis_metric.process( - {}, - [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) - eval_results = lvis_metric.evaluate(size=1) - target = {'lvis/AR@2': 0.5, 'lvis/AR@4': 1.0} - self.assertDictEqual(eval_results, target) + self.assertListEqual(list(lvis_metric.iou_thrs), [0.3, 0.6]) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_evaluate_proposal(self): @@ -303,10 +270,10 @@ def test_evaluate_proposal(self): [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { - 'lvis/AR@300': 1.0, - 'lvis/ARs@300': 1.0, - 'lvis/ARm@300': 1.0, - 'lvis/ARl@300': 1.0 + 'lvis/AR@300(%)': 100.0, + 'lvis/ARs@300(%)': 100.0, + 'lvis/ARm@300(%)': 100.0, + 'lvis/ARl@300(%)': 100.0 } self.assertDictEqual(eval_results, target) From d08cec3b9cd6c8380f1c99d99a73859641054b99 Mon Sep 17 00:00:00 2001 From: RangiLyu Date: Mon, 13 Mar 2023 20:30:09 +0800 Subject: [PATCH 5/5] sync mmeval --- mmdet/evaluation/metrics/lvis_metric.py | 64 +------------------------ 1 file changed, 2 insertions(+), 62 deletions(-) diff --git a/mmdet/evaluation/metrics/lvis_metric.py b/mmdet/evaluation/metrics/lvis_metric.py index 40dc249dae4..aa3ab913c85 100644 --- a/mmdet/evaluation/metrics/lvis_metric.py +++ b/mmdet/evaluation/metrics/lvis_metric.py @@ -1,12 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. -import itertools -import os.path as osp import warnings from typing import List, Optional, Sequence, Union -from mmengine.logging import MMLogger, print_log +from mmengine.logging import MMLogger from mmeval import LVISDetection -from terminaltables import AsciiTable from torch import Tensor from mmdet.registry import METRICS @@ -102,7 +99,7 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ - predictions, groundtruths = [], [] + predictions = [] for data_sample in data_samples: pred = dict() pred_instances = data_sample['pred_instances'] @@ -130,65 +127,8 @@ def evaluate(self, *args, **kwargs) -> dict: metric_results = self.compute(*args, **kwargs) self.reset() if self.format_only: - print_log( - 'Results are saved in ' - f'{osp.dirname(self.outfile_prefix)}', - logger='current') return metric_results - for metric in self.metrics: - result = metric_results.pop(f'{metric}_result', None) - if result is None: # empty results - break - if metric == 'proposal': - table_title = ' Recall Results (%)' - if self.metric_items is None: - assert len(result) == 4 - headers = [ - f'AR@{self.proposal_nums}', - f'AR_s@{self.proposal_nums}', - f'AR_m@{self.proposal_nums}', - f'AR_l@{self.proposal_nums}' - ] - else: - assert len(result) == len(self.metric_items) - headers = self.metric_items - else: - table_title = f' {metric} Results (%)' - if self.metric_items is None: - assert len(result) == 9 - headers = [ - f'{metric}_AP', f'{metric}_AP50', f'{metric}_AP75', - f'{metric}_APs', f'{metric}_APm', f'{metric}_APl', - f'{metric}_APr', f'{metric}_APc', f'{metric}_APf' - ] - else: - assert len(result) == len(self.metric_items) - headers = [ - f'{metric}_{item}' for item in self.metric_items - ] - table_data = [headers, result] - table = AsciiTable(table_data, title=table_title) - print_log('\n' + table.table, logger='current') - if self.classwise and \ - f'{metric}_classwise_result' in metric_results: - print_log( - f'Evaluating {metric} metric of each category...', - logger='current') - classwise_table_title = f' {metric} Classwise Results (%)' - classwise_result = metric_results.pop( - f'{metric}_classwise_result') - - num_columns = min(6, len(classwise_result) * 2) - results_flatten = list(itertools.chain(*classwise_result)) - headers = ['category', f'{metric}_AP'] * (num_columns // 2) - results_2d = itertools.zip_longest(*[ - results_flatten[i::num_columns] for i in range(num_columns) - ]) - table_data = [headers] - table_data += [result for result in results_2d] - table = AsciiTable(table_data, title=classwise_table_title) - print_log('\n' + table.table, logger='current') evaluate_results = { f'{self.prefix}/{k}(%)': round(float(v) * 100, 4) for k, v in metric_results.items()