From d4bbdfb7d1db68519124525dbda0811fcc8d0184 Mon Sep 17 00:00:00 2001 From: Toraudonn Date: Sun, 20 Nov 2022 07:49:16 +0000 Subject: [PATCH] update sbd evaluation script --- pyEdgeEval/evaluators/sbd.py | 14 ++- pyEdgeEval/helpers/evaluate_sbd.py | 194 ++++++++++++++++++++++++++--- pyEdgeEval/info.py | 2 +- 3 files changed, 189 insertions(+), 21 deletions(-) diff --git a/pyEdgeEval/evaluators/sbd.py b/pyEdgeEval/evaluators/sbd.py index 087fbac..792f210 100644 --- a/pyEdgeEval/evaluators/sbd.py +++ b/pyEdgeEval/evaluators/sbd.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os.path as osp +from warnings import warn from pyEdgeEval.common.multi_label import ( calculate_metrics, @@ -121,28 +122,31 @@ def set_eval_params( self.max_dist = 0.02 self.kill_internal = True self.skip_if_nonexistent = True + + if instance_sensitive: + warn("Pre-SEAL mode doesn't support instance sensitive") self.instance_sensitive = False elif eval_mode == "post-seal": print_log("Using Post-SEAL params", logger=self._logger) self.max_dist = 0.02 self.kill_internal = False self.skip_if_nonexistent = False - self.instance_sensitive = True - elif eval_mode == "high-quality": + self.instance_sensitive = instance_sensitive + elif eval_mode == "reanno": print_log( - "Using params for high-quality annotations", logger=self._logger + "Using params for re-annotated test split", logger=self._logger ) self.max_dist = 0.0075 self.kill_internal = False self.skip_if_nonexistent = False - self.instance_sensitive = True + self.instance_sensitive = instance_sensitive else: print_log("Using custom params", logger=self._logger) self.max_dist = max_dist self.kill_internal = kill_internal self.skip_if_nonexistent = skip_if_nonexistent - self.instance_sensitive = instance_sensitive + self.instance_sensitive = instance_sensitive if self.kill_internal and self.instance_sensitive: print_log( "kill_internal and instance_sensitive are both True which will conflict with each either", diff --git a/pyEdgeEval/helpers/evaluate_sbd.py b/pyEdgeEval/helpers/evaluate_sbd.py index 974d837..3bba68e 100644 --- a/pyEdgeEval/helpers/evaluate_sbd.py +++ b/pyEdgeEval/helpers/evaluate_sbd.py @@ -50,26 +50,37 @@ def parse_args(): type=str, help="the category number to evaluate; can be multiple values", ) - parser.add_argument( - "--thresholds", - type=str, - default="99", - help="the number of thresholds (could be a list of floats); use 99 for eval", - ) parser.add_argument( "--raw", action="store_true", help="option to remove the thinning process (i.e. uses raw predition)", ) parser.add_argument( - "--apply-nms", + "--pre-seal", action="store_true", - help="applies NMS before evaluation", + help="prior to SEAL, the evaluations were not as strict", + ) + parser.add_argument( + "--nonIS", + action="store_true", + help="non instance sensitive evaluation", + ) + parser.add_argument( + "--max-dist", + type=float, + default=0.02, + help="tolerance distance (default: 0.02)", + ) + parser.add_argument( + "--thresholds", + type=str, + default="99", + help="the number of thresholds (could be a list of floats); use 99 for eval", ) parser.add_argument( - "--kill-internal", + "--apply-nms", action="store_true", - help="kill internal contour", + help="applies NMS before evaluation", ) parser.add_argument( "--nproc", @@ -89,8 +100,11 @@ def evaluate( pred_suffix: str, output_path: str, categories: str, + pre_seal: bool, apply_thinning: bool, apply_nms: bool, + nonIS: bool, + max_dist: float, thresholds: str, nproc: int, ): @@ -160,13 +174,16 @@ def evaluate( log_file = osp.join(output_path, f"{timestamp}.log") logger = get_root_logger(log_file=log_file, log_level="INFO") logger.info("Running SBD Evaluation") + logger.info(f"cls_dir: \t{cls_dir}") + logger.info(f"inst_dir: \t{inst_dir}") + logger.info(f"pred_suffix:\t{pred_suffix}") logger.info(f"categories: \t{categories}") logger.info(f"thresholds: \t{thresholds}") + logger.info(f"pre-seal: \t{pre_seal}") logger.info(f"thin: \t{apply_thinning}") logger.info(f"nms: \t{apply_nms}") - logger.info(f"cls_dir: \t{cls_dir}") - logger.info(f"inst_dir: \t{inst_dir}") - logger.info(f"pred_suffix:\t{pred_suffix}") + logger.info(f"nonIS: \t{nonIS}") + logger.info(f"max_dist \t{max_dist}") print("\n\n") # initialize evaluator @@ -183,13 +200,15 @@ def evaluate( evaluator.set_sample_names() # set parameters - eval_mode = "pre-seal" # FIXME: hard-coded for now + eval_mode = "pre-seal" if pre_seal else "post-seal" + instance_sensitive = not nonIS evaluator.set_eval_params( eval_mode=eval_mode, scale=1.0, apply_thinning=apply_thinning, apply_nms=apply_nms, - instance_sensitive=False, + max_dist=max_dist, + instance_sensitive=instance_sensitive, ) # evaluate @@ -208,6 +227,151 @@ def evaluate_sbd(): apply_thinning = not args.raw evaluate( + sbd_path=args.sbd_path, + pred_path=args.pred_path, + cls_dir=args.cls_dir, + inst_dir=args.inst_dir, + pred_suffix=args.pred_suffix, + output_path=args.output_path, + categories=args.categories, + pre_seal=args.pre_seal, + apply_thinning=apply_thinning, + apply_nms=args.apply_nms, + thresholds=args.thresholds, + nproc=args.nproc, + ) + + +def evaluate_reanno( + sbd_path: str, + pred_path: str, + cls_dir: str, + inst_dir: str, + pred_suffix: str, + output_path: str, + categories: str, + apply_thinning: bool, + apply_nms: bool, + nonIS: bool, + thresholds: str, + nproc: int, +): + """Evaluate Re-annotated SBD""" + + if categories is None: + print("use all categories") + categories = list(range(1, len(SBDEvaluator.CLASSES) + 1)) + else: + # string evaluation for categories + categories = categories.strip() + try: + categories = [int(categories)] + except ValueError: + try: + if categories.startswith("[") and categories.endswith("]"): + categories = categories[1:-1] + categories = [ + int(cat.strip()) for cat in categories.split(",") + ] + else: + print( + "Bad categories format; should be a python list of floats (`[a, b, c]`)" + ) + return + except ValueError: + print( + "Bad categories format; should be a python list of ints (`[a, b, c]`)" + ) + return + + for cat in categories: + assert ( + 0 < cat < 21 + ), f"category needs to be between 1 ~ 19, but got {cat}" + + # string evaluation for thresholds + thresholds = thresholds.strip() + try: + n_thresholds = int(thresholds) + thresholds = n_thresholds + except ValueError: + try: + if thresholds.startswith("[") and thresholds.endswith("]"): + thresholds = thresholds[1:-1] + thresholds = [float(t.strip()) for t in thresholds.split(",")] + else: + print( + "Bad threshold format; should be a python list of floats (`[a, b, c]`)" + ) + return + except ValueError: + print( + "Bad threshold format; should be a python list of ints (`[a, b, c]`)" + ) + return + + if output_path is None: + timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) + output_path = osp.join( + osp.normpath(pred_path), f"edge_results_{timestamp}" + ) + + mkdir_or_exist(output_path) + + timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) + log_file = osp.join(output_path, f"{timestamp}.log") + logger = get_root_logger(log_file=log_file, log_level="INFO") + logger.info("Running Re-Annotated SBD Evaluation") + logger.info(f"cls_dir: \t{cls_dir}") + logger.info(f"inst_dir: \t{inst_dir}") + logger.info(f"pred_suffix:\t{pred_suffix}") + logger.info(f"categories: \t{categories}") + logger.info(f"thresholds: \t{thresholds}") + logger.info(f"thin: \t{apply_thinning}") + logger.info(f"nms: \t{apply_nms}") + logger.info(f"nonIS: \t{nonIS}") + print("\n\n") + + # initialize evaluator + evaluator = SBDEvaluator( + dataset_root=sbd_path, + pred_root=pred_path, + cls_dir=cls_dir, + inst_dir=inst_dir, + pred_suffix=pred_suffix, + ) + if evaluator.sample_names is None: + # load custom sample names + # SAMPLE_NAMES = ["2008_000051", "2008_000195"] + evaluator.set_sample_names() + + # set parameters + instance_sensitive = not nonIS + evaluator.set_eval_params( + eval_mode="reanno", # fixed to "reanno" mode + scale=1.0, + apply_thinning=apply_thinning, + apply_nms=apply_nms, + max_dist=0.0075, # fixed + instance_sensitive=instance_sensitive, + ) + + # evaluate + evaluator.evaluate( + categories=categories, + thresholds=thresholds, + nproc=nproc, + save_dir=output_path, + ) + + +def evaluate_reanno_sbd(): + args = parse_args() + + # by default, we apply thinning + apply_thinning = not args.raw + + evaluate_reanno( sbd_path=args.sbd_path, pred_path=args.pred_path, cls_dir=args.cls_dir, diff --git a/pyEdgeEval/info.py b/pyEdgeEval/info.py index 0ae77dc..14edbd3 100644 --- a/pyEdgeEval/info.py +++ b/pyEdgeEval/info.py @@ -2,7 +2,7 @@ """Information about this library""" -__version__ = "0.2.4" +__version__ = "0.2.5" __author__ = "haruishi43" __email__ = "haruyaishikawa@keio.jp" __description__ = "Python Edge Evaluation Tools"