update sbd evaluation script

pyEdgeEval/evaluators/sbd.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os.path as osp
+from warnings import warn
 
 from pyEdgeEval.common.multi_label import (
     calculate_metrics,
@@ -121,28 +122,31 @@ def set_eval_params(
             self.max_dist = 0.02
             self.kill_internal = True
             self.skip_if_nonexistent = True
+
+            if instance_sensitive:
+                warn("Pre-SEAL mode doesn't support instance sensitive")
             self.instance_sensitive = False
         elif eval_mode == "post-seal":
             print_log("Using Post-SEAL params", logger=self._logger)
             self.max_dist = 0.02
             self.kill_internal = False
             self.skip_if_nonexistent = False
-            self.instance_sensitive = True
-        elif eval_mode == "high-quality":
+            self.instance_sensitive = instance_sensitive
+        elif eval_mode == "reanno":
             print_log(
-                "Using params for high-quality annotations", logger=self._logger
+                "Using params for re-annotated test split", logger=self._logger
             )
             self.max_dist = 0.0075
             self.kill_internal = False
             self.skip_if_nonexistent = False
-            self.instance_sensitive = True
+            self.instance_sensitive = instance_sensitive
         else:
             print_log("Using custom params", logger=self._logger)
             self.max_dist = max_dist
             self.kill_internal = kill_internal
             self.skip_if_nonexistent = skip_if_nonexistent
-            self.instance_sensitive = instance_sensitive
 
+            self.instance_sensitive = instance_sensitive
         if self.kill_internal and self.instance_sensitive:
             print_log(
                 "kill_internal and instance_sensitive are both True which will conflict with each either",

pyEdgeEval/helpers/evaluate_sbd.py

@@ -50,26 +50,37 @@ def parse_args():
         type=str,
         help="the category number to evaluate; can be multiple values",
     )
-    parser.add_argument(
-        "--thresholds",
-        type=str,
-        default="99",
-        help="the number of thresholds (could be a list of floats); use 99 for eval",
-    )
     parser.add_argument(
         "--raw",
         action="store_true",
         help="option to remove the thinning process (i.e. uses raw predition)",
     )
     parser.add_argument(
-        "--apply-nms",
+        "--pre-seal",
         action="store_true",
-        help="applies NMS before evaluation",
+        help="prior to SEAL, the evaluations were not as strict",
+    )
+    parser.add_argument(
+        "--nonIS",
+        action="store_true",
+        help="non instance sensitive evaluation",
+    )
+    parser.add_argument(
+        "--max-dist",
+        type=float,
+        default=0.02,
+        help="tolerance distance (default: 0.02)",
+    )
+    parser.add_argument(
+        "--thresholds",
+        type=str,
+        default="99",
+        help="the number of thresholds (could be a list of floats); use 99 for eval",
     )
     parser.add_argument(
-        "--kill-internal",
+        "--apply-nms",
         action="store_true",
-        help="kill internal contour",
+        help="applies NMS before evaluation",
     )
     parser.add_argument(
         "--nproc",
@@ -89,8 +100,11 @@ def evaluate(
     pred_suffix: str,
     output_path: str,
     categories: str,
+    pre_seal: bool,
     apply_thinning: bool,
     apply_nms: bool,
+    nonIS: bool,
+    max_dist: float,
     thresholds: str,
     nproc: int,
 ):
@@ -160,13 +174,16 @@ def evaluate(
     log_file = osp.join(output_path, f"{timestamp}.log")
     logger = get_root_logger(log_file=log_file, log_level="INFO")
     logger.info("Running SBD Evaluation")
+    logger.info(f"cls_dir:    \t{cls_dir}")
+    logger.info(f"inst_dir:   \t{inst_dir}")
+    logger.info(f"pred_suffix:\t{pred_suffix}")
     logger.info(f"categories: \t{categories}")
     logger.info(f"thresholds: \t{thresholds}")
+    logger.info(f"pre-seal:   \t{pre_seal}")
     logger.info(f"thin:       \t{apply_thinning}")
     logger.info(f"nms:        \t{apply_nms}")
-    logger.info(f"cls_dir:    \t{cls_dir}")
-    logger.info(f"inst_dir:   \t{inst_dir}")
-    logger.info(f"pred_suffix:\t{pred_suffix}")
+    logger.info(f"nonIS:      \t{nonIS}")
+    logger.info(f"max_dist    \t{max_dist}")
     print("\n\n")
 
     # initialize evaluator
@@ -183,13 +200,15 @@ def evaluate(
         evaluator.set_sample_names()
 
     # set parameters
-    eval_mode = "pre-seal"  # FIXME: hard-coded for now
+    eval_mode = "pre-seal" if pre_seal else "post-seal"
+    instance_sensitive = not nonIS
     evaluator.set_eval_params(
         eval_mode=eval_mode,
         scale=1.0,
         apply_thinning=apply_thinning,
         apply_nms=apply_nms,
-        instance_sensitive=False,
+        max_dist=max_dist,
+        instance_sensitive=instance_sensitive,
     )
 
     # evaluate
@@ -208,6 +227,151 @@ def evaluate_sbd():
     apply_thinning = not args.raw
 
     evaluate(
+        sbd_path=args.sbd_path,
+        pred_path=args.pred_path,
+        cls_dir=args.cls_dir,
+        inst_dir=args.inst_dir,
+        pred_suffix=args.pred_suffix,
+        output_path=args.output_path,
+        categories=args.categories,
+        pre_seal=args.pre_seal,
+        apply_thinning=apply_thinning,
+        apply_nms=args.apply_nms,
+        thresholds=args.thresholds,
+        nproc=args.nproc,
+    )
+
+
+def evaluate_reanno(
+    sbd_path: str,
+    pred_path: str,
+    cls_dir: str,
+    inst_dir: str,
+    pred_suffix: str,
+    output_path: str,
+    categories: str,
+    apply_thinning: bool,
+    apply_nms: bool,
+    nonIS: bool,
+    thresholds: str,
+    nproc: int,
+):
+    """Evaluate Re-annotated SBD"""
+
+    if categories is None:
+        print("use all categories")
+        categories = list(range(1, len(SBDEvaluator.CLASSES) + 1))
+    else:
+        # string evaluation for categories
+        categories = categories.strip()
+        try:
+            categories = [int(categories)]
+        except ValueError:
+            try:
+                if categories.startswith("[") and categories.endswith("]"):
+                    categories = categories[1:-1]
+                    categories = [
+                        int(cat.strip()) for cat in categories.split(",")
+                    ]
+                else:
+                    print(
+                        "Bad categories format; should be a python list of floats (`[a, b, c]`)"
+                    )
+                    return
+            except ValueError:
+                print(
+                    "Bad categories format; should be a python list of ints (`[a, b, c]`)"
+                )
+                return
+
+    for cat in categories:
+        assert (
+            0 < cat < 21
+        ), f"category needs to be between 1 ~ 19, but got {cat}"
+
+    # string evaluation for thresholds
+    thresholds = thresholds.strip()
+    try:
+        n_thresholds = int(thresholds)
+        thresholds = n_thresholds
+    except ValueError:
+        try:
+            if thresholds.startswith("[") and thresholds.endswith("]"):
+                thresholds = thresholds[1:-1]
+                thresholds = [float(t.strip()) for t in thresholds.split(",")]
+            else:
+                print(
+                    "Bad threshold format; should be a python list of floats (`[a, b, c]`)"
+                )
+                return
+        except ValueError:
+            print(
+                "Bad threshold format; should be a python list of ints (`[a, b, c]`)"
+            )
+            return
+
+    if output_path is None:
+        timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime())
+        output_path = osp.join(
+            osp.normpath(pred_path), f"edge_results_{timestamp}"
+        )
+
+    mkdir_or_exist(output_path)
+
+    timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime())
+    log_file = osp.join(output_path, f"{timestamp}.log")
+    logger = get_root_logger(log_file=log_file, log_level="INFO")
+    logger.info("Running Re-Annotated SBD Evaluation")
+    logger.info(f"cls_dir:    \t{cls_dir}")
+    logger.info(f"inst_dir:   \t{inst_dir}")
+    logger.info(f"pred_suffix:\t{pred_suffix}")
+    logger.info(f"categories: \t{categories}")
+    logger.info(f"thresholds: \t{thresholds}")
+    logger.info(f"thin:       \t{apply_thinning}")
+    logger.info(f"nms:        \t{apply_nms}")
+    logger.info(f"nonIS:      \t{nonIS}")
+    print("\n\n")
+
+    # initialize evaluator
+    evaluator = SBDEvaluator(
+        dataset_root=sbd_path,
+        pred_root=pred_path,
+        cls_dir=cls_dir,
+        inst_dir=inst_dir,
+        pred_suffix=pred_suffix,
+    )
+    if evaluator.sample_names is None:
+        # load custom sample names
+        # SAMPLE_NAMES = ["2008_000051", "2008_000195"]
+        evaluator.set_sample_names()
+
+    # set parameters
+    instance_sensitive = not nonIS
+    evaluator.set_eval_params(
+        eval_mode="reanno",  # fixed to "reanno" mode
+        scale=1.0,
+        apply_thinning=apply_thinning,
+        apply_nms=apply_nms,
+        max_dist=0.0075,  # fixed
+        instance_sensitive=instance_sensitive,
+    )
+
+    # evaluate
+    evaluator.evaluate(
+        categories=categories,
+        thresholds=thresholds,
+        nproc=nproc,
+        save_dir=output_path,
+    )
+
+
+def evaluate_reanno_sbd():
+    args = parse_args()
+
+    # by default, we apply thinning
+    apply_thinning = not args.raw
+
+    evaluate_reanno(
         sbd_path=args.sbd_path,
         pred_path=args.pred_path,
         cls_dir=args.cls_dir,

pyEdgeEval/info.py

@@ -2,7 +2,7 @@
 
 """Information about this library"""
 
-__version__ = "0.2.4"
+__version__ = "0.2.5"
 __author__ = "haruishi43"
 __email__ = "haruyaishikawa@keio.jp"
 __description__ = "Python Edge Evaluation Tools"