harness: Detector only

docs/source/garak.harnesses.detectoronly.rst

@@ -0,0 +1,8 @@
+garak.harnesses.detectoronly
+====================
+
+.. automodule:: garak.harnesses.detectoronly
+   :members:
+   :undoc-members:
+   :show-inheritance:   
+

docs/source/harnesses.rst

@@ -6,5 +6,6 @@ garak.harnesses
 
    garak.harnesses
    garak.harnesses.base
+   garak.harnesses.detectoronly
    garak.harnesses.probewise
    garak.harnesses.pxd

garak/_config.py

@@ -28,7 +28,7 @@
 system_params = (
     "verbose narrow_output parallel_requests parallel_attempts skip_unknown".split()
 )
-run_params = "seed deprefix eval_threshold generations probe_tags interactive".split()
+run_params = "seed deprefix eval_threshold generations probe_tags interactive probed_report_path".split()
 plugins_params = "model_type model_name extended_detectors".split()
 reporting_params = "taxonomy report_prefix".split()
 project_dir_name = "garak"

garak/attempt.py

@@ -105,6 +105,24 @@ def as_dict(self) -> dict:
             "messages": self.messages,
         }
 
+    @classmethod
+    def from_dict(cls, dicti):
+        """Initializes an attempt object from dictionary"""
+        attempt_obj = cls()
+        attempt_obj.uuid = dicti['uuid']
+        attempt_obj.seq = dicti['seq']
+        attempt_obj.status = dicti['status']
+        attempt_obj.probe_classname = dicti['probe_classname']
+        attempt_obj.probe_params = dicti['probe_params']
+        attempt_obj.targets = dicti['targets']
+        attempt_obj.prompt = dicti['prompt']
+        attempt_obj.outputs = dicti['outputs']
+        attempt_obj.detector_results = dicti['detector_results']
+        attempt_obj.notes = dicti['notes']
+        attempt_obj.goal = dicti['goal']
+        attempt_obj.messages = dicti['messages']
+        return attempt_obj
+
     def __getattribute__(self, name: str) -> Any:
         """override prompt and outputs access to take from history"""
         if name == "prompt":

garak/cli.py

@@ -3,7 +3,7 @@
 
 """Flow for invoking garak from the command line"""
 
-command_options = "list_detectors list_probes list_generators list_buffs list_config plugin_info interactive report version".split()
+command_options = "list_detectors list_probes list_generators list_buffs list_config plugin_info interactive report version detector_only".split()
 
 
 def main(arguments=None) -> None:
@@ -107,6 +107,12 @@ def main(arguments=None) -> None:
     parser.add_argument(
         "--config", type=str, default=None, help="YAML config file for this run"
     )
+    parser.add_argument(
+        "--probed_report_path",
+        type=str,
+        default=None,
+        help="Path to jsonl report that stores the generators responses"
+    )
 
     ## PLUGINS
     # generator
@@ -247,6 +253,11 @@ def main(arguments=None) -> None:
         action="store_true",
         help="Launch garak in interactive.py mode",
     )
+    parser.add_argument(
+        "--detector_only",
+        action="store_true",
+        help="run detector on jsonl report"
+    )
 
     logging.debug("args - raw argument string received: %s", arguments)
 
@@ -512,6 +523,30 @@ def main(arguments=None) -> None:
                 )
 
             command.end_run()
+
+        elif args.detector_only:
+            # Run detector only detection
+            if not _config.plugins.detector_spec:
+                logging.error("Detector(s) not specified. Use --detectors")
+                raise ValueError("use --detectors to specify some detectors")
+
+            if not _config.run.probed_report_path:
+                logging.error("report path not specified")
+                raise ValueError("Specify jsonl report path using --probed_report_path")
+
+            evaluator = garak.evaluators.ThresholdEvaluator(_config.run.eval_threshold)
+            print(_config.plugins.detector_spec.split(","))
+
+            detector_names, detector_rejected = _config.parse_plugin_spec(
+                getattr(_config.plugins, "detector_spec", ""),
+                "detectors",
+                getattr(_config.run, "detector_tags", "")
+                )
+
+            command.start_run()
+            command.detector_only_run(_config.run.probed_report_path, detector_names, evaluator)
+            command.end_run()
+
         else:
             print("nothing to do 🤷  try --help")
             if _config.plugins.model_name and not _config.plugins.model_type:

garak/command.py

@@ -6,7 +6,6 @@
 import logging
 import json
 
-
 def start_logging():
     from garak import _config
 
@@ -255,3 +254,16 @@ def write_report_digest(report_filename, digest_filename):
     digest = report_digest.compile_digest(report_filename)
     with open(digest_filename, "w", encoding="utf-8") as f:
         f.write(digest)
+
+def detector_only_run(report_filename, detectors, evaluator):
+    import garak.harnesses.detectoronly
+    import garak.attempt
+
+    with open(report_filename) as f:
+        data = [json.loads(line) for line in f]
+
+    data = [d for d in data if d["entry_type"] == "attempt" and d["status"] == 1]
+    attempts = [garak.attempt.Attempt.from_dict(d) for d in data]
+
+    detector_only_h = garak.harnesses.detectoronly.DetectorOnly()
+    detector_only_h.run(attempts, detectors, evaluator)

garak/harnesses/base.py

@@ -109,28 +109,29 @@ def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None:
             assert isinstance(
                 attempt_results, (list, types.GeneratorType)
             ), "probing should always return an ordered iterable"
-
-            for d in detectors:
-                logging.debug("harness: run detector %s", d.detectorname)
-                attempt_iterator = tqdm.tqdm(attempt_results, leave=False)
-                detector_probe_name = d.detectorname.replace("garak.detectors.", "")
-                attempt_iterator.set_description("detectors." + detector_probe_name)
-                for attempt in attempt_iterator:
-                    attempt.detector_results[detector_probe_name] = list(
-                        d.detect(attempt)
-                    )
-
-            for attempt in attempt_results:
-                attempt.status = garak.attempt.ATTEMPT_COMPLETE
-                _config.transient.reportfile.write(json.dumps(attempt.as_dict()) + "\n")
-
-            if len(attempt_results) == 0:
-                logging.warning(
-                    "zero attempt results: probe %s, detector %s",
-                    probe.probename,
-                    detector_probe_name,
-                )
-            else:
-                evaluator.evaluate(attempt_results)
+            self.run_detectors(detectors, attempt_results, evaluator, probe)
 
         logging.debug("harness: probe list iteration completed")
+
+    def run_detectors(self, detectors, attempt_results, evaluator, probe=None):
+        for d in detectors:
+            logging.debug("harness: run detector %s", d.detectorname)
+            attempt_iterator = tqdm.tqdm(attempt_results, leave=False)
+            detector_probe_name = d.detectorname.replace("garak.detectors.", "")
+            attempt_iterator.set_description("detectors." + detector_probe_name)
+            for attempt in attempt_iterator:
+                attempt.detector_results[detector_probe_name] = list(
+                    d.detect(attempt)
+                )
+
+        for attempt in attempt_results:
+            attempt.status = garak.attempt.ATTEMPT_COMPLETE
+            _config.transient.reportfile.write(json.dumps(attempt.as_dict()) + "\n")
+
+        if len(attempt_results) == 0:
+            logging.warning(
+                "zero attempt results: probe %s",
+                probe.probename
+            )
+        else:
+            evaluator.evaluate(attempt_results)

garak/harnesses/detectoronly.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: Portions Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Detector only harness
+
+Runs specified detectors on already existing prompt-response pairs from parsing a report.jsonl file.
+"""
+
+import logging
+
+from garak import _config, _plugins
+from garak.harnesses import Harness
+from garak.detectors import Detector
+
+class DetectorOnly(Harness):
+    def __init__(self, config_root=_config):
+        super().__init__(config_root)
+
+    def _load_detector(self, detector_name: str) -> Detector:
+        detector = _plugins.load_plugin(
+            detector_name, break_on_fail=False
+        )
+        if detector:
+            return detector
+        else:
+            print(f" detector load failed: {detector_name}, skipping >>")
+            logging.error(f" detector load failed: {detector_name}, skipping >>")
+        return False
+
+    def run(self, attempts, detector_names, evaluator):
+        detectors = []
+        for detector in sorted(detector_names):
+            d = self._load_detector(detector)
+            if d:
+                detectors.append(d)
+
+        if len(detectors) == 0:
+            msg = "No detectors, nothing to do"
+            logging.warning(msg)
+            if hasattr(_config.system, "verbose") and _config.system.verbose >= 2:
+                print(msg)
+            raise ValueError(msg)
+
+        super().run_detectors(detectors, attempts, evaluator) # The probe is None, but hopefully no errors occur with probe.

garak/resources/garak.core.yaml

@@ -12,6 +12,7 @@ run:
   eval_threshold: 0.5
   generations: 10
   probe_tags:
+  probed_report_path:
 
 plugins:
   model_type: