Add new claude. (#695)

mlcommons · Nov 18, 2024 · e469bc8 · e469bc8
1 parent dc034d4
commit e469bc8
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/plugins/anthropic/modelgauge/suts/anthropic_api.py b/plugins/anthropic/modelgauge/suts/anthropic_api.py
@@ -87,7 +87,6 @@ def translate_response(self, request: AnthropicRequest, response: AnthropicMessa
 
 # TODO: Add claude 3.5 Haiku when it comes out later this month
 #  https://docs.anthropic.com/en/docs/about-claude/models#model-names
-model_names = ["claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022"]
-for model in model_names:
+for model in ["claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022"]:
     # UID is the model name.
     SUTS.register(AnthropicSUT, model, model, ANTHROPIC_SECRET)
diff --git a/src/modelbench/benchmark_runner.py b/src/modelbench/benchmark_runner.py
@@ -462,7 +462,7 @@ def check_annotator(annotator: CompletionAnnotator):
             annotators_worked = pool.map(check_annotator, annotators)
             if not all(annotators_worked):
                 raise RuntimeError(
-                    f"Not all SUTs are ready to go. Status: {dict(zip([a.uid for a in annotators], annotators_worked))}"
+                    f"Not all annotators are ready to go. Status: {dict(zip([a.uid for a in annotators], annotators_worked))}"
                 )
 
     def _calculate_test_results(self, test_run):