running swebench evaluation as subprocess

evals/deepeval_on_swe_bench.py

@@ -1,3 +1,8 @@
+from cognee.infrastructure.databases.vector import get_vector_engine
+from cognee.base_config import get_base_config
+import os
+import logging
+from cognee.infrastructure.llm.get_llm_client import get_llm_client
 from typing import List, Dict, Type
 from swebench.harness.utils import load_swebench_dataset
 from deepeval.dataset import EvaluationDataset
@@ -21,8 +26,6 @@ def convert_swe_to_deepeval(swe_dataset: List[Dict]):
         expected_output = datum["patch"]
         context = [datum["text"]]
         # retrieval_context = datum.get(retrieval_context_key_name)
-        # tools_called = datum.get(tools_called_key_name)
-        # expected_tools = json_obj.get(expected_tools_key_name)
 
         deepeval_dataset.add_test_case(
             LLMTestCase(
@@ -31,33 +34,32 @@ def convert_swe_to_deepeval(swe_dataset: List[Dict]):
                 expected_output=expected_output,
                 context=context,
                 # retrieval_context=retrieval_context,
-                # tools_called=tools_called,
-                # expected_tools=expected_tools,
             )
         )
     return deepeval_dataset
 
 
-from cognee.infrastructure.llm.get_llm_client import get_llm_client
-
-swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test')
+swe_dataset = load_swebench_dataset(
+    'princeton-nlp/SWE-bench_bm25_13K', split='test')
 deepeval_dataset = convert_swe_to_deepeval(swe_dataset)
 
-import logging
 
 logger = logging.getLogger(__name__)
 
+
 class AnswerModel(BaseModel):
-    response:str
+    response: str
 
-def get_answer_base(content: str, context:str, response_model: Type[BaseModel]):
+
+def get_answer_base(content: str, context: str, response_model: Type[BaseModel]):
     llm_client = get_llm_client()
 
     system_prompt = "THIS IS YOUR CONTEXT:" + str(context)
 
-    return  llm_client.create_structured_output(content, system_prompt, response_model)
+    return llm_client.create_structured_output(content, system_prompt, response_model)
+
 
-def get_answer(content: str,context, model: Type[BaseModel]= AnswerModel):
+def get_answer(content: str, context, model: Type[BaseModel] = AnswerModel):
 
     try:
         return (get_answer_base(
@@ -66,9 +68,11 @@ def get_answer(content: str,context, model: Type[BaseModel]= AnswerModel):
             model
         ))
     except Exception as error:
-        logger.error("Error extracting cognitive layers from content: %s", error, exc_info = True)
+        logger.error(
+            "Error extracting cognitive layers from content: %s", error, exc_info=True)
         raise error
 
+
 async def run_cognify_base_rag():
     from cognee.api.v1.add import add
     from cognee.api.v1.prune import prune
@@ -82,11 +86,7 @@ async def run_cognify_base_rag():
     pass
 
 
-import os
-from cognee.base_config import get_base_config
-from cognee.infrastructure.databases.vector import get_vector_engine
-
-async def cognify_search_base_rag(content:str, context:str):
+async def cognify_search_base_rag(content: str, context: str):
     base_config = get_base_config()
 
     cognee_directory_path = os.path.abspath(".cognee_system")
@@ -99,7 +99,8 @@ async def cognify_search_base_rag(content:str, context:str):
     print("results", return_)
     return return_
 
-async def cognify_search_graph(content:str, context:str):
+
+async def cognify_search_graph(content: str, context: str):
     from cognee.api.v1.search import search, SearchType
     params = {'query': 'Donald Trump'}
 
@@ -114,14 +115,16 @@ def convert_goldens_to_test_cases(test_cases_raw: List[LLMTestCase]) -> List[LLM
         test_case = LLMTestCase(
             input=case.input,
             # Generate actual output using the 'input' and 'additional_metadata'
-            actual_output= str(get_answer(case.input, case.context).model_dump()['response']),
+            actual_output=str(get_answer(
+                case.input, case.context).model_dump()['response']),
             expected_output=case.expected_output,
             context=case.context,
             retrieval_context=["retrieval_context"],
         )
         test_cases.append(test_case)
     return test_cases
 
+
 def convert_swe_to_deepeval_testcases(swe_dataset: List[Dict]):
     deepeval_dataset = EvaluationDataset()
     for datum in swe_dataset[:4]:
@@ -135,7 +138,8 @@ def convert_swe_to_deepeval_testcases(swe_dataset: List[Dict]):
         deepeval_dataset.add_test_case(
             LLMTestCase(
                 input=input,
-                actual_output= str(get_answer(input, context).model_dump()['response']),
+                actual_output=str(get_answer(
+                    input, context).model_dump()['response']),
                 expected_output=expected_output,
                 context=context,
                 # retrieval_context=retrieval_context,
@@ -145,9 +149,11 @@ def convert_swe_to_deepeval_testcases(swe_dataset: List[Dict]):
         )
     return deepeval_dataset
 
-swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test')
+
+swe_dataset = load_swebench_dataset(
+    'princeton-nlp/SWE-bench_bm25_13K', split='test')
 test_dataset = convert_swe_to_deepeval_testcases(swe_dataset)
-    
+
 if __name__ == "__main__":
 
     import asyncio
@@ -159,9 +165,10 @@ async def main():
     asyncio.run(main())
     # run_cognify_base_rag_and_search()
     # # Data preprocessing before setting the dataset test cases
-    swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test')
+    swe_dataset = load_swebench_dataset(
+        'princeton-nlp/SWE-bench_bm25_13K', split='test')
     test_dataset = convert_swe_to_deepeval_testcases(swe_dataset)
     from deepeval.metrics import HallucinationMetric
     metric = HallucinationMetric()
     evalresult = test_dataset.evaluate([metric])
-    pass
+    pass

evals/eval_swe_bench.py

@@ -1,38 +1,38 @@
-from swebench.harness.utils import load_swebench_dataset
-from swebench.harness.run_evaluation import get_dataset_from_preds
-from swebench.harness.run_evaluation import run_instances
-from swebench.harness.test_spec import make_test_spec, TestSpec
-
+import json
 import subprocess
+from pathlib import Path
+
+from swebench.harness.utils import load_swebench_dataset
 from swebench.inference.make_datasets.create_instance import PATCH_EXAMPLE
-from evals.eval_utils import download_instances
+
 import cognee
 from cognee.api.v1.cognify.code_graph_pipeline import code_graph_pipeline
 from cognee.api.v1.search import SearchType
-from pathlib import Path
 from cognee.infrastructure.databases.graph import get_graph_engine
 from cognee.infrastructure.llm.get_llm_client import get_llm_client
+from evals.eval_utils import download_instances
 
-async def cognee_and_llm(dataset, search_type = SearchType.CHUNKS):
+
+async def cognee_and_llm(dataset, search_type=SearchType.CHUNKS):
     await cognee.prune.prune_data()
-    await cognee.prune.prune_system(metadata = True)
+    await cognee.prune.prune_system(metadata=True)
 
     dataset_name = "SWE_test_data"
-    code_text = dataset[0]["text"][:100000]
+    code_text = dataset[0]["text"]
     await cognee.add([code_text], dataset_name)
     await code_graph_pipeline([dataset_name])
     graph_engine = await get_graph_engine()
     with open(graph_engine.filename, "r") as f:
-        graph_str  = f.read()
-    
+        graph_str = f.read()
+
     problem_statement = dataset[0]['problem_statement']
     instructions = (
-        f"I need you to solve this issue by looking at the provided knowledge graph and by "
-        + f"generating a single patch file that I can apply directly to this repository "
-        + f"using git apply. Please respond with a single patch "
-        + f"file in the following format."  
+        "I need you to solve this issue by looking at the provided knowledge graph and by "
+        + "generating a single patch file that I can apply directly to this repository "
+        + "using git apply. Please respond with a single patch "
+        + "file in the following format."
     )
-    
+
     prompt = "\n".join([
         instructions,
         "<patch>",
@@ -41,87 +41,71 @@ async def cognee_and_llm(dataset, search_type = SearchType.CHUNKS):
         "This is the knowledge graph:",
         graph_str
     ])
-    
+
     llm_client = get_llm_client()
     answer_prediction = llm_client.create_structured_output(
-                        text_input = problem_statement,
-                        system_prompt = prompt,
-                        response_model = str,
-                        )
+        text_input=problem_statement,
+        system_prompt=prompt,
+        response_model=str,
+    )
     return answer_prediction
 
 
 async def llm_on_preprocessed_data(dataset):
     problem_statement = dataset[0]['problem_statement']
     prompt = dataset[0]["text"]
-    
+
     llm_client = get_llm_client()
     answer_prediction = llm_client.create_structured_output(
-                        text_input = problem_statement,
-                        system_prompt = prompt, # TODO check if this is correct
-                        response_model = str,
-                        )
+        text_input=problem_statement,
+        system_prompt=prompt,
+        response_model=str,
+    )
     return answer_prediction
 
+
 async def get_preds(dataset, with_cognee=True):
     if with_cognee:
         text_output = await cognee_and_llm(dataset)
         model_name = "with_cognee"
     else:
         text_output = await llm_on_preprocessed_data(dataset)
         model_name = "without_cognee"
-
-    preds = {dataset[0]["instance_id"]:
-                {"instance_id": dataset[0]["instance_id"],
-                "model_patch": text_output,
-                "model_name_or_path": model_name}}
-
-    dataset_name = 'princeton-nlp/SWE-bench' if with_cognee else 'princeton-nlp/SWE-bench_bm25_13K'
-    preds_dataset = get_dataset_from_preds(dataset_name, 
-                                            "test", 
-                                            [dataset[0]["instance_id"]], 
-                                            preds, 
-                                            model_name)
-
-    return preds, preds_dataset
-
-async def evaluate(test_specs: list[TestSpec],
-                    preds: dict,
-                    ):
-    for test_spec in test_specs:
-        pred = preds[test_spec.instance_id]
-        log_dir = Path("logs")
-        log_dir.mkdir(parents=True, exist_ok=True)
-
-        patch_file = Path(log_dir / "patch.diff")
-        patch_file.write_text(pred["model_patch"] or "")
-        for command in test_spec.repo_script_list:
-            if "/testbed" in command:
-                command = command.replace("/testbed", "./testbed")
-            result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
-            print(result)
-
-        subprocess.run("git apply --allow-empty -v logs/patch.diff", shell=True, capture_output=True, text=True)
-
-
+
+    preds = [{"instance_id": dataset[0]["instance_id"],
+              "model_patch": text_output,
+              "model_name_or_path": model_name}]
+
+    return preds
+
 
 async def main():
-    swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench', split='test')
-    swe_dataset_preprocessed = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test')
-    test_data = swe_dataset[:1] 
-    test_data_preprocessed = swe_dataset_preprocessed[:1] 
+    swe_dataset = load_swebench_dataset(
+        'princeton-nlp/SWE-bench', split='test')
+    swe_dataset_preprocessed = load_swebench_dataset(
+        'princeton-nlp/SWE-bench_bm25_13K', split='test')
+    test_data = swe_dataset[:1]
+    test_data_preprocessed = swe_dataset_preprocessed[:1]
     assert test_data[0]["instance_id"] == test_data_preprocessed[0]["instance_id"]
     filepath = Path("SWE-bench_testsample")
     if filepath.exists():
         from datasets import Dataset
         dataset = Dataset.load_from_disk(filepath)
     else:
         dataset = download_instances(test_data, filepath)
-    
-    cognee_preds, cognee_preds_dataset = await get_preds(dataset, with_cognee=True)
+
+    cognee_preds = await get_preds(dataset, with_cognee=True)
     # nocognee_preds = await get_preds(dataset, with_cognee=False)
-    test_specs = list(map(make_test_spec, test_data))
-    results = await evaluate(test_specs, cognee_preds)
+    with open("withcognee.json", "w") as file:
+        json.dump(cognee_preds, file)
+
+    subprocess.run(["python", "-m", "swebench.harness.run_evaluation",
+                    "--dataset_name", 'princeton-nlp/SWE-bench',
+                    "--split", "test",
+                    "--predictions_path",  "withcognee.json",
+                    "--max_workers", "1",
+                    "--instance_ids", test_data[0]["instance_id"],
+                    "--run_id", "with_cognee"])
 
 if __name__ == "__main__":
     import asyncio