diff --git a/evals/deepeval_on_swe_bench.py b/evals/deepeval_on_swe_bench.py index b191a08d..8cb94abb 100644 --- a/evals/deepeval_on_swe_bench.py +++ b/evals/deepeval_on_swe_bench.py @@ -1,3 +1,8 @@ +from cognee.infrastructure.databases.vector import get_vector_engine +from cognee.base_config import get_base_config +import os +import logging +from cognee.infrastructure.llm.get_llm_client import get_llm_client from typing import List, Dict, Type from swebench.harness.utils import load_swebench_dataset from deepeval.dataset import EvaluationDataset @@ -21,8 +26,6 @@ def convert_swe_to_deepeval(swe_dataset: List[Dict]): expected_output = datum["patch"] context = [datum["text"]] # retrieval_context = datum.get(retrieval_context_key_name) - # tools_called = datum.get(tools_called_key_name) - # expected_tools = json_obj.get(expected_tools_key_name) deepeval_dataset.add_test_case( LLMTestCase( @@ -31,33 +34,32 @@ def convert_swe_to_deepeval(swe_dataset: List[Dict]): expected_output=expected_output, context=context, # retrieval_context=retrieval_context, - # tools_called=tools_called, - # expected_tools=expected_tools, ) ) return deepeval_dataset -from cognee.infrastructure.llm.get_llm_client import get_llm_client - -swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test') +swe_dataset = load_swebench_dataset( + 'princeton-nlp/SWE-bench_bm25_13K', split='test') deepeval_dataset = convert_swe_to_deepeval(swe_dataset) -import logging logger = logging.getLogger(__name__) + class AnswerModel(BaseModel): - response:str + response: str -def get_answer_base(content: str, context:str, response_model: Type[BaseModel]): + +def get_answer_base(content: str, context: str, response_model: Type[BaseModel]): llm_client = get_llm_client() system_prompt = "THIS IS YOUR CONTEXT:" + str(context) - return llm_client.create_structured_output(content, system_prompt, response_model) + return llm_client.create_structured_output(content, system_prompt, response_model) + -def get_answer(content: str,context, model: Type[BaseModel]= AnswerModel): +def get_answer(content: str, context, model: Type[BaseModel] = AnswerModel): try: return (get_answer_base( @@ -66,9 +68,11 @@ def get_answer(content: str,context, model: Type[BaseModel]= AnswerModel): model )) except Exception as error: - logger.error("Error extracting cognitive layers from content: %s", error, exc_info = True) + logger.error( + "Error extracting cognitive layers from content: %s", error, exc_info=True) raise error + async def run_cognify_base_rag(): from cognee.api.v1.add import add from cognee.api.v1.prune import prune @@ -82,11 +86,7 @@ async def run_cognify_base_rag(): pass -import os -from cognee.base_config import get_base_config -from cognee.infrastructure.databases.vector import get_vector_engine - -async def cognify_search_base_rag(content:str, context:str): +async def cognify_search_base_rag(content: str, context: str): base_config = get_base_config() cognee_directory_path = os.path.abspath(".cognee_system") @@ -99,7 +99,8 @@ async def cognify_search_base_rag(content:str, context:str): print("results", return_) return return_ -async def cognify_search_graph(content:str, context:str): + +async def cognify_search_graph(content: str, context: str): from cognee.api.v1.search import search, SearchType params = {'query': 'Donald Trump'} @@ -114,7 +115,8 @@ def convert_goldens_to_test_cases(test_cases_raw: List[LLMTestCase]) -> List[LLM test_case = LLMTestCase( input=case.input, # Generate actual output using the 'input' and 'additional_metadata' - actual_output= str(get_answer(case.input, case.context).model_dump()['response']), + actual_output=str(get_answer( + case.input, case.context).model_dump()['response']), expected_output=case.expected_output, context=case.context, retrieval_context=["retrieval_context"], @@ -122,6 +124,7 @@ def convert_goldens_to_test_cases(test_cases_raw: List[LLMTestCase]) -> List[LLM test_cases.append(test_case) return test_cases + def convert_swe_to_deepeval_testcases(swe_dataset: List[Dict]): deepeval_dataset = EvaluationDataset() for datum in swe_dataset[:4]: @@ -135,7 +138,8 @@ def convert_swe_to_deepeval_testcases(swe_dataset: List[Dict]): deepeval_dataset.add_test_case( LLMTestCase( input=input, - actual_output= str(get_answer(input, context).model_dump()['response']), + actual_output=str(get_answer( + input, context).model_dump()['response']), expected_output=expected_output, context=context, # retrieval_context=retrieval_context, @@ -145,9 +149,11 @@ def convert_swe_to_deepeval_testcases(swe_dataset: List[Dict]): ) return deepeval_dataset -swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test') + +swe_dataset = load_swebench_dataset( + 'princeton-nlp/SWE-bench_bm25_13K', split='test') test_dataset = convert_swe_to_deepeval_testcases(swe_dataset) - + if __name__ == "__main__": import asyncio @@ -159,9 +165,10 @@ async def main(): asyncio.run(main()) # run_cognify_base_rag_and_search() # # Data preprocessing before setting the dataset test cases - swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test') + swe_dataset = load_swebench_dataset( + 'princeton-nlp/SWE-bench_bm25_13K', split='test') test_dataset = convert_swe_to_deepeval_testcases(swe_dataset) from deepeval.metrics import HallucinationMetric metric = HallucinationMetric() evalresult = test_dataset.evaluate([metric]) - pass \ No newline at end of file + pass diff --git a/evals/eval_swe_bench.py b/evals/eval_swe_bench.py index 9acb176b..2cb22157 100644 --- a/evals/eval_swe_bench.py +++ b/evals/eval_swe_bench.py @@ -1,38 +1,38 @@ -from swebench.harness.utils import load_swebench_dataset -from swebench.harness.run_evaluation import get_dataset_from_preds -from swebench.harness.run_evaluation import run_instances -from swebench.harness.test_spec import make_test_spec, TestSpec - +import json import subprocess +from pathlib import Path + +from swebench.harness.utils import load_swebench_dataset from swebench.inference.make_datasets.create_instance import PATCH_EXAMPLE -from evals.eval_utils import download_instances + import cognee from cognee.api.v1.cognify.code_graph_pipeline import code_graph_pipeline from cognee.api.v1.search import SearchType -from pathlib import Path from cognee.infrastructure.databases.graph import get_graph_engine from cognee.infrastructure.llm.get_llm_client import get_llm_client +from evals.eval_utils import download_instances -async def cognee_and_llm(dataset, search_type = SearchType.CHUNKS): + +async def cognee_and_llm(dataset, search_type=SearchType.CHUNKS): await cognee.prune.prune_data() - await cognee.prune.prune_system(metadata = True) + await cognee.prune.prune_system(metadata=True) dataset_name = "SWE_test_data" - code_text = dataset[0]["text"][:100000] + code_text = dataset[0]["text"] await cognee.add([code_text], dataset_name) await code_graph_pipeline([dataset_name]) graph_engine = await get_graph_engine() with open(graph_engine.filename, "r") as f: - graph_str = f.read() - + graph_str = f.read() + problem_statement = dataset[0]['problem_statement'] instructions = ( - f"I need you to solve this issue by looking at the provided knowledge graph and by " - + f"generating a single patch file that I can apply directly to this repository " - + f"using git apply. Please respond with a single patch " - + f"file in the following format." + "I need you to solve this issue by looking at the provided knowledge graph and by " + + "generating a single patch file that I can apply directly to this repository " + + "using git apply. Please respond with a single patch " + + "file in the following format." ) - + prompt = "\n".join([ instructions, "", @@ -41,28 +41,29 @@ async def cognee_and_llm(dataset, search_type = SearchType.CHUNKS): "This is the knowledge graph:", graph_str ]) - + llm_client = get_llm_client() answer_prediction = llm_client.create_structured_output( - text_input = problem_statement, - system_prompt = prompt, - response_model = str, - ) + text_input=problem_statement, + system_prompt=prompt, + response_model=str, + ) return answer_prediction async def llm_on_preprocessed_data(dataset): problem_statement = dataset[0]['problem_statement'] prompt = dataset[0]["text"] - + llm_client = get_llm_client() answer_prediction = llm_client.create_structured_output( - text_input = problem_statement, - system_prompt = prompt, # TODO check if this is correct - response_model = str, - ) + text_input=problem_statement, + system_prompt=prompt, + response_model=str, + ) return answer_prediction + async def get_preds(dataset, with_cognee=True): if with_cognee: text_output = await cognee_and_llm(dataset) @@ -70,46 +71,21 @@ async def get_preds(dataset, with_cognee=True): else: text_output = await llm_on_preprocessed_data(dataset) model_name = "without_cognee" - - preds = {dataset[0]["instance_id"]: - {"instance_id": dataset[0]["instance_id"], - "model_patch": text_output, - "model_name_or_path": model_name}} - - dataset_name = 'princeton-nlp/SWE-bench' if with_cognee else 'princeton-nlp/SWE-bench_bm25_13K' - preds_dataset = get_dataset_from_preds(dataset_name, - "test", - [dataset[0]["instance_id"]], - preds, - model_name) - - return preds, preds_dataset - -async def evaluate(test_specs: list[TestSpec], - preds: dict, - ): - for test_spec in test_specs: - pred = preds[test_spec.instance_id] - log_dir = Path("logs") - log_dir.mkdir(parents=True, exist_ok=True) - - patch_file = Path(log_dir / "patch.diff") - patch_file.write_text(pred["model_patch"] or "") - for command in test_spec.repo_script_list: - if "/testbed" in command: - command = command.replace("/testbed", "./testbed") - result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True) - print(result) - - subprocess.run("git apply --allow-empty -v logs/patch.diff", shell=True, capture_output=True, text=True) - - + + preds = [{"instance_id": dataset[0]["instance_id"], + "model_patch": text_output, + "model_name_or_path": model_name}] + + return preds + async def main(): - swe_dataset = load_swebench_dataset('princeton-nlp/SWE-bench', split='test') - swe_dataset_preprocessed = load_swebench_dataset('princeton-nlp/SWE-bench_bm25_13K', split='test') - test_data = swe_dataset[:1] - test_data_preprocessed = swe_dataset_preprocessed[:1] + swe_dataset = load_swebench_dataset( + 'princeton-nlp/SWE-bench', split='test') + swe_dataset_preprocessed = load_swebench_dataset( + 'princeton-nlp/SWE-bench_bm25_13K', split='test') + test_data = swe_dataset[:1] + test_data_preprocessed = swe_dataset_preprocessed[:1] assert test_data[0]["instance_id"] == test_data_preprocessed[0]["instance_id"] filepath = Path("SWE-bench_testsample") if filepath.exists(): @@ -117,11 +93,19 @@ async def main(): dataset = Dataset.load_from_disk(filepath) else: dataset = download_instances(test_data, filepath) - - cognee_preds, cognee_preds_dataset = await get_preds(dataset, with_cognee=True) + + cognee_preds = await get_preds(dataset, with_cognee=True) # nocognee_preds = await get_preds(dataset, with_cognee=False) - test_specs = list(map(make_test_spec, test_data)) - results = await evaluate(test_specs, cognee_preds) + with open("withcognee.json", "w") as file: + json.dump(cognee_preds, file) + + subprocess.run(["python", "-m", "swebench.harness.run_evaluation", + "--dataset_name", 'princeton-nlp/SWE-bench', + "--split", "test", + "--predictions_path", "withcognee.json", + "--max_workers", "1", + "--instance_ids", test_data[0]["instance_id"], + "--run_id", "with_cognee"]) if __name__ == "__main__": import asyncio diff --git a/evals/eval_utils.py b/evals/eval_utils.py new file mode 100644 index 00000000..1c278573 --- /dev/null +++ b/evals/eval_utils.py @@ -0,0 +1,107 @@ +import json +import logging +import os +import traceback +from copy import deepcopy +from pathlib import Path +from tempfile import TemporaryDirectory + +import unidiff +from datasets import Dataset +from swebench.inference.make_datasets.create_instance import make_code_text +from swebench.inference.make_datasets.utils import (AutoContextManager, + ingest_directory_contents) +from tqdm.auto import tqdm + + +def ingest_files(filenames): + files_dict = dict() + for filename in filenames: + with open(filename) as f: + content = f.read() + files_dict[filename] = content + return files_dict + + +def ingest_repos(input_instances): + orig_dir = os.getcwd() + with TemporaryDirectory( + dir="/scratch" if os.path.exists("/scratch") else "/tmp" + ) as root_dir: + for instance in tqdm( + input_instances.values(), + total=len(input_instances), + desc="Downloading repos on specific commits", + ): + try: + with AutoContextManager( + instance, root_dir + ) as cm: + readmes = cm.get_readme_files() + instance["readmes"] = ingest_files(readmes) + instance["file_contents"] = ingest_directory_contents( + cm.repo_path + ) + finally: + # if AutoContextManager fails to exit properly future exits will return the wrong directory + os.chdir(orig_dir) + + return input_instances + + +def extract_fields(instance): + readmes_text = make_code_text(instance["readmes"]) + code_text = make_code_text( + instance["file_contents"], add_line_numbers=False) + + text_inputs = "\n".join([readmes_text, code_text]) + text_inputs = text_inputs.strip() + "\n\n" + # text_inputs = code_text + patch = "\n".join([f"", instance["patch"], ""]) + return {**instance, "text": text_inputs, "patch": patch} + + +def create_dataset(input_instances): + columns = [ + "instance_id", + "text", + "repo", + "base_commit", + "problem_statement", + "hints_text", + "created_at", + "patch", + "test_patch", + "version", + "FAIL_TO_PASS", + "PASS_TO_PASS", + "environment_setup_commit", + ] + + data_table = {key: list() for key in columns} + for instance in input_instances.values(): + datum = extract_fields(instance) + for key in columns: + data_table[key].append(datum[key] if key in datum else "") + dataset = Dataset.from_dict(data_table) + + return dataset + + +def download_instances( + input_data, + path=Path("SWE-bench_testsample"), + verbose=False, +): + """Downloads code from github. + + Args: + - input_data: dictionary with unprocessed input instances. + - verbose: set ContextManager verbose to True + """ + input_instances = {x["instance_id"]: x for x in input_data} + input_instances_copy = deepcopy(input_instances) + input_instances_with_text = ingest_repos(input_instances_copy) + dataset = create_dataset(input_instances_with_text) + dataset.save_to_disk(path) + return dataset