Eval fixes

cognesy · Oct 9, 2024 · b082abf · b082abf
1 parent 25c7d2c
commit b082abf
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 14 deletions.
diff --git a/src/Extras/Evals/Contracts/CanExecuteExperiment.php b/src/Extras/Evals/Contracts/CanExecuteExperiment.php
@@ -6,7 +6,8 @@
 
 interface CanExecuteExperiment
 {
-    public function executeFor(EvalInput $input): self;
+    public static function fromEvalInput(EvalInput $input): self;
+    public function execute(): void;
     public function getLLMResponse(): LLMResponse;
     public function getAnswer(): mixed;
 }
diff --git a/src/Extras/Evals/Evaluator.php b/src/Extras/Evals/Evaluator.php
@@ -15,14 +15,14 @@ class Evaluator {
     private Display $display;
     private string|array|object $schema;
     /** @var class-string */
-    private CanExecuteExperiment $executor;
+    private string $executor;
     private string|array $messages;
     private Closure $evalFn;
 
     public function __construct(
         string|array $messages,
         string|array|object $schema,
-        CanExecuteExperiment $executorClass,
+        string $executorClass,
         Closure $evalFn,
     ) {
         $this->messages = $messages;
@@ -74,11 +74,13 @@ private function executeSingle(
 
             // execute and measure time
             $time = microtime(true);
-            /** @var CanExecuteExperiment $execution */
-            $execution = $this->executor->executeFor($evalInput);
-            $llmResponse = $execution->getLLMResponse();
+
+            /** @var CanExecuteExperiment $experiment */
+            $experiment = ($this->executor)::fromEvalInput($evalInput);
+            $experiment->execute();
+            $llmResponse = $experiment->getLLMResponse();
             $evalInput->withResponse($llmResponse);
-            $answer = $execution->getAnswer();
+
             $timeElapsed = microtime(true) - $time;
             $isCorrect = ($this->evalFn)($evalInput);
 

diff --git a/src/Extras/Evals/Inference/RunInference.php b/src/Extras/Evals/Inference/RunInference.php
@@ -38,7 +38,7 @@ public function __construct(
         $this->isStreamed = $isStreamed;
     }
 
-    public function executeFor(EvalInput $input) : self {
+    public static function fromEvalInput(EvalInput $input) : self {
         $instance = new RunInference(
             query: $input->messages,
             schema: $input->evalSchema(),
@@ -47,11 +47,14 @@ public function executeFor(EvalInput $input) : self {
             isStreamed: $input->isStreamed,
             maxTokens: $input->maxTokens,
         );
-        $instance->response = $instance->makeLLMResponse();
-        $instance->answer = $instance->response->content();
         return $instance;
     }
 
+    public function execute() : void {
+        $this->response = $this->makeLLMResponse();
+        $this->answer = $this->response->content();
+    }
+
     public function getAnswer() : string {
         return $this->answer;
     }

diff --git a/src/Extras/Evals/Instructor/RunInstructor.php b/src/Extras/Evals/Instructor/RunInstructor.php
@@ -34,7 +34,7 @@ public function __construct(
         readonly public string              $model = '',
     ) {}
 
-    public static function executeFor(EvalInput $input) : self {
+    public static function fromEvalInput(EvalInput $input) : self {
         $instance = new RunInstructor(
             messages: $input->messages,
             responseModel: $input->schema,
@@ -46,12 +46,15 @@ public static function executeFor(EvalInput $input) : self {
             examples: [],
             model: '',
         );
-        $instance->instructorResponse = $instance->makeInstructorResponse();
-        $instance->llmResponse = $instance->instructorResponse->response();
-        $instance->answer = $instance->llmResponse->value();
         return $instance;
     }
 
+    public function execute() : void {
+        $this->instructorResponse = $this->makeInstructorResponse();
+        $this->llmResponse = $this->instructorResponse->response();
+        $this->answer = $this->llmResponse->value();
+    }
+
     public function getAnswer() : mixed {
         return $this->answer;
     }