Cleanup of RequestHandler, ResponseHandler, Scalar adapter

NOTES.md

@@ -1,17 +1,6 @@
 # NOTES
 
 
-## Observability
-
-> Priority: must have
-
-Requirements and solution - to be analyzed
-
- - How to track regular vs streamed responses? Streamed responses are unreadable / meaningless individually. Higher abstraction layer is needed to handle them - eg. "folder" with individual chunks of data. Completion ID allows to track incoming chunks under a single context.
- - Completion, if streamed, needs extra info on whether it has been completed or disrupted for any reason.
-
-
-
 ## Better control over deserialization
 
 > Priority: must have
@@ -28,6 +17,8 @@ Custom deserialization strategy is also needed for partial updates, maybe for st
 
 ## Validation
 
+> Priority: must have
+
 ### Returning errors - array vs typed object
 
 Array is simple and straightforward, but it's not type safe and does not provide a way to add custom methods to the error object.
@@ -271,7 +262,7 @@ Identify capabilities of the engine that could be parallelized, so we can speed
 ### Simple example
 
 ```cli
-iphp --messages "Jason is 35 years old" --respond-with UserDetails --response-format yaml
+instruct --messages "Jason is 35 years old" --respond-with UserDetails --response-format yaml
 ```
 It will search for UserFormat.php (PHP class) or UserFormat.json (JSONSchema) in current dir.
 We should be able to provide a path to class code / schema definitions directory.
@@ -280,7 +271,7 @@ Default response format is JSON, we can render it to YAML (or other supported fo
 ### Scalar example
 
 ```cli
-iphp --messages "Jason is 35 years old" --respond-with Scalar::bool('isAdult')
+instruct --messages "Jason is 35 years old" --respond-with Scalar::bool('isAdult')
 ```
 
 
@@ -376,3 +367,29 @@ Is it enough?
 ### Solution
 
 Validation can be also customized by implementing CanSelfValidate interface. It allows you to fully control how the data is validated. At the moment it skips built in Symfony Validator logic, so you have to deal with Symfony validation constraints manually.
+
+
+
+## Observability
+
+### Problem and ideas
+
+> Priority: must have
+
+Requirements and solution - to be analyzed
+
+- How to track regular vs streamed responses? Streamed responses are unreadable / meaningless individually. Higher abstraction layer is needed to handle them - eg. "folder" with individual chunks of data. Completion ID allows to track incoming chunks under a single context.
+- Completion, if streamed, needs extra info on whether it has been completed or disrupted for any reason.
+
+### Solution
+
+You can:
+- wiretap() to get stream of all internal events
+- connect to specific events via onEvent()
+
+This allows you plug in your preferred logging / monitoring system.
+
+- Performance - timestamps are available on events, which allows you to record performance of either full flow or individual steps.
+- Errors - can be done via onError()
+- Validation errors - can be done via onEvent()
+- Generated data models - can be done via onEvent()

config/autowire.php

@@ -1,6 +1,7 @@
 <?php
 namespace Cognesy\config;
 
+use Cognesy\Instructor\Configuration\Configuration;
 use Cognesy\Instructor\Contracts\CanCallFunction;
 use Cognesy\Instructor\Contracts\CanDeserializeResponse;
 use Cognesy\Instructor\Contracts\CanValidateResponse;
@@ -17,7 +18,6 @@
 use Cognesy\Instructor\Schema\SchemaMap;
 use Cognesy\Instructor\Schema\Utils\ReferenceQueue;
 use Cognesy\Instructor\Schema\Utils\SchemaBuilder;
-use Cognesy\Instructor\Utils\Configuration;
 use Cognesy\Instructor\Validators\Symfony\Validator;
 
 function autowire(Configuration $config) : Configuration

examples/OptionalFields/run.php

@@ -9,7 +9,7 @@
 
 use Cognesy\Instructor\Instructor;
 
-class Role
+class UserRole
 {
     public string $title = '';
 }
@@ -18,7 +18,7 @@ class UserDetail
 {
     public int $age;
     public string $name;
-    public ?Role $role = null;
+    public ?UserRole $role = null;
 }
 
 $user = (new Instructor)->respond(

src/Utils/ComponentConfig.php → src/Configuration/ComponentConfig.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Cognesy\Instructor\Utils;
+namespace Cognesy\Instructor\Configuration;
 
 use Cognesy\Instructor\Schema\Utils\ClassInfo;
 use Exception;

src/Utils/Configuration.php → src/Configuration/Configuration.php

@@ -1,5 +1,5 @@
 <?php
-namespace Cognesy\Instructor\Utils;
+namespace Cognesy\Instructor\Configuration;
 
 use Exception;
 use function Cognesy\config\autowire;

src/Contracts/CanDeserializeJson.php

@@ -4,5 +4,5 @@
 
 interface CanDeserializeJson
 {
-    public function fromJson(string $json) : self;
+    public function fromJson(string $json) : static;
 }

src/Contracts/CanSelfValidate.php

@@ -4,6 +4,5 @@
 
 interface CanSelfValidate
 {
-    public function validate(): bool;
-    public function errors(): string;
+    public function validate(): array;
 }

src/Contracts/CanValidateResponse.php

@@ -4,6 +4,10 @@
 
 interface CanValidateResponse
 {
-    public function validate(object $response) : bool;
-    public function errors() : string;
-}
+    /**
+     * Validate response object
+     * @param object $response
+     * @return array
+     */
+    public function validate(object $response) : array;
+}

src/Core/RequestHandler.php

@@ -3,16 +3,15 @@
 namespace Cognesy\Instructor\Core;
 
 use Cognesy\Instructor\Contracts\CanCallFunction;
-use Cognesy\Instructor\Contracts\CanTransformResponse;
 use Cognesy\Instructor\Events\RequestHandler\FunctionCallRequested;
-use Cognesy\Instructor\Events\RequestHandler\FunctionCallResultReady;
 use Cognesy\Instructor\Events\RequestHandler\ResponseGenerationFailed;
 use Cognesy\Instructor\Events\RequestHandler\ResponseModelBuilt;
 use Cognesy\Instructor\Events\RequestHandler\FunctionCallResponseReceived;
-use Cognesy\Instructor\Events\RequestHandler\FunctionCallResponseTransformed;
 use Cognesy\Instructor\Events\RequestHandler\FunctionCallResponseConvertedToObject;
 use Cognesy\Instructor\Events\RequestHandler\NewValidationRecoveryAttempt;
 use Cognesy\Instructor\Events\RequestHandler\ValidationRecoveryLimitReached;
+use Cognesy\Instructor\Exceptions\DeserializationException;
+use Cognesy\Instructor\Exceptions\ValidationException;
 use Exception;
 
 class RequestHandler
@@ -71,26 +70,30 @@ protected function tryRespond(
             );
             $this->eventDispatcher->dispatch(new FunctionCallResponseReceived($response));
             $json = $response->toolCalls[0]->functionArguments;
-            [$object, $errors] = $this->responseHandler->toResponse($responseModel, $json);
-            if (empty($errors)) {
-                $this->eventDispatcher->dispatch(new FunctionCallResponseConvertedToObject($object));
-                if ($object instanceof CanTransformResponse) {
-                    $result = $object->transform();
-                    $this->eventDispatcher->dispatch(new FunctionCallResponseTransformed($result));
-                } else {
-                    $result = $object;
+            try {
+                $result = $this->responseHandler->toResponse($responseModel, $json);
+                if ($result->isSuccess()) {
+                    $object = $result->value();
+                    $this->eventDispatcher->dispatch(new FunctionCallResponseConvertedToObject($object));
+                    return $object;
                 }
-                $this->eventDispatcher->dispatch(new FunctionCallResultReady($result));
-                return $result;
+                $errors = $result->errorValue();
+            } catch (ValidationException $e) {
+                $errors = [$e->getMessage()];
+            } catch (DeserializationException $e) {
+                $errors = [$e->getMessage()];
+            } catch (Exception $e) {
+                $this->eventDispatcher->dispatch(new ResponseGenerationFailed($request, $e->getMessage()));
+                throw $e;
             }
             $messages[] = ['role' => 'assistant', 'content' => $json];
-            $messages[] = ['role' => 'user', 'content' => $this->retryPrompt . '\n' . $errors];
+            $messages[] = ['role' => 'user', 'content' => $this->retryPrompt . ': ' . implode(", ", $errors)];
             $retries++;
             if ($retries <= $request->maxRetries) {
                 $this->eventDispatcher->dispatch(new NewValidationRecoveryAttempt($retries, $errors));
             }
         }
         $this->eventDispatcher->dispatch(new ValidationRecoveryLimitReached($retries, $errors));
-        throw new Exception("Failed to extract data due to validation constraints: " . $errors);
+        throw new Exception("Failed to extract data due to validation errors: " . implode(", ", $errors));
     }
 }

src/Core/ResponseHandler.php

@@ -5,16 +5,18 @@
 use Cognesy\Instructor\Contracts\CanDeserializeJson;
 use Cognesy\Instructor\Contracts\CanDeserializeResponse;
 use Cognesy\Instructor\Contracts\CanSelfValidate;
+use Cognesy\Instructor\Contracts\CanTransformResponse;
 use Cognesy\Instructor\Contracts\CanValidateResponse;
-use Cognesy\Instructor\Events\RequestHandler\ValidationRecoveryLimitReached;
 use Cognesy\Instructor\Events\ResponseHandler\CustomResponseDeserializationAttempt;
 use Cognesy\Instructor\Events\ResponseHandler\CustomResponseValidationAttempt;
 use Cognesy\Instructor\Events\ResponseHandler\ResponseDeserializationAttempt;
 use Cognesy\Instructor\Events\ResponseHandler\ResponseDeserializationFailed;
 use Cognesy\Instructor\Events\ResponseHandler\ResponseDeserialized;
+use Cognesy\Instructor\Events\ResponseHandler\ResponseTransformed;
 use Cognesy\Instructor\Events\ResponseHandler\ResponseValidated;
 use Cognesy\Instructor\Events\ResponseHandler\ResponseValidationAttempt;
 use Cognesy\Instructor\Events\ResponseHandler\ResponseValidationFailed;
+use Cognesy\Instructor\Utils\Result;
 
 class ResponseHandler
 {
@@ -36,55 +38,73 @@ public function __construct(
     /**
      * Deserialize JSON and validate response object
      */
-    public function toResponse(ResponseModel $responseModel, string $json) : array {
-        try {
-            $object = $this->deserialize($responseModel, $json);
-        } catch (\Exception $e) {
-            $this->eventDispatcher->dispatch(new ResponseDeserializationFailed($e->getMessage()));
-            return [null, $e->getMessage()];
+    public function toResponse(ResponseModel $responseModel, string $json) : Result {
+        // ...deserialize
+        $deserializationResult = $this->deserialize($responseModel, $json);
+        if ($deserializationResult->isFailure()) {
+            $this->eventDispatcher->dispatch(new ResponseDeserializationFailed($deserializationResult->errorMessage()));
+            return $deserializationResult;
         }
+        $object = $deserializationResult->value();
         $this->eventDispatcher->dispatch(new ResponseDeserialized($object));
-        if ($this->validate($object)) {
-            $this->eventDispatcher->dispatch(new ResponseValidated($object));
-            return [$object, null];
+
+        // ...validate
+        $validationResult = $this->validate($object);
+        if ($validationResult->isFailure()) {
+            $this->eventDispatcher->dispatch(new ResponseValidationFailed($validationResult->errorValue()));
+            return $validationResult;
         }
-        $this->eventDispatcher->dispatch(new ResponseValidationFailed($this->errors()));
-        return [null, $this->errors()];
+        $this->eventDispatcher->dispatch(new ResponseValidated($object));
+
+        // ...transform
+        $transformedObject = $this->transform($object);
+
+        return Result::success($transformedObject);
     }
 
     /**
      * Deserialize response JSON
      */
-    protected function deserialize(ResponseModel $responseModel, string $json) : mixed {
+    protected function deserialize(ResponseModel $responseModel, string $json) : Result {
         if ($responseModel->instance instanceof CanDeserializeJson) {
-            $this->eventDispatcher->dispatch(new CustomResponseDeserializationAttempt(
-                $responseModel->instance,
-                $json
-            ));
-            return $responseModel->instance->fromJson($json);
+            $this->eventDispatcher->dispatch(new CustomResponseDeserializationAttempt($responseModel->instance, $json));
+            return Result::try(fn() => $responseModel->instance->fromJson($json));
         }
         // else - use standard deserializer
         $this->eventDispatcher->dispatch(new ResponseDeserializationAttempt($responseModel, $json));
-        return $this->deserializer->deserialize($json, $responseModel->class);
+        return Result::try(fn() => $this->deserializer->deserialize($json, $responseModel->class));
     }
 
     /**
      * Validate deserialized response object
      */
-    protected function validate(object $response) : bool {
+    protected function validate(object $response) : Result {
         if ($response instanceof CanSelfValidate) {
             $this->eventDispatcher->dispatch(new CustomResponseValidationAttempt($response));
-            return $response->validate();
+            $errors = $response->validate();
+            return match(count($errors)) {
+                0 => Result::success($response),
+                default => Result::failure($errors)
+            };
         }
         // else - use standard validator
         $this->eventDispatcher->dispatch(new ResponseValidationAttempt($response));
-        return $this->validator->validate($response);
+        $errors = $this->validator->validate($response);
+        return match(count($errors)) {
+            0 => Result::success($response),
+            default => Result::failure($errors)
+        };
     }
 
     /**
-     * Get validation errors
+     * Transform response object
      */
-    public function errors() : string {
-        return $this->validator->errors();
+    protected function transform(object $object) : mixed {
+        if ($object instanceof CanTransformResponse) {
+            $result = $object->transform();
+            $this->eventDispatcher->dispatch(new ResponseTransformed($result));
+            return $result;
+        }
+        return $object;
     }
 }

src/Deserializers/Symfony/Deserializer.php

@@ -2,6 +2,7 @@
 namespace Cognesy\Instructor\Deserializers\Symfony;
 
 use Cognesy\Instructor\Contracts\CanDeserializeResponse;
+use Cognesy\Instructor\Exceptions\DeserializationException;
 use Cognesy\Instructor\Validators\Symfony\BackedEnumNormalizer;
 use Symfony\Component\PropertyInfo\Extractor\PhpDocExtractor;
 use Symfony\Component\PropertyInfo\Extractor\ReflectionExtractor;
@@ -36,6 +37,10 @@ public function deserialize(string $data, string $dataModelClass): object
             encoders: [new JsonEncoder()]
         );
 
-        return $serializer->deserialize($data, $dataModelClass, 'json');
+        try {
+            return $serializer->deserialize($data, $dataModelClass, 'json');
+        } catch (\Exception $e) {
+            throw new DeserializationException($e->getMessage(), $dataModelClass, $data);
+        }
     }
 }

src/Events/Instructor/InstructorReady.php

@@ -2,8 +2,8 @@
 
 namespace Cognesy\Instructor\Events\Instructor;
 
+use Cognesy\Instructor\Configuration\Configuration;
 use Cognesy\Instructor\Events\Event;
-use Cognesy\Instructor\Utils\Configuration;
 
 class InstructorReady extends Event
 {

src/Events/RequestHandler/NewValidationRecoveryAttempt.php

@@ -7,7 +7,7 @@ class NewValidationRecoveryAttempt extends Event
 {
     public function __construct(
         public int    $retry,
-        public string $errors,
+        public array $errors,
     )
     {
         parent::__construct();

src/Events/RequestHandler/ResponseGenerationFailed.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace Cognesy\Instructor\Events\RequestHandler;
+
+use Cognesy\Instructor\Core\Request;
+use Cognesy\Instructor\Events\Event;
+
+class ResponseGenerationFailed extends Event
+{
+    public function __construct(
+        public Request $request,
+        public string $error,
+    ) {
+        parent::__construct();
+    }
+
+    public function __toString(): string
+    {
+        return $this->error;
+    }
+}