diff --git a/evals/LLMModes/run.php b/evals/LLMModes/run.php index 04f76165..ca36667d 100644 --- a/evals/LLMModes/run.php +++ b/evals/LLMModes/run.php @@ -44,11 +44,7 @@ ); $experiment = new Experiment( - cases: InferenceCases::except( - connections: ['ollama'], - modes: [], - stream: [], - ), + cases: InferenceCases::all(), executor: new RunInference($data), processors: [ new CompanyEval( diff --git a/examples/A05_Extras/Embeddings/run.php b/examples/A05_Extras/Embeddings/run.php index cefeb44d..e184755e 100644 --- a/examples/A05_Extras/Embeddings/run.php +++ b/examples/A05_Extras/Embeddings/run.php @@ -1,57 +1,65 @@ ---- -title: 'Embeddings' -docname: 'embeddings' ---- - -## Overview - -`Embeddings` class offers access to embeddings APIs and convenient methods -to find top K vectors or documents most similar to provided query. - -`Embeddings` class supports following embeddings providers: - - Azure - - Cohere - - Gemini - - Jina - - Mistral - - OpenAI - -Embeddings providers access details can be found and modified via -`/config/embed.php`. - - -## Example - -```php -add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); - -use Cognesy\Instructor\Extras\Embeddings\Embeddings; - -$documents = [ - 'Computer vision models are used to analyze images and videos.', - 'The bakers at the Nashville Bakery baked 200 loaves of bread on Monday morning.', - 'The new movie starring Tom Hanks is now playing in theaters.', - 'Famous soccer player Lionel Messi has arrived in town.', - 'News about the latest iPhone model has been leaked.', - 'New car model by Tesla is now available for pre-order.', - 'Philip K. Dick is an author of many sci-fi novels.', -]; - -$query = "technology news"; - -$connections = ['azure', 'cohere1', 'gemini', 'jina', 'mistral', 'ollama', 'openai']; - -foreach($connections as $connection) { - $bestMatches = (new Embeddings)->withConnection($connection)->findSimilar( - query: $query, - documents: $documents, - topK: 3 - ); - - echo "\n[$connection]\n"; - dump($bestMatches); -} -?> -``` +--- +title: 'Embeddings' +docname: 'embeddings' +--- + +## Overview + +`Embeddings` class offers access to embeddings APIs and convenient methods +to find top K vectors or documents most similar to provided query. + +`Embeddings` class supports following embeddings providers: + - Azure + - Cohere + - Gemini + - Jina + - Mistral + - OpenAI + +Embeddings providers access details can be found and modified via +`/config/embed.php`. + + +## Example + +```php +add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); + +use Cognesy\Instructor\Extras\Embeddings\Embeddings; + +$documents = [ + 'Computer vision models are used to analyze images and videos.', + 'The bakers at the Nashville Bakery baked 200 loaves of bread on Monday morning.', + 'The new movie starring Tom Hanks is now playing in theaters.', + 'Famous soccer player Lionel Messi has arrived in town.', + 'News about the latest iPhone model has been leaked.', + 'New car model by Tesla is now available for pre-order.', + 'Philip K. Dick is an author of many sci-fi novels.', +]; + +$query = "technology news"; + +$connections = [ + 'azure', + 'cohere1', + 'gemini', + 'jina', + 'mistral', + //'ollama', + 'openai' +]; + +foreach($connections as $connection) { + $bestMatches = (new Embeddings)->withConnection($connection)->findSimilar( + query: $query, + documents: $documents, + topK: 3 + ); + + echo "\n[$connection]\n"; + dump($bestMatches); +} +?> +``` diff --git a/src/Extras/Embeddings/Drivers/AzureOpenAIDriver.php b/src/Extras/Embeddings/Drivers/AzureOpenAIDriver.php index 2f0f4384..14af9491 100644 --- a/src/Extras/Embeddings/Drivers/AzureOpenAIDriver.php +++ b/src/Extras/Embeddings/Drivers/AzureOpenAIDriver.php @@ -1,81 +1,84 @@ -httpClient = $httpClient ?? HttpClient::make(); - } - - public function vectorize(array $input, array $options = []): EmbeddingsResponse { - $response = $this->httpClient->handle( - $this->getEndpointUrl(), - $this->getRequestHeaders(), - $this->getRequestBody($input, $options), - ); - return $this->toResponse(json_decode($response->getContents(), true)); - } - - // INTERNAL ///////////////////////////////////////////////// - - protected function getEndpointUrl(): string { - return str_replace( - search: array_map(fn($key) => "{".$key."}", array_keys($this->config->metadata)), - replace: array_values($this->config->metadata), - subject: "{$this->config->apiUrl}{$this->config->endpoint}" - ) . $this->getUrlParams(); - } - - protected function getUrlParams(): string { - $params = array_filter([ - 'api-version' => $this->config->metadata['apiVersion'] ?? '', - ]); - if (!empty($params)) { - return '?' . http_build_query($params); - } - return ''; - } - - protected function getRequestHeaders(): array { - return [ - 'Api-Key' => $this->config->apiKey, - 'Content-Type' => 'application/json', - ]; - } - - protected function getRequestBody(array $input, array $options) : array { - return array_filter(array_merge([ - 'input' => $input, - 'model' => $this->config->model, - 'encoding_format' => 'float', - ], $options)); - } - - protected function toResponse(array $response) : EmbeddingsResponse { - return new EmbeddingsResponse( - vectors: array_map( - callback: fn($item) => new Vector(values: $item['embedding'], id: $item['index']), - array: $response['data'] - ), - usage: $this->makeUsage($response), - ); - } - - protected function makeUsage(array $response): Usage { - return new Usage( - inputTokens: $response['usage']['prompt_tokens'] ?? 0, - outputTokens: ($response['usage']['total_tokens'] ?? 0) - ($response['usage']['prompt_tokens'] ?? 0), - ); - } +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + public function vectorize(array $input, array $options = []): EmbeddingsResponse { + $response = $this->httpClient->handle( + $this->getEndpointUrl(), + $this->getRequestHeaders(), + $this->getRequestBody($input, $options), + ); + return $this->toResponse(json_decode($response->getContents(), true)); + } + + // INTERNAL ///////////////////////////////////////////////// + + protected function getEndpointUrl(): string { + return str_replace( + search: array_map(fn($key) => "{".$key."}", array_keys($this->config->metadata)), + replace: array_values($this->config->metadata), + subject: "{$this->config->apiUrl}{$this->config->endpoint}" + ) . $this->getUrlParams(); + } + + protected function getUrlParams(): string { + $params = array_filter([ + 'api-version' => $this->config->metadata['apiVersion'] ?? '', + ]); + if (!empty($params)) { + return '?' . http_build_query($params); + } + return ''; + } + + protected function getRequestHeaders(): array { + return [ + 'Api-Key' => $this->config->apiKey, + 'Content-Type' => 'application/json', + ]; + } + + protected function getRequestBody(array $input, array $options) : array { + return array_filter(array_merge([ + 'input' => $input, + 'model' => $this->config->model, + 'encoding_format' => 'float', + ], $options)); + } + + protected function toResponse(array $response) : EmbeddingsResponse { + return new EmbeddingsResponse( + vectors: array_map( + callback: fn($item) => new Vector(values: $item['embedding'], id: $item['index']), + array: $response['data'] + ), + usage: $this->makeUsage($response), + ); + } + + protected function makeUsage(array $response): Usage { + return new Usage( + inputTokens: $response['usage']['prompt_tokens'] ?? 0, + outputTokens: ($response['usage']['total_tokens'] ?? 0) - ($response['usage']['prompt_tokens'] ?? 0), + ); + } } \ No newline at end of file diff --git a/src/Extras/Embeddings/Drivers/CohereDriver.php b/src/Extras/Embeddings/Drivers/CohereDriver.php index ef02a7a3..4a852d43 100644 --- a/src/Extras/Embeddings/Drivers/CohereDriver.php +++ b/src/Extras/Embeddings/Drivers/CohereDriver.php @@ -1,71 +1,74 @@ -httpClient = $httpClient ?? HttpClient::make(); - } - - public function vectorize(array $input, array $options = []): EmbeddingsResponse { - $response = $this->httpClient->handle( - $this->getEndpointUrl(), - $this->getRequestHeaders(), - $this->getRequestBody($input, $options), - ); - return $this->toResponse(json_decode($response->getContents(), true)); - } - - // INTERNAL ///////////////////////////////////////////////// - - protected function getEndpointUrl(): string { - return "{$this->config->apiUrl}{$this->config->endpoint}"; - } - - protected function getRequestHeaders(): array { - return [ - 'Authorization' => "Bearer {$this->config->apiKey}", - 'Content-Type' => 'application/json', - ]; - } - - protected function getRequestBody(array $input, array $options) : array { - $options['input_type'] = $options['input_type'] ?? 'search_document'; - return array_filter(array_merge([ - 'texts' => $input, - 'model' => $this->config->model, - 'embedding_types' => ['float'], - 'truncate' => 'END', - ], $options)); - } - - protected function toResponse(array $response) : EmbeddingsResponse { - $vectors = []; - foreach ($response['embeddings']['float'] as $key => $item) { - $vectors[] = new Vector(values: $item, id: $key); - } - return new EmbeddingsResponse( - vectors: $vectors, - usage: $this->makeUsage($response), - ); - } - - private function makeUsage(array $response) : Usage { - return new Usage( - inputTokens: $response['meta']['billed_units']['input_tokens'] ?? 0, - outputTokens: $response['meta']['billed_units']['output_tokens'] ?? 0, - ); - } -} +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + public function vectorize(array $input, array $options = []): EmbeddingsResponse { + $response = $this->httpClient->handle( + $this->getEndpointUrl(), + $this->getRequestHeaders(), + $this->getRequestBody($input, $options), + ); + return $this->toResponse(json_decode($response->getContents(), true)); + } + + // INTERNAL ///////////////////////////////////////////////// + + protected function getEndpointUrl(): string { + return "{$this->config->apiUrl}{$this->config->endpoint}"; + } + + protected function getRequestHeaders(): array { + return [ + 'Authorization' => "Bearer {$this->config->apiKey}", + 'Content-Type' => 'application/json', + ]; + } + + protected function getRequestBody(array $input, array $options) : array { + $options['input_type'] = $options['input_type'] ?? 'search_document'; + return array_filter(array_merge([ + 'texts' => $input, + 'model' => $this->config->model, + 'embedding_types' => ['float'], + 'truncate' => 'END', + ], $options)); + } + + protected function toResponse(array $response) : EmbeddingsResponse { + $vectors = []; + foreach ($response['embeddings']['float'] as $key => $item) { + $vectors[] = new Vector(values: $item, id: $key); + } + return new EmbeddingsResponse( + vectors: $vectors, + usage: $this->makeUsage($response), + ); + } + + private function makeUsage(array $response) : Usage { + return new Usage( + inputTokens: $response['meta']['billed_units']['input_tokens'] ?? 0, + outputTokens: $response['meta']['billed_units']['output_tokens'] ?? 0, + ); + } +} diff --git a/src/Extras/Embeddings/Drivers/GeminiDriver.php b/src/Extras/Embeddings/Drivers/GeminiDriver.php index 3ef574aa..f45e2f88 100644 --- a/src/Extras/Embeddings/Drivers/GeminiDriver.php +++ b/src/Extras/Embeddings/Drivers/GeminiDriver.php @@ -1,82 +1,85 @@ -httpClient = $httpClient ?? HttpClient::make(); - } - - public function vectorize(array $input, array $options = []): EmbeddingsResponse { - $this->inputCharacters = $this->countCharacters($input); - $response = $this->httpClient->handle( - $this->getEndpointUrl(), - $this->getRequestHeaders(), - $this->getRequestBody($input, $options), - ); - return $this->toResponse(json_decode($response->getContents(), true)); - } - - // INTERNAL ///////////////////////////////////////////////// - - protected function getEndpointUrl(): string { - return str_replace( - "{model}", - $this->config->model, - "{$this->config->apiUrl}{$this->config->endpoint}?key={$this->config->apiKey}" - ); - } - - protected function getRequestHeaders(): array { - return [ - 'Content-Type' => 'application/json', - ]; - } - - protected function getRequestBody(array $input, array $options) : array { - return array_merge([ - 'requests' => array_map( - fn($item) => [ - 'model' => $this->config->model, - 'content' => ['parts' => [['text' => $item]]] - ], - $input - ), - ], $options); - } - - protected function toResponse(array $response) : EmbeddingsResponse { - $vectors = []; - foreach ($response['embeddings'] as $key => $item) { - $vectors[] = new Vector(values: $item['values'], id: $key); - } - return new EmbeddingsResponse( - vectors: $vectors, - usage: $this->makeUsage($response), - ); - } - - private function countCharacters(array $input) : int { - return array_sum(array_map(fn($item) => strlen($item), $input)); - } - - private function makeUsage(array $response) : Usage { - return new Usage( - inputTokens: $this->inputCharacters, - outputTokens: 0, - ); - } -} +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + public function vectorize(array $input, array $options = []): EmbeddingsResponse { + $this->inputCharacters = $this->countCharacters($input); + $response = $this->httpClient->handle( + $this->getEndpointUrl(), + $this->getRequestHeaders(), + $this->getRequestBody($input, $options), + ); + return $this->toResponse(json_decode($response->getContents(), true)); + } + + // INTERNAL ///////////////////////////////////////////////// + + protected function getEndpointUrl(): string { + return str_replace( + "{model}", + $this->config->model, + "{$this->config->apiUrl}{$this->config->endpoint}?key={$this->config->apiKey}" + ); + } + + protected function getRequestHeaders(): array { + return [ + 'Content-Type' => 'application/json', + ]; + } + + protected function getRequestBody(array $input, array $options) : array { + return array_merge([ + 'requests' => array_map( + fn($item) => [ + 'model' => $this->config->model, + 'content' => ['parts' => [['text' => $item]]] + ], + $input + ), + ], $options); + } + + protected function toResponse(array $response) : EmbeddingsResponse { + $vectors = []; + foreach ($response['embeddings'] as $key => $item) { + $vectors[] = new Vector(values: $item['values'], id: $key); + } + return new EmbeddingsResponse( + vectors: $vectors, + usage: $this->makeUsage($response), + ); + } + + private function countCharacters(array $input) : int { + return array_sum(array_map(fn($item) => strlen($item), $input)); + } + + private function makeUsage(array $response) : Usage { + return new Usage( + inputTokens: $this->inputCharacters, + outputTokens: 0, + ); + } +} diff --git a/src/Extras/Embeddings/Drivers/JinaDriver.php b/src/Extras/Embeddings/Drivers/JinaDriver.php index 95e81e8e..3de41f5b 100644 --- a/src/Extras/Embeddings/Drivers/JinaDriver.php +++ b/src/Extras/Embeddings/Drivers/JinaDriver.php @@ -1,74 +1,77 @@ -httpClient = $httpClient ?? HttpClient::make(); - } - - public function vectorize(array $input, array $options = []): EmbeddingsResponse { - $response = $this->httpClient->handle( - $this->getEndpointUrl(), - $this->getRequestHeaders(), - $this->getRequestBody($input, $options), - ); - return $this->toResponse(json_decode($response->getContents(), true)); - } - - // INTERNAL ///////////////////////////////////////////////// - - protected function getEndpointUrl(): string { - return "{$this->config->apiUrl}{$this->config->endpoint}"; - } - - protected function getRequestHeaders(): array { - return [ - 'Content-Type' => 'application/json', - 'Authorization' => "Bearer {$this->config->apiKey}", - ]; - } - - protected function getRequestBody(array $input, array $options) : array { - $body = array_filter(array_merge([ - 'model' => $this->config->model, - 'normalized' => true, - 'embedding_type' => 'float', - 'input' => $input, - ], $options)); - if ($this->config->model === 'jina-colbert-v2') { - $body['input_type'] = $options['input_type'] ?? 'document'; - $body['dimensions'] = $options['dimensions'] ?? 128; - } - return $body; - } - - protected function toResponse(array $response) : EmbeddingsResponse { - return new EmbeddingsResponse( - vectors: array_map( - fn($item) => new Vector(values: $item['embedding'], id: $item['index']), - $response['data'] - ), - usage: $this->makeUsage($response), - ); - } - - private function makeUsage(array $response) : Usage { - return new Usage( - inputTokens: $response['usage']['prompt_tokens'] ?? 0, - outputTokens: ($response['usage']['total_tokens'] ?? 0) - ($response['usage']['prompt_tokens'] ?? 0), - ); - } -} +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + public function vectorize(array $input, array $options = []): EmbeddingsResponse { + $response = $this->httpClient->handle( + $this->getEndpointUrl(), + $this->getRequestHeaders(), + $this->getRequestBody($input, $options), + ); + return $this->toResponse(json_decode($response->getContents(), true)); + } + + // INTERNAL ///////////////////////////////////////////////// + + protected function getEndpointUrl(): string { + return "{$this->config->apiUrl}{$this->config->endpoint}"; + } + + protected function getRequestHeaders(): array { + return [ + 'Content-Type' => 'application/json', + 'Authorization' => "Bearer {$this->config->apiKey}", + ]; + } + + protected function getRequestBody(array $input, array $options) : array { + $body = array_filter(array_merge([ + 'model' => $this->config->model, + 'normalized' => true, + 'embedding_type' => 'float', + 'input' => $input, + ], $options)); + if ($this->config->model === 'jina-colbert-v2') { + $body['input_type'] = $options['input_type'] ?? 'document'; + $body['dimensions'] = $options['dimensions'] ?? 128; + } + return $body; + } + + protected function toResponse(array $response) : EmbeddingsResponse { + return new EmbeddingsResponse( + vectors: array_map( + fn($item) => new Vector(values: $item['embedding'], id: $item['index']), + $response['data'] + ), + usage: $this->makeUsage($response), + ); + } + + private function makeUsage(array $response) : Usage { + return new Usage( + inputTokens: $response['usage']['prompt_tokens'] ?? 0, + outputTokens: ($response['usage']['total_tokens'] ?? 0) - ($response['usage']['prompt_tokens'] ?? 0), + ); + } +} diff --git a/src/Extras/Embeddings/Drivers/OpenAIDriver.php b/src/Extras/Embeddings/Drivers/OpenAIDriver.php index d2630e08..e62a179c 100644 --- a/src/Extras/Embeddings/Drivers/OpenAIDriver.php +++ b/src/Extras/Embeddings/Drivers/OpenAIDriver.php @@ -1,68 +1,71 @@ -httpClient = $httpClient ?? HttpClient::make(); - } - - public function vectorize(array $input, array $options = []): EmbeddingsResponse { - $response = $this->httpClient->handle( - $this->getEndpointUrl(), - $this->getRequestHeaders(), - $this->getRequestBody($input, $options), - ); - return $this->toResponse(json_decode($response->getContents(), true)); - } - - // INTERNAL ///////////////////////////////////////////////// - - protected function getEndpointUrl(): string { - return "{$this->config->apiUrl}{$this->config->endpoint}"; - } - - protected function getRequestHeaders(): array { - return [ - 'Authorization' => "Bearer {$this->config->apiKey}", - 'Content-Type' => 'application/json', - ]; - } - - protected function getRequestBody(array $input, array $options) : array { - return array_filter(array_merge([ - 'input' => $input, - 'model' => $this->config->model, - 'encoding_format' => 'float', - ], $options)); - } - - protected function toResponse(array $response) : EmbeddingsResponse { - return new EmbeddingsResponse( - vectors: array_map( - callback: fn($item) => new Vector(values: $item['embedding'], id: $item['index']), - array: $response['data'] - ), - usage: $this->toUsage($response), - ); - } - - private function toUsage(array $response) : Usage { - return new Usage( - inputTokens: $response['usage']['prompt_tokens'] ?? 0, - outputTokens: ($response['usage']['total_tokens'] ?? 0) - ($response['usage']['prompt_tokens'] ?? 0), - ); - } +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + public function vectorize(array $input, array $options = []): EmbeddingsResponse { + $response = $this->httpClient->handle( + $this->getEndpointUrl(), + $this->getRequestHeaders(), + $this->getRequestBody($input, $options), + ); + return $this->toResponse(json_decode($response->getContents(), true)); + } + + // INTERNAL ///////////////////////////////////////////////// + + protected function getEndpointUrl(): string { + return "{$this->config->apiUrl}{$this->config->endpoint}"; + } + + protected function getRequestHeaders(): array { + return [ + 'Authorization' => "Bearer {$this->config->apiKey}", + 'Content-Type' => 'application/json', + ]; + } + + protected function getRequestBody(array $input, array $options) : array { + return array_filter(array_merge([ + 'input' => $input, + 'model' => $this->config->model, + 'encoding_format' => 'float', + ], $options)); + } + + protected function toResponse(array $response) : EmbeddingsResponse { + return new EmbeddingsResponse( + vectors: array_map( + callback: fn($item) => new Vector(values: $item['embedding'], id: $item['index']), + array: $response['data'] + ), + usage: $this->toUsage($response), + ); + } + + private function toUsage(array $response) : Usage { + return new Usage( + inputTokens: $response['usage']['prompt_tokens'] ?? 0, + outputTokens: ($response['usage']['total_tokens'] ?? 0) - ($response['usage']['prompt_tokens'] ?? 0), + ); + } } \ No newline at end of file diff --git a/src/Extras/Embeddings/Embeddings.php b/src/Extras/Embeddings/Embeddings.php index 8288f751..76f43ec4 100644 --- a/src/Extras/Embeddings/Embeddings.php +++ b/src/Extras/Embeddings/Embeddings.php @@ -1,97 +1,97 @@ -events = $events ?? new EventDispatcher(); - $this->config = $config ?? EmbeddingsConfig::load($connection - ?: Settings::get('embed', "defaultConnection") - ); - $this->httpClient = $httpClient ?? HttpClient::make($this->config->httpClient); - $this->driver = $driver ?? $this->getDriver($this->config, $this->httpClient); - } - - // PUBLIC /////////////////////////////////////////////////// - - public function withConnection(string $connection) : self { - $this->config = EmbeddingsConfig::load($connection); - $this->driver = $this->getDriver($this->config, $this->httpClient); - return $this; - } - - public function withConfig(EmbeddingsConfig $config) : self { - $this->config = $config; - $this->driver = $this->getDriver($this->config, $this->httpClient); - return $this; - } - - public function withModel(string $model) : self { - $this->config->model = $model; - return $this; - } - - public function withHttpClient(CanHandleHttp $httpClient) : self { - $this->httpClient = $httpClient; - $this->driver = $this->getDriver($this->config, $this->httpClient); - return $this; - } - - public function withDriver(CanVectorize $driver) : self { - $this->driver = $driver; - return $this; - } - - public function create(string|array $input, array $options = []) : EmbeddingsResponse { - if (is_string($input)) { - $input = [$input]; - } - if (count($input) > $this->config->maxInputs) { - throw new InvalidArgumentException("Number of inputs exceeds the limit of {$this->config->maxInputs}"); - } - return $this->driver->vectorize($input, $options); - } - - // INTERNAL ///////////////////////////////////////////////// - - protected function getDriver(EmbeddingsConfig $config, CanHandleHttp $httpClient) : CanVectorize { - return match ($config->providerType) { - LLMProviderType::Azure => new AzureOpenAIDriver($config, $httpClient), - LLMProviderType::CohereV1 => new CohereDriver($config, $httpClient), - LLMProviderType::Gemini => new GeminiDriver($config, $httpClient), - LLMProviderType::Mistral => new OpenAIDriver($config, $httpClient), - LLMProviderType::OpenAI => new OpenAIDriver($config, $httpClient), - LLMProviderType::Ollama => new OpenAIDriver($config, $httpClient), - LLMProviderType::Jina => new JinaDriver($config, $httpClient), - default => throw new InvalidArgumentException("Unknown client: {$config->providerType->value}"), - }; - } -} +events = $events ?? new EventDispatcher(); + $this->config = $config ?? EmbeddingsConfig::load($connection + ?: Settings::get('embed', "defaultConnection") + ); + $this->httpClient = $httpClient ?? HttpClient::make(client: $this->config->httpClient, events: $this->events); + $this->driver = $driver ?? $this->getDriver($this->config, $this->httpClient); + } + + // PUBLIC /////////////////////////////////////////////////// + + public function withConnection(string $connection) : self { + $this->config = EmbeddingsConfig::load($connection); + $this->driver = $this->getDriver($this->config, $this->httpClient); + return $this; + } + + public function withConfig(EmbeddingsConfig $config) : self { + $this->config = $config; + $this->driver = $this->getDriver($this->config, $this->httpClient); + return $this; + } + + public function withModel(string $model) : self { + $this->config->model = $model; + return $this; + } + + public function withHttpClient(CanHandleHttp $httpClient) : self { + $this->httpClient = $httpClient; + $this->driver = $this->getDriver($this->config, $this->httpClient); + return $this; + } + + public function withDriver(CanVectorize $driver) : self { + $this->driver = $driver; + return $this; + } + + public function create(string|array $input, array $options = []) : EmbeddingsResponse { + if (is_string($input)) { + $input = [$input]; + } + if (count($input) > $this->config->maxInputs) { + throw new InvalidArgumentException("Number of inputs exceeds the limit of {$this->config->maxInputs}"); + } + return $this->driver->vectorize($input, $options); + } + + // INTERNAL ///////////////////////////////////////////////// + + protected function getDriver(EmbeddingsConfig $config, CanHandleHttp $httpClient) : CanVectorize { + return match ($config->providerType) { + LLMProviderType::Azure => new AzureOpenAIDriver($config, $httpClient, $this->events), + LLMProviderType::CohereV1 => new CohereDriver($config, $httpClient, $this->events), + LLMProviderType::Gemini => new GeminiDriver($config, $httpClient, $this->events), + LLMProviderType::Mistral => new OpenAIDriver($config, $httpClient, $this->events), + LLMProviderType::OpenAI => new OpenAIDriver($config, $httpClient, $this->events), + LLMProviderType::Ollama => new OpenAIDriver($config, $httpClient, $this->events), + LLMProviderType::Jina => new JinaDriver($config, $httpClient, $this->events), + default => throw new InvalidArgumentException("Unknown client: {$config->providerType->value}"), + }; + } +} diff --git a/src/Extras/Evals/Executors/Data/InferenceCaseParams.php b/src/Extras/Evals/Executors/Data/InferenceCaseParams.php index 8d4bd3f9..338e7dd8 100644 --- a/src/Extras/Evals/Executors/Data/InferenceCaseParams.php +++ b/src/Extras/Evals/Executors/Data/InferenceCaseParams.php @@ -1,32 +1,32 @@ -mode = $values['mode'] ?? Mode::Text; - $instance->connection = $values['connection'] ?? 'openai'; - $instance->isStreaming = $values['isStreaming'] ?? false; - return $instance; - } - - public function toArray() : array { - return [ - 'connection' => $this->connection, - 'isStreaming' => $this->isStreaming, - 'mode' => $this->mode, - ]; - } - - public function __toString() : string { - return $this->connection.'::'.$this->mode->value.'::'.($this->isStreaming ? 'streamed' : 'sync'); - } -} +mode = $values['mode'] ?? Mode::Text; + $instance->connection = $values['connection'] ?? 'openai'; + $instance->isStreamed = $values['isStreamed'] ?? false; + return $instance; + } + + public function toArray() : array { + return [ + 'connection' => $this->connection, + 'isStreamed' => $this->isStreamed, + 'mode' => $this->mode, + ]; + } + + public function __toString() : string { + return $this->connection.'::'.$this->mode->value.'::'.($this->isStreamed ? 'streamed' : 'sync'); + } +} diff --git a/src/Extras/Evals/Executors/Data/InferenceCases.php b/src/Extras/Evals/Executors/Data/InferenceCases.php index 1a558f6d..efcadaa4 100644 --- a/src/Extras/Evals/Executors/Data/InferenceCases.php +++ b/src/Extras/Evals/Executors/Data/InferenceCases.php @@ -1,130 +1,130 @@ -connections = $connections; - $this->modes = $modes; - $this->stream = $stream; - } - - public static function all() : Generator { - return (new self)->initiateWithAll()->make(); - } - - public static function except( - array $connections = [], - array $modes = [], - array $stream = [], - ) : Generator { - $instance = (new self)->initiateWithAll(); - $instance->connections = match(true) { - [] === $connections => $instance->connections, - default => array_diff($instance->connections, $connections), - }; - $instance->modes = match(true) { - [] === $modes => $instance->modes, - default => array_filter($instance->modes, fn($mode) => !$mode->isIn($modes)), - }; - $instance->stream = match(true) { - [] === $stream => $instance->stream, - default => array_diff($instance->stream, $stream), - }; - return $instance->make(); - } - - public static function only( - array $connections = [], - array $modes = [], - array $stream = [], - ) : Generator { - $instance = (new self)->initiateWithAll(); - $instance->connections = match(true) { - [] === $connections => $instance->connections, - default => array_intersect($instance->connections, $connections), - }; - $instance->modes = match(true) { - [] === $modes => $instance->modes, - default => array_filter($instance->modes, fn($mode) => $mode->isIn($modes)), - }; - $instance->stream = match(true) { - [] === $stream => $instance->stream, - default => array_intersect($instance->stream, $stream), - }; - return $instance->make(); - } - - // INTERNAL ////////////////////////////////////////////////// - - private function initiateWithAll() : self { - return new self( - connections: $this->connections(), - modes: $this->modes(), - stream: $this->streamingModes(), - ); - } - - /** - * @return Generator - */ - private function make() : Generator { - return Combination::generator( - mapping: InferenceCaseParams::class, - sources: [ - 'isStreaming' => $this->stream ?: $this->streamingModes(), - 'mode' => $this->modes ?: $this->modes(), - 'connection' => $this->connections ?: $this->connections(), - ], - ); - } - - private function connections() : array { - $connections = Settings::get('llm', 'connections', []); - return array_keys($connections); -// return [ -// 'azure', -// 'cohere1', -// 'cohere2', -// 'fireworks', -// 'gemini', -// 'groq', -// 'mistral', -// 'ollama', -// 'openai', -// 'openrouter', -// 'together', -// ]; - } - - private function streamingModes() : array { - return [ - false, - true, - ]; - } - - private function modes() : array { - return [ - Mode::Text, - Mode::MdJson, - Mode::Json, - Mode::JsonSchema, - Mode::Tools, - ]; - } +connections = $connections; + $this->modes = $modes; + $this->stream = $stream; + } + + public static function all() : Generator { + return (new self)->initiateWithAll()->make(); + } + + public static function except( + array $connections = [], + array $modes = [], + array $stream = [], + ) : Generator { + $instance = (new self)->initiateWithAll(); + $instance->connections = match(true) { + [] === $connections => $instance->connections, + default => array_diff($instance->connections, $connections), + }; + $instance->modes = match(true) { + [] === $modes => $instance->modes, + default => array_filter($instance->modes, fn($mode) => !$mode->isIn($modes)), + }; + $instance->stream = match(true) { + [] === $stream => $instance->stream, + default => array_diff($instance->stream, $stream), + }; + return $instance->make(); + } + + public static function only( + array $connections = [], + array $modes = [], + array $stream = [], + ) : Generator { + $instance = (new self)->initiateWithAll(); + $instance->connections = match(true) { + [] === $connections => $instance->connections, + default => array_intersect($instance->connections, $connections), + }; + $instance->modes = match(true) { + [] === $modes => $instance->modes, + default => array_filter($instance->modes, fn($mode) => $mode->isIn($modes)), + }; + $instance->stream = match(true) { + [] === $stream => $instance->stream, + default => array_intersect($instance->stream, $stream), + }; + return $instance->make(); + } + + // INTERNAL ////////////////////////////////////////////////// + + private function initiateWithAll() : self { + return new self( + connections: $this->connections(), + modes: $this->modes(), + stream: $this->streamingModes(), + ); + } + + /** + * @return Generator + */ + private function make() : Generator { + return Combination::generator( + mapping: InferenceCaseParams::class, + sources: [ + 'isStreamed' => $this->stream ?: $this->streamingModes(), + 'mode' => $this->modes ?: $this->modes(), + 'connection' => $this->connections ?: $this->connections(), + ], + ); + } + + private function connections() : array { + $connections = Settings::get('llm', 'connections', []); + return array_keys($connections); +// return [ +// 'azure', +// 'cohere1', +// 'cohere2', +// 'fireworks', +// 'gemini', +// 'groq', +// 'mistral', +// 'ollama', +// 'openai', +// 'openrouter', +// 'together', +// ]; + } + + private function streamingModes() : array { + return [ + false, + true, + ]; + } + + private function modes() : array { + return [ + Mode::Text, + Mode::MdJson, + Mode::Json, + Mode::JsonSchema, + Mode::Tools, + ]; + } } \ No newline at end of file diff --git a/src/Extras/Evals/Executors/RunInference.php b/src/Extras/Evals/Executors/RunInference.php index 002251b0..158f77a2 100644 --- a/src/Extras/Evals/Executors/RunInference.php +++ b/src/Extras/Evals/Executors/RunInference.php @@ -1,37 +1,37 @@ -inferenceAdapter = new InferenceAdapter(); - $this->inferenceData = $data; - } - - public function run(Execution $execution) : Execution { - $execution->data()->set('response', $this->makeLLMResponse($execution)); - return $execution; - } - - // INTERNAL ///////////////////////////////////////////////// - - private function makeLLMResponse(Execution $execution) : LLMResponse { - return $this->inferenceAdapter->callInferenceFor( - connection: $execution->get('case.connection'), - mode: $execution->get('case.mode'), - isStreamed: $execution->get('case.isStreaming'), - messages: $this->inferenceData->messages, - evalSchema: $this->inferenceData->inferenceSchema(), - maxTokens: $this->inferenceData->maxTokens, - ); - } +inferenceAdapter = new InferenceAdapter(); + $this->inferenceData = $data; + } + + public function run(Execution $execution) : Execution { + $execution->data()->set('response', $this->makeLLMResponse($execution)); + return $execution; + } + + // INTERNAL ///////////////////////////////////////////////// + + private function makeLLMResponse(Execution $execution) : LLMResponse { + return $this->inferenceAdapter->callInferenceFor( + connection: $execution->get('case.connection'), + mode: $execution->get('case.mode'), + isStreamed: $execution->get('case.isStreamed'), + messages: $this->inferenceData->messages, + evalSchema: $this->inferenceData->inferenceSchema(), + maxTokens: $this->inferenceData->maxTokens, + ); + } } \ No newline at end of file diff --git a/src/Features/LLM/Drivers/AnthropicDriver.php b/src/Features/LLM/Drivers/AnthropicDriver.php index 093b378f..b7b728de 100644 --- a/src/Features/LLM/Drivers/AnthropicDriver.php +++ b/src/Features/LLM/Drivers/AnthropicDriver.php @@ -1,300 +1,300 @@ -events = $events ?? new EventDispatcher(); - $this->httpClient = $httpClient ?? HttpClient::make(); - } - - // REQUEST ////////////////////////////////////////////// - - public function handle(InferenceRequest $request) : CanAccessResponse { - $request = $this->withCachedContext($request); - return $this->httpClient->handle( - url: $this->getEndpointUrl($request), - headers: $this->getRequestHeaders(), - body: $this->getRequestBody( - $request->messages, - $request->model, - $request->tools, - $request->toolChoice, - $request->responseFormat, - $request->options, - $request->mode, - ), - streaming: $request->options['stream'] ?? false, - ); - } - - public function getEndpointUrl(InferenceRequest $request) : string { - return "{$this->config->apiUrl}{$this->config->endpoint}"; - } - - public function getRequestHeaders() : array { - return array_filter([ - 'x-api-key' => $this->config->apiKey, - 'content-type' => 'application/json', - 'accept' => 'application/json', - 'anthropic-version' => $this->config->metadata['apiVersion'] ?? '', - 'anthropic-beta' => $this->config->metadata['beta'] ?? '', - ]); - } - - public function getRequestBody( - array $messages = [], - string $model = '', - array $tools = [], - string|array $toolChoice = '', - array $responseFormat = [], - array $options = [], - Mode $mode = Mode::Text, - ) : array { - $request = array_filter(array_merge([ - 'model' => $model ?: $this->config->model, - 'max_tokens' => $options['max_tokens'] ?? $this->config->maxTokens, - 'system' => Messages::fromArray($messages) - ->forRoles(['system']) - ->toString(), - 'messages' => $this->toNativeMessages(Messages::fromArray($messages) - ->exceptRoles(['system']) - ->toMergedPerRole() - ->toArray() - ), - ], $options)); - - return $this->applyMode($request, $mode, $tools, $toolChoice, $responseFormat); - } - - // RESPONSE ///////////////////////////////////////////// - - public function toLLMResponse(array $data): ?LLMResponse { - return new LLMResponse( - content: $this->makeContent($data), - responseData: $data, -// toolName: $data['content'][0]['name'] ?? '', -// toolArgs: Json::encode($data['content'][0]['input'] ?? ''), - toolsData: $this->mapToolsData($data), - finishReason: $data['stop_reason'] ?? '', - toolCalls: $this->makeToolCalls($data), - usage: $this->makeUsage($data), - ); - } - - public function toPartialLLMResponse(array $data) : ?PartialLLMResponse { - if (empty($data)) { - return null; - } - return new PartialLLMResponse( - contentDelta: $this->makeContentDelta($data), - responseData: $data, - toolName: $data['content_block']['name'] ?? '', - toolArgs: $data['delta']['partial_json'] ?? '', - finishReason: $data['delta']['stop_reason'] ?? $data['message']['stop_reason'] ?? '', - usage: $this->makeUsage($data), - ); - } - - public function getData(string $data): string|bool { - if (!str_starts_with($data, 'data:')) { - return ''; - } - $data = trim(substr($data, 5)); - return match(true) { - $data === 'event: message_stop' => false, - default => $data, - }; - } - - // PRIVATE ////////////////////////////////////////////// - - private function applyMode( - array $request, - Mode $mode, - array $tools, - string|array $toolChoice, - array $responseFormat - ) : array { - if ($mode->is(Mode::Tools)) { - $request['tools'] = $this->toTools($tools); - $request['tool_choice'] = $this->toToolChoice($toolChoice, $tools); - } - return $request; - } - - private function toTools(array $tools) : array { - $result = []; - foreach ($tools as $tool) { - $result[] = [ - 'name' => $tool['function']['name'], - 'description' => $tool['function']['description'] ?? '', - 'input_schema' => $tool['function']['parameters'], - ]; - } - return $result; - } - - private function toToolChoice(string|array $toolChoice, array $tools) : array|string { - return match(true) { - empty($tools) => '', - is_array($toolChoice) => [ - 'type' => 'tool', - 'name' => $toolChoice['function']['name'], - ], - empty($toolChoice) => [ - 'type' => 'auto', - ], - default => [ - 'type' => $toolChoice, - ], - }; - } - - private function toNativeMessages(array $messages) : array { - return array_map( - fn($message) => [ - 'role' => $this->mapRole($message['role'] ?? 'user'), - 'content' => $this->toNativeContent($message['content']), - ], - $messages - ); - } - - private function mapRole(string $role) : string { - $roles = ['user' => 'user', 'assistant' => 'assistant', 'system' => 'user', 'tool' => 'user']; - return $roles[$role] ?? $role; - } - - private function toNativeContent(string|array $content) : string|array { - if (is_string($content)) { - return $content; - } - // if content is array - process each part - $transformed = []; - foreach ($content as $contentPart) { - $transformed[] = $this->contentPartToNative($contentPart); - } - return $transformed; - } - - private function contentPartToNative(array $contentPart) : array { - $type = $contentPart['type'] ?? 'text'; - if ($type === 'image_url') { - $contentPart = $this->toNativeImage($contentPart); - } - return $contentPart; - } - - private function toNativeImage(array $contentPart) : array { - $mimeType = Str::between($contentPart['image_url']['url'], 'data:', ';base64,'); - $base64content = Str::after($contentPart['image_url']['url'], ';base64,'); - $contentPart = [ - 'type' => 'image', - 'source' => [ - 'type' => 'base64', - 'media_type' => $mimeType, - 'data' => $base64content, - ], - ]; - return $contentPart; - } - private function makeToolCalls(array $data) : ToolCalls { - return ToolCalls::fromMapper(array_map( - callback: fn(array $call) => $call, - array: $data['content'] ?? [] - ), fn($call) => ToolCall::fromArray(['name' => $call['name'] ?? '', 'arguments' => $call['input'] ?? ''])); - } - - private function mapToolsData(array $data) : array { - return array_map( - fn($tool) => [ - 'name' => $tool['name'] ?? '', - 'arguments' => $tool['input'] ?? '', - ], - array_filter($data['content'] ?? [], fn($part) => 'tool_use' === ($part['type'] ?? '')) - ); - } - - private function makeContent(array $data) : string { - return $data['content'][0]['text'] ?? Json::encode($data['content'][0]['input']) ?? ''; - } - - private function makeContentDelta(array $data) : string { - return $data['delta']['text'] ?? $data['delta']['partial_json'] ?? ''; - } - - private function withCachedContext(InferenceRequest $request): InferenceRequest { - if (!isset($request->cachedContext)) { - return $request; - } - - $cloned = clone $request; - - $cloned->messages = empty($request->cachedContext->messages) - ? $request->messages - : array_merge($this->setCacheMarker($request->cachedContext->messages), $request->messages); - $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; - $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; - $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; - return $cloned; - } - - private function setCacheMarker(array $messages): array { - $lastIndex = count($messages) - 1; - $lastMessage = $messages[$lastIndex]; - - if (is_array($lastMessage['content'])) { - $subIndex = count($lastMessage['content']) - 1; - $lastMessage['content'][$subIndex]['cache_control'] = ["type" => "ephemeral"]; - } else { - $lastMessage['content'] = [[ - 'type' => $lastMessage['type'] ?? 'text', - 'text' => $lastMessage['content'] ?? '', - 'cache_control' => ["type" => "ephemeral"], - ]]; - } - - $messages[$lastIndex] = $lastMessage; - return $messages; - } - - private function makeUsage(array $data) : Usage { - return new Usage( - inputTokens: $data['usage']['input_tokens'] - ?? $data['message']['usage']['input_tokens'] - ?? 0, - outputTokens: $data['usage']['output_tokens'] - ?? $data['message']['usage']['output_tokens'] - ?? 0, - cacheWriteTokens: $data['usage']['cache_creation_input_tokens'] - ?? $data['message']['usage']['cache_creation_input_tokens'] - ?? 0, - cacheReadTokens: $data['usage']['cache_read_input_tokens'] - ?? $data['message']['usage']['cache_read_input_tokens'] - ?? 0, - reasoningTokens: 0, - ); - } -} +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + // REQUEST ////////////////////////////////////////////// + + public function handle(InferenceRequest $request) : CanAccessResponse { + $request = $this->withCachedContext($request); + return $this->httpClient->handle( + url: $this->getEndpointUrl($request), + headers: $this->getRequestHeaders(), + body: $this->getRequestBody( + $request->messages, + $request->model, + $request->tools, + $request->toolChoice, + $request->responseFormat, + $request->options, + $request->mode, + ), + streaming: $request->options['stream'] ?? false, + ); + } + + public function getEndpointUrl(InferenceRequest $request) : string { + return "{$this->config->apiUrl}{$this->config->endpoint}"; + } + + public function getRequestHeaders() : array { + return array_filter([ + 'x-api-key' => $this->config->apiKey, + 'content-type' => 'application/json', + 'accept' => 'application/json', + 'anthropic-version' => $this->config->metadata['apiVersion'] ?? '', + 'anthropic-beta' => $this->config->metadata['beta'] ?? '', + ]); + } + + public function getRequestBody( + array $messages = [], + string $model = '', + array $tools = [], + string|array $toolChoice = '', + array $responseFormat = [], + array $options = [], + Mode $mode = Mode::Text, + ) : array { + $request = array_filter(array_merge([ + 'model' => $model ?: $this->config->model, + 'max_tokens' => $options['max_tokens'] ?? $this->config->maxTokens, + 'system' => Messages::fromArray($messages) + ->forRoles(['system']) + ->toString(), + 'messages' => $this->toNativeMessages(Messages::fromArray($messages) + ->exceptRoles(['system']) + ->toMergedPerRole() + ->toArray() + ), + ], $options)); + + return $this->applyMode($request, $mode, $tools, $toolChoice, $responseFormat); + } + + // RESPONSE ///////////////////////////////////////////// + + public function toLLMResponse(array $data): ?LLMResponse { + return new LLMResponse( + content: $this->makeContent($data), + responseData: $data, +// toolName: $data['content'][0]['name'] ?? '', +// toolArgs: Json::encode($data['content'][0]['input'] ?? ''), + toolsData: $this->mapToolsData($data), + finishReason: $data['stop_reason'] ?? '', + toolCalls: $this->makeToolCalls($data), + usage: $this->makeUsage($data), + ); + } + + public function toPartialLLMResponse(array $data) : ?PartialLLMResponse { + if (empty($data)) { + return null; + } + return new PartialLLMResponse( + contentDelta: $this->makeContentDelta($data), + responseData: $data, + toolName: $data['content_block']['name'] ?? '', + toolArgs: $data['delta']['partial_json'] ?? '', + finishReason: $data['delta']['stop_reason'] ?? $data['message']['stop_reason'] ?? '', + usage: $this->makeUsage($data), + ); + } + + public function getData(string $data): string|bool { + if (!str_starts_with($data, 'data:')) { + return ''; + } + $data = trim(substr($data, 5)); + return match(true) { + $data === 'event: message_stop' => false, + default => $data, + }; + } + + // PRIVATE ////////////////////////////////////////////// + + private function applyMode( + array $request, + Mode $mode, + array $tools, + string|array $toolChoice, + array $responseFormat + ) : array { + if ($mode->is(Mode::Tools)) { + $request['tools'] = $this->toTools($tools); + $request['tool_choice'] = $this->toToolChoice($toolChoice, $tools); + } + return $request; + } + + private function toTools(array $tools) : array { + $result = []; + foreach ($tools as $tool) { + $result[] = [ + 'name' => $tool['function']['name'], + 'description' => $tool['function']['description'] ?? '', + 'input_schema' => $tool['function']['parameters'], + ]; + } + return $result; + } + + private function toToolChoice(string|array $toolChoice, array $tools) : array|string { + return match(true) { + empty($tools) => '', + is_array($toolChoice) => [ + 'type' => 'tool', + 'name' => $toolChoice['function']['name'], + ], + empty($toolChoice) => [ + 'type' => 'auto', + ], + default => [ + 'type' => $toolChoice, + ], + }; + } + + private function toNativeMessages(array $messages) : array { + return array_map( + fn($message) => [ + 'role' => $this->mapRole($message['role'] ?? 'user'), + 'content' => $this->toNativeContent($message['content']), + ], + $messages + ); + } + + private function mapRole(string $role) : string { + $roles = ['user' => 'user', 'assistant' => 'assistant', 'system' => 'user', 'tool' => 'user']; + return $roles[$role] ?? $role; + } + + private function toNativeContent(string|array $content) : string|array { + if (is_string($content)) { + return $content; + } + // if content is array - process each part + $transformed = []; + foreach ($content as $contentPart) { + $transformed[] = $this->contentPartToNative($contentPart); + } + return $transformed; + } + + private function contentPartToNative(array $contentPart) : array { + $type = $contentPart['type'] ?? 'text'; + if ($type === 'image_url') { + $contentPart = $this->toNativeImage($contentPart); + } + return $contentPart; + } + + private function toNativeImage(array $contentPart) : array { + $mimeType = Str::between($contentPart['image_url']['url'], 'data:', ';base64,'); + $base64content = Str::after($contentPart['image_url']['url'], ';base64,'); + $contentPart = [ + 'type' => 'image', + 'source' => [ + 'type' => 'base64', + 'media_type' => $mimeType, + 'data' => $base64content, + ], + ]; + return $contentPart; + } + private function makeToolCalls(array $data) : ToolCalls { + return ToolCalls::fromMapper(array_map( + callback: fn(array $call) => $call, + array: $data['content'] ?? [] + ), fn($call) => ToolCall::fromArray(['name' => $call['name'] ?? '', 'arguments' => $call['input'] ?? ''])); + } + + private function mapToolsData(array $data) : array { + return array_map( + fn($tool) => [ + 'name' => $tool['name'] ?? '', + 'arguments' => $tool['input'] ?? '', + ], + array_filter($data['content'] ?? [], fn($part) => 'tool_use' === ($part['type'] ?? '')) + ); + } + + private function makeContent(array $data) : string { + return $data['content'][0]['text'] ?? Json::encode($data['content'][0]['input']) ?? ''; + } + + private function makeContentDelta(array $data) : string { + return $data['delta']['text'] ?? $data['delta']['partial_json'] ?? ''; + } + + private function withCachedContext(InferenceRequest $request): InferenceRequest { + if (!isset($request->cachedContext)) { + return $request; + } + + $cloned = clone $request; + + $cloned->messages = empty($request->cachedContext->messages) + ? $request->messages + : array_merge($this->setCacheMarker($request->cachedContext->messages), $request->messages); + $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; + $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; + $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; + return $cloned; + } + + private function setCacheMarker(array $messages): array { + $lastIndex = count($messages) - 1; + $lastMessage = $messages[$lastIndex]; + + if (is_array($lastMessage['content'])) { + $subIndex = count($lastMessage['content']) - 1; + $lastMessage['content'][$subIndex]['cache_control'] = ["type" => "ephemeral"]; + } else { + $lastMessage['content'] = [[ + 'type' => $lastMessage['type'] ?? 'text', + 'text' => $lastMessage['content'] ?? '', + 'cache_control' => ["type" => "ephemeral"], + ]]; + } + + $messages[$lastIndex] = $lastMessage; + return $messages; + } + + private function makeUsage(array $data) : Usage { + return new Usage( + inputTokens: $data['usage']['input_tokens'] + ?? $data['message']['usage']['input_tokens'] + ?? 0, + outputTokens: $data['usage']['output_tokens'] + ?? $data['message']['usage']['output_tokens'] + ?? 0, + cacheWriteTokens: $data['usage']['cache_creation_input_tokens'] + ?? $data['message']['usage']['cache_creation_input_tokens'] + ?? 0, + cacheReadTokens: $data['usage']['cache_read_input_tokens'] + ?? $data['message']['usage']['cache_read_input_tokens'] + ?? 0, + reasoningTokens: 0, + ); + } +} diff --git a/src/Features/LLM/Drivers/CohereV1Driver.php b/src/Features/LLM/Drivers/CohereV1Driver.php index 156c8a2a..bf783d77 100644 --- a/src/Features/LLM/Drivers/CohereV1Driver.php +++ b/src/Features/LLM/Drivers/CohereV1Driver.php @@ -1,257 +1,257 @@ -events = $events ?? new EventDispatcher(); - $this->httpClient = $httpClient ?? HttpClient::make(); - } - - // REQUEST ////////////////////////////////////////////// - - public function handle(InferenceRequest $request) : CanAccessResponse { - $request = $this->withCachedContext($request); - return $this->httpClient->handle( - url: $this->getEndpointUrl($request), - headers: $this->getRequestHeaders(), - body: $this->getRequestBody( - $request->messages, - $request->model, - $request->tools, - $request->toolChoice, - $request->responseFormat, - $request->options, - $request->mode, - ), - streaming: $request->options['stream'] ?? false, - ); - } - - public function getEndpointUrl(InferenceRequest $request) : string { - return "{$this->config->apiUrl}{$this->config->endpoint}"; - } - - public function getRequestHeaders() : array { - return [ - 'Authorization' => "Bearer {$this->config->apiKey}", - 'Content-Type' => 'application/json', - ]; - } - - public function getRequestBody( - array $messages = [], - string $model = '', - array $tools = [], - string|array $toolChoice = '', - array $responseFormat = [], - array $options = [], - Mode $mode = Mode::Text, - ) : array { - $system = ''; - $chatHistory = []; - - $request = array_filter(array_merge([ - 'model' => $model ?: $this->config->model, - 'preamble' => $system, - 'chat_history' => $chatHistory, - 'message' => Messages::asString($messages), - ], $options)); - - return $this->applyMode($request, $mode, $tools, $toolChoice, $responseFormat); - } - - // RESPONSE ///////////////////////////////////////////// - - public function toLLMResponse(array $data): LLMResponse { - return new LLMResponse( - content: $this->makeContent($data), - responseData: $data, - toolsData: $this->mapToolsData($data), - finishReason: $data['finish_reason'] ?? '', - toolCalls: $this->makeToolCalls($data), - usage: $this->makeUsage($data), - ); - } - - public function toPartialLLMResponse(array $data) : PartialLLMResponse { - return new PartialLLMResponse( - contentDelta: $this->makeContentDelta($data), - responseData: $data, - toolName: $this->makeToolNameDelta($data), - toolArgs: $this->makeToolArgsDelta($data), - finishReason: $data['response']['finish_reason'] ?? $data['delta']['finish_reason'] ?? '', - usage: $this->makeUsage($data), - ); - } - - public function getData(string $data): string|bool { - $data = trim($data); - return match(true) { - $data === '[DONE]' => false, - default => $data, - }; - } - - // PRIVATE ////////////////////////////////////////////// - - private function applyMode( - array $request, - Mode $mode, - array $tools, - string|array $toolChoice, - array $responseFormat - ) : array { - switch($mode) { - case Mode::Tools: - $request['tools'] = $this->toTools($tools); - break; - case Mode::Json: - $request['response_format'] = [ - 'type' => 'json_object', - 'schema' => $responseFormat['schema'] ?? [], - ]; - break; - case Mode::JsonSchema: - $request['response_format'] = [ - 'type' => 'json_object', - 'schema' => $responseFormat['json_schema']['schema'] ?? [], - ]; - break; - } - return $request; - } - - private function toTools(array $tools): array { - $result = []; - foreach ($tools as $tool) { - $parameters = []; - foreach ($tool['function']['parameters']['properties'] as $name => $param) { - $parameters[$name] = array_filter([ - 'description' => $param['description'] ?? '', - 'type' => $this->toCohereType($param), - 'required' => in_array( - needle: $name, - haystack: $tool['function']['parameters']['required'] ?? [], - ), - ]); - } - $result[] = [ - 'name' => $tool['function']['name'], - 'description' => $tool['function']['description'] ?? '', - 'parameterDefinitions' => $parameters, - ]; - } - return $result; - } - - private function toCohereType(array $param) : string { - return match($param['type']) { - 'string' => 'str', - 'number' => 'float', - 'integer' => 'int', - 'boolean' => 'bool', - 'array' => throw new \Exception('Array type not supported by Cohere'), - 'object' => throw new \Exception('Object type not supported by Cohere'), - default => throw new \Exception('Unknown type'), - }; - } - - private function makeToolCalls(array $data) : ToolCalls { - return ToolCalls::fromMapper( - $data['tool_calls'] ?? [], - fn($call) => ToolCall::fromArray(['name' => $call['name'] ?? '', 'arguments' => $call['parameters'] ?? '']) - ); - } - - private function mapToolsData(array $data) : array { - return array_map( - fn($tool) => [ - 'name' => $tool['name'] ?? '', - 'arguments' => $tool['parameters'] ?? '', - ], - $data['tool_calls'] ?? [] - ); - } - - private function makeContent(array $data) : string { - return ($data['text'] ?? '') . (!empty($data['tool_calls']) - ? ("\n" . Json::encode($data['tool_calls'])) - : '' - ); - } - - private function makeContentDelta(array $data) : string { - if (!$this->isStreamChunk($data)) { - return ''; - } - return $data['tool_call_delta']['parameters'] ?? $data['text'] ?? ''; - } - - private function makeToolArgsDelta(array $data) : string { - if (!$this->isStreamChunk($data)) { - return ''; - } - $toolArgs = $data['tool_calls'][0]['parameters'] ?? ''; - return ('' === $toolArgs) ? '' : Json::encode($toolArgs); - } - - private function makeToolNameDelta(array $data) : string { - if (!$this->isStreamChunk($data)) { - return ''; - } - return $data['tool_calls'][0]['name'] ?? ''; - } - - private function isStreamChunk(array $data) : bool { - return in_array(($data['event_type'] ?? ''), ['text-generation', 'tool-calls-chunk']); - } - - private function withCachedContext(InferenceRequest $request): InferenceRequest { - if (!isset($request->cachedContext)) { - return $request; - } - $cloned = clone $request; - $cloned->messages = array_merge($request->cachedContext->messages, $request->messages); - $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; - $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; - $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; - return $cloned; - } - - private function makeUsage(array $data) : Usage { - return new Usage( - inputTokens: $data['meta']['tokens']['input_tokens'] - ?? $data['response']['meta']['tokens']['input_tokens'] - ?? $data['delta']['tokens']['input_tokens'] - ?? 0, - outputTokens: $data['meta']['tokens']['output_tokens'] - ?? $data['response']['meta']['tokens']['output_tokens'] - ?? $data['delta']['tokens']['input_tokens'] - ?? 0, - cacheWriteTokens: 0, - cacheReadTokens: 0, - reasoningTokens: 0, - ); - } -} +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + // REQUEST ////////////////////////////////////////////// + + public function handle(InferenceRequest $request) : CanAccessResponse { + $request = $this->withCachedContext($request); + return $this->httpClient->handle( + url: $this->getEndpointUrl($request), + headers: $this->getRequestHeaders(), + body: $this->getRequestBody( + $request->messages, + $request->model, + $request->tools, + $request->toolChoice, + $request->responseFormat, + $request->options, + $request->mode, + ), + streaming: $request->options['stream'] ?? false, + ); + } + + public function getEndpointUrl(InferenceRequest $request) : string { + return "{$this->config->apiUrl}{$this->config->endpoint}"; + } + + public function getRequestHeaders() : array { + return [ + 'Authorization' => "Bearer {$this->config->apiKey}", + 'Content-Type' => 'application/json', + ]; + } + + public function getRequestBody( + array $messages = [], + string $model = '', + array $tools = [], + string|array $toolChoice = '', + array $responseFormat = [], + array $options = [], + Mode $mode = Mode::Text, + ) : array { + $system = ''; + $chatHistory = []; + + $request = array_filter(array_merge([ + 'model' => $model ?: $this->config->model, + 'preamble' => $system, + 'chat_history' => $chatHistory, + 'message' => Messages::asString($messages), + ], $options)); + + return $this->applyMode($request, $mode, $tools, $toolChoice, $responseFormat); + } + + // RESPONSE ///////////////////////////////////////////// + + public function toLLMResponse(array $data): LLMResponse { + return new LLMResponse( + content: $this->makeContent($data), + responseData: $data, + toolsData: $this->mapToolsData($data), + finishReason: $data['finish_reason'] ?? '', + toolCalls: $this->makeToolCalls($data), + usage: $this->makeUsage($data), + ); + } + + public function toPartialLLMResponse(array $data) : PartialLLMResponse { + return new PartialLLMResponse( + contentDelta: $this->makeContentDelta($data), + responseData: $data, + toolName: $this->makeToolNameDelta($data), + toolArgs: $this->makeToolArgsDelta($data), + finishReason: $data['response']['finish_reason'] ?? $data['delta']['finish_reason'] ?? '', + usage: $this->makeUsage($data), + ); + } + + public function getData(string $data): string|bool { + $data = trim($data); + return match(true) { + $data === '[DONE]' => false, + default => $data, + }; + } + + // PRIVATE ////////////////////////////////////////////// + + private function applyMode( + array $request, + Mode $mode, + array $tools, + string|array $toolChoice, + array $responseFormat + ) : array { + switch($mode) { + case Mode::Tools: + $request['tools'] = $this->toTools($tools); + break; + case Mode::Json: + $request['response_format'] = [ + 'type' => 'json_object', + 'schema' => $responseFormat['schema'] ?? [], + ]; + break; + case Mode::JsonSchema: + $request['response_format'] = [ + 'type' => 'json_object', + 'schema' => $responseFormat['json_schema']['schema'] ?? [], + ]; + break; + } + return $request; + } + + private function toTools(array $tools): array { + $result = []; + foreach ($tools as $tool) { + $parameters = []; + foreach ($tool['function']['parameters']['properties'] as $name => $param) { + $parameters[$name] = array_filter([ + 'description' => $param['description'] ?? '', + 'type' => $this->toCohereType($param), + 'required' => in_array( + needle: $name, + haystack: $tool['function']['parameters']['required'] ?? [], + ), + ]); + } + $result[] = [ + 'name' => $tool['function']['name'], + 'description' => $tool['function']['description'] ?? '', + 'parameterDefinitions' => $parameters, + ]; + } + return $result; + } + + private function toCohereType(array $param) : string { + return match($param['type']) { + 'string' => 'str', + 'number' => 'float', + 'integer' => 'int', + 'boolean' => 'bool', + 'array' => throw new \Exception('Array type not supported by Cohere'), + 'object' => throw new \Exception('Object type not supported by Cohere'), + default => throw new \Exception('Unknown type'), + }; + } + + private function makeToolCalls(array $data) : ToolCalls { + return ToolCalls::fromMapper( + $data['tool_calls'] ?? [], + fn($call) => ToolCall::fromArray(['name' => $call['name'] ?? '', 'arguments' => $call['parameters'] ?? '']) + ); + } + + private function mapToolsData(array $data) : array { + return array_map( + fn($tool) => [ + 'name' => $tool['name'] ?? '', + 'arguments' => $tool['parameters'] ?? '', + ], + $data['tool_calls'] ?? [] + ); + } + + private function makeContent(array $data) : string { + return ($data['text'] ?? '') . (!empty($data['tool_calls']) + ? ("\n" . Json::encode($data['tool_calls'])) + : '' + ); + } + + private function makeContentDelta(array $data) : string { + if (!$this->isStreamChunk($data)) { + return ''; + } + return $data['tool_call_delta']['parameters'] ?? $data['text'] ?? ''; + } + + private function makeToolArgsDelta(array $data) : string { + if (!$this->isStreamChunk($data)) { + return ''; + } + $toolArgs = $data['tool_calls'][0]['parameters'] ?? ''; + return ('' === $toolArgs) ? '' : Json::encode($toolArgs); + } + + private function makeToolNameDelta(array $data) : string { + if (!$this->isStreamChunk($data)) { + return ''; + } + return $data['tool_calls'][0]['name'] ?? ''; + } + + private function isStreamChunk(array $data) : bool { + return in_array(($data['event_type'] ?? ''), ['text-generation', 'tool-calls-chunk']); + } + + private function withCachedContext(InferenceRequest $request): InferenceRequest { + if (!isset($request->cachedContext)) { + return $request; + } + $cloned = clone $request; + $cloned->messages = array_merge($request->cachedContext->messages, $request->messages); + $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; + $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; + $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; + return $cloned; + } + + private function makeUsage(array $data) : Usage { + return new Usage( + inputTokens: $data['meta']['tokens']['input_tokens'] + ?? $data['response']['meta']['tokens']['input_tokens'] + ?? $data['delta']['tokens']['input_tokens'] + ?? 0, + outputTokens: $data['meta']['tokens']['output_tokens'] + ?? $data['response']['meta']['tokens']['output_tokens'] + ?? $data['delta']['tokens']['input_tokens'] + ?? 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + reasoningTokens: 0, + ); + } +} diff --git a/src/Features/LLM/Drivers/GeminiDriver.php b/src/Features/LLM/Drivers/GeminiDriver.php index 5fbf6624..d218bb56 100644 --- a/src/Features/LLM/Drivers/GeminiDriver.php +++ b/src/Features/LLM/Drivers/GeminiDriver.php @@ -1,317 +1,317 @@ -events = $events ?? new EventDispatcher(); - $this->httpClient = $httpClient ?? HttpClient::make(); - } - - // REQUEST ////////////////////////////////////////////// - - public function handle(InferenceRequest $request) : CanAccessResponse { - $request = $this->withCachedContext($request); - return $this->httpClient->handle( - url: $this->getEndpointUrl($request), - headers: $this->getRequestHeaders(), - body: $this->getRequestBody( - $request->messages, - $request->model, - $request->tools, - $request->toolChoice, - $request->responseFormat, - $request->options, - $request->mode, - ), - streaming: $request->options['stream'] ?? false, - ); - } - - public function getEndpointUrl(InferenceRequest $request): string { - $urlParams = ['key' => $this->config->apiKey]; - - if ($request->options['stream'] ?? false) { - $this->config->endpoint = '/models/{model}:streamGenerateContent'; - $urlParams['alt'] = 'sse'; - } else { - $this->config->endpoint = '/models/{model}:generateContent'; - } - - return str_replace( - search: "{model}", - replace: $request->model ?: $this->config->model, - subject: "{$this->config->apiUrl}{$this->config->endpoint}?" . http_build_query($urlParams)); - } - - public function getRequestHeaders() : array { - return [ - 'Content-Type' => 'application/json', - ]; - } - - public function getRequestBody( - array $messages = [], - string $model = '', - array $tools = [], - string|array $toolChoice = '', - array $responseFormat = [], - array $options = [], - Mode $mode = Mode::Text, - ) : array { - $request = array_filter([ - 'systemInstruction' => $this->toSystem($messages), - 'contents' => $this->toMessages($messages), - 'generationConfig' => $this->toOptions($options, $responseFormat, $mode), - ]); - - if ($mode == Mode::Tools) { - $request['tools'] = $this->toTools($tools); - $request['tool_config'] = $this->toToolChoice($toolChoice); - } - return $request; - } - - // RESPONSE ///////////////////////////////////////////// - - public function toLLMResponse(array $data): ?LLMResponse { - return new LLMResponse( - content: $this->makeContent($data), - responseData: $data, -// toolName: $data['candidates'][0]['content']['parts'][0]['functionCall']['name'] ?? '', -// toolArgs: Json::encode($data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? []), - toolsData: $this->mapToolsData($data), - finishReason: $data['candidates'][0]['finishReason'] ?? '', - toolCalls: $this->makeToolCalls($data), - usage: $this->makeUsage($data), - ); - } - - public function toPartialLLMResponse(array $data) : ?PartialLLMResponse { - if (empty($data)) { - return null; - } - return new PartialLLMResponse( - contentDelta: $this->makeContentDelta($data), - responseData: $data, - toolName: $this->makeToolName($data), - toolArgs: $this->makeToolArgs($data), - finishReason: $data['candidates'][0]['finishReason'] ?? '', - usage: $this->makeUsage($data), - ); - } - - public function getData(string $data): string|bool { - if (!str_starts_with($data, 'data:')) { - return ''; - } - $data = trim(substr($data, 5)); - return match(true) { - $data === '[DONE]' => false, - default => $data, - }; - } - - // PRIVATE ////////////////////////////////////////////// - - private function toSystem(array $messages) : array { - $system = Messages::fromArray($messages) - ->forRoles(['system']) - ->toString(); - - return empty($system) ? [] : ['parts' => ['text' => $system]]; - } - - private function toMessages(array $messages) : array { - return $this->toNativeMessages(Messages::fromArray($messages) - ->exceptRoles(['system']) - //->toMergedPerRole() - ->toArray()); - } - - protected function toOptions( - array $options, - array $responseFormat, - Mode $mode, - ) : array { - return array_filter([ - "responseMimeType" => $this->toResponseMimeType($mode), - "responseSchema" => $this->toResponseSchema($responseFormat, $mode), - "candidateCount" => 1, - "maxOutputTokens" => $options['max_tokens'] ?? $this->config->maxTokens, - "temperature" => $options['temperature'] ?? 1.0, - ]); - } - - protected function toTools(array $tools) : array { - return ['function_declarations' => array_map( - callback: fn($tool) => $this->removeDisallowedEntries($tool['function']), - array: $tools - )]; - } - - protected function toToolChoice(array $toolChoice): string|array { - return match(true) { - empty($toolChoice) => ["function_calling_config" => ["mode" => "ANY"]], - is_array($toolChoice) => [ - "function_calling_config" => array_filter([ - "mode" => "ANY", - "allowed_function_names" => $toolChoice['function']['name'] ?? [], - ]), - ], - default => ["function_calling_config" => ["mode" => "ANY"]], - }; - } - - protected function toResponseMimeType(Mode $mode): string { - return match($mode) { - Mode::Text => "text/plain", - Mode::MdJson => "text/plain", - Mode::Tools => "text/plain", - default => "application/json", - }; - } - - protected function toResponseSchema(array $responseFormat, Mode $mode) : array { - return match($mode) { - Mode::MdJson => $this->removeDisallowedEntries($responseFormat['schema'] ?? []), - Mode::Json => $this->removeDisallowedEntries($responseFormat['schema'] ?? []), - Mode::JsonSchema => $this->removeDisallowedEntries($responseFormat['schema'] ?? []), - default => [], - }; - } - - protected function removeDisallowedEntries(array $jsonSchema) : array { - return Arrays::removeRecursively($jsonSchema, [ - 'x-title', - 'x-php-class', - 'additionalProperties', - ]); - } - - protected function toNativeMessages(string|array $messages) : array { - if (is_string($messages)) { - return [["text" => $messages]]; - } - $transformed = []; - foreach ($messages as $message) { - $transformed[] = [ - 'role' => $this->mapRole($message['role']), - 'parts' => $this->contentPartsToNative($message['content']), - ]; - } - return $transformed; - } - - protected function mapRole(string $role) : string { - $roles = ['user' => 'user', 'assistant' => 'model', 'system' => 'user', 'tool' => 'tool']; - return $roles[$role] ?? $role; - } - - protected function contentPartsToNative(string|array $contentParts) : array { - if (is_string($contentParts)) { - return [["text" => $contentParts]]; - } - $transformed = []; - foreach ($contentParts as $contentPart) { - $transformed[] = $this->contentPartToNative($contentPart); - } - return $transformed; - } - - protected function contentPartToNative(array $contentPart) : array { - $type = $contentPart['type'] ?? 'text'; - return match($type) { - 'text' => ['text' => $contentPart['text'] ?? ''], - 'image_url' => [ - 'inlineData' => [ - 'mimeType' => Str::between($contentPart['image_url']['url'], 'data:', ';base64,'), - 'data' => Str::after($contentPart['image_url']['url'], ';base64,'), - ], - ], - default => $contentPart, - }; - } - - private function makeToolCalls(array $data) : ToolCalls { - return ToolCalls::fromMapper(array_map( - callback: fn(array $call) => $call['functionCall'] ?? [], - array: $data['candidates'][0]['content']['parts'] ?? [] - ), fn($call) => ToolCall::fromArray(['name' => $call['name'] ?? '', 'arguments' => $call['args'] ?? ''])); - } - - private function mapToolsData(array $data) : array { - return array_map( - fn($tool) => [ - 'name' => $tool['functionCall']['name'] ?? '', - 'arguments' => $tool['functionCall']['args'] ?? '', - ], - $data['candidates'][0]['content']['parts'] ?? [] - ); - } - - private function makeContent(array $data) : string { - return $data['candidates'][0]['content']['parts'][0]['text'] - ?? Json::encode($data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? '') - ?? ''; - } - - private function makeContentDelta(array $data): string { - return $data['candidates'][0]['content']['parts'][0]['text'] - ?? Json::encode($data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? '') - ?? ''; - } - - private function withCachedContext(InferenceRequest $request): InferenceRequest { - if (!isset($request->cachedContext)) { - return $request; - } - $cloned = clone $request; - $cloned->messages = array_merge($request->cachedContext->messages, $request->messages); - $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; - $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; - $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; - return $cloned; - } - - private function makeToolName(array $data) : string { - return $data['candidates'][0]['content']['parts'][0]['functionCall']['name'] ?? ''; - } - - private function makeToolArgs(array $data) : string { - $value = $data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? ''; - return is_array($value) ? Json::encode($value) : ''; - } - - private function makeUsage(array $data) : Usage { - return new Usage( - inputTokens: $data['usageMetadata']['promptTokenCount'] ?? 0, - outputTokens: $data['usageMetadata']['candidatesTokenCount'] ?? 0, - cacheWriteTokens: 0, - cacheReadTokens: 0, - reasoningTokens: 0, - ); - } -} +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + // REQUEST ////////////////////////////////////////////// + + public function handle(InferenceRequest $request) : CanAccessResponse { + $request = $this->withCachedContext($request); + return $this->httpClient->handle( + url: $this->getEndpointUrl($request), + headers: $this->getRequestHeaders(), + body: $this->getRequestBody( + $request->messages, + $request->model, + $request->tools, + $request->toolChoice, + $request->responseFormat, + $request->options, + $request->mode, + ), + streaming: $request->options['stream'] ?? false, + ); + } + + public function getEndpointUrl(InferenceRequest $request): string { + $urlParams = ['key' => $this->config->apiKey]; + + if ($request->options['stream'] ?? false) { + $this->config->endpoint = '/models/{model}:streamGenerateContent'; + $urlParams['alt'] = 'sse'; + } else { + $this->config->endpoint = '/models/{model}:generateContent'; + } + + return str_replace( + search: "{model}", + replace: $request->model ?: $this->config->model, + subject: "{$this->config->apiUrl}{$this->config->endpoint}?" . http_build_query($urlParams)); + } + + public function getRequestHeaders() : array { + return [ + 'Content-Type' => 'application/json', + ]; + } + + public function getRequestBody( + array $messages = [], + string $model = '', + array $tools = [], + string|array $toolChoice = '', + array $responseFormat = [], + array $options = [], + Mode $mode = Mode::Text, + ) : array { + $request = array_filter([ + 'systemInstruction' => $this->toSystem($messages), + 'contents' => $this->toMessages($messages), + 'generationConfig' => $this->toOptions($options, $responseFormat, $mode), + ]); + + if ($mode == Mode::Tools) { + $request['tools'] = $this->toTools($tools); + $request['tool_config'] = $this->toToolChoice($toolChoice); + } + return $request; + } + + // RESPONSE ///////////////////////////////////////////// + + public function toLLMResponse(array $data): ?LLMResponse { + return new LLMResponse( + content: $this->makeContent($data), + responseData: $data, +// toolName: $data['candidates'][0]['content']['parts'][0]['functionCall']['name'] ?? '', +// toolArgs: Json::encode($data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? []), + toolsData: $this->mapToolsData($data), + finishReason: $data['candidates'][0]['finishReason'] ?? '', + toolCalls: $this->makeToolCalls($data), + usage: $this->makeUsage($data), + ); + } + + public function toPartialLLMResponse(array $data) : ?PartialLLMResponse { + if (empty($data)) { + return null; + } + return new PartialLLMResponse( + contentDelta: $this->makeContentDelta($data), + responseData: $data, + toolName: $this->makeToolName($data), + toolArgs: $this->makeToolArgs($data), + finishReason: $data['candidates'][0]['finishReason'] ?? '', + usage: $this->makeUsage($data), + ); + } + + public function getData(string $data): string|bool { + if (!str_starts_with($data, 'data:')) { + return ''; + } + $data = trim(substr($data, 5)); + return match(true) { + $data === '[DONE]' => false, + default => $data, + }; + } + + // PRIVATE ////////////////////////////////////////////// + + private function toSystem(array $messages) : array { + $system = Messages::fromArray($messages) + ->forRoles(['system']) + ->toString(); + + return empty($system) ? [] : ['parts' => ['text' => $system]]; + } + + private function toMessages(array $messages) : array { + return $this->toNativeMessages(Messages::fromArray($messages) + ->exceptRoles(['system']) + //->toMergedPerRole() + ->toArray()); + } + + protected function toOptions( + array $options, + array $responseFormat, + Mode $mode, + ) : array { + return array_filter([ + "responseMimeType" => $this->toResponseMimeType($mode), + "responseSchema" => $this->toResponseSchema($responseFormat, $mode), + "candidateCount" => 1, + "maxOutputTokens" => $options['max_tokens'] ?? $this->config->maxTokens, + "temperature" => $options['temperature'] ?? 1.0, + ]); + } + + protected function toTools(array $tools) : array { + return ['function_declarations' => array_map( + callback: fn($tool) => $this->removeDisallowedEntries($tool['function']), + array: $tools + )]; + } + + protected function toToolChoice(array $toolChoice): string|array { + return match(true) { + empty($toolChoice) => ["function_calling_config" => ["mode" => "ANY"]], + is_array($toolChoice) => [ + "function_calling_config" => array_filter([ + "mode" => "ANY", + "allowed_function_names" => $toolChoice['function']['name'] ?? [], + ]), + ], + default => ["function_calling_config" => ["mode" => "ANY"]], + }; + } + + protected function toResponseMimeType(Mode $mode): string { + return match($mode) { + Mode::Text => "text/plain", + Mode::MdJson => "text/plain", + Mode::Tools => "text/plain", + default => "application/json", + }; + } + + protected function toResponseSchema(array $responseFormat, Mode $mode) : array { + return match($mode) { + Mode::MdJson => $this->removeDisallowedEntries($responseFormat['schema'] ?? []), + Mode::Json => $this->removeDisallowedEntries($responseFormat['schema'] ?? []), + Mode::JsonSchema => $this->removeDisallowedEntries($responseFormat['schema'] ?? []), + default => [], + }; + } + + protected function removeDisallowedEntries(array $jsonSchema) : array { + return Arrays::removeRecursively($jsonSchema, [ + 'x-title', + 'x-php-class', + 'additionalProperties', + ]); + } + + protected function toNativeMessages(string|array $messages) : array { + if (is_string($messages)) { + return [["text" => $messages]]; + } + $transformed = []; + foreach ($messages as $message) { + $transformed[] = [ + 'role' => $this->mapRole($message['role']), + 'parts' => $this->contentPartsToNative($message['content']), + ]; + } + return $transformed; + } + + protected function mapRole(string $role) : string { + $roles = ['user' => 'user', 'assistant' => 'model', 'system' => 'user', 'tool' => 'tool']; + return $roles[$role] ?? $role; + } + + protected function contentPartsToNative(string|array $contentParts) : array { + if (is_string($contentParts)) { + return [["text" => $contentParts]]; + } + $transformed = []; + foreach ($contentParts as $contentPart) { + $transformed[] = $this->contentPartToNative($contentPart); + } + return $transformed; + } + + protected function contentPartToNative(array $contentPart) : array { + $type = $contentPart['type'] ?? 'text'; + return match($type) { + 'text' => ['text' => $contentPart['text'] ?? ''], + 'image_url' => [ + 'inlineData' => [ + 'mimeType' => Str::between($contentPart['image_url']['url'], 'data:', ';base64,'), + 'data' => Str::after($contentPart['image_url']['url'], ';base64,'), + ], + ], + default => $contentPart, + }; + } + + private function makeToolCalls(array $data) : ToolCalls { + return ToolCalls::fromMapper(array_map( + callback: fn(array $call) => $call['functionCall'] ?? [], + array: $data['candidates'][0]['content']['parts'] ?? [] + ), fn($call) => ToolCall::fromArray(['name' => $call['name'] ?? '', 'arguments' => $call['args'] ?? ''])); + } + + private function mapToolsData(array $data) : array { + return array_map( + fn($tool) => [ + 'name' => $tool['functionCall']['name'] ?? '', + 'arguments' => $tool['functionCall']['args'] ?? '', + ], + $data['candidates'][0]['content']['parts'] ?? [] + ); + } + + private function makeContent(array $data) : string { + return $data['candidates'][0]['content']['parts'][0]['text'] + ?? Json::encode($data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? '') + ?? ''; + } + + private function makeContentDelta(array $data): string { + return $data['candidates'][0]['content']['parts'][0]['text'] + ?? Json::encode($data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? '') + ?? ''; + } + + private function withCachedContext(InferenceRequest $request): InferenceRequest { + if (!isset($request->cachedContext)) { + return $request; + } + $cloned = clone $request; + $cloned->messages = array_merge($request->cachedContext->messages, $request->messages); + $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; + $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; + $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; + return $cloned; + } + + private function makeToolName(array $data) : string { + return $data['candidates'][0]['content']['parts'][0]['functionCall']['name'] ?? ''; + } + + private function makeToolArgs(array $data) : string { + $value = $data['candidates'][0]['content']['parts'][0]['functionCall']['args'] ?? ''; + return is_array($value) ? Json::encode($value) : ''; + } + + private function makeUsage(array $data) : Usage { + return new Usage( + inputTokens: $data['usageMetadata']['promptTokenCount'] ?? 0, + outputTokens: $data['usageMetadata']['candidatesTokenCount'] ?? 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + reasoningTokens: 0, + ); + } +} diff --git a/src/Features/LLM/Drivers/OpenAIDriver.php b/src/Features/LLM/Drivers/OpenAIDriver.php index 1f956cfb..cdb0afd1 100644 --- a/src/Features/LLM/Drivers/OpenAIDriver.php +++ b/src/Features/LLM/Drivers/OpenAIDriver.php @@ -1,224 +1,224 @@ -events = $events ?? new EventDispatcher(); - $this->httpClient = $httpClient ?? HttpClient::make(); - } - - // REQUEST ////////////////////////////////////////////// - - public function handle(InferenceRequest $request) : CanAccessResponse { - $request = $this->withCachedContext($request); - return $this->httpClient->handle( - url: $this->getEndpointUrl($request), - headers: $this->getRequestHeaders(), - body: $this->getRequestBody( - $request->messages, - $request->model, - $request->tools, - $request->toolChoice, - $request->responseFormat, - $request->options, - $request->mode, - ), - streaming: $request->options['stream'] ?? false, - ); - } - - public function getEndpointUrl(InferenceRequest $request): string { - return "{$this->config->apiUrl}{$this->config->endpoint}"; - } - - public function getRequestHeaders() : array { - $extras = array_filter([ - "OpenAI-Organization" => $this->config->metadata['organization'] ?? '', - "OpenAI-Project" => $this->config->metadata['project'] ?? '', - ]); - return array_merge([ - 'Authorization' => "Bearer {$this->config->apiKey}", - 'Content-Type' => 'application/json', - ], $extras); - } - - public function getRequestBody( - array $messages = [], - string $model = '', - array $tools = [], - string|array $toolChoice = '', - array $responseFormat = [], - array $options = [], - Mode $mode = Mode::Text, - ) : array { - $request = array_filter(array_merge([ - 'model' => $model ?: $this->config->model, - 'max_tokens' => $this->config->maxTokens, - 'messages' => $messages, - ], $options)); - - if ($options['stream'] ?? false) { - $request['stream_options']['include_usage'] = true; - } - - return $this->applyMode($request, $mode, $tools, $toolChoice, $responseFormat); - } - - // RESPONSE ///////////////////////////////////////////// - - public function toLLMResponse(array $data): ?LLMResponse { - return new LLMResponse( - content: $this->makeContent($data), - responseData: $data, - toolsData: $this->makeToolsData($data), - finishReason: $data['choices'][0]['finish_reason'] ?? '', - toolCalls: $this->makeToolCalls($data), - usage: $this->makeUsage($data), - ); - } - - public function toPartialLLMResponse(array|null $data) : ?PartialLLMResponse { - if ($data === null || empty($data)) { - return null; - } - return new PartialLLMResponse( - contentDelta: $this->makeContentDelta($data), - responseData: $data, - toolName: $this->makeToolNameDelta($data), - toolArgs: $this->makeToolArgsDelta($data), - finishReason: $data['choices'][0]['finish_reason'] ?? '', - usage: $this->makeUsage($data), - ); - } - - public function getData(string $data): string|bool { - if (!str_starts_with($data, 'data:')) { - return ''; - } - $data = trim(substr($data, 5)); - return match(true) { - $data === '[DONE]' => false, - default => $data, - }; - } - - // PRIVATE ////////////////////////////////////////////// - - private function applyMode( - array $request, - Mode $mode, - array $tools, - string|array $toolChoice, - array $responseFormat - ) : array { - switch($mode) { - case Mode::Tools: - $request['tools'] = $tools; - $request['tool_choice'] = $toolChoice; - break; - case Mode::Json: - $request['response_format'] = ['type' => 'json_object']; - break; - case Mode::JsonSchema: - $request['response_format'] = $responseFormat; - break; - case Mode::Text: - case Mode::MdJson: - $request['response_format'] = ['type' => 'text']; - break; - } - return $request; - } - - private function withCachedContext(InferenceRequest $request): InferenceRequest { - if (!isset($request->cachedContext)) { - return $request; - } - - $cloned = clone $request; - $cloned->messages = array_merge($request->cachedContext->messages, $request->messages); - $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; - $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; - $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; - return $cloned; - } - - private function makeToolCalls(array $data) : ToolCalls { - return ToolCalls::fromArray(array_map( - callback: fn(array $call) => $call['function'] ?? [], - array: $data['choices'][0]['message']['tool_calls'] ?? [] - )); - } - - private function makeToolsData(array $data) : array { - return array_map( - fn($tool) => [ - 'name' => $tool['function']['name'] ?? '', - 'arguments' => Json::decode($tool['function']['arguments']) ?? '', - ], - $data['choices'][0]['message']['tool_calls'] ?? [] - ); - } - - private function makeContent(array $data): string { - $contentMsg = $data['choices'][0]['message']['content'] ?? ''; - $contentFnArgs = $data['choices'][0]['message']['tool_calls'][0]['function']['arguments'] ?? ''; - return match(true) { - !empty($contentMsg) => $contentMsg, - !empty($contentFnArgs) => $contentFnArgs, - default => '' - }; - } - - private function makeContentDelta(array $data): string { - $deltaContent = $data['choices'][0]['delta']['content'] ?? ''; - $deltaFnArgs = $data['choices'][0]['delta']['tool_calls'][0]['function']['arguments'] ?? ''; - return match(true) { - ('' !== $deltaContent) => $deltaContent, - ('' !== $deltaFnArgs) => $deltaFnArgs, - default => '' - }; - } - - private function makeToolNameDelta(array $data) : string { - return $data['choices'][0]['delta']['tool_calls'][0]['function']['name'] ?? ''; - } - - private function makeToolArgsDelta(array $data) : string { - return $data['choices'][0]['delta']['tool_calls'][0]['function']['arguments'] ?? ''; - } - - private function makeUsage(array $data): Usage { - return new Usage( - inputTokens: $data['usage']['prompt_tokens'] - ?? $data['x_groq']['usage']['prompt_tokens'] - ?? 0, - outputTokens: $data['usage']['completion_tokens'] - ?? $data['x_groq']['usage']['completion_tokens'] - ?? 0, - cacheWriteTokens: 0, - cacheReadTokens: $data['usage']['prompt_tokens_details']['cached_tokens'] ?? 0, - reasoningTokens: $data['usage']['prompt_tokens_details']['reasoning_tokens'] ?? 0, - ); - } -} +events = $events ?? new EventDispatcher(); + $this->httpClient = $httpClient ?? HttpClient::make(events: $this->events); + } + + // REQUEST ////////////////////////////////////////////// + + public function handle(InferenceRequest $request) : CanAccessResponse { + $request = $this->withCachedContext($request); + return $this->httpClient->handle( + url: $this->getEndpointUrl($request), + headers: $this->getRequestHeaders(), + body: $this->getRequestBody( + $request->messages, + $request->model, + $request->tools, + $request->toolChoice, + $request->responseFormat, + $request->options, + $request->mode, + ), + streaming: $request->options['stream'] ?? false, + ); + } + + public function getEndpointUrl(InferenceRequest $request): string { + return "{$this->config->apiUrl}{$this->config->endpoint}"; + } + + public function getRequestHeaders() : array { + $extras = array_filter([ + "OpenAI-Organization" => $this->config->metadata['organization'] ?? '', + "OpenAI-Project" => $this->config->metadata['project'] ?? '', + ]); + return array_merge([ + 'Authorization' => "Bearer {$this->config->apiKey}", + 'Content-Type' => 'application/json', + ], $extras); + } + + public function getRequestBody( + array $messages = [], + string $model = '', + array $tools = [], + string|array $toolChoice = '', + array $responseFormat = [], + array $options = [], + Mode $mode = Mode::Text, + ) : array { + $request = array_filter(array_merge([ + 'model' => $model ?: $this->config->model, + 'max_tokens' => $this->config->maxTokens, + 'messages' => $messages, + ], $options)); + + if ($options['stream'] ?? false) { + $request['stream_options']['include_usage'] = true; + } + + return $this->applyMode($request, $mode, $tools, $toolChoice, $responseFormat); + } + + // RESPONSE ///////////////////////////////////////////// + + public function toLLMResponse(array $data): ?LLMResponse { + return new LLMResponse( + content: $this->makeContent($data), + responseData: $data, + toolsData: $this->makeToolsData($data), + finishReason: $data['choices'][0]['finish_reason'] ?? '', + toolCalls: $this->makeToolCalls($data), + usage: $this->makeUsage($data), + ); + } + + public function toPartialLLMResponse(array|null $data) : ?PartialLLMResponse { + if ($data === null || empty($data)) { + return null; + } + return new PartialLLMResponse( + contentDelta: $this->makeContentDelta($data), + responseData: $data, + toolName: $this->makeToolNameDelta($data), + toolArgs: $this->makeToolArgsDelta($data), + finishReason: $data['choices'][0]['finish_reason'] ?? '', + usage: $this->makeUsage($data), + ); + } + + public function getData(string $data): string|bool { + if (!str_starts_with($data, 'data:')) { + return ''; + } + $data = trim(substr($data, 5)); + return match(true) { + $data === '[DONE]' => false, + default => $data, + }; + } + + // PRIVATE ////////////////////////////////////////////// + + private function applyMode( + array $request, + Mode $mode, + array $tools, + string|array $toolChoice, + array $responseFormat + ) : array { + switch($mode) { + case Mode::Tools: + $request['tools'] = $tools; + $request['tool_choice'] = $toolChoice; + break; + case Mode::Json: + $request['response_format'] = ['type' => 'json_object']; + break; + case Mode::JsonSchema: + $request['response_format'] = $responseFormat; + break; + case Mode::Text: + case Mode::MdJson: + $request['response_format'] = ['type' => 'text']; + break; + } + return $request; + } + + private function withCachedContext(InferenceRequest $request): InferenceRequest { + if (!isset($request->cachedContext)) { + return $request; + } + + $cloned = clone $request; + $cloned->messages = array_merge($request->cachedContext->messages, $request->messages); + $cloned->tools = empty($request->tools) ? $request->cachedContext->tools : $request->tools; + $cloned->toolChoice = empty($request->toolChoice) ? $request->cachedContext->toolChoice : $request->toolChoice; + $cloned->responseFormat = empty($request->responseFormat) ? $request->cachedContext->responseFormat : $request->responseFormat; + return $cloned; + } + + private function makeToolCalls(array $data) : ToolCalls { + return ToolCalls::fromArray(array_map( + callback: fn(array $call) => $call['function'] ?? [], + array: $data['choices'][0]['message']['tool_calls'] ?? [] + )); + } + + private function makeToolsData(array $data) : array { + return array_map( + fn($tool) => [ + 'name' => $tool['function']['name'] ?? '', + 'arguments' => Json::decode($tool['function']['arguments']) ?? '', + ], + $data['choices'][0]['message']['tool_calls'] ?? [] + ); + } + + private function makeContent(array $data): string { + $contentMsg = $data['choices'][0]['message']['content'] ?? ''; + $contentFnArgs = $data['choices'][0]['message']['tool_calls'][0]['function']['arguments'] ?? ''; + return match(true) { + !empty($contentMsg) => $contentMsg, + !empty($contentFnArgs) => $contentFnArgs, + default => '' + }; + } + + private function makeContentDelta(array $data): string { + $deltaContent = $data['choices'][0]['delta']['content'] ?? ''; + $deltaFnArgs = $data['choices'][0]['delta']['tool_calls'][0]['function']['arguments'] ?? ''; + return match(true) { + ('' !== $deltaContent) => $deltaContent, + ('' !== $deltaFnArgs) => $deltaFnArgs, + default => '' + }; + } + + private function makeToolNameDelta(array $data) : string { + return $data['choices'][0]['delta']['tool_calls'][0]['function']['name'] ?? ''; + } + + private function makeToolArgsDelta(array $data) : string { + return $data['choices'][0]['delta']['tool_calls'][0]['function']['arguments'] ?? ''; + } + + private function makeUsage(array $data): Usage { + return new Usage( + inputTokens: $data['usage']['prompt_tokens'] + ?? $data['x_groq']['usage']['prompt_tokens'] + ?? 0, + outputTokens: $data['usage']['completion_tokens'] + ?? $data['x_groq']['usage']['completion_tokens'] + ?? 0, + cacheWriteTokens: 0, + cacheReadTokens: $data['usage']['prompt_tokens_details']['cached_tokens'] ?? 0, + reasoningTokens: $data['usage']['prompt_tokens_details']['reasoning_tokens'] ?? 0, + ); + } +} diff --git a/src/Features/LLM/Inference.php b/src/Features/LLM/Inference.php index 290b6cd9..7c6d5e7b 100644 --- a/src/Features/LLM/Inference.php +++ b/src/Features/LLM/Inference.php @@ -58,7 +58,7 @@ public function __construct( $this->config = $config ?? LLMConfig::load( connection: $connection ?: Settings::get('llm', "defaultConnection") ); - $this->httpClient = $httpClient ?? HttpClient::make($this->config->httpClient); + $this->httpClient = $httpClient ?? HttpClient::make(client: $this->config->httpClient, events: $this->events); $this->driver = $driver ?? $this->makeDriver($this->config, $this->httpClient); } diff --git a/src/Utils/Str.php b/src/Utils/Str.php index 5428b21a..a7fc5e8f 100644 --- a/src/Utils/Str.php +++ b/src/Utils/Str.php @@ -4,6 +4,10 @@ class Str { + static public function split(string $input, string $delimiter = ' ') : array { + return explode($delimiter, $input); + } + static public function pascal(string $input) : string { // turn any case into pascal case $normalized = self::spaceSeparated($input);