diff --git a/newrelic/config.py b/newrelic/config.py index 28f0f4f60..f4ccd7483 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2085,6 +2085,11 @@ def _process_trace_cache_import_hooks(): def _process_module_builtin_defaults(): + _process_module_definition( + "google.generativeai", + "newrelic.hooks.mlmodel_gemini", + "instrument_gemini_api_resources_chat_completion", + ) _process_module_definition( "openai.api_resources.embedding", "newrelic.hooks.mlmodel_openai", diff --git a/newrelic/hooks/external_botocore.py b/newrelic/hooks/external_botocore.py index 37bb65d79..e30ee0f55 100644 --- a/newrelic/hooks/external_botocore.py +++ b/newrelic/hooks/external_botocore.py @@ -202,6 +202,7 @@ def create_chat_completion_message_event( def extract_bedrock_titan_text_model_request(request_body, bedrock_attrs): request_body = json.loads(request_body) + request_config = request_body.get("textGenerationConfig", {}) input_message_list = [{"role": "user", "content": request_body.get("inputText")}] @@ -224,7 +225,7 @@ def extract_bedrock_mistral_text_model_request(request_body, bedrock_attrs): def extract_bedrock_titan_text_model_response(response_body, bedrock_attrs): if response_body: response_body = json.loads(response_body) - + output_message_list = [ {"role": "assistant", "content": result["outputText"]} for result in response_body.get("results", []) ] @@ -314,10 +315,10 @@ def extract_bedrock_ai21_j2_model_response(response_body, bedrock_attrs): def extract_bedrock_claude_model_request(request_body, bedrock_attrs): request_body = json.loads(request_body) - + if "messages" in request_body: input_message_list = [ - {"role": message.get("role", "user"), "content": message.get("content")} + {"role": message.get("role", "user"), "content": message.get("content")[0].get("text")} for message in request_body.get("messages") ] else: @@ -332,8 +333,9 @@ def extract_bedrock_claude_model_request(request_body, bedrock_attrs): def extract_bedrock_claude_model_response(response_body, bedrock_attrs): if response_body: response_body = json.loads(response_body) + role = response_body.get("role", "assistant") - content = response_body.get("content") or response_body.get("completion") + content = response_body.get("content")[0].get("text") or response_body.get("completion") output_message_list = [{"role": role, "content": content}] bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") bedrock_attrs["output_message_list"] = output_message_list diff --git a/newrelic/hooks/mlmodel_gemini.py b/newrelic/hooks/mlmodel_gemini.py new file mode 100644 index 000000000..087dbe9dc --- /dev/null +++ b/newrelic/hooks/mlmodel_gemini.py @@ -0,0 +1,1041 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import sys +import traceback +import uuid + +import google.generativeai as genai + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings + +GEMINI_VERSION = get_package_version("google.generativeai") +GEMINI_VERSION_TUPLE = tuple(map(int, GEMINI_VERSION.split("."))) +GEMINI_V1 = GEMINI_VERSION_TUPLE >= (1,) +EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE = "Exception occurred in gemini instrumentation: While reporting an exception in gemini, another exception occurred. Report this issue to New Relic Support.\n%s" +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Gemini instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" +STREAM_PARSING_FAILURE_LOG_MESSAGE = "Exception occurred in Gemini instrumentation: Failed to process event stream information. Please report this issue to New Relic Support.\n%s" + +_logger = logging.getLogger(__name__) + + +def wrap_embedding_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if ( + not transaction + or kwargs.get("stream", False) + or (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream" + ): + return wrapped(*args, **kwargs) + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + # Obtain attributes to be stored on embedding events regardless of whether we hit an error + embedding_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/embedding/Gemini") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = wrapped(*args, **kwargs) + except Exception as exc: + _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft, response) + return response + + +def wrap_chat_completion_sync(wrapped, instance, args, kwargs): + #_logger.debug("Wrapping chat completion sync") + #_logger.debug("Wrapped: %s", wrapped) + #_logger.debug("Instance: %s", instance) + #_logger.debug("Args: %s", args) + #_logger.debug("Kwargs: %s", kwargs) + + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + # If `.with_streaming_response.` wrapper used, switch to streaming + # For now, we will exit and instrument this later + if (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream": + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + request_message_list = kwargs.get("messages", []) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/Gemini") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + raise + + kwargs["prompt"] = args[0] + #_logger.debug("model_name: %s", instance.model_name); + kwargs["model"] = instance.model_name.replace("models/", "") + + _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + return return_val + + +def check_rate_limit_header(response_headers, header_name, is_int): + if not response_headers: + return None + + if header_name in response_headers: + header_value = response_headers.get(header_name) + if is_int: + try: + header_value = int(header_value) + except Exception: + pass + return header_value + else: + return None + + +def create_chat_completion_message_event( + transaction, + input_message_list, + chat_completion_id, + span_id, + trace_id, + response_model, + request_model, + response_id, + request_id, + llm_metadata, + output_message_list, + prompt_token_count, + candidates_token_count +): + settings = transaction.settings if transaction.settings is not None else global_settings() + # Loop through all input messages received from the create request and emit a custom event for each one + for index, message in enumerate(input_message_list): + message_content = "" + if "content" in message: + message_content = message.get("content") + + # Response ID was set, append message index to it. + if response_id: + message_id = f"{response_id}-{int(index)}" + # No response IDs, use random UUID + else: + message_id = str(uuid.uuid4()) + + chat_completion_input_message_dict = { + "id": message_id, + "request_id": request_id, + "span_id": span_id, + "trace_id": trace_id, + #"token_count": ( + # settings.ai_monitoring.llm_token_count_callback(request_model, message_content) + # if settings.ai_monitoring.llm_token_count_callback + # else None + #), + "token_count": prompt_token_count, + "role": message.get("role"), + "completion_id": chat_completion_id, + "sequence": index, + "response.model": response_model, + "vendor": "gemini", + "ingest_source": "Python", + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_input_message_dict["content"] = message_content + + chat_completion_input_message_dict.update(llm_metadata) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_input_message_dict) + + if output_message_list: + # Loop through all output messages received from the LLM response and emit a custom event for each one + for index, message in enumerate(output_message_list): + message_content = message.get("content") + + # Add offset of input_message_length so we don't receive any duplicate index values that match the input message IDs + index += len(input_message_list) + + # Response ID was set, append message index to it. + if response_id: + message_id = f"{response_id}-{int(index)}" + # No response IDs, use random UUID + else: + message_id = str(uuid.uuid4()) + + chat_completion_output_message_dict = { + "id": message_id, + "request_id": request_id, + "span_id": span_id, + "trace_id": trace_id, + #"token_count": ( + # settings.ai_monitoring.llm_token_count_callback(response_model, message_content) + # if settings.ai_monitoring.llm_token_count_callback + # else None + #), + "token_count": candidates_token_count, + "role": message.get("role"), + "completion_id": chat_completion_id, + "sequence": index, + "response.model": response_model, + "vendor": "gemini", + "ingest_source": "Python", + "is_response": True, + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_output_message_dict["content"] = message_content + + chat_completion_output_message_dict.update(llm_metadata) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_output_message_dict) + + +async def wrap_embedding_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if ( + not transaction + or kwargs.get("stream", False) + or (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream" + ): + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + # Obtain attributes to be stored on embedding events regardless of whether we hit an error + embedding_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/embedding/Gemini") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = await wrapped(*args, **kwargs) + except Exception as exc: + _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft, response) + return response + + +def _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft, response): + settings = transaction.settings if transaction.settings is not None else global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + try: + response_headers = getattr(response, "_nr_response_headers", {}) + input = kwargs.get("input") + + attribute_response = response + # In v1, response objects are pydantic models so this function call converts the + # object back to a dictionary for backwards compatibility. + if GEMINI_V1: + if hasattr(response, "model_dump"): + attribute_response = response.model_dump() + elif hasattr(response, "http_response") and hasattr(response.http_response, "text"): + # This is for the .with_raw_response. wrapper. This is expected + # to change, but the return type for now is the following: + # openai._legacy_response.LegacyAPIResponse + attribute_response = json.loads(response.http_response.text.strip()) + + request_id = response_headers.get("x-request-id") + response_model = attribute_response.get("model") + response_usage = attribute_response.get("usage", {}) or {} + organization = ( + response_headers.get("gemini-organization") + if GEMINI_V1 + else getattr(attribute_response, "organization", None) + ) + + full_embedding_response_dict = { + "id": embedding_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(response_model, input) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "request.model": kwargs.get("model") or kwargs.get("engine"), + "request_id": request_id, + "duration": ft.duration * 1000, + "response.model": response_model, + "response.organization": organization, + "response.headers.llmVersion": response_headers.get("gemini-version"), + "response.headers.ratelimitLimitRequests": check_rate_limit_header( + response_headers, "x-ratelimit-limit-requests", True + ), + "response.headers.ratelimitLimitTokens": check_rate_limit_header( + response_headers, "x-ratelimit-limit-tokens", True + ), + "response.headers.ratelimitResetTokens": check_rate_limit_header( + response_headers, "x-ratelimit-reset-tokens", False + ), + "response.headers.ratelimitResetRequests": check_rate_limit_header( + response_headers, "x-ratelimit-reset-requests", False + ), + "response.headers.ratelimitRemainingTokens": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-tokens", True + ), + "response.headers.ratelimitRemainingRequests": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-requests", True + ), + "vendor": "gemini", + "ingest_source": "Python", + } + if settings.ai_monitoring.record_content.enabled: + full_embedding_response_dict["input"] = input + full_embedding_response_dict.update(_get_llm_attributes(transaction)) + transaction.record_custom_event("LlmEmbedding", full_embedding_response_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc): + settings = transaction.settings if transaction.settings is not None else global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + model = kwargs.get("model") or kwargs.get("engine") + input = kwargs.get("input") + + exc_organization = None + notice_error_attributes = {} + try: + if GEMINI_V1: + response = getattr(exc, "response", None) + response_headers = getattr(response, "headers", None) or {} + exc_organization = response_headers.get("gemini-organization") + # There appears to be a bug here in openai v1 where despite having code, + # param, etc in the error response, they are not populated on the exception + # object so grab them from the response body object instead. + body = getattr(exc, "body", None) or {} + notice_error_attributes = { + "http.statusCode": getattr(exc, "status_code", None), + "error.message": body.get("message"), + "error.code": body.get("code"), + "error.param": body.get("param"), + "embedding_id": embedding_id, + } + else: + exc_organization = getattr(exc, "organization", None) + notice_error_attributes = { + "http.statusCode": getattr(exc, "http_status", None), + "error.message": getattr(exc, "_message", None), + "error.code": getattr(getattr(exc, "error", None), "code", None), + "error.param": getattr(exc, "param", None), + "embedding_id": embedding_id, + } + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + message = notice_error_attributes.pop("error.message", None) + if message: + exc._nr_message = message + ft.notice_error( + attributes=notice_error_attributes, + ) + # Exit the trace now so that the duration is calculated. + ft.__exit__(*sys.exc_info()) + + try: + error_embedding_dict = { + "id": embedding_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(model, input) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "request.model": model, + "vendor": "gemini", + "ingest_source": "Python", + "response.organization": exc_organization, + "duration": ft.duration * 1000, + "error": True, + } + if settings.ai_monitoring.record_content.enabled: + error_embedding_dict["input"] = input + error_embedding_dict.update(_get_llm_attributes(transaction)) + transaction.record_custom_event("LlmEmbedding", error_embedding_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +async def wrap_chat_completion_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + # If `.with_streaming_response.` wrapper used, switch to streaming + # For now, we will exit and instrument this later + if (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream": + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Gemini", GEMINI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/Gemini") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = await wrapped(*args, **kwargs) + except Exception as exc: + _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + raise + + kwargs["prompt"] = args[0] + #_logger.debug("model_name: %s", instance.model_name); + kwargs["model"] = instance.model_name.replace("models/", "") + + _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + return return_val + + +def _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val): + settings = transaction.settings if transaction.settings is not None else global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + request_message_list = kwargs.get("messages") or [] + stream = kwargs.get("stream", False) + # Only if streaming and streaming monitoring is enabled and the response is not empty + # do we not exit the function trace. + if not stream or not settings.ai_monitoring.streaming.enabled or not return_val: + ft.__exit__(None, None, None) + + # If the return value is empty or stream monitoring is disabled exit early. + if not return_val or (stream and not settings.ai_monitoring.streaming.enabled): + return + if stream: + try: + # The function trace will be exited when in the final iteration of the response + # generator. + setattr(return_val, "_nr_ft", ft) + setattr(return_val, "_nr_gemini_attrs", getattr(return_val, "_nr_gemini_attrs", {})) + return_val._nr_gemini_attrs["messages"] = kwargs.get("messages", []) + return_val._nr_gemini_attrs["temperature"] = kwargs.get("temperature") + return_val._nr_gemini_attrs["max_tokens"] = kwargs.get("max_tokens") + return_val._nr_gemini_attrs["model"] = kwargs.get("model") or kwargs.get("engine") + return + except Exception: + _logger.warning(STREAM_PARSING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + try: + # If response is not a stream generator, record the event data. + # At this point, we have a response so we can grab attributes only available on the response object + response_headers = getattr(return_val, "_nr_response_headers", {}) + response = return_val + + # In v1, response objects are pydantic models so this function call converts the + # object back to a dictionary for backwards compatibility. + if GEMINI_V1: + if hasattr(response, "model_dump"): + response = response.model_dump() + elif hasattr(response, "http_response") and hasattr(response.http_response, "text"): + # This is for the .with_raw_response. wrapper. This is expected + # to change, but the return type for now is the following: + # openai._legacy_response.LegacyAPIResponse + response = json.loads(response.http_response.text.strip()) + + _record_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response): + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + responseDict = response.to_dict() + try: + response_model = kwargs.get("model") + response_id = "" + response_usage = {} + finish_reason = "" + prompt_token_count = 0 + candidates_token_count = 0 + if response: + if "model" in response: + response_model = responseDict.get("model") + if "id" in response: + response_id = responseDict.get("id") + if "usage" in response: + response_usage = responseDict.get("usage") or {} + output_message_list = [] + finish_reason = None + candidates = responseDict.get("candidates") or [] + if candidates: + output_message_list = [ + candidates[0].get("message") or {"content": candidates[0].get("content").get("parts")[0].get("text"), "role": "assistant"} + ] + finish_reason = candidates[0].get("finish_reason") + + prompt_token_count = responseDict.get("usage_metadata").get("prompt_token_count") + candidates_token_count = responseDict.get("usage_metadata").get("candidates_token_count") + else: + response_model = kwargs.get("response.model") + response_id = kwargs.get("id") + response_usage = {} + output_message_list = [] + finish_reason = None + if "content" in kwargs: + output_message_list = [{"content": kwargs.candidates.get("content"), "role": kwargs.get("role")}] + finish_reason = kwargs.get("finish_reason") + prompt_token_count = responseDict.get("usage_metadata").get("prompt_token_count") + candidates_token_count = responseDict.get("usage_metadata").get("candidates_token_count") + request_model = kwargs.get("model") or kwargs.get("engine") + + request_id = response_headers.get("x-request-id") + organization = response_headers.get("gemini-organization") or getattr(response, "organization", None) + messages = kwargs.get("messages") or [{"content": kwargs.get("prompt"), "role": "user"}] + input_message_list = list(messages) + full_chat_completion_summary_dict = { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "request.model": request_model, + "request.temperature": kwargs.get("temperature"), + "request.max_tokens": kwargs.get("max_tokens"), + "vendor": "gemini", + "ingest_source": "Python", + "request_id": request_id, + "duration": ft.duration * 1000, + "response.model": response_model, + "response.organization": organization, + "response.choices.finish_reason": finish_reason, + "response.candidates.finish_reason": finish_reason, + "response.headers.llmVersion": response_headers.get("gemini-version"), + "response.headers.ratelimitLimitRequests": check_rate_limit_header( + response_headers, "x-ratelimit-limit-requests", True + ), + "response.headers.ratelimitLimitTokens": check_rate_limit_header( + response_headers, "x-ratelimit-limit-tokens", True + ), + "response.headers.ratelimitResetTokens": check_rate_limit_header( + response_headers, "x-ratelimit-reset-tokens", False + ), + "response.headers.ratelimitResetRequests": check_rate_limit_header( + response_headers, "x-ratelimit-reset-requests", False + ), + "response.headers.ratelimitRemainingTokens": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-tokens", True + ), + "response.headers.ratelimitRemainingRequests": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-requests", True + ), + "response.headers.ratelimitLimitTokensUsageBased": check_rate_limit_header( + response_headers, "x-ratelimit-limit-tokens_usage_based", True + ), + "response.headers.ratelimitResetTokensUsageBased": check_rate_limit_header( + response_headers, "x-ratelimit-reset-tokens_usage_based", False + ), + "response.headers.ratelimitRemainingTokensUsageBased": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-tokens_usage_based", True + ), + "response.number_of_messages": len(input_message_list) + len(output_message_list), + } + llm_metadata = _get_llm_attributes(transaction) + full_chat_completion_summary_dict.update(llm_metadata) + transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) + create_chat_completion_message_event( + transaction, + input_message_list, + completion_id, + span_id, + trace_id, + response_model, + request_model, + response_id, + request_id, + llm_metadata, + output_message_list, + prompt_token_count, + candidates_token_count + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc): + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + request_message_list = kwargs.get("messages", None) or [] + notice_error_attributes = {} + try: + if GEMINI_V1: + response = getattr(exc, "response", None) + response_headers = getattr(response, "headers", None) or {} + exc_organization = response_headers.get("gemini-organization") + # There appears to be a bug here in openai v1 where despite having code, + # param, etc in the error response, they are not populated on the exception + # object so grab them from the response body object instead. + body = getattr(exc, "body", None) or {} + notice_error_attributes = { + "http.statusCode": getattr(exc, "status_code", None), + "error.message": body.get("message"), + "error.code": body.get("code"), + "error.param": body.get("param"), + "completion_id": completion_id, + } + else: + exc_organization = getattr(exc, "organization", None) + notice_error_attributes = { + "http.statusCode": getattr(exc, "http_status", None), + "error.message": getattr(exc, "_message", None), + "error.code": getattr(getattr(exc, "error", None), "code", None), + "error.param": getattr(exc, "param", None), + "completion_id": completion_id, + } + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + # Override the default message if it is not empty. + message = notice_error_attributes.pop("error.message", None) + if message: + exc._nr_message = message + + ft.notice_error( + attributes=notice_error_attributes, + ) + # Stop the span now so we compute the duration before we create the events. + ft.__exit__(*sys.exc_info()) + + try: + # In a rare case where we are streaming the response and we do get back a request + # and response id, even though an error was encountered, record them. + response_headers = kwargs.get("response_headers") or {} + request_id = response_headers.get("x-request-id") + response_id = kwargs.get("id") + request_model = kwargs.get("model") or kwargs.get("engine") + error_chat_completion_dict = { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "response.number_of_messages": len(request_message_list), + "request.model": request_model, + "request.temperature": kwargs.get("temperature"), + "request.max_tokens": kwargs.get("max_tokens"), + "vendor": "gemini", + "ingest_source": "Python", + "response.organization": exc_organization, + "duration": ft.duration * 1000, + "error": True, + } + llm_metadata = _get_llm_attributes(transaction) + error_chat_completion_dict.update(llm_metadata) + transaction.record_custom_event("LlmChatCompletionSummary", error_chat_completion_dict) + output_message_list = [] + if "content" in kwargs: + output_message_list = [{"content": kwargs.get("content"), "role": kwargs.get("role")}] + create_chat_completion_message_event( + transaction, + request_message_list, + completion_id, + span_id, + trace_id, + kwargs.get("response.model"), + request_model, + response_id, + request_id, + llm_metadata, + output_message_list, + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def wrap_convert_to_gemini_object(wrapped, instance, args, kwargs): + """Obtain reponse headers for v0.""" + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + resp = args[0] + returned_response = wrapped(*args, **kwargs) + + if isinstance(returned_response, genai.gemini_object.GeminiObject) and isinstance( + resp, genai.gemini_response.GeminiResponse + ): + setattr(returned_response, "_nr_response_headers", getattr(resp, "_headers", {})) + + return returned_response + + +def wrap_base_client_process_response_sync(wrapped, instance, args, kwargs): + """Obtain response headers for v1.""" + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + nr_response_headers = getattr(bound_args["response"], "headers", None) or {} + + return_val = wrapped(*args, **kwargs) + return_val._nr_response_headers = nr_response_headers + return return_val + + +async def wrap_base_client_process_response_async(wrapped, instance, args, kwargs): + """Obtain response headers for v1.""" + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + nr_response_headers = getattr(bound_args["response"], "headers", None) or {} + return_val = await wrapped(*args, **kwargs) + return_val._nr_response_headers = nr_response_headers + return return_val + + +class GeneratorProxy(ObjectProxy): + def __init__(self, wrapped): + super(GeneratorProxy, self).__init__(wrapped) + + def __iter__(self): + return self + + def __next__(self): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.__next__() + + return_val = None + try: + return_val = self.__wrapped__.__next__() + _record_stream_chunk(self, return_val) + except StopIteration as e: + _record_events_on_stop_iteration(self, transaction) + raise + except Exception as exc: + _handle_streaming_completion_error(self, transaction, exc) + raise + return return_val + + def close(self): + return super(GeneratorProxy, self).close() + + +def _record_stream_chunk(self, return_val): + if return_val: + try: + if GEMINI_V1: + if getattr(return_val, "data", "").startswith("[DONE]"): + return + return_val = return_val.json() + self._nr_gemini_attrs["response_headers"] = getattr(self, "_nr_response_headers", {}) + else: + self._nr_gemini_attrs["response_headers"] = getattr(return_val, "_nr_response_headers", {}) + candidates = return_val.get("candidates") or [] + self._nr_gemini_attrs["response.model"] = return_val.get("model") + self._nr_gemini_attrs["id"] = return_val.get("id") + self._nr_gemini_attrs["response.organization"] = return_val.get("organization") + if candidates: + delta = candidates[0].get("delta") or {} + if delta: + self._nr_gemini_attrs["content"] = self._nr_gemini_attrs.get("content", "") + ( + delta.get("content") or "" + ) + self._nr_gemini_attrs["role"] = self._nr_gemini_attrs.get("role") or delta.get("role") + self._nr_gemini_attrs["finish_reason"] = candidates[0].get("finish_reason") + except Exception: + _logger.warning(STREAM_PARSING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_events_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + linking_metadata = get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + try: + gemini_attrs = getattr(self, "_nr_gemini_attrs", {}) + + # If there are no openai attrs exit early as there's no data to record. + if not gemini_attrs: + return + + completion_id = str(uuid.uuid4()) + response_headers = gemini_attrs.get("response_headers") or {} + _record_completion_success( + transaction, linking_metadata, completion_id, gemini_attrs, self._nr_ft, response_headers, None + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + finally: + # Clear cached data as this can be very large. + # Note this is also important for not reporting the events twice. In gemini v1 + # there are two loops around the iterator, the second is meant to clear the + # stream since there is a condition where the iterator may exit before all the + # stream contents is read. This results in StopIteration being raised twice + # instead of once at the end of the loop. + if hasattr(self, "_nr_gemini_attrs"): + self._nr_gemini_attrs.clear() + + +def _handle_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + gemini_attrs = getattr(self, "_nr_gemini_attrs", {}) + + # If there are no gemini attrs exit early as there's no data to record. + if not gemini_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + linking_metadata = get_trace_linking_metadata() + completion_id = str(uuid.uuid4()) + _record_completion_error(transaction, linking_metadata, completion_id, gemini_attrs, self._nr_ft, exc) + + +class AsyncGeneratorProxy(ObjectProxy): + def __init__(self, wrapped): + super(AsyncGeneratorProxy, self).__init__(wrapped) + + def __aiter__(self): + self._nr_wrapped_iter = self.__wrapped__.__aiter__() + return self + + async def __anext__(self): + transaction = current_transaction() + if not transaction: + return await self._nr_wrapped_iter.__anext__() + + return_val = None + try: + return_val = await self._nr_wrapped_iter.__anext__() + _record_stream_chunk(self, return_val) + except StopAsyncIteration as e: + _record_events_on_stop_iteration(self, transaction) + raise + except Exception as exc: + _handle_streaming_completion_error(self, transaction, exc) + raise + return return_val + + async def aclose(self): + return await super(AsyncGeneratorProxy, self).aclose() + + +def wrap_stream_iter_events_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled or not settings.ai_monitoring.streaming.enabled: + return wrapped(*args, **kwargs) + + return_val = wrapped(*args, **kwargs) + proxied_return_val = GeneratorProxy(return_val) + set_attrs_on_generator_proxy(proxied_return_val, instance) + return proxied_return_val + + +def wrap_stream_iter_events_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled or not settings.ai_monitoring.streaming.enabled: + return wrapped(*args, **kwargs) + + return_val = wrapped(*args, **kwargs) + proxied_return_val = AsyncGeneratorProxy(return_val) + set_attrs_on_generator_proxy(proxied_return_val, instance) + return proxied_return_val + + +def set_attrs_on_generator_proxy(proxy, instance): + """Pass the nr attributes to the generator proxy.""" + if hasattr(instance, "_nr_ft"): + proxy._nr_ft = instance._nr_ft + if hasattr(instance, "_nr_response_headers"): + proxy._nr_response_headers = instance._nr_response_headers + if hasattr(instance, "_nr_gemini_attrs"): + proxy._nr_gemini_attrs = instance._nr_gemini_attrs + + +def wrap_engine_api_resource_create_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + stream = is_stream(wrapped, args, kwargs) + return_val = wrapped(*args, **kwargs) + if stream and settings.ai_monitoring.streaming.enabled: + return GeneratorProxy(return_val) + else: + return return_val + + +async def wrap_engine_api_resource_create_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + stream = is_stream(wrapped, args, kwargs) + return_val = await wrapped(*args, **kwargs) + if stream and settings.ai_monitoring.streaming.enabled: + return AsyncGeneratorProxy(return_val) + else: + return return_val + + +def is_stream(wrapped, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + return bound_args["params"].get("stream", False) + + +def _get_llm_attributes(transaction): + """Returns llm.* custom attributes off of the transaction.""" + custom_attrs_dict = transaction._custom_params + llm_metadata = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + return llm_metadata + + +def instrument_gemini_api_resources_embedding(module): + if hasattr(module, "Embedding"): + if hasattr(module.Embedding, "create"): + wrap_function_wrapper(module, "Embedding.create", wrap_embedding_sync) + if hasattr(module.Embedding, "acreate"): + wrap_function_wrapper(module, "Embedding.acreate", wrap_embedding_async) + # This is to mark where we instrument so the SDK knows not to instrument them + # again. + setattr(module.Embedding, "_nr_wrapped", True) + + +def instrument_gemini_api_resources_chat_completion(module): + if hasattr(module, "GenerativeModel"): + if hasattr(module.GenerativeModel, "generate_content"): + wrap_function_wrapper(module, "GenerativeModel.generate_content", wrap_chat_completion_sync) + if hasattr(module.GenerativeModel, "generate_content_async"): + wrap_function_wrapper(module, "GenerativeModel.generate_content_async", wrap_chat_completion_async) + # This is to mark where we instrument so the SDK knows not to instrument them + # again. + setattr(module.GenerativeModel, "_nr_wrapped", True) + + +def instrument_gemini_resources_chat_completions(module): + if hasattr(module.Completions, "create"): + wrap_function_wrapper(module, "Completions.create", wrap_chat_completion_sync) + if hasattr(module.AsyncCompletions, "create"): + wrap_function_wrapper(module, "AsyncCompletions.create", wrap_chat_completion_async) + + +def instrument_gemini_resources_embeddings(module): + if hasattr(module, "Embeddings"): + if hasattr(module.Embeddings, "create"): + wrap_function_wrapper(module, "Embeddings.create", wrap_embedding_sync) + + if hasattr(module, "AsyncEmbeddings"): + if hasattr(module.AsyncEmbeddings, "create"): + wrap_function_wrapper(module, "AsyncEmbeddings.create", wrap_embedding_async) + + +def instrument_gemini_util(module): + if hasattr(module, "convert_to_gemini_object"): + wrap_function_wrapper(module, "convert_to_gemini_object", wrap_convert_to_gemini_object) + # This is to mark where we instrument so the SDK knows not to instrument them + # again. + setattr(module.convert_to_gemini_object, "_nr_wrapped", True) + + +def instrument_gemini_base_client(module): + if hasattr(module, "BaseClient") and hasattr(module.BaseClient, "_process_response"): + wrap_function_wrapper(module, "BaseClient._process_response", wrap_base_client_process_response_sync) + else: + if hasattr(module, "SyncAPIClient") and hasattr(module.SyncAPIClient, "_process_response"): + wrap_function_wrapper(module, "SyncAPIClient._process_response", wrap_base_client_process_response_sync) + if hasattr(module, "AsyncAPIClient") and hasattr(module.AsyncAPIClient, "_process_response"): + wrap_function_wrapper(module, "AsyncAPIClient._process_response", wrap_base_client_process_response_async) + + +def instrument_gemini_api_resources_abstract_engine_api_resource(module): + if hasattr(module, "EngineAPIResource"): + if hasattr(module.EngineAPIResource, "create"): + wrap_function_wrapper(module, "EngineAPIResource.create", wrap_engine_api_resource_create_sync) + if hasattr(module.EngineAPIResource, "acreate"): + wrap_function_wrapper(module, "EngineAPIResource.acreate", wrap_engine_api_resource_create_async) + + +def instrument_รง__streaming(module): + if hasattr(module, "Stream"): + if hasattr(module.Stream, "_iter_events"): + wrap_function_wrapper(module, "Stream._iter_events", wrap_stream_iter_events_sync) + if hasattr(module, "AsyncStream"): + if hasattr(module.AsyncStream, "_iter_events"): + wrap_function_wrapper(module, "AsyncStream._iter_events", wrap_stream_iter_events_async) diff --git a/tests/mlmodel_gemini/_mock_external_gemini_server.py b/tests/mlmodel_gemini/_mock_external_gemini_server.py new file mode 100644 index 000000000..5bfa69745 --- /dev/null +++ b/tests/mlmodel_gemini/_mock_external_gemini_server.py @@ -0,0 +1,769 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest +from testing_support.mock_external_http_server import MockExternalHTTPServer + +from newrelic.common.package_version_utils import get_package_version_tuple + +# This defines an external server test apps can make requests to instead of +# the real Gemini backend. This provides 3 features: +# +# 1) This removes dependencies on external websites. +# 2) Provides a better mechanism for making an external call in a test app than +# simple calling another endpoint the test app makes available because this +# server will not be instrumented meaning we don't have to sort through +# transactions to separate the ones created in the test app and the ones +# created by an external call. +# 3) This app runs on a separate thread meaning it won't block the test app. + +# STREAMED_RESPONSES = { +# "Stream parsing error.": [ +# { +# "Content-Type": "text/event-stream", +# "openai-model": "gpt-3.5-turbo-0613", +# "openai-organization": "new-relic-nkmd8b", +# "openai-processing-ms": "516", +# "openai-version": "2020-10-01", +# "x-ratelimit-limit-requests": "200", +# "x-ratelimit-limit-tokens": "40000", +# "x-ratelimit-remaining-requests": "199", +# "x-ratelimit-remaining-tokens": "39940", +# "x-ratelimit-reset-requests": "7m12s", +# "x-ratelimit-reset-tokens": "90ms", +# "x-request-id": "49dbbffbd3c3f4612aa48def69059ccd", +# }, +# 200, +# [ +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [ +# {"index": 0, "delta": {"role": "assistant", "content": ""}, "logprobs": None, "finish_reason": None} +# ], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": "212"}, "logprobs": None, "finish_reason": None}], +# }, +# ], +# ], +# "Invalid API key.": [ +# {"Content-Type": "application/json; charset=utf-8", "x-request-id": "4f8f61a7d0401e42a6760ea2ca2049f6"}, +# 401, +# { +# "error": { +# "message": "Incorrect API key provided: DEADBEEF. You can find your API key at https://platform.openai.com/account/api-keys.", +# "type": "invalid_request_error", +# "param": None, +# "code": "invalid_api_key", +# } +# }, +# ], +# "Model does not exist.": [ +# { +# "Content-Type": "application/json; charset=utf-8", +# "x-request-id": "cfdf51fb795362ae578c12a21796262c", +# }, +# 404, +# { +# "error": { +# "message": "The model `does-not-exist` does not exist", +# "type": "invalid_request_error", +# "param": None, +# "code": "model_not_found", +# } +# }, +# ], +# "You are a scientist.": [ +# { +# "Content-Type": "text/event-stream", +# "openai-model": "gpt-3.5-turbo-0613", +# "openai-organization": "new-relic-nkmd8b", +# "openai-processing-ms": "516", +# "openai-version": "2020-10-01", +# "x-ratelimit-limit-requests": "200", +# "x-ratelimit-limit-tokens": "40000", +# "x-ratelimit-remaining-requests": "199", +# "x-ratelimit-remaining-tokens": "39940", +# "x-ratelimit-reset-requests": "7m12s", +# "x-ratelimit-reset-tokens": "90ms", +# "x-request-id": "49dbbffbd3c3f4612aa48def69059ccd", +# }, +# 200, +# [ +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [ +# {"index": 0, "delta": {"role": "assistant", "content": ""}, "logprobs": None, "finish_reason": None} +# ], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": "212"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " degrees"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " Fahrenheit"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " is"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " equal"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " to"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " "}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": "100"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " degrees"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " Celsius"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": "."}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", +# "object": "chat.completion.chunk", +# "created": 1706565311, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {}, "logprobs": None, "finish_reason": "stop"}], +# }, +# ], +# ], +# } + +RESPONSES_V1 = { + "Model does not exist.": [ + {"content-type": "application/json; charset=utf-8", "x-request-id": "req_715be6580ab5bf4eef8d2b0893926ec9"}, + 404, + { + "error": { + "message": "The model `does-not-exist` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + } + }, + ], + "Invalid API key.": [ + {"content-type": "application/json; charset=utf-8", "x-request-id": "req_7ffd0e41c0d751be15275b1df6b2644c"}, + 401, + { + "error": { + "message": "Incorrect API key provided: DEADBEEF. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } + }, + ], + "You are a scientist.": [ + { + "content-type": "application/json", + "openai-organization": "new-relic-nkmd8b", + "openai-processing-ms": "1676", + "openai-version": "2020-10-01", + "x-ratelimit-limit-requests": "10000", + "x-ratelimit-limit-tokens": "60000", + "x-ratelimit-remaining-requests": "9993", + "x-ratelimit-remaining-tokens": "59880", + "x-ratelimit-reset-requests": "54.889s", + "x-ratelimit-reset-tokens": "120ms", + "x-request-id": "req_25be7e064e0c590cd65709c85385c796", + }, + 200, + { + "id": "chatcmpl-9NPYxI4Zk5ztxNwW5osYdpevgoiBQ", + "object": "chat.completion", + "created": 1715366835, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "212 degrees Fahrenheit is equivalent to 100 degrees Celsius. \n\nThe formula to convert Fahrenheit to Celsius is: \n\n\\[Celsius = (Fahrenheit - 32) \\times \\frac{5}{9}\\]\n\nSo, for 212 degrees Fahrenheit:\n\n\\[Celsius = (212 - 32) \\times \\frac{5}{9} = 100\\]", + }, + "logprobs": None, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 26, "completion_tokens": 75, "total_tokens": 101}, + "system_fingerprint": None, + }, + ], + "No usage data": [ + { + "content-type": "application/json", + "openai-organization": "new-relic-nkmd8b", + "openai-processing-ms": "324", + "openai-version": "2020-10-01", + "x-ratelimit-limit-requests": "10000", + "x-ratelimit-limit-tokens": "60000", + "x-ratelimit-remaining-requests": "9986", + "x-ratelimit-remaining-tokens": "59895", + "x-ratelimit-reset-requests": "1m55.869s", + "x-ratelimit-reset-tokens": "105ms", + "x-request-id": "req_2c8bb96fe67d2ccfa8305923f04759a2", + }, + 200, + { + "id": "chatcmpl-9NPZEmq5Loals5BA3Uw2GsSLhmlNH", + "object": "chat.completion", + "created": 1715366852, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello! How can I assist you today?"}, + "logprobs": None, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 9, "total_tokens": 19}, + "system_fingerprint": None, + }, + ], + "This is an embedding test.": [ + { + "content-type": "application/json", + "openai-model": "text-embedding-ada-002", + "openai-organization": "new-relic-nkmd8b", + "openai-processing-ms": "17", + "openai-version": "2020-10-01", + "x-ratelimit-limit-requests": "3000", + "x-ratelimit-limit-tokens": "1000000", + "x-ratelimit-remaining-requests": "2999", + "x-ratelimit-remaining-tokens": "999994", + "x-ratelimit-reset-requests": "20ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_eb2b9f2d23a671ad0d69545044437d68", + }, + 200, + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": "/PewvOJoiTsN5zg7gDeTOxfbo7tzJus7JK3uu3QArbyKlLe8FL7mvOAljruL17I87jgFvDDiBTqEmZq7PQicPJDQgDz0M6I7x91/PMwqmbxwStq7vX6MO7JJdbsNk+27GWEavNIlNrycs5s8HYL1vPa5GDzuOIW8gOPHOy5eXrxUzAK8BlMGvb8Z8bvoqPA5+YIKPEV2EL2sTtg8MSfQu6/BLzyhAgS7cEgLO+MD7ryJUTy7ikBsOz6hsTeuKJq86KYhvIrqUTyhrrg8hdyVu4RDAL1jzik7zNZNO0JZUzzFqQW7dplCPHwrpjtA0Y287fWJvK/BrzzCNi68/9PBO7jZCbwfBp26vDuRO7ukSjxX6448nLMbPLv65Dz7Xps8A4qUu4d1K7q1vMw87V7DutssQLwSjLu7Rg8mPPg/DzyKlLc6AbDSvLw9YLx3Mtg8ugu1OUmXa7szA2G8ZgDVPGkdEryNB4+5DxcVPAv4iLod1sA7UkjbO+osGLyrC908x4WWu2v5ojzBSU08QllTvHpR5Lu/w1Y7oNT2ulhBKbwfsAK90YwgvISZmjp/oEy8FL7mPAGugzxeK/a7rigauXGLBrwG/zq9xuwAO5EVSzzoUlY7P5DhvL+wN7xiN2O7rKTyPEy29zyVdQM8e5KQvDsu2jwYHp+5NJz2uw8XFb2olra8ul+AvJDl7jvOsI88b1uqPKkvTDzJY/a8wAbSPDF7m7yOoKQ87+Q5vaxOWLwILxc8ZkGBPExejry6tRq8SZfru9wZoTzUqyy64mrYOnyBwDusoqO8X2yiOfteG7sxfWq6Zqo6PFEDkbsZCwA9weATPKZkC7wbUMq87Uuku8PPQ7o3Y5k8rPg9vCWYALwusqk8gxXzPIwaLjycsxu62pMqPGl1e7xN96O802ixPOnpHL2lIRA88/L1Oq9rFbv1eOw7GvqvuwyRnry4hb68gY/8vFOJhzx7kpA8xBI/PPTdB7zz8Ka8g6w5PMuT0rs2zFI77AipvLJJ9TyKQOw8jkoKPKykcjoPbS+/fNfauWFIMz1O5IS8N7mzPAfu6jwWRF08UzW8vCzYZ7xdPMY8BM0PvFBsSrzbLMC8GN1yvNwZoTvFqYW7sfGLuzsu2rwv93M8iLr1PPf8E7znY6Y8U4tWPBHzpbuNXSm7z/MKPMbsADwp+oe8/PewPCzY5zoYyIS8Mr6WPHMmazzGRGo85EQaPZyzm7wLt9y8jkoKO+732DqZlt48Bv+6vM9cxDj4P488jMSTugGug7xLca08m8Y6uyd0kbtt1bO6s4qhOgw7BLwsgs08SxsTvHJ6trvxarA796jIvNuC2jwD4C48n9KnPMwqGTxHUqG7n3yNPHaZQrw+obG6QWzyvKa6pTyHdau8eqUvPVrHnzsIhTG9UQVgPIYyMDtRW/q7YFtSu3aZwjzOBio9pXl5PMAGUrwl7hq8ul+APLdCwzuqcke81K17vNMUZjyqcsc7ZpcbO/OaDL2V3rw6s+C7O4NWnzvKpKI8tqmtO/QzIr0eb9a8mtcKvAlyEjxac1Q8yMiRPHZF9zymuqW7cTe7uxcxvrsqpjw8v8PWO/XMtzzhffe7uIU+uuMBnzxD8ui7gDeTvKjqAb1P0zS8vSrBu/hB3jondJG8C04jvVkwWTxgWYM8WwxqPGjcZbz3/uI7m8a6O4K9iTw/5ns87LKOu19sIrvzmow63BmhvOimIbys+D08jMZiuycgxrvTEpc8G/x+vPz3sDyz4Ds8ebhOPBS8l7vNbRQ8fluCvByTxbqIYow8HSzbOmtPPbwFaPS81yDTOglyEr3k8E47B+5quopAbLs6Qfm4VbuyOyGOYrsv9aQ7x9swvLfsKLx9cPC7aranvIhiDDs8xaC8riiaug/DyTwPGWS8mKl9u6tfKLznDYw7cyZrvN5e6zzAnZi8XxYIvfJZYLvGQpu8W7QAO2HyGDylzUS7Kj2DPPiVqbyZlI+8HdbAu5y16ryPj1S7OKaUugUQizpwSlo7YosuPKf/bzxGDyY87fUJPD2ygTp279w7+dgkuxkLgDzc2PS8ghMkPB2C9btml5s7U98hvETfyTttf5k74CUOPOmTAj1pycY7PHHVuqFYHjutO7m6wUnNvAykvbxpcyy8EFzfPOhSVjx6+8k80PVZu2LhyLzbgto7ABe9O598jTzjA+67a/kiPAUQC7ytj4S8tWayvJtwIDsWmKg8RXYQOYY0f7xYLgo8ybdBPLM2Vjy429g8MDggvYqUt7uPj9Q8Vv6tPOK+ozxBbHI8cs6BvE99mjyIYgw84X33PMs9uDt/9Bc8WYbzPMGf5zzKToi8eMvtPFOJhzww4gU9YZ5NPAyRnrsFvL88/uZgvDZ2ODsyass7reUePAZTBj2429i7PqGxvFR4tzuqxpI8QWojPZ7lxjseb9Y7mKl9ur3UpjpFzKo88RSWvMkNXLyvKuk54CWOvBAGxTvjq4S8E882vN/ikjuViKK72LnoO34HNzubGgY9lw4ZPIi6dTzUrfs7eHVTO/2j5byANxM9YZ5Nuy0ZFLz897C8OajjvL8Z8Tsb55C82VCvO6Tg47lGD6Y8UQXgPEwKw7wSOPA7elHkPD6hsTy/sLc7TaGJO1DAlbztXkO6lTTXuzF96jvCOP26Ff+SPPFs/7thns268llgvD1etrxWZ+c75PDOO5U0VzyXDpm8BRLau2AFuDxco7A8jV/4u7ZTEzyOoKQ8xLwkPK8qabyrCY68Bby/u4cfkTyj3hQ9OuvePBX/kjtdkuC7pODju62PBLwfBh29/icNPEYPJruhWB48h3WrvJV1g7xEia88dkX3u4rqUTwiJ/g7tRAYunNnF7ynUzs8JFdUO3zVC7wFElq8QH3Cu9GO77rXdB48TjzuvFM1PDy8PeA6QNGNOr1+DL0CR5k7TjqfPDosC7zc2PQ4EuAGvDimFD0sgs07H7LRPP46rDxvW6o6t5hdO/z3MDwV/xK8+dgkPN5e6zu42Qm8uRyFvJnqKbp81Qu7ld68u1yjsDzenxe6sp3AvIMAhbxrowg8aR2SOw/DSbxHUiG8xzFLvJ32Fr3wfc+8e+p5vAJJ6Lt/9ua8M63GvBbuwrx9cHC8NyJtvNaHvbqIuCa8ek8Vu1JGjLxh8hi97jgFvTNXLDr/Kdw77AipPEPwGbxr+/G7EfMlvB+y0TuIDsG7oQIEPJDQgLyHH5G7qh58PO97gLuIuvU6vheiuoxwSLx2mcI8pc1EPAGuAzvujp+868UtPIY0/7oZYRo88WqwPFfrjrvGQpu8YLHsO85v47wQXF87pSGQu+k/NzyziqE5QNNcuwBrCDzYt5m87k3zOnYwibybxjq8y+lsu8WpBboCSeg8wyMPvNGMILzrb5M8L/dzvMgerLuqHK26GMgEvCBJGDyUm0E8skl1O1jt3TwUvma8bOhSuzwbu7ya1wo8wozIvL/D1jz5goq8TuSEvMMjD7wYytO8aranvE2j2DohjuI7VHg3O6f/b7wUErI7f6DMuldUSLw8G7u8B5jQvKrGEr1LHeI8lJvBvHvqeTthSDM8rtROPGtPvbzbgAu9CIUxO6Pxs7xUeLe8qsaSPAsN9zxYl0M8c70xPWBbUryK6II8k+8MPDe5MzrNbZQ8UQMROSMSirzGmLW6f/QXO3XtjTz/08G7HSxbO2v5ojtbChs82VCvPNzDBr3cw4Y7c2cXvC/387tLG5O80xTmPGZBgbxNo9g77zrUvAPgLrxBFAm5Wh06PLKdQDzc2HQ8iuiCPFsM6rtWEc06Nna4O+6OH7w8bwa8s+A7vDRGXLxac1S8a6OIPC/387ozrcY8TuSEvP2j5Tw6QXk76KhwPGfaFrxqYA28pXeqOy9LP7qXDpm8he80vasJDr0FZiW852OmuqGuuDyMxuK8sUcmPIRDgLwDNsm7hjT/u4chYDyQ0k895TNKO0Fs8jxBFtg8EZ/auvXMN7zGROo8BqkgOis/0joD4v08ebjOPHzVi7xEia+8EFzfuKV3KrymuqU7zrCPvKTgYzyNBw889mXNvL4XorumZlo7xzFLvd5e6zr1dp28eMvtu1TO0bylzUS7yvq8vGzmAz0yasu8Q0a0PGI1FLvsx/y7DKS9u2nJxjsjFFm7bwUQO3LOAT0H7uo8KA0nvPIDxrtlvVk8JK3uO7b/x7qP4588pmZavEcR9TwcPau8cEiLvHZFdzw4/K48IPVMPGpgjTshjmI7lJvBvKdTu7v03Ye7YLHsPFDCZLw2IB67jMbiu/SJPLzmIno8nUyxu3HhoLylIZC7kwT7vMCdmLsfXLc7u6TKuvB9zzxqtie8Ses2vMgerDxT36E8AkeZvNd0Hrtk0Hg8+7Q1PJDQgLuDrDm8WC4KPGl1e7xl/oU8Is+OvK8q6brnDQy9A4oUvDTwQTy7+BW7SxuTvO97gLwWmCi8t+53PJYhuLun/SC8DxcVO6Zki7vB87I7jMZiPDYgHrviFL46TaNYujzFIDsX2yM7wyXevMMl3rtWqBM8Wdq+PIF6Drt+W4K8VOHwvI+NhbsYdDm8F4WJPHgfubuqHny8WwobPF08xjuGNP86Ff8SO1kw2byiWm27PMdvvJe6zbr5l/i8y+nsukV2kLtU4fA7AbBSPD86x7x3Mti8sfELOpbLHb1819q8fXDwu73UJrw2zFI6KpOdPADDcbx6TxU9Oy5aPMHzsjz5Lr+8M63GOzSaJzuzdwK9AGuIvJtwIDz+kMa8WYQkvHeGIzss2Oc6FavHO6ZkCz0Y3fI69XhsvL1+jLqej6w7bYHoOy+fijvv5Dk8Y86pu/3kETwtby671yDTNxN5HLuoQBy8DJGeO5dksznq2Ew8wZ/nvOA4LTxXAH2880ZBvMjIkTy7+uS7CC8XvJDQAD0Abdc8MmpLu7znRbxs5oM8OemPvEOczryNX3g7MXubvOhQB70fCOy8DT3TujzHb7xWEU250iW2O4DjR7zPCPm7XimnPE7kBDwqUvG846sEvGfttbxU4fA7e+iqvO97gLtWZ2c7VqiTuZWKcTxCV4Q8xakFO2v5Irv4Qd47njmSPIQCVLo/jpI8TGBdPsZEarwH7Bu7BqmgPP4njTuUMgg8s3eCOEXMKjsMO4S7GQsAPBKMuzzdBgI87UskulBsSrtWqBO8du9cvNYzcrwS4Ia8XearvLyRq7xOPO67GB6fvJG/MDwX26O8A4qUPDF7G7xP0zS8lxDouy9LPzxGeN88iLgmvP8p3DvGROo8k+8MOx8GnbzF/5+74r6jPGySODzLPbg8Z+21Oz+OEj1sPJ68vX4MPH5bArvheyg8U4kHPIuBmLyNCV67IxIKuzsu2jxGDyY7he80vFKcpjzbgIs8BHlEu0xgXbz7tLU8Gg1PO/g/jzyT7wy8xpg1OuIUPjxEMxW89XhsPP2jZbzQNoY8GN3yvHm4zjsRnQs8K+k3PAw7BDwUvua7cJ4lPAFauLs2ygO9/aPlvINWHz3W3dc83+ISPcCdmDzb1qW7ryrpuwGuA7w+obG8kb8wvYn7Ib0PGWQ8cTe7vCfKqzotxci8cY1Vu0sdYju0edG7hEOAvJmW3jpdkBE8sfGLPJOu4DxP07Q7hAJUvAphwrynpwY9X2wiPZmW3rtc+co6iGRbPKpyx7xXAH28n3yNOwJJ6LqCEyS8TuQEvbDD/jt9xLu8pmSLu3aZQjwLDfc8wjj9u3uUXzz8oRa8SoTMvF7TDDwdgnW7WYZzPO6hvjrEvnO8TpC5vDosC7xPJwA88NNpvGl1ezwi0d27cYuGPG1/mTvw0+m6UMJkPGI1lDsxe5s89DOivMgeLDyTruA8PMUgvEvHR7xRW/q6+sWFvKQ0r7y6Yc88jBouujWHiDvyV5G8mxoGvSfKK7zenxc7bYFouxGf2jyK6IK8hJkavKPxM70xJ9C7F4UJvAW8P72ySXW7jBquPPpxurznDYy8QlcEvMHzMr4S4tU7opsZPNGMIL2DVp+7H7LRug6ATjxTiYe8FGhMvI/jH7wxJ1A8riiavA2T7byJp1a8sfGLu6ZkizyQ5W67cY3VvNSrLD1OPG48nyjCPD2yAb3YuWg8nLXquhh0ubsO1Jm7aXV7u12QET0kATo8cEgLvDUzPTwV/xK8cYsGPVxNlrssLLM70J8/POTwTj3dsja8fl3Ruh8I7Dua14o8EjYhPIzG4rxHUiE8+OvDOzA4oDwxJQE9T9O0u/yhFj1lvVm84mgJPNO+y7wjvj48hjKwO9zDhjymZIs7mZQPu1odujwkrW48c2eXu4WbaTxGuQu95w/bOhtQyrz6G6C8oVievMWpBb0+TeY7H7ACOkIDuTrIdEa83VwcuEWLfjxBwL07iGKMOwsN9zxket67+dikPPf8E7tmQQE8ifuhuzREDT1mqjo8fRiHO5cQ6DsG/zq8r2sVPPkuPzinqdW8QWzyu9bdVzwUEjK9dzLYuiLRXbyJ+6E79SCDPOREGrvOsI87aIbLO9ceBLu5HIW82GPOu0cRdbx3Mtg81yDTPPSJPDtGZUC8/pBGPMcxSz1BwD257fWJOi6yqbyBJsM87V7DPC6yqTte1ds8uXTuOrHxi7yzdwI8HSxbO3aZQj1LG5O7a/vxvOHRwjyKPh27tlOTvE8pz70nyqu8t5aOPKZkCz0aY2m7uNkJPff+YjzxarA891KuvNo/Xzy2qa28s3cCvWZBgbxmVm+8xu7PvLkcBTvTFOY8Xn/BO5Z30rwSNiE9pXl5u/ZlTbxFdhA9RN/JvIzGYrpgBTg85PDOvN/iEjymuiU830tMOhjd8rq9KsG8rPg9udfKuLz1dh26KpMdvB4ZPLx1Qyi80YwgPb2Terxk0Hg8rZFTOjRGXLyfKEK8ZVSgux3WwDpBFIm8xakFuz+Q4bsFaPS8s+C7vLPgOzvpP7e8JjGWPIvXsjwh4q08zm9jPO31Cbwew6G8a089Oa9rFb1Vu7K76y7nvABtV7seb9Y7mKcuO9/1Mb1OOh+80xTmu/OcWzzakyq8LNYYOz6hsbvKpKI89/7ivOhQhzqRaZa8k+8MvQb/ujwAwSK6WC6KvLeY3bvZ+pQ85iL6vC4IRLnNw647dBNMvFQiHTz+5mA61FUSvcVVujgiJSk8OkH5PBeFCb2lefm8YyREO7Pgu7tYl0O8qOzQO5B8NTy+F6K8KVAivGW9Wb0izw48c70xO9A2Br2pg5e8cEpau18WCLxac9S77bTdOmZBAT3xFBY7kNAAPFYRzbu6X4A8wyOPvLZTE7y9k/o8RrkLOwZThjy8PeA8oa64uwOKlLy9k3o6vOfFPBw9K7w8xSC8YLFsvDDkVDwmM+W8IPXMux8GnTvdBgK9+Zd4vPTdhzyww367JjPlu51MMTxIqLs8PqExPbkeVDz4lSm8opsZvbCukDxZ2j67VWUYvH6xnDw/Oke8U4tWun1w8DvEvvM7JjNlPH/0l7uWITi8kNAAPKjs0LteKae7ifshPBGdCzlVuzI8LrIpvLFaRT1xi4Y86KhwPFylfzvt9Qk82yxAPAEEHrxD8Jm70ntQPD2yAbwQXN+71ZpcOzUzPTzyVxE8svNaPOsbSDz5goo8cY1Vu7OKIbyOoKQ8ZLuKPJqDvzsG/zq8phBAPF08RjwYHp88njmSOoSZmjyzd4K7L/UkvIK9ibzkmjQ8jqAkPGQRJbtaHbq8uC8kvAb/ujlwnqU8vDsRPVEDkTyNXSk9s+A7PP2j5bv2DzO9vSpBvEUixTu2VeK7KHbgvJEVyzuww345bOjSOpPvjLusTti8RrkLPPa5GLzjAR+8iugCvHyBwLv1IAO9h8vFPL0qQbxO5IQ7OVLJPPJZYDy/GfE8NEQNu2S7CjyySfW7YfKYPKQ0L7tgsey7elHkO+hQh7zYDTQ7YFmDvPrFBTq7+uS8o94UPKHB1zvNbZQ9qdmxPF/CPLwkV1Q8YK8dvDzH7zuiBNM8rEwJPOYierzGQpu8a0+9O8pOCDsvSz+8O4Ilvaa6pbz3Uq67umFPO6V3KjzUV2G8Nd2iPK9rFTybcKA8xzHLPGYAVTyXEOi7a/txPBS+5jy+F6K83MOGvL0qQbytkVM8LNjnO1yjML3dXJy88H1PPLHxi7vQNoY6bwWQuvYPszzzRkE8BqmgPFfrDj29gNu8Wscfu2KLLjxmqjo7hohKvK8qabzVmA28", + } + ], + "model": "text-embedding-ada-002", + "usage": {"prompt_tokens": 6, "total_tokens": 6}, + }, + ], +} + +# STREAMED_RESPONSES_V1 = { +# "Invalid API key.": [ +# {"content-type": "application/json; charset=utf-8", "x-request-id": "req_a78a2cb09e3c7f224e78bfbf0841e38a"}, +# 401, +# { +# "error": { +# "message": "Incorrect API key provided: DEADBEEF. You can find your API key at https://platform.openai.com/account/api-keys.", +# "type": "invalid_request_error", +# "param": None, +# "code": "invalid_api_key", +# } +# }, +# ], +# "Model does not exist.": [ +# {"content-type": "application/json; charset=utf-8", "x-request-id": "req_a03714410fba92532c7de2532d8cf46c"}, +# 404, +# { +# "error": { +# "message": "The model `does-not-exist` does not exist", +# "type": "invalid_request_error", +# "param": None, +# "code": "model_not_found", +# } +# }, +# ], +# "You are a scientist.": [ +# { +# "content-type": "text/event-stream", +# "openai-model": "gpt-3.5-turbo-0613", +# "openai-organization": "new-relic-nkmd8b", +# "openai-processing-ms": "6326", +# "openai-version": "2020-10-01", +# "x-ratelimit-limit-requests": "200", +# "x-ratelimit-limit-tokens": "40000", +# "x-ratelimit-remaining-requests": "198", +# "x-ratelimit-remaining-tokens": "39880", +# "x-ratelimit-reset-requests": "11m32.334s", +# "x-ratelimit-reset-tokens": "180ms", +# "x-request-id": "f8d0f53b6881c5c0a3698e55f8f410ac", +# }, +# 200, +# [ +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [ +# {"index": 0, "delta": {"role": "assistant", "content": ""}, "logprobs": None, "finish_reason": None} +# ], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": "212"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " degrees"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " Fahrenheit"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " is"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " equal"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " to"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " "}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": "100"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " degrees"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": " Celsius"}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {"content": "."}, "logprobs": None, "finish_reason": None}], +# }, +# { +# "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", +# "object": "chat.completion.chunk", +# "created": 1707867026, +# "model": "gpt-3.5-turbo-0613", +# "system_fingerprint": None, +# "choices": [{"index": 0, "delta": {}, "logprobs": None, "finish_reason": "stop"}], +# }, +# ], +# ], +# } +RESPONSES = { + "Invalid API key.": ( + {"Content-Type": "application/json; charset=utf-8", "x-request-id": "4f8f61a7d0401e42a6760ea2ca2049f6"}, + 401, + { + "error": { + "message": "Incorrect API key provided: invalid. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": "null", + "code": "invalid_api_key", + } + }, + ), + "Embedded: Invalid API key.": ( + {"Content-Type": "application/json; charset=utf-8", "x-request-id": "4f8f61a7d0401e42a6760ea2ca2049f6"}, + 401, + { + "error": { + "message": "Incorrect API key provided: DEADBEEF. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": "null", + "code": "invalid_api_key", + } + }, + ), + "Model does not exist.": ( + { + "Content-Type": "application/json", + "x-request-id": "cfdf51fb795362ae578c12a21796262c", + }, + 404, + { + "error": { + "message": "The model `does-not-exist` does not exist", + "type": "invalid_request_error", + "param": "null", + "code": "model_not_found", + } + }, + ), + "No usage data": [ + { + "content-type": "application/json", + "openai-model": "gpt-3.5-turbo-0613", + "openai-organization": "new-relic-nkmd8b", + "openai-processing-ms": "6326", + "openai-version": "2020-10-01", + "x-ratelimit-limit-requests": "200", + "x-ratelimit-limit-tokens": "40000", + "x-ratelimit-limit-tokens_usage_based": "40000", + "x-ratelimit-remaining-requests": "198", + "x-ratelimit-remaining-tokens": "39880", + "x-ratelimit-remaining-tokens_usage_based": "39880", + "x-ratelimit-reset-requests": "11m32.334s", + "x-ratelimit-reset-tokens": "180ms", + "x-ratelimit-reset-tokens_usage_based": "180ms", + "x-request-id": "f8d0f53b6881c5c0a3698e55f8f410ac", + }, + 200, + { + "id": "chatcmpl-8TJ9dS50zgQM7XicE8PLnCyEihRug", + "object": "chat.completion", + "created": 1701995833, + "model": "gpt-3.5-turbo-0613", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "212 degrees Fahrenheit is equal to 100 degrees Celsius.", + }, + "finish_reason": "stop", + } + ], + "usage": None, + "system_fingerprint": None, + }, + ], + "This is an embedding test.": ( + { + "Content-Type": "application/json", + "openai-organization": "new-relic-nkmd8b", + "openai-processing-ms": "54", + "openai-version": "2020-10-01", + "x-ratelimit-limit-requests": "200", + "x-ratelimit-limit-tokens": "150000", + "x-ratelimit-remaining-requests": "197", + "x-ratelimit-remaining-tokens": "149994", + "x-ratelimit-reset-requests": "19m45.394s", + "x-ratelimit-reset-tokens": "2ms", + "x-request-id": "c70828b2293314366a76a2b1dcb20688", + }, + 200, + { + "data": [ + { + "embedding": "SLewvFF6iztXKj07UOCQO41IorspWOk79KHuu12FrbwjqLe8FCTnvBKqj7sz6bM8qqUEvFSfITpPrJu7uOSbPM8agzyYYqM7YJl/PBF2mryNN967uRiRO9lGcbszcuq7RZIavAnnNLwWA5s8mnb1vG+UGTyqpYS846PGO2M1X7wIxAO8HfgFvc8s8LuQXPQ5qgsKPOinEL15ndY8/MrOu1LRMTxCbQS7PEYJOyMx7rwDJj+79dVjO5P4UzmoPZq8jUgivL36UjzA/Lc8Jt6Ru4bKAL1jRiM70i5VO4neUjwneAy7mlNEPBVpoDuayo28TO2KvAmBrzzwvyy8B3/KO0ZgCry3sKa6QTmPO0a1Szz46Iw87AAcPF0O5DyJVZw8Ac+Yu1y3Pbqzesw8DUDAuq8hQbyALLy7TngmPL6lETxXxLc6TzXSvKJrYLy309c8OHa0OU3NZ7vru2K8mIXUPCxrErxLU5C5s/EVPI+wjLp7BcE74TvcO+2aFrx4A9w80j+Zu/aAojwmzU08k/hTvBpL4rvHFFQ76YftutrxL7wyxgK9BsIevLkYkTq4B028OZnlPPkcgjxhzfS79oCiuB34BbwITTq97nrzOugwRzwGS1U7CqTgvFxROLx4aWG7E/DxPA3J9jwd+AU8dVWPvGlc2jzwWae57nrzu569E72GU7e8Vn9+vFLA7TtVbZE8eOCqPG+3Sjxr5/W8s+DRPE+sm7wFKKQ8A8A5vUSBVryeIxk8hsqAPAeQjryeIxm8gU/tuxVpoDxVXM250GDlOlEDwjs0t6O8Tt6rOVrGHLvmyFy6dhI7PLPxlbv3YP88B/YTPEZgCrxqKsq8Xh+ou96wQLp5rpo8LSg+vL63/rsFjqk8E/DxPEi3MDzTcw66PjcqPNgSfLwqnaK85QuxPI7iHL2+pRE8Z+ICOxzEELvph+07jHqyu2ltnrwNQMC82BL8vAOdiDwSqo88CLM/PCKFBrzmP6a85Nc7PBaM0bvh1VY7NB2pvMkF9Tx3New87mgGPAoKZjo+nS+/Rk/GucqwMz3fwYS8yrCzPMo56jyDHV08XLe9vB4+aLwXwMY8dVUPvCFATbx2eMC8V7NzvEnrpTsIxIO7yVmNu2lc2ryGQnM8A6/1PH/VFbySO6g80i5VPOY/prv6cyi7W5QMPJVP+jsyLIi84H6wPKM50DrZNIS8UEaWPPrIaTzvrmg8rcoaPRuQm7ysH9y8OxIUO7ss4zq3Od08paG6vAPAuTjYAI88/qmCuuROhbzBMK08R4M7u67+j7uClKa6/KedOsqNArzysM08QJ8UvMD8t7v5P7M799fIvAWx2jxiEi48ja6nPL0LFzxFkpq7LAWNPA1AQLyWlLO6qrfxvOGypTxJUau8aJ8uPceLnTtS0TG9omtgPO7xPDvzbfm7FfJWu2CqwzwAASk96FN4PLPgUbwRdhq8Vn9+PLk7wjs8NUW84yx9vHJCZjzysM079hodO/NbDL2BxrY6CE26OzpEpDv7DaM8y0quO41IIr1+Kte8QdMJvKlxDzy9+lI8hfyQPA3J9jzWmKS7z6O5u4a5vLtXKj088XzYO1fEtzwY4/e7Js1NugbCnjymxOu7906SvPSPAb1ieDO8dnjAu/EW0zp/b5C8mGIjvWTPWTwIxIM8YgFqPKvrZrwKpOA7/jK5O2vViDyfaXs8DR2Pu0AFGrvTc446IIOhvDreHrxRnTw8ROdbu55Gyrsht5Y8tVmAvHK5rzzZvTo8bx1QPMglmLvigBU8oIuDvAFYz7pblIw8OZnlOsTvPbxhzfS8BxnFOpkwE72E60w7cNp7utp6ZrtvHdC4uwmyO5dRX7sAm6M7kqEtvElRK7yWg++7JHanvM6ACDvrZqG8Xh+oupQsyTwkZWO8VzuBu5xVKbzEZoc7wB9pvA796zyZlpi8YbsHvQs+W7u9cZy8gKMFOxYDGzyu7Uu71KeDPJxVqbxwyI68VpDCu9VT67xKqFG7KWmtuvNteTocs0w7aJ8uPMUSbzz6cyg8MiwIPEtlfTo+wOA75tkgu7VZgDw8WPa8mGIjPKq38bsr0Zc7Ot4evNNiyju9C5c7YCENPP6pAj3uV8I7X3bOusfxIjvpZLy655bMvL9ivbxO3iu8NKbfPNe7VTz9ZMk88RZTu5QsybxeQtk7qpTAOzGSjTxSwO27mGIjPO7OC7x7FoW8wJayvI2uJzttxqk84H4wOUtlfbxblAw8uTtCPIO3Vzxkz9k8ENwfvfQYuLvHFNQ8LvatPF65ojzPLHA8+RyCvK3Kmjx27wk8Dcn2PARatDv3tBc8hkLzPEOz5jyQSoe8gU/tPMRmhzzp2wU90shPPBv2oLsNQMA8jTdevIftMTt/Xsw7MMQdPICjBT012tS7SLewvJBtuDuevZM8LyojPa6HxjtOAd07v9mGusZXqDoPqKo8qdeUvETnW7y5occ5pOSOvPPkwjsDN4O8Mk85vKnXlDtp06O7kZDpO6GuNDtRFAY9lAkYPGHNdDx2Afc7RRtROy5/5LyUoxI9mu0+u/dOEryrYrC867vivJp29TtVbZG8SVGrO0im7LnhsqU80frfPL/IwryBT+07/+/kPLZ8sTwoNbg7ZkiIOxadlbxlnUm68RbTuxkX7Tu/cwG7aqGTPO8CAbzTYsq6AIpfvA50tbzllOc7s3rMO0SBVjzXzJm8eZ3Wu4vgtzwPDrA8W6b5uwJpEzwLtaQ81pgkPJuqarxmro288369u48WkjwREBU9JP/dPJ69kzvw4t27h3bouxhrBbwrNx29F9EKPFmSJ7v8px08Tt6rvEJthLxon648UYz4u61TUTz4lPQ7ERAVuhwqFrzfSjs8RRtRO6lxD7zHelm87lfCu10O5LrXMh886YftvL9iPTxCf/E6MZKNOmAhDb2diZ47eRSgPBfRCrznlsw5MiwIvHW7FD3tI807uG3SPE7eqzx1VY864TtcO3zTMDw7EhS8c+0kPLr47TvUDQm8domEvEi3MLruaAa7tUi8u4FgsTwbkBu6pQfAvEJthLwDnQg8S1OQO55GSrxZLCK8nkZKvFXTFr01dM+8W6Z5vO+u6Luh0eW8rofGvFsdw7x7KHK8sN5svCFAzbo/0SS8f9UVu7Qli7wr0Re95E4FvSg1ODok/907AAGpPHQhGrwtS++71pgkvCtazjsSzcC7exYFPLVZgLzZmom7W6Z5PHr0fLtn9O86oUivukvcRrzjPcE8a8REPAei+zoBNZ685aUrPNBg5bqeIxk8FJuwPPdOkrtUOZy8GRftO4KD4rz/72Q7ERCVu8WJODy5O8I5L7NZuxJECjxFkpq8Uq4AOy2fh7wY9Du8GRdtu48o/7mHdug803MOvCUQIrw2hZM8v+tzvE54pruyI6a6exYFvDXrGDwNQEA8zyxwO7c53TwUJGe8Wk9Tu6ouu7yqCwo8vi7IvNe71TxB04m8domEvKTkDrzsidK8+nOovLfT1zr11eM7SVErO3EOcbzqMqw74Tvcut4WRrz5pbi8oznQvMi/Er0aS+I87lfCvK+qdztd6zI83eJQPFy3vbyACQu9/8wzO/k/s7weG7e8906SPA3J9jw8NUU8TUQxPfEWU7wjH4E8J3gMPC72LTp6SJU8exaFOXBiibyf4MS6EXYaO3DIjjy61by7ACRaO5NvnTvMGB48Dw6wPFEUBr30j4E7niMZvIZC87s7EpS8OZnlPJZxgrxug9U7/DDUvNrxL7yV14e3E2c7PBdaQTwT8HE8oIuDPGIB6rvMB9o6cR+1OwbCHrylfgm8z6M5vIiqXbxFG1G8a9WIPItp7rpGT8Y838GEvAoK5jyAG3g7xRJvPPxBGLzJWQ28XYWtO85vRLp0IZq8cR81vc7mDb28PSe89LKyuig1uDyxEuK8GlwmPIbKgLwHGcW7/qkCvC8ZXzzSyE89F8BGOxPw8Tx+Ktc8BkvVurXiNryRkOk8jyj/OcKH0zp69Pw8apDPPFuUjLwPDrC8xuBeuD43KrxuYKQ7qXGPvF0OZDx1VQ88VVzNvD9rn7ushWE7EZlLvSL9+DrHi528dzXsu3k30bzeFka7hrm8vD3gAz1/Xsy80D20PNPZE7sorAG86WS8u2Y3xDtvHVC7PKwOO5DkAT3KOeo8c+0kvI+fyLuY61k8SKbsO4TrzLrrZqE87O9XvMkF9Tynb6q847SKvBjjdzyhSK88zTtPPNNzjjsvGV87UQPCvMD8t7stn4e7GRftPBQkZ7x4eiW7sqzcu3ufO7yAG3g8OHa0u0T4n7wcxJC7r6r3vAbCnrth3rg7BxnFumqQzzyXyCi8V8Q3vEPEqjyIu6E8Ac+YvGR6GLulkHY8um83PMqNgrv5pTi8N7kIPOhTeLy6TIY8B5COvDLGArvEzAy9IbcWvIUfQjxQ4BC7B/aTvCfwfrz15ie8ucR4PD1pursLtSS8AgMOOzIsiLv0srI7Q01hPCvRF7vySsg6O5tKunh6JTvCZCI7xuDevLc53btvLhQ8/pi+PJU9Dbugi4O8Qn/xvLpMhrth3ji8n/GIPKouu7tBS3y853MbPGAQyTt27wk7iokRO8d62bzZRnG7sN5svAG+1Lqvqve8JGXjur0Ll7tCf/E75/xRPIWFx7wgDNi8ucT4OZNvHb2nktu8qrfxuyR2J7zWh2A6juKcPDhlcLx/1RU9IAxYPGJ4szylB8C8qfrFO276HjuWcQK9QdOJvCUQIjzjo8a8SeslvBrCKztCf/E66MrBOx1eCz2Xt+Q66YdtvKg9mrrLSq47fFznO1uUjDsoNTg8QyqwuzH4Ejz/Zi67A8A5uKg9GrtFkhq862ahOzSmXzkMDEs8q+vmvNVkLzwc1n28mu0+vCbekTyCg+K7ekgVvO8CAT2yRtc8apBPu1b2R7zUp4M8VW2RvPc9zrx69Hw753ObvCcSB71sG+u8OwHQuv67b7zLSi65HrWxO0ZPRrxmwPq7t7CmPGxvAzygnfC8oIsDvKY7tbwZF+07p2+qvOnbhbv0oW47/2auuThlcDwIxIM8n/EIO6ijH7vHetk7uRiRPGUDT7pgh5I85shcPpGQabykShS7FWmgPPjojDvJ8wc8mlPEOY2uJzt7FoW7HNb9O7rVvDzKjQI80NcuuqvINbvNTBO8TgFdvEJ/cbzEZoe8SVGrvMvkqLyHdui7P2ufvBSbMDw0t6O82GaUPOLmGrxSNze8KVjpuwizPzwqjN48Xh8ovE4B3TtiAeo8azsOO8eLnbyO4py7x/GiPIvgNzzvi7c8BFq0O/dOEj1fU5282ZoJPCL9+LqyIyY8IoUGPNI/mbwKpGC7EkQKuzrN2jwVzyU7QpA1vLIjpjwi64s8HYE8u6eSW7yryLU8yK5OOzysjjwi6wu8GsIrOu7xPDwCaRO8dzVsPP/vZLwT3oQ8cQ7xvOJv0TtWBww8hlM3PBPeBDxT9OK71pgkPPSysrugiwO90GDlvHOHHz3xfNg8904SPVpglzzmP6a7Cgrmu9/BBLyH7bG85QsxvVSfIb2Xt2Q8paG6vOqYsTos9Mi8nqxPu8wHWjuYhdS7GAWAvCIOvTp/bxA8j7CMPG1P4Dxd67I7xxRUvOM9wbxMhwU9Kp0iPfF82LvQYOU6XkJZPBxNx7y0nX28B5COO8FT3rp4eiW8R/oEvSfw/jtC9rq8n/GIux3nQTw8WPY8LBf6uzSmXzzSPxm88rDNvDysDjwyPnW7tdFyPBLNwDo8WHa8bPi5vOO0CrylGAQ8YgFqvEFLfDy7LOO7TIeFPAHPmDv3YP+6/+9kPBKqjzt5rpo8VJ+hvE7eKzyc3t88P2sfvLQUR7wJ1vC6exaFvD6dr7zNO888i+A3ulwuhzuF/JC8gKMFveoyLLxqBxk7YgFquws+2zwOUYS8agcZvGJ4M71AjtC747QKvAizP73UH3a7LvatPJBtuLzEzIy8bG8DvJEHM75E59s7zbIYPObZIL2uZJW7WRveugblTzy6TIa802JKvD9rH7xlA088QAWavIFP7bwL2FW8vqWRu0ZgijyRkGm7ZGnUvIeHLD1c2m48THbBPPkcAr1NzWc8+JT0uulkvLvXMp+7lU96u7kYET1xhTo8e3wKvItGPTxb+hG87mgGPWqhk7uhrrQ73rBAPCbNTT13rDW8K8DTus8s8DsNt4k8gpQmPLES4ryyvSA8lcbDO60woDyLVwE9BFq0u+cNFj3C7Vi8UXoLPDYOyryQ0z083+S1Ox34hTzEzIw7pX4Ju6ouuzxIpmw8w5iXuylYaTy5sgu9Js3NOo+fyLyjFp+8MMSdvOROBb2n+OA7b7fKOeIJzDoNpkW8WsYct7SdfTxXxLc7TO2KO3YB9zynktu7OkSkPKnXFLvtRv47AJujuzGSDT0twjg8AgOOO4d26DvpZDy8lAkYPI5r0zcGS9W8OGXwu9xIVjyH7TG9IUDNuiqMXrwb9qA79I+BPL1xHLuVPY07MOfOO0ztCruvMoW8BuXPu4AbeLyIRNg8uG3SPO5XQjuFH0K8zm9EPEAoSz0tKL652ZqJOgABqbwsjsM8mlPEPLewpjsVWNw8OGXwOlYHjLzfwQQ81iFbOyJ0Qj3d85S7cQ7xvIqswjxKhSC7906SvAFYz72xiau8LAWNPB1eCz09jGu72ZoJPfDiXTwPDrA8CYGvvNH6XzxTa6y8+RwCvY8of7xxDnG8Ef/QvJ9p+zqh0eU8a16/OzBN1LyDLiE9PFh2u+0jTbxLUxA9ZZ3JvItXgbqL4Dc8BuXPvKnXFDzmPyY8k/hTOlum+bqAksG8OZnluPmluLxRnTy6/KcdvKAUOrzRcSm8fqEgPcTeebzeOXc8KCR0OnN2W7xRA0K8Wsacu+M9wToyLIi8mTATu21P4LuadvW8Dtq6vPmlODsjqLe88ieXPJEHszySoa08U/RiPNQNCbwb9qC8bG+DOXW7FL0OdLW7Tc3nvG8dULsAJNo7fNMwO7sJMr2O4hy85ZTnuwAkWjw+Nyq8rcoaO+8lsrvx86E8U/TivGUUkzp6SJW8lT0NvWz4uTzeFka6qguKvIKD4rt/1ZU8LBf6vD6dr7es/Ko7qWBLvIlVHDxwUUU6Jt4RvRJEijnRcSk88235PGvVCL3zbfm8DaZFO+7xvLs3qES8oznQO9XKNDxZLKK8IIMhvComWb0CAw48fDk2O+nbBb29C5e8ogVbu1EUBryYhdS7OTPgOul1AD25sgs7i1cBPBYmzLtSroA8hfyQvP3bErz9h/o82ZoJO7/ZhjxtT+A8UZ28uzaFk7wJ1nA6dd7FPGg5Kbwb9iC8psRrvBXyVjzGRuS8uAfNu0+smzvFAAK96FN4vC2fhzy65oC7tgXou/9mLjxMELw8GSgxPRBlVjxDxCq80j8ZveinkDxHgzu70j8ZvPGNnDyPn0i8Vn9+urXR8ju10fI7sRJiPDBemLt8OTa8tJ39O4ne0rsaXKa7t0ohPHQhGrdYXjI824sqvDw1RT2/2YY8E/BxPIUOfjv9dQ08PM8/PMwYHrwwXpi7nqxPPM8aA7w+wOC7ROdbO79iPTxVbRE8U45dPOOjRjxwYok8ME1Uu1SfIbyifKQ8UXqLPI85wzsITTq8R+lAPMRVQzzcv58892B/Oqg9mjw3MXu7P9EkvM6AiLyx7zA8eHolPLYWLLugFLq8AJsjvEOzZjk6RKQ8uRgRPXVVjzw0HSk9PWk6PLss47spzzK93rBAvJpTxDun+OC7OTPgvEa1yzvAH+k5fZDcOid4jLuN0di8N7kIPPe0F7wVaSC8zxoDvJVgvrvUpwO9dd7FPKUHQLxn4oI7Ng7KPIydYzzZRvE8LTkCu3bvCTy10fK7QAWaPGHeOLu6+O27omvgO8Rmh7xrXj87AzeDvORg8jnGRuS8UEYWPLPg0TvYZpQ9FJuwPLC7O7xug1U8bvoevAnW8DvxFtM8kEoHPDxYdrzcWZq8n3q/O94nCjvZI0C82yUlvayWpbyHh6y7ME1UO9b+KTzbFGG89oCiPFpgFzzhTKA84gnMPKgsVjyia+C7XNpuPHxc5zyDLqG8ukyGvKqUQLwG5U88wB/pO+B+ML2O4py8MOdOPHt8irsDnYg6rv6PumJ4szzuV0I80qWePKTkDj14A9y8fqEgu9DXLjykbUU7yEhJvLYFaLyfVw68", + "index": 0, + "object": "embedding", + } + ], + "model": "text-embedding-ada-002-v2", + "object": "list", + "usage": {"prompt_tokens": 6, "total_tokens": 6}, + }, + ), + "You are a scientist.": ( + { + "Content-Type": "application/json", + "openai-model": "gpt-3.5-turbo-0613", + "openai-organization": "new-relic-nkmd8b", + "openai-processing-ms": "1469", + "openai-version": "2020-10-01", + "x-ratelimit-limit-requests": "200", + "x-ratelimit-limit-tokens": "40000", + "x-ratelimit-remaining-requests": "199", + "x-ratelimit-remaining-tokens": "39940", + "x-ratelimit-reset-requests": "7m12s", + "x-ratelimit-reset-tokens": "90ms", + "x-request-id": "49dbbffbd3c3f4612aa48def69059ccd", + }, + 200, + { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "212 degrees " "Fahrenheit is " "equal to 100 " "degrees " "Celsius.", + "role": "assistant", + }, + } + ], + "created": 1696888863, + "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv", + "model": "gpt-3.5-turbo-0613", + "object": "chat.completion", + "usage": {"completion_tokens": 11, "prompt_tokens": 53, "total_tokens": 64}, + }, + ), + "You are a mathematician.": ( + { + "Content-Type": "application/json", + "openai-model": "gpt-3.5-turbo-0613", + "openai-organization": "new-relic-nkmd8b", + "openai-processing-ms": "1469", + "openai-version": "2020-10-01", + "x-ratelimit-limit-requests": "200", + "x-ratelimit-limit-tokens": "40000", + "x-ratelimit-remaining-requests": "199", + "x-ratelimit-remaining-tokens": "39940", + "x-ratelimit-reset-requests": "7m12s", + "x-ratelimit-reset-tokens": "90ms", + "x-request-id": "49dbbffbd3c3f4612aa48def69059aad", + }, + 200, + { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "1 plus 2 is 3.", + "role": "assistant", + }, + } + ], + "created": 1696888865, + "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTeat", + "model": "gpt-3.5-turbo-0613", + "object": "chat.completion", + "usage": {"completion_tokens": 11, "prompt_tokens": 53, "total_tokens": 64}, + }, + ), +} + + +@pytest.fixture(scope="session") +def simple_get(gemini_version, extract_shortened_prompt): + def _simple_get(self): + content_len = int(self.headers.get("content-length")) + content = json.loads(self.rfile.read(content_len).decode("utf-8")) + stream = content.get("stream", False) + prompt = extract_shortened_prompt(content) + if not prompt: + self.send_response(500) + self.end_headers() + self.wfile.write("Could not parse prompt.".encode("utf-8")) + return + + headers, response = ({}, "") + + mocked_responses = RESPONSES_V1 + # if stream: + # mocked_responses = STREAMED_RESPONSES_V1 + + for k, v in mocked_responses.items(): + if prompt.startswith(k): + headers, status_code, response = v + break + else: # If no matches found + self.send_response(500) + self.end_headers() + self.wfile.write(f"Unknown Prompt:\n{prompt}".encode("utf-8")) + return + + # Send response code + self.send_response(status_code) + + # Send headers + for k, v in headers.items(): + self.send_header(k, v) + self.end_headers() + + # Send response body + # if stream and status_code < 400: + # for resp in response: + # data = json.dumps(resp).encode("utf-8") + # if prompt == "Stream parsing error.": + # # Force a parsing error by writing an invalid streamed response. + # self.wfile.write(b"data: %s" % data) + # else: + # self.wfile.write(b"data: %s\n\n" % data) + # else: + self.wfile.write(json.dumps(response).encode("utf-8")) + return + + return _simple_get + + +@pytest.fixture(scope="session") +def MockExternalGeminiServer(simple_get): + class _MockExternalGeminiServer(MockExternalHTTPServer): + # To use this class in a test one needs to start and stop this server + # before and after making requests to the test app that makes the external + # calls. + + def __init__(self, handler=simple_get, port=None, *args, **kwargs): + super(_MockExternalGeminiServer, self).__init__(handler=handler, port=port, *args, **kwargs) + + return _MockExternalGeminiServer + + +@pytest.fixture(scope="session") +def extract_shortened_prompt(gemini_version): + def _extract_shortened_prompt(content): + prompt = content.get("input", None) or content.get("messages")[0]["content"] + return prompt + + return _extract_shortened_prompt + + +def get_gemini_version(): + # Import Gemini so that get package version can catpure the version from the + # system module. Gemini does not have a package version in v0. + import google.generativeai as genai # noqa: F401; pylint: disable=W0611 + + return get_package_version_tuple("genai") + + +@pytest.fixture(scope="session") +def gemini_version(): + return get_gemini_version() + + +if __name__ == "__main__": + _MockExternalGeminiServer = MockExternalGeminiServer() + with MockExternalGeminiServer() as server: + print(f"MockExternalGeminiServer serving on port {str(server.port)}") + while True: + pass # Serve forever diff --git a/tests/mlmodel_gemini/conftest.py b/tests/mlmodel_gemini/conftest.py new file mode 100644 index 000000000..ef4d55505 --- /dev/null +++ b/tests/mlmodel_gemini/conftest.py @@ -0,0 +1,329 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import pytest +from mlmodel_gemini._mock_external_gemini_server import ( # noqa: F401; pylint: disable=W0611 + MockExternalGeminiServer, + extract_shortened_prompt, + get_gemini_version, + gemini_version, + simple_get, +) +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, + override_application_settings, +) + +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.common.signature import bind_args + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "ml_insights_events.enabled": True, + "ai_monitoring.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (mlmodel_gemini)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (mlmodel_gemini)"], +) + +# if get_gemini_version() < (1, 0): +# collect_ignore = [ +# "test_chat_completion_v1.py", +# "test_chat_completion_error_v1.py", +# ] +# else: +# collect_ignore = [ +# "test_chat_completion.py", +# "test_chat_completion_error.py", +# ] +collect_ignore = [ + "test_chat_completion_error.py" +] + +GEMINI_AUDIT_LOG_FILE = os.path.join(os.path.realpath(os.path.dirname(__file__)), "gemini_audit.log") +GEMINI_AUDIT_LOG_CONTENTS = {} +# Intercept outgoing requests and log to file for mocking +RECORDED_HEADERS = set(["x-request-id", "content-type"]) + + +@pytest.fixture(scope="session") +def gemini_clients(gemini_version, MockExternalGeminiServer): # noqa: F811 + """ + This configures the gemini client and returns it for gemini v1 and only configures + gemini for v0 since there is no client. + """ + import google.generativeai as genai + + from newrelic.core.config import _environ_as_bool + + gemini_model = "gemini-1.5-flash" + gemini_api_key = "GEMINI_API_KEY" + #gemini_api_key = os.environ.get("GEMINI_API_KEY") + #gemini_model = os.environ.get("GEMINI_MODEL") + if not _environ_as_bool("NEW_RELIC_TESTING_RECORD_GEMINI_RESPONSES", False): + with MockExternalGeminiServer() as server: + if not gemini_api_key: + raise RuntimeError("GEMINI_API_KEY environment variable required.") + genai.configure(api_key=gemini_api_key) + model = genai.GenerativeModel(gemini_model) + else: + if not gemini_api_key: + raise RuntimeError("GEMINI_API_KEY environment variable required.") + + genai.configure(api_key=gemini_api_key) + model = genai.GenerativeModel(gemini_model) + yield (model) + +@pytest.fixture(scope="session") +def sync_gemini_client(gemini_clients): + sync_client = gemini_clients + return sync_client + +@pytest.fixture(autouse=True, scope="session") +def gemini_server( + gemini_version, # noqa: F811 + gemini_clients, + wrap_gemini_api_requestor_request, + wrap_gemini_api_requestor_interpret_response, + wrap_httpx_client_send, + wrap_engine_api_resource_create, + wrap_stream_iter_events, +): + """ + This fixture will either create a mocked backend for testing purposes, or will + set up an audit log file to log responses of the real Gemini backend to a file. + The behavior can be controlled by setting NEW_RELIC_TESTING_RECORD_gemini_RESPONSES=1 as + an environment variable to run using the real Gemini backend. (Default: mocking) + """ + from newrelic.core.config import _environ_as_bool + + if _environ_as_bool("NEW_RELIC_TESTING_RECORD_gemini_RESPONSES", False): + # Apply function wrappers to record data + wrap_function_wrapper("httpx._client", "Client.send", wrap_httpx_client_send) + wrap_function_wrapper( + "gemini._streaming", + "Stream._iter_events", + wrap_stream_iter_events, + ) + yield # Run tests + # Write responses to audit log + with open(GEMINI_AUDIT_LOG_FILE, "w") as audit_log_fp: + json.dump(GEMINI_AUDIT_LOG_CONTENTS, fp=audit_log_fp, indent=4) + else: + # We are mocking gemini responses so we don't need to do anything in this case. + yield + + +@pytest.fixture(scope="session") +def wrap_httpx_client_send(extract_shortened_prompt): # noqa: F811 + def _wrap_httpx_client_send(wrapped, instance, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + stream = bound_args.get("stream", False) + request = bound_args["request"] + if not request: + return wrapped(*args, **kwargs) + + params = json.loads(request.content.decode("utf-8")) + prompt = extract_shortened_prompt(params) + + # Send request + response = wrapped(*args, **kwargs) + + if response.status_code >= 500 or response.status_code < 200: + prompt = "error" + + rheaders = getattr(response, "headers") + + headers = dict( + filter( + lambda k: k[0].lower() in RECORDED_HEADERS + or k[0].lower().startswith("gemini") + or k[0].lower().startswith("x-ratelimit"), + rheaders.items(), + ) + ) + if stream: + GEMINI_AUDIT_LOG_CONTENTS[prompt] = [headers, response.status_code, []] # Append response data to log + if prompt == "error": + GEMINI_AUDIT_LOG_CONTENTS[prompt][2] = json.loads(response.read()) + else: + body = json.loads(response.content.decode("utf-8")) + GEMINI_AUDIT_LOG_CONTENTS[prompt] = headers, response.status_code, body # Append response data to log + return response + + return _wrap_httpx_client_send + + +@pytest.fixture(scope="session") +def wrap_gemini_api_requestor_interpret_response(): + def _wrap_gemini_api_requestor_interpret_response(wrapped, instance, args, kwargs): + rbody, rcode, rheaders = bind_request_interpret_response_params(*args, **kwargs) + headers = dict( + filter( + lambda k: k[0].lower() in RECORDED_HEADERS + or k[0].lower().startswith("gemini") + or k[0].lower().startswith("x-ratelimit"), + rheaders.items(), + ) + ) + + if rcode >= 400 or rcode < 200: + rbody = json.loads(rbody) + GEMINI_AUDIT_LOG_CONTENTS["error"] = headers, rcode, rbody # Append response data to audit log + return wrapped(*args, **kwargs) + + return _wrap_gemini_api_requestor_interpret_response + + +@pytest.fixture(scope="session") +def wrap_gemini_api_requestor_request(extract_shortened_prompt): # noqa: F811 + def _wrap_gemini_api_requestor_request(wrapped, instance, args, kwargs): + params = bind_request_params(*args, **kwargs) + if not params: + return wrapped(*args, **kwargs) + + prompt = extract_shortened_prompt(params) + + # Send request + result = wrapped(*args, **kwargs) + + # Append response data to audit log + if not kwargs.get("stream", False): + # Clean up data + data = result[0].data + headers = result[0]._headers + headers = dict( + filter( + lambda k: k[0].lower() in RECORDED_HEADERS + or k[0].lower().startswith("gemini") + or k[0].lower().startswith("x-ratelimit"), + headers.items(), + ) + ) + GEMINI_AUDIT_LOG_CONTENTS[prompt] = headers, 200, data + else: + GEMINI_AUDIT_LOG_CONTENTS[prompt] = [None, 200, []] + return result + + return _wrap_gemini_api_requestor_request + + +def bind_request_params(method, url, params=None, *args, **kwargs): + return params + + +def bind_request_interpret_response_params(result, stream): + return result.content.decode("utf-8"), result.status_code, result.headers + + +@pytest.fixture(scope="session") +def generator_proxy(gemini_version): + class GeneratorProxy(ObjectProxy): + def __init__(self, wrapped): + super(GeneratorProxy, self).__init__(wrapped) + + def __iter__(self): + return self + + # Make this Proxy a pass through to our instrumentation's proxy by passing along + # get attr and set attr calls to our instrumentation's proxy. + def __getattr__(self, attr): + return self.__wrapped__.__getattr__(attr) + + def __setattr__(self, attr, value): + return self.__wrapped__.__setattr__(attr, value) + + def __next__(self): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.__next__() + + try: + return_val = self.__wrapped__.__next__() + if return_val: + prompt = [k for k in GEMINI_AUDIT_LOG_CONTENTS.keys()][-1] + if gemini_version < (1, 0): + headers = dict( + filter( + lambda k: k[0].lower() in RECORDED_HEADERS + or k[0].lower().startswith("gemini") + or k[0].lower().startswith("x-ratelimit"), + return_val._nr_response_headers.items(), + ) + ) + GEMINI_AUDIT_LOG_CONTENTS[prompt][0] = headers + GEMINI_AUDIT_LOG_CONTENTS[prompt][2].append(return_val.to_dict_recursive()) + else: + if not getattr(return_val, "data", "").startswith("[DONE]"): + GEMINI_AUDIT_LOG_CONTENTS[prompt][2].append(return_val.json()) + return return_val + except Exception as e: + raise + + def close(self): + return super(GeneratorProxy, self).close() + + return GeneratorProxy + + +@pytest.fixture(scope="session") +def wrap_engine_api_resource_create(generator_proxy): + def _wrap_engine_api_resource_create(wrapped, instance, args, kwargs): + transaction = current_transaction() + + if not transaction: + return wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + stream = bound_args["params"].get("stream", False) + + return_val = wrapped(*args, **kwargs) + + if stream: + return generator_proxy(return_val) + else: + return return_val + + return _wrap_engine_api_resource_create + + +@pytest.fixture(scope="session") +def wrap_stream_iter_events(generator_proxy): + def _wrap_stream_iter_events(wrapped, instance, args, kwargs): + transaction = current_transaction() + + if not transaction: + return wrapped(*args, **kwargs) + + return_val = wrapped(*args, **kwargs) + proxied_return_val = generator_proxy(return_val) + return proxied_return_val + + return _wrap_stream_iter_events diff --git a/tests/mlmodel_gemini/test_chat_completion.py b/tests/mlmodel_gemini/test_chat_completion.py new file mode 100644 index 000000000..088f39f2c --- /dev/null +++ b/tests/mlmodel_gemini/test_chat_completion.py @@ -0,0 +1,283 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import google.generativeai as genai +from testing_support.fixtures import ( + override_llm_token_callback_settings, + reset_core_stats_engine, + validate_attributes, +) +from testing_support.ml_testing_utils import ( # noqa: F401 + add_token_count_to_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + disabled_ai_monitoring_streaming_settings, + events_sans_content, + events_sans_llm_metadata, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute + +_test_gemini_chat_completion_messages = ( + "You are a scientist.", + "What is 212 degrees Fahrenheit converted to Celsius?", +) + +chat_completion_recorded_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "49dbbffbd3c3f4612aa48def69059ccd", + "duration": None, # Response time varies each test run + "request.model": "gemini-1.5-flash", + "response.model": "gemini-1.5-flash", + "response.organization": "new-relic-nkmd8b", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "stop", + "response.headers.llmVersion": "2020-10-01", + "response.headers.ratelimitLimitRequests": 200, + "response.headers.ratelimitLimitTokens": 40000, + "response.headers.ratelimitResetTokens": "90ms", + "response.headers.ratelimitResetRequests": "7m12s", + "response.headers.ratelimitRemainingTokens": 39940, + "response.headers.ratelimitRemainingRequests": 199, + "vendor": "gemini", + "ingest_source": "Python", + "response.number_of_messages": 3, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv-0", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "49dbbffbd3c3f4612aa48def69059ccd", + "span_id": None, + "trace_id": "trace-id", + "content": "You are a scientist.", + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": "gemini-1.5-flash", + "vendor": "gemini", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv-1", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "49dbbffbd3c3f4612aa48def69059ccd", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": "gemini-1.5-flash", + "vendor": "gemini", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTemv-2", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "49dbbffbd3c3f4612aa48def69059ccd", + "span_id": None, + "trace_id": "trace-id", + "content": "212 degrees Fahrenheit is equal to 100 degrees Celsius.", + "role": "assistant", + "completion_id": None, + "sequence": 2, + "response.model": "gemini-1.5-flash", + "vendor": "gemini", + "is_response": True, + "ingest_source": "Python", + }, + ), +] + + +@reset_core_stats_engine() +#@validate_custom_events(events_with_context_attrs(chat_completion_recorded_events)) +# One summary event, one system message, one user message, and one response message from the assistant +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_chat_completion:test_gemini_chat_completion_sync_with_llm_metadata", + custom_metrics=[ + (f"Supportability/Python/ML/Gemini/{genai.__version__}", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_chat_completion_sync_with_llm_metadata(set_trace_info, sync_gemini_client): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + sync_gemini_client.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +#@validate_custom_events(events_sans_content(chat_completion_recorded_events)) +# One summary event, one system message, one user message, and one response message from the assistant +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_chat_completion:test_gemini_chat_completion_sync_no_content", + custom_metrics=[ + (f"Supportability/Python/ML/Gemini/{genai.__version__}", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_chat_completion_sync_no_content(set_trace_info, sync_gemini_client): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + + sync_gemini_client.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +#@validate_custom_events(add_token_count_to_events(chat_completion_recorded_events)) +# One summary event, one system message, one user message, and one response message from the assistant +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_chat_completion:test_gemini_chat_completion_sync_with_token_count", + custom_metrics=[ + (f"Supportability/Python/ML/Gemini/{genai.__version__}", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_chat_completion_sync_with_token_count(set_trace_info, sync_gemini_client): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + + #genai.configure(api_key=os.environ["GEMINI_API_KEY"]) + #model = genai.GenerativeModel("gemini-1.5-flash") + sync_gemini_client.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +@reset_core_stats_engine() +#@validate_custom_events(events_sans_llm_metadata(chat_completion_recorded_events)) +# One summary event, one system message, one user message, and one response message from the assistant +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + "test_chat_completion:test_gemini_chat_completion_sync_no_llm_metadata", + scoped_metrics=[("Llm/completion/Gemini/generate_content", 1)], + rollup_metrics=[("Llm/completion/Gemini/generate_content", 1)], + background_task=True, +) +@background_task() +def test_gemini_chat_completion_sync_no_llm_metadata(set_trace_info, sync_gemini_client): + set_trace_info() + + sync_gemini_client.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_gemini_chat_completion_sync_outside_txn(sync_gemini_client): + sync_gemini_client.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_gemini_chat_completion_sync_ai_monitoring_disabled(sync_gemini_client): + sync_gemini_client.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +@reset_core_stats_engine() +# One summary event, one system message, one user message, and one response message from the assistant +@validate_custom_event_count(count=3) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_gemini_chat_completion_sync_no_usage_data(set_trace_info, sync_gemini_client): + # Only testing that there are events, and there was no exception raised + set_trace_info() + + sync_gemini_client.generate_content( + "No usage data", + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +#@reset_core_stats_e +#def test_gemini_chat_completion_functions_marked_as_wrapped_for_sdk_compatibility(): +# assert genai._nr_wrapped +# assert genai.util.convert_to_gemini_object._nr_wrapped diff --git a/tests/mlmodel_gemini/test_chat_completion_error.py b/tests/mlmodel_gemini/test_chat_completion_error.py new file mode 100644 index 000000000..bbc46e06c --- /dev/null +++ b/tests/mlmodel_gemini/test_chat_completion_error.py @@ -0,0 +1,726 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import google.auth as gauth +import google.generativeai as genai +import pytest +from testing_support.fixtures import ( + dt_enabled, + override_llm_token_callback_settings, + reset_core_stats_engine, +) +from testing_support.ml_testing_utils import ( # noqa: F401 + add_token_count_to_events, + disabled_ai_monitoring_record_content_settings, + events_sans_content, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import ( + validate_error_trace_attributes, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name + +_test_gemini_chat_completion_messages = ( + {"role": "system", "content": "You are a scientist."}, + {"role": "user", "content": "What is 212 degrees Fahrenheit converted to Celsius?"}, +) + +expected_events_on_no_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 2, + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": "You are a scientist.", + "role": "system", + "completion_id": None, + "sequence": 0, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 1, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), +] + + +# No model provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.param": "engine", + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Must provide an 'engine' or 'model' parameter to create a ", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_no_model", + scoped_metrics=[("Llm/completion/Gemini/create", 1)], + rollup_metrics=[("Llm/completion/Gemini/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_no_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_invalid_request_error_no_model(set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + with WithLlmCustomAttributes({"context": "attr"}): + model.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +@dt_enabled +@disabled_ai_monitoring_record_content_settings +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.param": "engine", + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Must provide an 'engine' or 'model' parameter to create a ", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_no_model_no_content", + scoped_metrics=[("Llm/completion/Gemini/create", 1)], + rollup_metrics=[("Llm/completion/Gemini/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_no_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_invalid_request_error_no_model_no_content(set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + model.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +expected_events_on_invalid_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "does-not-exist", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": "Model does not exist.", + "role": "user", + "completion_id": None, + "sequence": 0, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), +] + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.code": "model_not_found", + "http.statusCode": 404, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "The model `does-not-exist` does not exist", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_invalid_model_with_token_count", + scoped_metrics=[("Llm/completion/Gemini/create", 1)], + rollup_metrics=[("Llm/completion/Gemini/create", 1)], + background_task=True, +) +@validate_custom_events(add_token_count_to_events(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_invalid_request_error_invalid_model_with_token_count(set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + model.generate_content( + model="does-not-exist", + message="Model does not exist.", + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +# Invalid model provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.code": "model_not_found", + "http.statusCode": 404, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "The model `does-not-exist` does not exist", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_invalid_model", + scoped_metrics=[("Llm/completion/Gemini/create", 1)], + rollup_metrics=[("Llm/completion/Gemini/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_invalid_model_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_invalid_request_error_invalid_model(set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + model.generate_content( + model="does-not-exist", + message="Model does not exist.", + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +expected_events_on_auth_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "gemini-1.5-flash", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 2, + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": "You are a scientist.", + "role": "system", + "completion_id": None, + "sequence": 0, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 1, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), +] + + +# No api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(gauth.exceptions.DefaultCredentialsError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": {}, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "No API key provided. You can set your API key in code using 'openai.api_key = ', or you can set the environment variable OPENAI_API_KEY=). If your API key is stored in a file, you can point the openai module at it with 'openai.api_key_path = '. You can generate API keys in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details.", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_authentication_error", + scoped_metrics=[("Llm/completion/Gemini/create", 1)], + rollup_metrics=[("Llm/completion/Gemini/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_auth_error) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_authentication_error(monkeypatch, set_trace_info): + with pytest.raises(gauth.exceptions.DefaultCredentialsError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + monkeypatch.setenv("API_KEY", None, prepend=None) + genai.configure(api_key=os.environ["API_KEY"]) + #monkeypatch.setattr(genai.configure, "api_key", os.environ["API_KEY"]) # openai.api_key = None + model = genai.GenerativeModel("gemini-1.5-flash") + model.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +expected_events_on_wrong_api_key_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "gemini-1.5-flash", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "gemini", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "span_id": None, + "trace_id": "trace-id", + "content": "Invalid API key.", + "role": "user", + "completion_id": None, + "sequence": 0, + "vendor": "gemini", + "ingest_source": "Python", + }, + ), +] + + +# Wrong api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(gauth.exceptions.DefaultCredentialsError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 401, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Incorrect API key provided: invalid. You can find your API key at https://cloud.google.com/vertex-ai/generative-ai.", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_wrong_api_key_error", + scoped_metrics=[("Llm/completion/Gemini/create", 1)], + rollup_metrics=[("Llm/completion/Gemini/create", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_wrong_api_key_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_wrong_api_key_error(monkeypatch, set_trace_info): + with pytest.raises(gauth.exceptions.DefaultCredentialsError): + set_trace_info() + monkeypatch.setenv("API_KEY", None, prepend=None) + genai.configure(api_key=os.environ["API_KEY"]) + #monkeypatch.setattr(genai.configure, "api_key", "DEADBEEF") # openai.api_key = "DEADBEEF" + model = genai.GenerativeModel("gemini-1.5-flash") + model.generate_content( + "Invalid API key.", + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + + +# Async tests: +# No model provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.param": "engine", + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Must provide an 'engine' or 'model' parameter to create a ", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_no_model_async", + scoped_metrics=[("Llm/completion/Gemini/acreate", 1)], + rollup_metrics=[("Llm/completion/Gemini/acreate", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_no_model_error) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_invalid_request_error_no_model_async(loop, set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + loop.run_until_complete( + model.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + ) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.param": "engine", + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Must provide an 'engine' or 'model' parameter to create a ", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_no_model_async_no_content", + scoped_metrics=[("Llm/completion/Gemini/acreate", 1)], + rollup_metrics=[("Llm/completion/Gemini/acreate", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_no_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_invalid_request_error_no_model_async_no_content(loop, set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + loop.run_until_complete( + model.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + ) + + +@dt_enabled +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.code": "model_not_found", + "http.statusCode": 404, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "The model `does-not-exist` does not exist", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_invalid_model_with_token_count_async", + scoped_metrics=[("Llm/completion/Gemini/acreate", 1)], + rollup_metrics=[("Llm/completion/Gemini/acreate", 1)], + background_task=True, +) +@validate_custom_events(add_token_count_to_events(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_invalid_request_error_invalid_model_with_token_count_async(loop, set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + loop.run_until_complete( + model.generate_content( + "Model does not exist.", + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + ) + + +# Invalid model provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(genai.types.IncompleteIterationError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.code": "model_not_found", + "http.statusCode": 404, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "The model `does-not-exist` does not exist", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_invalid_request_error_invalid_model_async", + scoped_metrics=[("Llm/completion/Gemini/acreate", 1)], + rollup_metrics=[("Llm/completion/Gemini/acreate", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_invalid_model_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_invalid_request_error_invalid_model_async(loop, set_trace_info): + with pytest.raises(genai.types.IncompleteIterationError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + model = genai.GenerativeModel("gemini-1.5-flash") + loop.run_until_complete( + model.generate_content( + "Model does not exist.", + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + ) + + +# No api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(gauth.exceptions.DefaultCredentialsError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": {}, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "No API key provided. You can set your API key in code using 'gemini.api_key = ', or you can set the environment variable OPENAI_API_KEY=). If your API key is stored in a file, you can point the openai module at it with 'openai.api_key_path = '. You can generate API keys in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details.", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_authentication_error_async", + scoped_metrics=[("Llm/completion/Gemini/acreate", 1)], + rollup_metrics=[("Llm/completion/Gemini/acreate", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_auth_error) +@validate_custom_event_count(count=3) +@background_task() +def test_chat_completion_authentication_error_async(loop, monkeypatch, set_trace_info): + with pytest.raises(gauth.exceptions.DefaultCredentialsError): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + monkeypatch.setenv("API_KEY", None, prepend=None) + genai.configure(api_key=os.environ["API_KEY"]) + #monkeypatch.setattr(genai.configure, "api_key", None) # openai.api_key = None + model = genai.GenerativeModel("gemini-1.5-flash") + loop.run_until_complete( + model.generate_content( + _test_gemini_chat_completion_messages, + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + ) + + +# Wrong api_key provided +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(gauth.exceptions.DefaultCredentialsError), + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 401, + }, + }, +) +@validate_span_events( + exact_agents={ + "error.message": "Incorrect API key provided: invalid. You can find your API key at https://cloud.google.com/vertex-ai/generative-ai.", + } +) +@validate_transaction_metrics( + "test_chat_completion_error:test_chat_completion_wrong_api_key_error_async", + scoped_metrics=[("Llm/completion/Gemini/acreate", 1)], + rollup_metrics=[("Llm/completion/Gemini/acreate", 1)], + background_task=True, +) +@validate_custom_events(expected_events_on_wrong_api_key_error) +@validate_custom_event_count(count=2) +@background_task() +def test_chat_completion_wrong_api_key_error_async(loop, monkeypatch, set_trace_info): + with pytest.raises(gauth.exceptions.DefaultCredentialsError): + set_trace_info() + monkeypatch.setenv("API_KEY", None, prepend=None) + genai.configure(api_key=os.environ["API_KEY"]) + #monkeypatch.setattr(genai.configure, "api_key", "DEADBEEF") # openai.api_key = "DEADBEEF" + model = genai.GenerativeModel("gemini-1.5-flash") + loop.run_until_complete( + model.generate_content( + "Invalid API key.", + generation_config=genai.types.GenerationConfig( + temperature=0.7 + ) + ) + ) diff --git a/tox.ini b/tox.ini index 335ea0776..0ea1b8d75 100644 --- a/tox.ini +++ b/tox.ini @@ -152,6 +152,7 @@ envlist = python-mlmodel_langchain-{py39,py310,py311,py312}, ;; Package not ready for Python 3.13 (uses an older version of numpy) ; python-mlmodel_langchain-py313, + python-mlmodel_gemini-{py37,py38,py39,py310,py311,py312}-geminilatest, python-mlmodel_openai-openai0-{py37,py38,py39,py310,py311,py312}, python-mlmodel_openai-openai107-py312, python-mlmodel_openai-openailatest-{py37,py38,py39,py310,py311,py312,py313}, @@ -375,6 +376,8 @@ deps = framework_tornado: pycurl framework_tornado-tornadolatest: tornado framework_tornado-tornadomaster: https://github.com/tornadoweb/tornado/archive/master.zip + mlmodel_gemini-geminilatest: google.generativeai[datalib] + mlmodel_gemini: protobuf mlmodel_openai-openai0: openai[datalib]<1.0 mlmodel_openai-openai107: openai[datalib]<1.8 mlmodel_openai-openailatest: openai[datalib] @@ -440,7 +443,7 @@ commands = allowlist_externals={toxinidir}/.github/scripts/* install_command= - {toxinidir}/.github/scripts/retry.sh 3 pip install {opts} {packages} + pip install {opts} {packages} extras = agent_streaming: infinite-tracing @@ -519,6 +522,7 @@ changedir = messagebroker_kafkapython: tests/messagebroker_kafkapython messagebroker_pika: tests/messagebroker_pika mlmodel_langchain: tests/mlmodel_langchain + mlmodel_gemini: tests/mlmodel_gemini mlmodel_openai: tests/mlmodel_openai mlmodel_sklearn: tests/mlmodel_sklearn template_genshi: tests/template_genshi