From a31d4eb1b81011d442d16426a8cd7c9aaa3d77ac Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 4 Aug 2026 16:04:37 -0700 Subject: [PATCH 1/9] Add OpenAI Responses API instrumentation --- newrelic/config.py | 3 + newrelic/hooks/mlmodel_openai.py | 279 ++++++++++++++++++++++++++++--- 2 files changed, 262 insertions(+), 20 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 568900363c..78795f2b09 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2308,6 +2308,9 @@ def _process_module_builtin_defaults(): _process_module_definition( "openai.resources.completions", "newrelic.hooks.mlmodel_openai", "instrument_openai_resources_chat_completions" ) + _process_module_definition( + "openai.resources.responses", "newrelic.hooks.mlmodel_openai", "instrument_openai_resources_responses" + ) _process_module_definition("openai._base_client", "newrelic.hooks.mlmodel_openai", "instrument_openai_base_client") _process_module_definition("google.genai.models", "newrelic.hooks.mlmodel_gemini", "instrument_genai_models") diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index 3b0a9e810c..610e8c5298 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -494,7 +494,7 @@ async def wrap_chat_completion_async(wrapped, instance, args, kwargs): def _handle_completion_success( - *, transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp=None + *, transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp=None, responses_api=False ): settings = transaction.settings if transaction.settings is not None else global_settings() stream = kwargs.get("stream", False) @@ -513,6 +513,8 @@ def _handle_completion_success( return_val._nr_ft = ft return_val._nr_metadata = linking_metadata return_val._nr_openai_attrs = getattr(return_val, "_nr_openai_attrs", {}) + # Marker so _record_stream_chunk parses Responses events instead of Chat Completions chunks. + return_val._nr_openai_attrs["_nr_response_api"] = responses_api return_val._nr_openai_attrs["messages"] = kwargs.get("messages", []) return_val._nr_openai_attrs["temperature"] = kwargs.get("temperature") return_val._nr_openai_attrs["max_tokens"] = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens") @@ -538,6 +540,13 @@ def _handle_completion_success( # openai._legacy_response.LegacyAPIResponse response = json.loads(response.http_response.text.strip()) + if responses_api: + # Merge the extracted response attributes into kwargs and record via the shared + # response=None branch, exactly as the streaming path does. Set the + # response to None here to indicate attr extraction has already happened. + kwargs.update(_responses_api_response_attrs(response)) + response = None # Avoids later attr extraction logic. + _record_completion_success( transaction=transaction, linking_metadata=linking_metadata, @@ -790,6 +799,172 @@ def _record_completion_error(*, transaction, linking_metadata, completion_id, kw _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) +def _responses_api_request_attrs(kwargs): + """Extract attributes from the Responses API's request kwargs. Shared by streaming and non-streaming paths.""" + input_ = kwargs.get("input") + messages = [] + if kwargs.get("instructions"): + messages.append({"content": kwargs["instructions"], "role": "system"}) + if isinstance(input_, str): + messages.append({"content": input_, "role": "user"}) + elif isinstance(input_, (list, tuple)): + for item in input_: + if isinstance(item, dict) and item.get("role") is not None: + content = item.get("content") + # Content is a string, or a list of typed parts each carrying `text`. + if isinstance(content, (list, tuple)): + content = "".join(part.get("text") or "" for part in content if isinstance(part, dict)) + messages.append({"content": content, "role": item["role"]}) + + return { + "messages": messages, + "model": kwargs.get("model"), + "temperature": kwargs.get("temperature"), + "max_tokens": kwargs.get("max_output_tokens"), + "stream": kwargs.get("stream", False), + } + + +def _responses_api_response_attrs(response): + """Extract attributes from the Responses API's Response object. Shared by streaming and non-streaming paths.""" + response = response or {} + + # Response content exists in output items of type "message", inside content parts of type "output_text". + content = "".join( + part.get("text") + for item in response.get("output") or [] + if isinstance(item, dict) and item.get("type") == "message" + for part in item.get("content") or [] + if isinstance(part, dict) and part.get("type") == "output_text" and part.get("text") + ) + + # The Responses API has no per choice finish_reason. + # Use incomplete_details.reason if it exists, or fall back to status. + incomplete_details = response.get("incomplete_details") or {} + + # Pull usage object once and unpack below + usage = response.get("usage") or {} + + return { + "response.model": response.get("model"), + "id": response.get("id"), + "response.usage": { + "prompt_tokens": usage.get("input_tokens"), + "completion_tokens": usage.get("output_tokens"), + "total_tokens": usage.get("total_tokens"), + }, + "content": content, + "role": "assistant", + "finish_reason": incomplete_details.get("reason") or response.get("status"), + } + + +def wrap_responses_sync(wrapped, instance, args, kwargs): + """Instrument openai.resources.responses.Responses.create() and .parse().""" + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + # The .with_streaming_response. wrapper is not yet instrumented. Skip it for now. + if _is_streaming_response_wrapper(kwargs): + return wrapped(*args, **kwargs) + + request_timestamp = int(1000.0 * time.time()) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("OpenAI", OPENAI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + nr_kwargs = _responses_api_request_attrs(kwargs) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/OpenAI") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _record_completion_error( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=completion_id, + kwargs=nr_kwargs, + ft=ft, + exc=exc, + request_timestamp=request_timestamp, + ) + raise + + _handle_completion_success( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=completion_id, + kwargs=nr_kwargs, + ft=ft, + return_val=return_val, + request_timestamp=request_timestamp, + responses_api=True, + ) + return return_val + + +async def wrap_responses_async(wrapped, instance, args, kwargs): + """Instrument openai.resources.responses.AsyncResponses.create() and .parse().""" + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + # The .with_streaming_response. wrapper is not yet instrumented. Skip it for now. + if _is_streaming_response_wrapper(kwargs): + return await wrapped(*args, **kwargs) + + request_timestamp = int(1000.0 * time.time()) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("OpenAI", OPENAI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + nr_kwargs = _responses_api_request_attrs(kwargs) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/OpenAI") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = await wrapped(*args, **kwargs) + except Exception as exc: + _record_completion_error( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=completion_id, + kwargs=nr_kwargs, + ft=ft, + exc=exc, + request_timestamp=request_timestamp, + ) + raise + + _handle_completion_success( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=completion_id, + kwargs=nr_kwargs, + ft=ft, + return_val=return_val, + request_timestamp=request_timestamp, + responses_api=True, + ) + return return_val + + def wrap_convert_to_openai_object(wrapped, instance, args, kwargs): """Obtain reponse headers for v0.""" transaction = current_transaction() @@ -879,6 +1054,11 @@ def close(self): def _record_stream_chunk(stream_proxy, return_val): + """ + Record a stream chunk emitted by OpenAI. + + Dispatches the bulk of the work to separate functions for OpenAI ChatCompletions vs Responses APIs. + """ transaction = current_transaction() if return_val: try: @@ -890,29 +1070,75 @@ def _record_stream_chunk(stream_proxy, return_val): stream_proxy._nr_openai_attrs["response_headers"] = getattr(stream_proxy, "_nr_response_headers", {}) else: stream_proxy._nr_openai_attrs["response_headers"] = getattr(return_val, "_nr_response_headers", {}) - choices = return_val.get("choices") or [] - stream_proxy._nr_openai_attrs["response.model"] = return_val.get("model") - stream_proxy._nr_openai_attrs["id"] = return_val.get("id") - stream_proxy._nr_openai_attrs["response.organization"] = return_val.get("organization") - stream_proxy._nr_openai_attrs["response.usage"] = return_val.get("usage") - if choices: - delta = choices[0].get("delta") or {} - if delta: - if delta.get("content") and "time_to_first_token" not in stream_proxy._nr_openai_attrs: - stream_proxy._nr_openai_attrs["time_to_first_token"] = ( - int(1000.0 * time.time()) - stream_proxy._nr_request_timestamp - ) - stream_proxy._nr_openai_attrs["content"] = stream_proxy._nr_openai_attrs.get("content", "") + ( - delta.get("content") or "" - ) - stream_proxy._nr_openai_attrs["role"] = stream_proxy._nr_openai_attrs.get("role") or delta.get( - "role" - ) - stream_proxy._nr_openai_attrs["finish_reason"] = choices[0].get("finish_reason") + + # Responses API stream events use a different schema than Chat Completions chunks. + if stream_proxy._nr_openai_attrs.get("_nr_response_api"): + _record_responses_stream_event(stream_proxy, return_val) + else: + _record_chat_completion_stream_event(stream_proxy, return_val) except Exception: _logger.warning(STREAM_PARSING_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) +def _record_chat_completion_stream_event(stream_proxy, event): + """ + Record a stream event emitted by the Chat Completion API. + + We keep a record of stream events as they arrive, and when the stream finishes we emit the events + from the agent as LLMChatCompletionMessage events using the shared _record_completion_success + function. + + This function helps to stores these attributes in a common format for the two different + OpenAI endpoints which produce different event shapes. + """ + + choices = event.get("choices") or [] + stream_proxy._nr_openai_attrs["response.model"] = event.get("model") + stream_proxy._nr_openai_attrs["id"] = event.get("id") + stream_proxy._nr_openai_attrs["response.organization"] = event.get("organization") + stream_proxy._nr_openai_attrs["response.usage"] = event.get("usage") + if choices: + delta = choices[0].get("delta") or {} + if delta: + if delta.get("content") and "time_to_first_token" not in stream_proxy._nr_openai_attrs: + stream_proxy._nr_openai_attrs["time_to_first_token"] = ( + int(1000.0 * time.time()) - stream_proxy._nr_request_timestamp + ) + stream_proxy._nr_openai_attrs["content"] = stream_proxy._nr_openai_attrs.get("content", "") + ( + delta.get("content") or "" + ) + stream_proxy._nr_openai_attrs["role"] = stream_proxy._nr_openai_attrs.get("role") or delta.get("role") + stream_proxy._nr_openai_attrs["finish_reason"] = choices[0].get("finish_reason") + + +def _record_responses_stream_event(stream_proxy, event): + """ + Record a stream event emitted by the Responses API. + + We keep a record of stream events as they arrive, and when the stream finishes we emit the events + from the agent as LLMChatCompletionMessage events using the shared _record_completion_success + function. + + This function helps to stores these attributes in a common format for the two different + OpenAI endpoints which produce different event shapes. + """ + event_type = event.get("type") + if event_type == "response.output_text.delta": + delta = event.get("delta") or "" + if delta and "time_to_first_token" not in stream_proxy._nr_openai_attrs: + stream_proxy._nr_openai_attrs["time_to_first_token"] = ( + int(1000.0 * time.time()) - stream_proxy._nr_request_timestamp + ) + stream_proxy._nr_openai_attrs["content"] = stream_proxy._nr_openai_attrs.get("content", "") + delta + stream_proxy._nr_openai_attrs["role"] = stream_proxy._nr_openai_attrs.get("role") or "assistant" + elif event_type in ("response.completed", "response.incomplete", "response.failed"): + response = _responses_api_response_attrs(event.get("response") or {}) + stream_proxy._nr_openai_attrs["response.model"] = response["response.model"] + stream_proxy._nr_openai_attrs["id"] = response["id"] + stream_proxy._nr_openai_attrs["response.usage"] = response["response.usage"] + stream_proxy._nr_openai_attrs["finish_reason"] = response["finish_reason"] + + def _record_events_on_stop_iteration(stream_proxy, transaction, request_timestamp=None): if hasattr(stream_proxy, "_nr_ft"): # We first check for our saved linking metadata before making a new call to get_trace_linking_metadata @@ -1127,6 +1353,19 @@ def instrument_openai_resources_chat_completions(module): wrap_function_wrapper(module, "AsyncCompletions.create", wrap_chat_completion_async) +def instrument_openai_resources_responses(module): + if hasattr(module, "Responses"): + if hasattr(module.Responses, "create"): + wrap_function_wrapper(module, "Responses.create", wrap_responses_sync) + if hasattr(module.Responses, "parse"): + wrap_function_wrapper(module, "Responses.parse", wrap_responses_sync) + if hasattr(module, "AsyncResponses"): + if hasattr(module.AsyncResponses, "create"): + wrap_function_wrapper(module, "AsyncResponses.create", wrap_responses_async) + if hasattr(module.AsyncResponses, "parse"): + wrap_function_wrapper(module, "AsyncResponses.parse", wrap_responses_async) + + def instrument_openai_resources_embeddings(module): if hasattr(module, "Embeddings"): if hasattr(module.Embeddings, "create"): From 77d42f4582f2f167fa4a43b671a6915997539c16 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 4 Aug 2026 16:29:29 -0700 Subject: [PATCH 2/9] Add tests for the OpenAI Responses API --- tests/mlmodel_openai/test_responses.py | 256 +++++++++++++++ tests/mlmodel_openai/test_responses_error.py | 185 +++++++++++ tests/mlmodel_openai/test_responses_parse.py | 276 ++++++++++++++++ tests/mlmodel_openai/test_responses_stream.py | 300 ++++++++++++++++++ .../test_responses_stream_error.py | 222 +++++++++++++ 5 files changed, 1239 insertions(+) create mode 100644 tests/mlmodel_openai/test_responses.py create mode 100644 tests/mlmodel_openai/test_responses_error.py create mode 100644 tests/mlmodel_openai/test_responses_parse.py create mode 100644 tests/mlmodel_openai/test_responses_stream.py create mode 100644 tests/mlmodel_openai/test_responses_stream_error.py diff --git a/tests/mlmodel_openai/test_responses.py b/tests/mlmodel_openai/test_responses.py new file mode 100644 index 0000000000..1e59bfaebc --- /dev/null +++ b/tests/mlmodel_openai/test_responses.py @@ -0,0 +1,256 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import openai +import pytest +from pydantic import BaseModel +from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + add_token_counts_to_chat_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_sans_content, + events_sans_llm_metadata, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute + +MODEL = "gpt-5.1" +INSTRUCTIONS = "You are a text parser." +PROMPT = [ + {"role": "user", "content": "How many letters are in the word Python? Answer in one word with no formatting."} +] + +responses_recorded_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "req_d1266ca570484342b5fd1b897dde95c3", + "duration": None, # Response time varies each test run + "request.model": MODEL, + "response.model": "gpt-5.1-2025-11-13", + "response.organization": "nr-test-org", + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.usage.completion_tokens": 11, + "response.usage.prompt_tokens": 33, + "response.usage.total_tokens": 44, + "response.choices.finish_reason": "completed", + "response.headers.llmVersion": "2020-10-01", + "response.headers.ratelimitLimitRequests": 10000, + "response.headers.ratelimitLimitTokens": 50000000, + "response.headers.ratelimitResetTokens": "0s", + "response.headers.ratelimitResetRequests": "6ms", + "response.headers.ratelimitRemainingTokens": 49999975, + "response.headers.ratelimitRemainingRequests": 9999, + "vendor": "openai", + "ingest_source": "Python", + "response.number_of_messages": 3, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "resp_052fa11ffa9ab45a006a5eb79243f48198a3778dfab4bf1742-0", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "req_d1266ca570484342b5fd1b897dde95c3", + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "resp_052fa11ffa9ab45a006a5eb79243f48198a3778dfab4bf1742-1", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "req_d1266ca570484342b5fd1b897dde95c3", + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "resp_052fa11ffa9ab45a006a5eb79243f48198a3778dfab4bf1742-2", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "req_d1266ca570484342b5fd1b897dde95c3", + "span_id": None, + "trace_id": "trace-id", + "content": "six", + "role": "assistant", + "completion_id": None, + "sequence": 2, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "is_response": True, + "ingest_source": "Python", + }, + ), +] + + +@pytest.fixture(params=("sync", "async")) +def exercise(request, loop, async_openai_client, sync_openai_client): + def _exercise(): + if request.param == "sync": + sync_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + else: + loop.run_until_complete( + async_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + ) + + return _exercise + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses:test_openai_responses_with_llm_metadata", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_with_llm_metadata(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + exercise() + + +@reset_core_stats_engine() +@validate_custom_events(responses_recorded_events) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses:test_openai_responses_with_raw_response", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_with_raw_response(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + exercise() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(events_sans_content(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses:test_openai_responses_no_content", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_no_content(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + exercise() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_custom_events(add_token_counts_to_chat_events(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses:test_openai_responses_with_token_count", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_with_token_count(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + exercise() + + +@reset_core_stats_engine() +@validate_custom_events(events_sans_llm_metadata(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses:test_openai_responses_no_llm_metadata", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_no_llm_metadata(set_trace_info, exercise): + set_trace_info() + exercise() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_settings +@validate_custom_event_count(count=0) +@validate_transaction_metrics(name="test_responses:test_openai_responses_disabled_ai_monitoring", background_task=True) +@background_task() +def test_openai_responses_disabled_ai_monitoring(set_trace_info, exercise): + set_trace_info() + exercise() + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_openai_responses_outside_transaction(set_trace_info, exercise): + exercise() diff --git a/tests/mlmodel_openai/test_responses_error.py b/tests/mlmodel_openai/test_responses_error.py new file mode 100644 index 0000000000..1beed10a9d --- /dev/null +++ b/tests/mlmodel_openai/test_responses_error.py @@ -0,0 +1,185 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import openai +import pytest +from testing_support.fixtures import dt_enabled, reset_core_stats_engine +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + events_sans_content, + events_with_context_attrs, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name + +INVALID_MODEL = "does-not-exist-nr-test" +EXPECTED_ERROR = openai.BadRequestError +INSTRUCTIONS = "You are a text parser." +PROMPT = [ + {"role": "user", "content": "How many letters are in the word Python? Answer in one word with no formatting."} +] + +expected_events_on_invalid_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": INVALID_MODEL, + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.number_of_messages": 2, + "response.organization": "nr-test-org", + "vendor": "openai", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), +] + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_invalid_model_error", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_invalid_model_error(set_trace_info, sync_openai_client): + set_trace_info() + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + sync_openai_client.responses.create( + model=INVALID_MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_invalid_model_error_no_content", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_invalid_model_error_no_content(set_trace_info, sync_openai_client): + set_trace_info() + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + sync_openai_client.responses.create( + model=INVALID_MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_invalid_model_error_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_invalid_model_error_async(loop, set_trace_info, async_openai_client): + set_trace_info() + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete( + async_openai_client.responses.create( + model=INVALID_MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_invalid_model_error_no_content_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_invalid_model_error_no_content_async(loop, set_trace_info, async_openai_client): + set_trace_info() + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + loop.run_until_complete( + async_openai_client.responses.create( + model=INVALID_MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + ) diff --git a/tests/mlmodel_openai/test_responses_parse.py b/tests/mlmodel_openai/test_responses_parse.py new file mode 100644 index 0000000000..375ac5b66a --- /dev/null +++ b/tests/mlmodel_openai/test_responses_parse.py @@ -0,0 +1,276 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""responses.parse() is a distinct entry point that does not route through create which returns structured output.""" + +import openai +import pytest +from pydantic import BaseModel +from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + add_token_counts_to_chat_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_sans_content, + events_sans_llm_metadata, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute + +MODEL = "gpt-5.1" +INSTRUCTIONS = "You are a text parser." +PROMPT = [ + {"role": "user", "content": "How many letters are in the word Python? Answer in one word with no formatting."} +] + + +responses_recorded_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "req_a23af6d131df4e958e44fd0fee47af24", + "duration": None, # Response time varies each test run + "request.model": MODEL, + "response.model": "gpt-5.1-2025-11-13", + "response.organization": "nr-test-org", + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.usage.completion_tokens": 19, + "response.usage.prompt_tokens": 64, + "response.usage.total_tokens": 83, + "response.choices.finish_reason": "completed", + "response.headers.llmVersion": "2020-10-01", + "response.headers.ratelimitLimitRequests": 10000, + "response.headers.ratelimitLimitTokens": 50000000, + "response.headers.ratelimitResetTokens": "0s", + "response.headers.ratelimitResetRequests": "6ms", + "response.headers.ratelimitRemainingTokens": 49999975, + "response.headers.ratelimitRemainingRequests": 9999, + "vendor": "openai", + "ingest_source": "Python", + "response.number_of_messages": 3, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "resp_0464cf6d565af4e8006a73d0e80c50819889e3cb5edfd6b6a3-0", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "req_a23af6d131df4e958e44fd0fee47af24", + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "resp_0464cf6d565af4e8006a73d0e80c50819889e3cb5edfd6b6a3-1", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "req_a23af6d131df4e958e44fd0fee47af24", + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "resp_0464cf6d565af4e8006a73d0e80c50819889e3cb5edfd6b6a3-2", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "req_a23af6d131df4e958e44fd0fee47af24", + "span_id": None, + "trace_id": "trace-id", + "content": '{"count":1}', + "role": "assistant", + "completion_id": None, + "sequence": 2, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "is_response": True, + "ingest_source": "Python", + }, + ), +] + + +class LetterCount(BaseModel): + count: int + + +@pytest.fixture(params=("sync", "async")) +def exercise(request, loop, async_openai_client, sync_openai_client): + def _exercise(): + if request.param == "sync": + sync_openai_client.responses.parse( + model=MODEL, + instructions=INSTRUCTIONS, + input=PROMPT, + text_format=LetterCount, + temperature=0.7, + max_output_tokens=500, + ) + else: + loop.run_until_complete( + async_openai_client.responses.parse( + model=MODEL, + instructions=INSTRUCTIONS, + input=PROMPT, + text_format=LetterCount, + temperature=0.7, + max_output_tokens=500, + ) + ) + + return _exercise + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_parse:test_openai_responses_parse_with_llm_metadata", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_parse_with_llm_metadata(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + exercise() + + +@reset_core_stats_engine() +@validate_custom_events(responses_recorded_events) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_parse:test_openai_responses_parse_with_raw_response", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_parse_with_raw_response(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + exercise() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(events_sans_content(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_parse:test_openai_responses_parse_no_content", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_parse_no_content(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + exercise() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +@validate_custom_events(add_token_counts_to_chat_events(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_parse:test_openai_responses_parse_with_token_count", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_parse_with_token_count(set_trace_info, exercise): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + exercise() + + +@reset_core_stats_engine() +@validate_custom_events(events_sans_llm_metadata(responses_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_parse:test_openai_responses_parse_no_llm_metadata", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_parse_no_llm_metadata(set_trace_info, exercise): + set_trace_info() + exercise() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_settings +@validate_custom_event_count(count=0) +@validate_transaction_metrics( + name="test_responses_parse:test_openai_responses_parse_disabled_ai_monitoring", background_task=True +) +@background_task() +def test_openai_responses_parse_disabled_ai_monitoring(set_trace_info, exercise): + set_trace_info() + exercise() + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_openai_responses_parse_outside_transaction(set_trace_info, exercise): + exercise() diff --git a/tests/mlmodel_openai/test_responses_stream.py b/tests/mlmodel_openai/test_responses_stream.py new file mode 100644 index 0000000000..5dc91165a0 --- /dev/null +++ b/tests/mlmodel_openai/test_responses_stream.py @@ -0,0 +1,300 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Streaming tests for OpenAI Responses API instrumentation (``responses.create(stream=True)``). + +See test_responses.py for the recording workflow and the note on wildcarded (``None``) expected +values. Streaming summaries additionally carry ``time_to_first_token``; token usage is reconstructed +from the terminal ``response.completed`` event. +""" + +import openai +import pytest +from conftest import OPENAI_VERSION +from testing_support.fixtures import reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + disabled_ai_monitoring_streaming_settings, + events_sans_content, + events_with_context_attrs, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute + +SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE = pytest.mark.skipif( + OPENAI_VERSION < (1, 8), reason="OpenAI does not support .with_streaming_response. until v1.8" +) + +MODEL = "gpt-5.1" +INSTRUCTIONS = "You are a text parser." +PROMPT = [ + {"role": "user", "content": "How many letters are in the word Python? Answer in one word with no formatting."} +] + +responses_stream_recorded_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": None, + "duration": None, + "request.model": MODEL, + "response.model": "gpt-5.1-2025-11-13", + "response.organization": "nr-test-org", + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.usage.completion_tokens": 11, + "response.usage.prompt_tokens": 33, + "response.usage.total_tokens": 44, + "response.choices.finish_reason": "completed", + "response.headers.llmVersion": "2020-10-01", + "response.headers.ratelimitLimitRequests": 10000, + "response.headers.ratelimitLimitTokens": 50000000, + "response.headers.ratelimitResetTokens": "0s", + "response.headers.ratelimitResetRequests": "6ms", + "response.headers.ratelimitRemainingTokens": 49999975, + "response.headers.ratelimitRemainingRequests": 9999, + "vendor": "openai", + "ingest_source": "Python", + "response.number_of_messages": 3, + "time_to_first_token": None, # varies each test run + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": None, + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": None, + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": None, + "span_id": None, + "trace_id": "trace-id", + "content": "six", + "role": "assistant", + "completion_id": None, + "sequence": 2, + "response.model": "gpt-5.1-2025-11-13", + "token_count": 0, + "vendor": "openai", + "is_response": True, + "ingest_source": "Python", + }, + ), +] + + +def _consume_sync(client): + generator = client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500, stream=True + ) + for _ in generator: + pass + + +def _consume_async(client, loop): + async def _run(): + generator = await client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500, stream=True + ) + async for _ in generator: + pass + + loop.run_until_complete(_run()) + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(responses_stream_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_stream:test_openai_responses_stream_sync_with_llm_metadata", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_stream_sync_with_llm_metadata(set_trace_info, sync_openai_client): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + _consume_sync(sync_openai_client) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(events_sans_content(responses_stream_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_stream:test_openai_responses_stream_sync_no_content", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_stream_sync_no_content(set_trace_info, sync_openai_client): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + _consume_sync(sync_openai_client) + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(responses_stream_recorded_events)) +@validate_custom_event_count(count=4) +@validate_transaction_metrics( + name="test_responses_stream:test_openai_responses_stream_async_with_llm_metadata", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_stream_async_with_llm_metadata(set_trace_info, async_openai_client, loop): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + _consume_async(async_openai_client, loop) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_streaming_settings +@validate_custom_event_count(count=0) +@background_task() +def test_openai_responses_stream_sync_disabled_streaming(set_trace_info, sync_openai_client): + # With streaming monitoring disabled we do not proxy the stream, so no LLM events are recorded. + set_trace_info() + _consume_sync(sync_openai_client) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_settings +@validate_custom_event_count(count=0) +@background_task() +def test_openai_responses_stream_sync_disabled_ai_monitoring(set_trace_info, sync_openai_client): + set_trace_info() + _consume_sync(sync_openai_client) + + +def _with_streaming_response_create_kwargs(stream_set, stream_val): + kwargs = { + "model": MODEL, + "instructions": INSTRUCTIONS, + "input": PROMPT, + "temperature": 0.7, + "max_output_tokens": 500, + } + if stream_set: + kwargs["stream"] = stream_val + return kwargs + + +# `.with_streaming_response.` is not yet instrumented, so it emits no LLM events. The tests below +# assert that current behavior (count=0). Instrumenting it is future work. +@SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE +@reset_core_stats_engine() +@pytest.mark.parametrize("iter_method", ["iter_lines", "iter_bytes", "iter_text"]) +@pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) +@validate_custom_event_count(count=0) +@validate_transaction_metrics( + name="test_responses_stream:test_openai_responses_sync_with_streaming_response", background_task=True +) +@background_task() +def test_openai_responses_sync_with_streaming_response( + set_trace_info, sync_openai_client, iter_method, stream_set, stream_val +): + set_trace_info() + create_kwargs = _with_streaming_response_create_kwargs(stream_set, stream_val) + with sync_openai_client.responses.with_streaming_response.create(**create_kwargs) as response: + for _ in getattr(response, iter_method)(): + pass + + +@SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE +@reset_core_stats_engine() +@pytest.mark.parametrize("iter_method", ["iter_lines", "iter_bytes", "iter_text"]) +@pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) +@validate_custom_event_count(count=0) +@validate_transaction_metrics( + name="test_responses_stream:test_openai_responses_async_with_streaming_response", background_task=True +) +@background_task() +def test_openai_responses_async_with_streaming_response( + set_trace_info, async_openai_client, loop, iter_method, stream_set, stream_val +): + set_trace_info() + create_kwargs = _with_streaming_response_create_kwargs(stream_set, stream_val) + + async def _run(): + async with async_openai_client.responses.with_streaming_response.create(**create_kwargs) as response: + async for _ in getattr(response, iter_method)(): + pass + + loop.run_until_complete(_run()) diff --git a/tests/mlmodel_openai/test_responses_stream_error.py b/tests/mlmodel_openai/test_responses_stream_error.py new file mode 100644 index 0000000000..2d5d173e18 --- /dev/null +++ b/tests/mlmodel_openai/test_responses_stream_error.py @@ -0,0 +1,222 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import openai +import pytest +from testing_support.fixtures import dt_enabled, reset_core_stats_engine +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + events_sans_content, + events_with_context_attrs, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name + +INVALID_MODEL = "does-not-exist-nr-test" +EXPECTED_ERROR = openai.BadRequestError +INSTRUCTIONS = "You are a text parser." +PROMPT = [ + {"role": "user", "content": "How many letters are in the word Python? Answer in one word with no formatting."} +] + +expected_events_on_invalid_model_error = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, + "timestamp": None, + "llm.context": "attr", + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, + "request.model": INVALID_MODEL, + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.number_of_messages": 2, + "response.organization": "nr-test-org", + "vendor": "openai", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "timestamp": None, + "llm.context": "attr", + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "timestamp": None, + "llm.context": "attr", + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), +] + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_invalid_model_error", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_invalid_model_error(set_trace_info, sync_openai_client): + set_trace_info() + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + generator = sync_openai_client.responses.create( + model=INVALID_MODEL, + instructions=INSTRUCTIONS, + input=PROMPT, + temperature=0.7, + max_output_tokens=500, + stream=True, + ) + for _ in generator: + pass + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_invalid_model_error_no_content", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_invalid_model_error_no_content(set_trace_info, sync_openai_client): + set_trace_info() + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + generator = sync_openai_client.responses.create( + model=INVALID_MODEL, + instructions=INSTRUCTIONS, + input=PROMPT, + temperature=0.7, + max_output_tokens=500, + stream=True, + ) + for _ in generator: + pass + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_invalid_model_error_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_invalid_model_error_async(loop, set_trace_info, async_openai_client): + set_trace_info() + + async def _run(): + generator = await async_openai_client.responses.create( + model=INVALID_MODEL, + instructions=INSTRUCTIONS, + input=PROMPT, + temperature=0.7, + max_output_tokens=500, + stream=True, + ) + async for _ in generator: + pass + + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete(_run()) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes(callable_name(EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_invalid_model_error_no_content_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_invalid_model_error)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_invalid_model_error_no_content_async(loop, set_trace_info, async_openai_client): + set_trace_info() + + async def _run(): + generator = await async_openai_client.responses.create( + model=INVALID_MODEL, + instructions=INSTRUCTIONS, + input=PROMPT, + temperature=0.7, + max_output_tokens=500, + stream=True, + ) + async for _ in generator: + pass + + with pytest.raises(EXPECTED_ERROR): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete(_run()) From 4d742f0886bf041ab9961fbef1cebf1ef5887fca Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 4 Aug 2026 16:29:36 -0700 Subject: [PATCH 3/9] Record Responses API cassettes for VCR replay --- tests/mlmodel_openai/cassette.yaml | 175 +++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/tests/mlmodel_openai/cassette.yaml b/tests/mlmodel_openai/cassette.yaml index 0b26ea5197..02683e9bd6 100644 --- a/tests/mlmodel_openai/cassette.yaml +++ b/tests/mlmodel_openai/cassette.yaml @@ -1926,4 +1926,179 @@ interactions: status: code: 200 message: OK +- request: + body: '{"input":[{"role":"user","content":"How many letters are in the word Python? + Answer in one word with no formatting."}],"instructions":"You are a text parser.","max_output_tokens":500,"model":"gpt-5.1","stream":false,"temperature":0.7}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - XXXXXX + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + x-stainless-read-timeout: + - '600' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: '{"id": "resp_081ea74341470ade006a725d6656348199bd2fadddedab5ed1", "object": + "response", "created_at": 1785879910, "status": "completed", "background": + false, "billing": {"payer": "developer"}, "completed_at": 1785879911, "error": + null, "frequency_penalty": 0.0, "incomplete_details": null, "instructions": + "You are a text parser.", "max_output_tokens": 500, "max_tool_calls": null, + "model": "gpt-5.1-2025-11-13", "moderation": null, "output": [{"id": "msg_081ea74341470ade006a725d66d4688199972a545cb42ae2ed", + "type": "message", "status": "completed", "content": [{"type": "output_text", + "annotations": [], "logprobs": [], "text": "six"}], "role": "assistant"}], + "parallel_tool_calls": true, "presence_penalty": 0.0, "previous_response_id": + null, "prompt_cache_key": null, "prompt_cache_retention": "in_memory", "reasoning": + {"context": "current_turn", "effort": "none", "mode": "standard", "summary": + null}, "safety_identifier": null, "service_tier": "default", "store": true, + "temperature": 0.7, "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", "tool_usage": {"image_gen": {"input_tokens": 0, "input_tokens_details": + {"image_tokens": 0, "text_tokens": 0}, "output_tokens": 0, "output_tokens_details": + {"image_tokens": 0, "text_tokens": 0}, "total_tokens": 0}, "web_search": {"num_requests": + 0}}, "tools": [], "top_logprobs": 0, "top_p": 1.0, "truncation": "disabled", + "usage": {"input_tokens": 33, "input_tokens_details": {"cache_write_tokens": + 0, "cached_tokens": 0}, "output_tokens": 11, "output_tokens_details": {"reasoning_tokens": + 0}, "total_tokens": 44}, "user": null, "metadata": {}}' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + cf-cache-status: + - DYNAMIC + cf-ray: + - a260bf5e9cc70d87-PDX + connection: + - keep-alive + content-type: + - application/json + date: + - Tue, 04 Aug 2026 21:45:11 GMT + openai-organization: + - nr-test-org + openai-processing-ms: + - '1018' + openai-project: + - nr-test-project + openai-version: + - '2020-10-01' + server: + - cloudflare + transfer-encoding: + - chunked + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '50000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '49999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_68cab68b5b5543fb87a72b0dcca278f8 + status: + code: 200 + message: OK +- request: + body: '{"input":[{"role":"user","content":"How many letters are in the word Python? + Answer in one word with no formatting."}],"instructions":"You are a text parser.","max_output_tokens":500,"model":"gpt-5.1","temperature":0.7,"text":{"format":{"type":"json_schema","strict":true,"name":"LetterCount","schema":{"properties":{"count":{"title":"Count","type":"integer"}},"required":["count"],"title":"LetterCount","type":"object","additionalProperties":false}}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - XXXXXX + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + x-stainless-read-timeout: + - '600' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: '{"id": "resp_0464cf6d565af4e8006a73d0e80c50819889e3cb5edfd6b6a3", "object": + "response", "created_at": 1785975016, "status": "completed", "background": + false, "billing": {"payer": "developer"}, "completed_at": 1785975017, "error": + null, "frequency_penalty": 0.0, "incomplete_details": null, "instructions": + "You are a text parser.", "max_output_tokens": 500, "max_tool_calls": null, + "model": "gpt-5.1-2025-11-13", "moderation": null, "output": [{"id": "msg_0464cf6d565af4e8006a73d0e9117081989140609510e0639d", + "type": "message", "status": "completed", "content": [{"type": "output_text", + "annotations": [], "logprobs": [], "text": "{\"count\":1}"}], "role": "assistant"}], + "parallel_tool_calls": true, "presence_penalty": 0.0, "previous_response_id": + null, "prompt_cache_key": null, "prompt_cache_retention": "in_memory", "reasoning": + {"context": "current_turn", "effort": "none", "mode": "standard", "summary": + null}, "safety_identifier": null, "service_tier": "default", "store": true, + "temperature": 0.7, "text": {"format": {"type": "json_schema", "description": + null, "name": "LetterCount", "schema": {"properties": {"count": {"title": + "Count", "type": "integer"}}, "required": ["count"], "title": "LetterCount", + "type": "object", "additionalProperties": false}, "strict": true}, "verbosity": + "medium"}, "tool_choice": "auto", "tool_usage": {"image_gen": {"input_tokens": + 0, "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, "output_tokens": + 0, "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, "total_tokens": + 0}, "web_search": {"num_requests": 0}}, "tools": [], "top_logprobs": 0, "top_p": + 1.0, "truncation": "disabled", "usage": {"input_tokens": 64, "input_tokens_details": + {"cache_write_tokens": 0, "cached_tokens": 0}, "output_tokens": 19, "output_tokens_details": + {"reasoning_tokens": 0}, "total_tokens": 83}, "user": null, "metadata": {}}' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + cf-cache-status: + - DYNAMIC + cf-ray: + - a269d149cda6dcdd-PDX + connection: + - keep-alive + content-type: + - application/json + date: + - Thu, 06 Aug 2026 00:10:17 GMT + openai-organization: + - nr-test-org + openai-processing-ms: + - '1725' + openai-project: + - nr-test-project + openai-version: + - '2020-10-01' + server: + - cloudflare + transfer-encoding: + - chunked + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '50000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '49999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_a23af6d131df4e958e44fd0fee47af24 + status: + code: 200 + message: OK version: 1 From c864e0edd9cad6598c6f04a72be23b3431444c9f Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Wed, 5 Aug 2026 17:21:58 -0700 Subject: [PATCH 4/9] Cover the uninstrumented with_streaming_response path with count=0 tests --- .../test_chat_completion_stream.py | 16 ++++++++++++++-- tests/mlmodel_openai/test_responses_stream.py | 6 ++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/mlmodel_openai/test_chat_completion_stream.py b/tests/mlmodel_openai/test_chat_completion_stream.py index 5d15427dd9..e5e3caf104 100644 --- a/tests/mlmodel_openai/test_chat_completion_stream.py +++ b/tests/mlmodel_openai/test_chat_completion_stream.py @@ -36,8 +36,8 @@ from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes from newrelic.api.transaction import add_custom_attribute -# TODO: Once instrumentation support is added for `.with_streaming_response.` -# the validator checks can be uncommented/active. +# TODO: Once instrumentation support is added for with_streaming_response +# the validators can be uncommented/active. SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE = pytest.mark.skipif( OPENAI_VERSION < (1, 8), reason="OpenAI does not support .with_streaming_response. until v1.8" @@ -189,6 +189,8 @@ def test_openai_chat_completion_sync_with_llm_metadata(set_trace_info, sync_open @SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE @reset_core_stats_engine() @pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) +# TODO: Activate validators for with_streaming_response once instrumented. +@validate_custom_event_count(count=0) @validate_transaction_metrics( name="test_chat_completion_stream:test_openai_chat_completion_sync_with_llm_metadata_with_streaming_response_lines", # custom_metrics=[ @@ -223,6 +225,8 @@ def test_openai_chat_completion_sync_with_llm_metadata_with_streaming_response_l @SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE @reset_core_stats_engine() @pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) +# TODO: Activate validators for with_streaming_response once instrumented. +@validate_custom_event_count(count=0) @validate_transaction_metrics( name="test_chat_completion_stream:test_openai_chat_completion_sync_with_llm_metadata_with_streaming_response_bytes", # custom_metrics=[ @@ -257,6 +261,8 @@ def test_openai_chat_completion_sync_with_llm_metadata_with_streaming_response_b @SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE @reset_core_stats_engine() @pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) +# TODO: Activate validators for with_streaming_response once instrumented. +@validate_custom_event_count(count=0) @validate_transaction_metrics( name="test_chat_completion_stream:test_openai_chat_completion_sync_with_llm_metadata_with_streaming_response_text", # custom_metrics=[ @@ -522,6 +528,8 @@ async def consumer(): @pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) # @validate_custom_events(chat_completion_recorded_events) # @validate_custom_event_count(count=4) +# TODO: Activate validators for with_streaming_response once instrumented. +@validate_custom_event_count(count=0) @validate_transaction_metrics( "test_chat_completion_stream:test_openai_chat_completion_async_with_llm_metadata_with_streaming_response_lines", # scoped_metrics=[("Llm/completion/OpenAI/create", 1)], @@ -562,6 +570,8 @@ async def consumer(): @pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) # @validate_custom_events(chat_completion_recorded_events) # @validate_custom_event_count(count=4) +# # TODO: Activate validators for with_streaming_response once instrumented. +@validate_custom_event_count(count=0) @validate_transaction_metrics( "test_chat_completion_stream:test_openai_chat_completion_async_with_llm_metadata_with_streaming_response_bytes", # scoped_metrics=[("Llm/completion/OpenAI/create", 1)], @@ -602,6 +612,8 @@ async def consumer(): @pytest.mark.parametrize("stream_set, stream_val", [(False, None), (True, True), (True, False)]) # @validate_custom_events(chat_completion_recorded_events) # @validate_custom_event_count(count=4) +# TODO: Activate validators for with_streaming_response once instrumented. +@validate_custom_event_count(count=0) @validate_transaction_metrics( "test_chat_completion_stream:test_openai_chat_completion_async_with_llm_metadata_with_streaming_response_text", # scoped_metrics=[("Llm/completion/OpenAI/create", 1)], diff --git a/tests/mlmodel_openai/test_responses_stream.py b/tests/mlmodel_openai/test_responses_stream.py index 5dc91165a0..629306d37d 100644 --- a/tests/mlmodel_openai/test_responses_stream.py +++ b/tests/mlmodel_openai/test_responses_stream.py @@ -256,8 +256,10 @@ def _with_streaming_response_create_kwargs(stream_set, stream_val): return kwargs -# `.with_streaming_response.` is not yet instrumented, so it emits no LLM events. The tests below -# assert that current behavior (count=0). Instrumenting it is future work. +# TODO: Once instrumentation support is added for .with_streaming_response +# the validators can be rewritten. + + @SKIP_IF_NO_OPENAI_WITH_STREAMING_RESPONSE @reset_core_stats_engine() @pytest.mark.parametrize("iter_method", ["iter_lines", "iter_bytes", "iter_text"]) From bb1e7407f1588dd9eaf6f9342072e4a4fb2223fd Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Fri, 7 Aug 2026 14:24:31 -0700 Subject: [PATCH 5/9] Fix collect_ignore for openai v0 --- tests/mlmodel_openai/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/mlmodel_openai/conftest.py b/tests/mlmodel_openai/conftest.py index 9b152f0739..d0bd274395 100644 --- a/tests/mlmodel_openai/conftest.py +++ b/tests/mlmodel_openai/conftest.py @@ -58,8 +58,9 @@ # The Responses API only exists in openai v1+. "test_responses.py", "test_responses_error.py", - "test_responses_stream.py", + "test_responses_parse.py", "test_responses_stream_error.py", + "test_responses_stream.py", ] else: collect_ignore = [ From f156a2bd516a25cb2dc54c100fe20a97708178a9 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Mon, 12 Oct 2026 15:05:15 -0400 Subject: [PATCH 6/9] Testing for midband failures --- tests/mlmodel_openai/test_responses.py | 145 ++++++++++ tests/mlmodel_openai/test_responses_error.py | 256 ++++++++++++++++++ .../test_responses_stream_error.py | 246 +++++++++++++++++ 3 files changed, 647 insertions(+) diff --git a/tests/mlmodel_openai/test_responses.py b/tests/mlmodel_openai/test_responses.py index 1e59bfaebc..7ed2f8000a 100644 --- a/tests/mlmodel_openai/test_responses.py +++ b/tests/mlmodel_openai/test_responses.py @@ -13,6 +13,8 @@ # limitations under the License. import openai import pytest +from openai.types.responses import Response +from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from pydantic import BaseModel from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes from testing_support.ml_testing_utils import ( @@ -254,3 +256,146 @@ def test_openai_responses_disabled_ai_monitoring(set_trace_info, exercise): @validate_custom_event_count(count=0) def test_openai_responses_outside_transaction(set_trace_info, exercise): exercise() + + +# === Tool-call-only turns === +# When the model responds with only a function_call (no assistant message text), the Responses API +# returns a completed response whose output carries no "message" item. The agent must not record an +# empty-content LlmChatCompletionMessage for it — only the summary and the input messages. This is +# mocked with a real SDK Response so the tool-call-only output shape is exercised deterministically. + +TOOL_CALL_RESPONSE_ID = "resp_nr_test_tool_call" +TOOL_CALL_RESPONSE_MODEL = "gpt-5.1-2025-11-13" + + +def _tool_call_response_post(*args, **kwargs): + return Response.model_construct( + id=TOOL_CALL_RESPONSE_ID, + status="completed", + model=TOOL_CALL_RESPONSE_MODEL, + output=[ + ResponseFunctionToolCall.model_construct( + id="fc_nr_test", + call_id="call_nr_test", + name="get_temperature", + arguments='{"location": "Paris"}', + type="function_call", + status="completed", + ) + ], + usage={"input_tokens": 40, "output_tokens": 8, "total_tokens": 48}, + ) + +async def _async_tool_call_response_post(*args, **kwargs): + return _tool_call_response_post() + + +# Only the summary and the two input messages are recorded — crucially, no empty assistant message. +expected_events_on_tool_call_response = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "duration": None, + "request.model": MODEL, + "response.model": TOOL_CALL_RESPONSE_MODEL, + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.usage.completion_tokens": 8, + "response.usage.prompt_tokens": 40, + "response.usage.total_tokens": 48, + "response.choices.finish_reason": "completed", + "vendor": "openai", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": f"{TOOL_CALL_RESPONSE_ID}-0", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": TOOL_CALL_RESPONSE_MODEL, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": f"{TOOL_CALL_RESPONSE_ID}-1", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": TOOL_CALL_RESPONSE_MODEL, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), +] + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(expected_events_on_tool_call_response)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_responses:test_openai_responses_tool_call_only_sync", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_tool_call_only_sync(monkeypatch, set_trace_info, sync_openai_client): + set_trace_info() + monkeypatch.setattr(sync_openai_client.responses, "_post", _tool_call_response_post) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + with WithLlmCustomAttributes({"context": "attr"}): + sync_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(expected_events_on_tool_call_response)) +@validate_custom_event_count(count=3) +@validate_transaction_metrics( + name="test_responses:test_openai_responses_tool_call_only_async", + custom_metrics=[(f"Supportability/Python/ML/OpenAI/{openai.__version__}", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_openai_responses_tool_call_only_async(monkeypatch, loop, set_trace_info, async_openai_client): + set_trace_info() + + monkeypatch.setattr(async_openai_client.responses, "_post", _async_tool_call_response_post) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete( + async_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + ) diff --git a/tests/mlmodel_openai/test_responses_error.py b/tests/mlmodel_openai/test_responses_error.py index 1beed10a9d..acfc019e92 100644 --- a/tests/mlmodel_openai/test_responses_error.py +++ b/tests/mlmodel_openai/test_responses_error.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json + import openai +import openai._base_client as _openai_base_client import pytest +from openai.types.responses import Response +from openai.types.responses.response_error import ResponseError from testing_support.fixtures import dt_enabled, reset_core_stats_engine from testing_support.ml_testing_utils import ( disabled_ai_monitoring_record_content_settings, @@ -38,6 +43,97 @@ {"role": "user", "content": "How many letters are in the word Python? Answer in one word with no formatting."} ] +# A valid model for the in-band failure case, where the request itself is well-formed but the +# server reports the failure in the response body rather than as an HTTP error. +MODEL = "gpt-5.1" +FAILED_RESPONSE_ID = "resp_nr_test_failed" +FAILED_RESPONSE_MODEL = "gpt-5.1-2025-11-13" +FAILED_ERROR_CODE = "server_error" +FAILED_ERROR_MESSAGE = "The model failed to generate a response." +# An in-band failure surfaces as a synthesized openai.APIError, not the SDK's HTTP error types. +FAILED_EXPECTED_ERROR = openai.APIError + +# The openai SDK depends on a vendored httpx (imported as "httpx2" in recent versions); resolve it +# from the SDK rather than importing httpx directly so this stays correct across SDK versions. +_httpx = getattr(_openai_base_client, "httpx2", None) or _openai_base_client.httpx + +# Served by a mock transport, so VCR must not try to replay or record requests to this host. +MOCK_HOST = "localhost" + + +def _failed_post(*args, **kwargs): + """A Responses API Response whose status is 'failed' with a nested error, as the server + returns for an in-band failure (e.g. a background response that terminates in failure).""" + return Response.model_construct( + id=FAILED_RESPONSE_ID, + status="failed", + error=ResponseError(code=FAILED_ERROR_CODE, message=FAILED_ERROR_MESSAGE), + model=FAILED_RESPONSE_MODEL, + output=[], + ) + + +async def _async_failed_post(*args, **kwargs): + return _failed_post() + + +expected_events_on_failed_response = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, + "request.model": MODEL, + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.number_of_messages": 2, + "vendor": "openai", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": f"{FAILED_RESPONSE_ID}-0", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": FAILED_RESPONSE_MODEL, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": f"{FAILED_RESPONSE_ID}-1", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": FAILED_RESPONSE_MODEL, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), +] + expected_events_on_invalid_model_error = [ ( {"type": "LlmChatCompletionSummary"}, @@ -183,3 +279,163 @@ def test_openai_responses_invalid_model_error_no_content_async(loop, set_trace_i model=INVALID_MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 ) ) + + +# === In-band failures: a 200 response whose status is "failed" === +# The SDK does not raise for these, so create() returns normally; the agent must still record the +# turn as an error. These are mocked because a synchronous responses.create() essentially never +# returns status "failed" against the live API (failures come back as HTTP errors), so there is no +# real interaction to record. + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_failed_status_error", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_failed_status_error(monkeypatch, set_trace_info, sync_openai_client): + set_trace_info() + monkeypatch.setattr(sync_openai_client.responses, "_post", _failed_post) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + # No pytest.raises: the SDK returns a Response with status "failed" rather than raising. + with WithLlmCustomAttributes({"context": "attr"}): + sync_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_failed_status_error_no_content", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_failed_status_error_no_content(monkeypatch, set_trace_info, sync_openai_client): + set_trace_info() + monkeypatch.setattr(sync_openai_client.responses, "_post", _failed_post) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + sync_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_failed_status_error_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_failed_status_error_async(monkeypatch, loop, set_trace_info, async_openai_client): + set_trace_info() + + monkeypatch.setattr(async_openai_client.responses, "_post", _async_failed_post) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete( + async_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + ) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_failed_status_error_no_content_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_failed_status_error_no_content_async(monkeypatch, loop, set_trace_info, async_openai_client): + set_trace_info() + + monkeypatch.setattr(async_openai_client.responses, "_post", _async_failed_post) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + loop.run_until_complete( + async_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) + ) + + +# === In-band failure returned through .with_raw_response === +# Unlike the cases above, this goes through the real SDK response parsing (a LegacyAPIResponse +# wrapping the HTTP response) instead of patching _post, so it covers the raw-response unwrapping. + + +@pytest.mark.vcr(ignore_hosts=[MOCK_HOST]) +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_error:test_openai_responses_failed_status_error_with_raw_response", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_failed_status_error_with_raw_response(set_trace_info, openai_api_key): + set_trace_info() + body = json.dumps( + { + "id": FAILED_RESPONSE_ID, + "object": "response", + "created_at": 0, + "model": FAILED_RESPONSE_MODEL, + "status": "failed", + "error": {"code": FAILED_ERROR_CODE, "message": FAILED_ERROR_MESSAGE}, + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + } + ).encode() + + def handler(request): + return _httpx.Response(200, content=body, headers={"content-type": "application/json"}) + + client = openai.OpenAI( + api_key=openai_api_key, + base_url=f"http://{MOCK_HOST}/v1", + http_client=_httpx.Client(transport=_httpx.MockTransport(handler)), + ) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + client.responses.with_raw_response.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500 + ) diff --git a/tests/mlmodel_openai/test_responses_stream_error.py b/tests/mlmodel_openai/test_responses_stream_error.py index 2d5d173e18..9aa7375971 100644 --- a/tests/mlmodel_openai/test_responses_stream_error.py +++ b/tests/mlmodel_openai/test_responses_stream_error.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json + import openai +import openai._base_client as _openai_base_client import pytest +from openai._streaming import AsyncStream, Stream +from openai.types.responses import ResponseStreamEvent from testing_support.fixtures import dt_enabled, reset_core_stats_engine from testing_support.ml_testing_utils import ( disabled_ai_monitoring_record_content_settings, @@ -38,6 +43,116 @@ {"role": "user", "content": "How many letters are in the word Python? Answer in one word with no formatting."} ] +# A valid model for the in-band failure case: the request is well-formed, but the stream reports +# the failure via a response.failed event rather than an HTTP error. +MODEL = "gpt-5.1" +FAILED_RESPONSE_ID = "resp_nr_test_failed_stream" +FAILED_RESPONSE_MODEL = "gpt-5.1-2025-11-13" +FAILED_ERROR_CODE = "server_error" +FAILED_ERROR_MESSAGE = "The model failed to generate a response." +# An in-band failure surfaces as a synthesized openai.APIError, not the SDK's HTTP error types. +FAILED_EXPECTED_ERROR = openai.APIError + +# The openai SDK depends on a vendored httpx (imported as "httpx2" in recent versions); resolve it +# from the SDK rather than importing httpx directly so this stays correct across SDK versions. +_httpx = getattr(_openai_base_client, "httpx2", None) or _openai_base_client.httpx + +_FAILED_STREAM_SSE = ( + b"event: response.failed\ndata: " + + json.dumps( + { + "type": "response.failed", + "sequence_number": 1, + "response": { + "id": FAILED_RESPONSE_ID, + "object": "response", + "created_at": 1784592277, + "status": "failed", + "error": {"code": FAILED_ERROR_CODE, "message": FAILED_ERROR_MESSAGE}, + "model": FAILED_RESPONSE_MODEL, + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 33, "output_tokens": 0, "total_tokens": 33}, + }, + } + ).encode() + + b"\n\n" +) + + +def _failed_sse_response(): + return _httpx.Response(200, content=_FAILED_STREAM_SSE, headers={"content-type": "text/event-stream"}) + + +def _build_failed_stream(client): + """A real openai.Stream that yields a single response.failed event, as the server sends for an + in-band failure. Building a real Stream keeps the agent's Stream._iter_events wrapper in play.""" + return Stream(cast_to=ResponseStreamEvent, response=_failed_sse_response(), client=client) + + +def _build_failed_async_stream(client): + return AsyncStream(cast_to=ResponseStreamEvent, response=_failed_sse_response(), client=client) + + +expected_events_on_failed_response = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "duration": None, + "request.model": MODEL, + "request.temperature": 0.7, + "request.max_tokens": 500, + "response.number_of_messages": 2, + "vendor": "openai", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": f"{FAILED_RESPONSE_ID}-0", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": INSTRUCTIONS, + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": FAILED_RESPONSE_MODEL, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": f"{FAILED_RESPONSE_ID}-1", + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "span_id": None, + "trace_id": "trace-id", + "content": PROMPT[0]["content"], + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": FAILED_RESPONSE_MODEL, + "token_count": 0, + "vendor": "openai", + "ingest_source": "Python", + }, + ), +] + expected_events_on_invalid_model_error = [ ( {"type": "LlmChatCompletionSummary"}, @@ -220,3 +335,134 @@ async def _run(): add_custom_attribute("llm.conversation_id", "my-awesome-id") with WithLlmCustomAttributes({"context": "attr"}): loop.run_until_complete(_run()) + + +# === In-band failures: a response.failed event mid-stream === +# The stream ends normally after a response.failed event (no exception is raised), so the agent +# must record the turn as an error rather than a success. These are mocked because a live stream +# does not reliably terminate in a failed state on demand, leaving no real interaction to record. + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_failed_status_error", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_failed_status_error(monkeypatch, set_trace_info, sync_openai_client): + set_trace_info() + monkeypatch.setattr( + sync_openai_client.responses, "_post", lambda *args, **kwargs: _build_failed_stream(sync_openai_client) + ) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + # No pytest.raises: the stream ends normally after the response.failed event. + with WithLlmCustomAttributes({"context": "attr"}): + generator = sync_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500, stream=True + ) + for _ in generator: + pass + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_failed_status_error_no_content", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_failed_status_error_no_content(monkeypatch, set_trace_info, sync_openai_client): + set_trace_info() + monkeypatch.setattr( + sync_openai_client.responses, "_post", lambda *args, **kwargs: _build_failed_stream(sync_openai_client) + ) + add_custom_attribute("llm.conversation_id", "my-awesome-id") + generator = sync_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500, stream=True + ) + for _ in generator: + pass + + +@dt_enabled +@reset_core_stats_engine() +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_failed_status_error_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_with_context_attrs(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_failed_status_error_async(monkeypatch, loop, set_trace_info, async_openai_client): + set_trace_info() + + async def _failed_post(*args, **kwargs): + return _build_failed_async_stream(async_openai_client) + + monkeypatch.setattr(async_openai_client.responses, "_post", _failed_post) + + async def _run(): + generator = await async_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500, stream=True + ) + async for _ in generator: + pass + + add_custom_attribute("llm.conversation_id", "my-awesome-id") + with WithLlmCustomAttributes({"context": "attr"}): + loop.run_until_complete(_run()) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_error_trace_attributes( + callable_name(FAILED_EXPECTED_ERROR), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}} +) +@validate_transaction_metrics( + "test_responses_stream_error:test_openai_responses_stream_failed_status_error_no_content_async", + scoped_metrics=[("Llm/completion/OpenAI/create", 1)], + rollup_metrics=[("Llm/completion/OpenAI/create", 1)], + background_task=True, +) +@validate_custom_events(events_sans_content(expected_events_on_failed_response)) +@validate_custom_event_count(count=3) +@background_task() +def test_openai_responses_stream_failed_status_error_no_content_async( + monkeypatch, loop, set_trace_info, async_openai_client +): + set_trace_info() + + async def _failed_post(*args, **kwargs): + return _build_failed_async_stream(async_openai_client) + + monkeypatch.setattr(async_openai_client.responses, "_post", _failed_post) + + async def _run(): + generator = await async_openai_client.responses.create( + model=MODEL, instructions=INSTRUCTIONS, input=PROMPT, temperature=0.7, max_output_tokens=500, stream=True + ) + async for _ in generator: + pass + + add_custom_attribute("llm.conversation_id", "my-awesome-id") + loop.run_until_complete(_run()) From 7477e7d832ecfadd109b9ee20710988fed2038ef Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Mon, 12 Oct 2026 15:05:25 -0400 Subject: [PATCH 7/9] Instrumentation updates for midband failures --- newrelic/hooks/mlmodel_openai.py | 152 ++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 21 deletions(-) diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index 610e8c5298..d5fa3c8aeb 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -498,6 +498,13 @@ def _handle_completion_success( ): settings = transaction.settings if transaction.settings is not None else global_settings() stream = kwargs.get("stream", False) + + # For a non-streaming call, coerce the response object to a dict up front so it can be reused + # by the success path below. + response = None + if not stream and return_val is not None: + response = _coerce_openai_response_to_dict(return_val) + # Only if streaming and streaming monitoring is enabled and the response is not empty # do we not exit the function trace. if not stream or not settings.ai_monitoring.streaming.enabled or not return_val: @@ -525,20 +532,9 @@ def _handle_completion_success( try: # If response is not a stream generator, record the event data. - # At this point, we have a response so we can grab attributes only available on the response object + # At this point, we have a response (already coerced to a dict above) so we can grab + # attributes only available on the response object. response_headers = getattr(return_val, "_nr_response_headers", {}) - response = return_val - - # In v1, response objects are pydantic models so this function call converts the - # object back to a dictionary for backwards compatibility. - if OPENAI_V1: - if hasattr(response, "model_dump"): - response = response.model_dump() - elif hasattr(response, "http_response") and hasattr(response.http_response, "text"): - # This is for the .with_raw_response. wrapper. This is expected - # to change, but the return type for now is the following: - # openai._legacy_response.LegacyAPIResponse - response = json.loads(response.http_response.text.strip()) if responses_api: # Merge the extracted response attributes into kwargs and record via the shared @@ -799,6 +795,24 @@ def _record_completion_error(*, transaction, linking_metadata, completion_id, kw _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) +def _coerce_openai_response_to_dict(return_val): + """Convert an OpenAI v1 response object into a plain dict for backwards-compatible attr extraction. + + v1 response objects are pydantic models; the .with_raw_response. wrapper instead returns a + LegacyAPIResponse carrying the raw JSON text. Anything else (e.g. v0 objects) is returned as-is. + """ + response = return_val + if OPENAI_V1: + if hasattr(response, "model_dump"): + response = response.model_dump() + elif hasattr(response, "http_response") and hasattr(response.http_response, "text"): + # This is for the .with_raw_response. wrapper. This is expected + # to change, but the return type for now is the following: + # openai._legacy_response.LegacyAPIResponse + response = json.loads(response.http_response.text.strip()) + return response + + def _responses_api_request_attrs(kwargs): """Extract attributes from the Responses API's request kwargs. Shared by streaming and non-streaming paths.""" input_ = kwargs.get("input") @@ -828,16 +842,32 @@ def _responses_api_request_attrs(kwargs): def _responses_api_response_attrs(response): """Extract attributes from the Responses API's Response object. Shared by streaming and non-streaming paths.""" response = response or {} + output = response.get("output") or [] # Response content exists in output items of type "message", inside content parts of type "output_text". - content = "".join( - part.get("text") - for item in response.get("output") or [] - if isinstance(item, dict) and item.get("type") == "message" - for part in item.get("content") or [] - if isinstance(part, dict) and part.get("type") == "output_text" and part.get("text") + # Tool-call-only responses have no message items; content stays None so no empty output message is recorded. + message_items = [ + item for item in response.get("output") or [] if isinstance(item, dict) and item.get("type") == "message" + ] + content = ( + "".join( + part.get("text") + for item in message_items + for part in item.get("content") or [] + if isinstance(part, dict) and part.get("type") == "output_text" and part.get("text") + ) + if message_items + else None ) + # A tool-call-only turn produces a function_call output item with no assistant message text. + # The Responses API has no "tool_calls" finish_reason for _record_completion_success to key + # off of (as the Chat Completions path does), so signal suppression here with content=None, + # which _record_completion_success already treats as "no output message to record". The tool + # call itself is echoed back in the input on the following turn, so nothing is lost. + if not content and any(isinstance(item, dict) and item.get("type") == "function_call" for item in output): + content = None + # The Responses API has no per choice finish_reason. # Use incomplete_details.reason if it exists, or fall back to status. incomplete_details = response.get("incomplete_details") or {} @@ -856,9 +886,54 @@ def _responses_api_response_attrs(response): "content": content, "role": "assistant", "finish_reason": incomplete_details.get("reason") or response.get("status"), + "error": (response.get("error") or {}) if response.get("status") == "failed" else None, + } + + +def _failed_response_kwargs(return_val, kwargs): + """Return kwargs for error recording if a non-streamed Response has status "failed", otherwise None.""" + if kwargs.get("stream"): + return None + try: + # .with_raw_response returns a LegacyAPIResponse; parse() caches its result for the caller. + response = return_val.parse() if hasattr(return_val, "http_response") else return_val + if getattr(response, "status", None) != "failed": + return None + attrs = _responses_api_response_attrs(response.model_dump()) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) + return None + return { + **kwargs, + "id": attrs["id"], + "response.model": attrs["response.model"], + "response_headers": getattr(return_val, "_nr_response_headers", {}), + "error": attrs["error"], } +def _record_responses_failure(*, transaction, linking_metadata, completion_id, kwargs, ft, request_timestamp): + # A failed Response arrives as a normal return value rather than an exception, so raise the + # APIError the SDK uses for other failures to give notice_error an active exception to capture. + try: + error = kwargs.get("error") + error = error if isinstance(error, dict) else {} + try: + raise openai.APIError(message=error.get("message") or "Response failed", request=None, body=error) + except openai.APIError as exc: + _record_completion_error( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=completion_id, + kwargs=kwargs, + ft=ft, + exc=exc, + request_timestamp=request_timestamp, + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) + + def wrap_responses_sync(wrapped, instance, args, kwargs): """Instrument openai.resources.responses.Responses.create() and .parse().""" transaction = current_transaction() @@ -899,6 +974,18 @@ def wrap_responses_sync(wrapped, instance, args, kwargs): ) raise + failure_kwargs = _failed_response_kwargs(return_val, nr_kwargs) + if failure_kwargs is not None: + _record_responses_failure( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=completion_id, + kwargs=failure_kwargs, + ft=ft, + request_timestamp=request_timestamp, + ) + return return_val + _handle_completion_success( transaction=transaction, linking_metadata=linking_metadata, @@ -952,6 +1039,18 @@ async def wrap_responses_async(wrapped, instance, args, kwargs): ) raise + failure_kwargs = _failed_response_kwargs(return_val, nr_kwargs) + if failure_kwargs is not None: + _record_responses_failure( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=completion_id, + kwargs=failure_kwargs, + ft=ft, + request_timestamp=request_timestamp, + ) + return return_val + _handle_completion_success( transaction=transaction, linking_metadata=linking_metadata, @@ -1137,6 +1236,7 @@ def _record_responses_stream_event(stream_proxy, event): stream_proxy._nr_openai_attrs["id"] = response["id"] stream_proxy._nr_openai_attrs["response.usage"] = response["response.usage"] stream_proxy._nr_openai_attrs["finish_reason"] = response["finish_reason"] + stream_proxy._nr_openai_attrs["error"] = response["error"] def _record_events_on_stop_iteration(stream_proxy, transaction, request_timestamp=None): @@ -1145,10 +1245,20 @@ def _record_events_on_stop_iteration(stream_proxy, transaction, request_timestam # Directly calling get_trace_linking_metadata() causes the incorrect span ID to be captured and associated with the LLM call # This leads to incorrect linking of the LLM call in the UI linking_metadata = stream_proxy._nr_metadata or get_trace_linking_metadata() + openai_attrs = getattr(stream_proxy, "_nr_openai_attrs", {}) + if openai_attrs.get("error") is not None: + _record_responses_failure( + transaction=transaction, + linking_metadata=linking_metadata, + completion_id=str(uuid.uuid4()), + kwargs=openai_attrs, + ft=stream_proxy._nr_ft, + request_timestamp=request_timestamp, + ) + openai_attrs.clear() + return stream_proxy._nr_ft.__exit__(None, None, None) try: - openai_attrs = getattr(stream_proxy, "_nr_openai_attrs", {}) - # If there are no openai attrs exit early as there's no data to record. if not openai_attrs: return From abe381e4ea4fd7c6f41760bc3723e6b56b6a1f2a Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Mon, 12 Oct 2026 15:09:36 -0400 Subject: [PATCH 8/9] Fixes from code review --- newrelic/hooks/mlmodel_openai.py | 27 ++++++++++---------------- tests/mlmodel_openai/test_responses.py | 1 + 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index d5fa3c8aeb..e2b36d07ad 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -499,12 +499,6 @@ def _handle_completion_success( settings = transaction.settings if transaction.settings is not None else global_settings() stream = kwargs.get("stream", False) - # For a non-streaming call, coerce the response object to a dict up front so it can be reused - # by the success path below. - response = None - if not stream and return_val is not None: - response = _coerce_openai_response_to_dict(return_val) - # Only if streaming and streaming monitoring is enabled and the response is not empty # do we not exit the function trace. if not stream or not settings.ai_monitoring.streaming.enabled or not return_val: @@ -532,8 +526,9 @@ def _handle_completion_success( try: # If response is not a stream generator, record the event data. - # At this point, we have a response (already coerced to a dict above) so we can grab - # attributes only available on the response object. + # Coercion happens inside the try block so a malformed response body can't raise into the + # user's OpenAI call. + response = None if stream else _coerce_openai_response_to_dict(return_val) response_headers = getattr(return_val, "_nr_response_headers", {}) if responses_api: @@ -846,9 +841,7 @@ def _responses_api_response_attrs(response): # Response content exists in output items of type "message", inside content parts of type "output_text". # Tool-call-only responses have no message items; content stays None so no empty output message is recorded. - message_items = [ - item for item in response.get("output") or [] if isinstance(item, dict) and item.get("type") == "message" - ] + message_items = [item for item in output if isinstance(item, dict) and item.get("type") == "message"] content = ( "".join( part.get("text") @@ -860,12 +853,12 @@ def _responses_api_response_attrs(response): else None ) - # A tool-call-only turn produces a function_call output item with no assistant message text. - # The Responses API has no "tool_calls" finish_reason for _record_completion_success to key - # off of (as the Chat Completions path does), so signal suppression here with content=None, - # which _record_completion_success already treats as "no output message to record". The tool - # call itself is echoed back in the input on the following turn, so nothing is lost. - if not content and any(isinstance(item, dict) and item.get("type") == "function_call" for item in output): + # A tool-call turn may include an empty-text message item alongside the function_call item, which + # would yield content="". The Responses API has no "tool_calls" finish_reason for + # _record_completion_success to key off of (as the Chat Completions path does), so normalize that + # empty string to None, which it treats as "no output message to record". The tool call itself is + # echoed back in the input on the following turn, so nothing is lost. + if content == "" and any(isinstance(item, dict) and item.get("type") == "function_call" for item in output): content = None # The Responses API has no per choice finish_reason. diff --git a/tests/mlmodel_openai/test_responses.py b/tests/mlmodel_openai/test_responses.py index 7ed2f8000a..3d1216aec8 100644 --- a/tests/mlmodel_openai/test_responses.py +++ b/tests/mlmodel_openai/test_responses.py @@ -286,6 +286,7 @@ def _tool_call_response_post(*args, **kwargs): usage={"input_tokens": 40, "output_tokens": 8, "total_tokens": 48}, ) + async def _async_tool_call_response_post(*args, **kwargs): return _tool_call_response_post() From 6ce08d9341102ba5a91a3dd3fd23f577099b4823 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 13 Oct 2026 16:30:58 -0700 Subject: [PATCH 9/9] Add clarifying comment for role check --- newrelic/hooks/mlmodel_openai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index e2b36d07ad..faec7fe65d 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -818,6 +818,8 @@ def _responses_api_request_attrs(kwargs): messages.append({"content": input_, "role": "user"}) elif isinstance(input_, (list, tuple)): for item in input_: + # Only items with a role are messages. function_call / function_call_output items carry no role, + # so they are skipped here, mirroring chat_completion suppression of content-less tool-call messages. if isinstance(item, dict) and item.get("role") is not None: content = item.get("content") # Content is a string, or a list of typed parts each carrying `text`.