Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions sentry_sdk/integrations/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -759,15 +759,15 @@ def _set_common_output_data(
if record_outputs:
for output in response.output:
if output.type == "function_call":
output_messages["tool"].append(output.dict())
output_messages["tool"].append(output.model_dump())
elif output.type == "message":
for output_message in output.content:
try:
output_messages["response"].append(output_message.text) # type: ignore[union-attr]
except AttributeError:
# Unknown output message type, just return the json
output_messages["response"].append(
output_message.dict()
output_message.model_dump()
)

if len(output_messages["tool"]) > 0:
Expand All @@ -790,14 +790,16 @@ def _set_common_output_data(
):
for output in response.output:
if output.type == "function_call":
output_messages["tool"].append(output.dict())
output_messages["tool"].append(output.model_dump())
elif output.type == "message":
for output_message in output.content:
try:
output_messages["response"].append(output_message.text) # type: ignore[union-attr]
except AttributeError:
# Unknown output message type, just return the json
output_messages["response"].append(output_message.dict())
output_messages["response"].append(
output_message.model_dump()
)

if len(output_messages["tool"]) > 0:
set_data_normalized(
Expand Down
74 changes: 74 additions & 0 deletions tests/integrations/openai/test_openai.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import warnings

import pytest

Expand Down Expand Up @@ -46,6 +47,11 @@
except ImportError:
pass

try:
from pydantic.warnings import PydanticDeprecatedSince20
except ImportError:
PydanticDeprecatedSince20 = None

SKIP_RESPONSES_TESTS = False

try:
Expand Down Expand Up @@ -6248,6 +6254,74 @@ def test_responses_api_data_collection_outputs_shapes(
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data


@pytest.mark.parametrize(
"data_collection,send_default_pii",
[
pytest.param({"gen_ai": {"outputs": True}}, False, id="data-collection"),
pytest.param(None, True, id="send-default-pii"),
],
)
@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available")
@pytest.mark.skipif(
PydanticDeprecatedSince20 is None,
reason="Pydantic v1 does not deprecate `dict()`",
)
def test_responses_api_output_serialization_emits_no_pydantic_deprecation_warning(
sentry_init,
capture_events,
capture_items,
data_collection,
send_default_pii,
):
init_kwargs = {
"integrations": [OpenAIIntegration()],
"disabled_integrations": [StdlibIntegration],
"traces_sample_rate": 1.0,
"send_default_pii": send_default_pii,
"stream_gen_ai_spans": False,
}
if data_collection is not None:
init_kwargs["data_collection"] = data_collection

sentry_init(**init_kwargs)

client = OpenAI(api_key="z")
client.responses._post = mock.Mock(
return_value=_make_responses_api_response(
output=[
_make_responses_api_output_message(
content=[
ResponseOutputRefusal(
refusal="I cannot help with that.",
type="refusal",
),
]
),
_make_responses_api_function_call(),
]
)
)

with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
span_data = _collect_responses_span_data(
capture_events,
capture_items,
False,
False,
lambda: client.responses.create(model="gpt-4o", input="hello"),
)

assert not [
warning
for warning in caught
if issubclass(warning.category, PydanticDeprecatedSince20)
]

assert "I cannot help with that." in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT]
assert "get_current_weather" in span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]


@pytest.mark.parametrize("span_streaming", [True, False])
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
@pytest.mark.parametrize(
Expand Down
97 changes: 97 additions & 0 deletions tests/integrations/openai_agents/test_openai_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import json
import logging
import os
import warnings
from unittest import mock
from unittest.mock import MagicMock, patch

Expand All @@ -27,6 +28,7 @@
from agents.version import __version__ as OPENAI_AGENTS_VERSION
from openai import AsyncOpenAI, InternalServerError
from openai.types.responses.tool_param import CodeInterpreter, ImageGeneration
from pydantic.warnings import PydanticDeprecatedSince20

from sentry_sdk.integrations import DidNotEnable

Expand Down Expand Up @@ -82,6 +84,7 @@
Response,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseOutputRefusal,
ResponseTextDeltaEvent,
ResponseUsage,
)
Expand Down Expand Up @@ -3561,6 +3564,100 @@ async def test_tool_execution_span(
assert ai_client_span2["data"]["gen_ai.usage.total_tokens"] == 25


@pytest.mark.asyncio
async def test_response_output_serialization_emits_no_pydantic_deprecation_warning(
sentry_init,
capture_items,
test_agent,
simple_test_tool,
get_model_response,
nonstreaming_responses_tool_call_model_responses,
):
client = AsyncOpenAI(api_key="test-key")
model = OpenAIResponsesModel(model="gpt-4", openai_client=client)
agent_with_tool = test_agent.clone(tools=[simple_test_tool], model=model)

usage = ResponseUsage(
input_tokens=10,
input_tokens_details=InputTokensDetails(
cached_tokens=0,
cache_write_tokens=0,
),
output_tokens=5,
output_tokens_details=OutputTokensDetails(
reasoning_tokens=0,
),
total_tokens=15,
)
responses = nonstreaming_responses_tool_call_model_responses(
tool_name="simple_test_tool",
arguments='{"message": "hello"}',
response_model="gpt-4",
response_text="Task completed using the tool",
response_ids=iter(["resp_tool_123", "resp_final_123"]),
usages=iter([usage, usage]),
)
tool_call_response = next(responses)
# `openai-agents` can raise `ModelRefusalError` when a refusal ends the run.
tool_call_response.output.append(
ResponseOutputMessage(
id="msg_refusal",
type="message",
status="completed",
content=[
ResponseOutputRefusal(
refusal="Refused by the model",
type="refusal",
)
],
role="assistant",
)
)
final_response = next(responses)

sentry_init(
integrations=[OpenAIAgentsIntegration()],
disabled_integrations=[StdlibIntegration],
traces_sample_rate=1.0,
send_default_pii=True,
)
items = capture_items("span")

with patch.object(
agent_with_tool.model._client._client,
"send",
side_effect=[
get_model_response(tool_call_response, serialize_pydantic=True),
get_model_response(final_response, serialize_pydantic=True),
],
):
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
await agents.Runner.run(
agent_with_tool,
"Please use the simple test tool",
run_config=test_run_config,
)

assert not [
warning
for warning in caught
if issubclass(warning.category, PydanticDeprecatedSince20)
]

tool_call_span = next(
item.payload
for item in items
if item.payload["attributes"]["sentry.op"] == OP.GEN_AI_CHAT
)
assert (
"simple_test_tool" in tool_call_span["attributes"]["gen_ai.response.tool_calls"]
)
assert (
"Refused by the model" in tool_call_span["attributes"]["gen_ai.response.text"]
)


@pytest.mark.parametrize("user_hooks", [True, False])
@pytest.mark.parametrize("span_streaming", [True, False])
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
Expand Down